@fireemu/darwin-x64 0.43.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/fireemu +2 -2
- package/bin/runner-python/fireemu_runner/appcheck.py +255 -0
- package/bin/runner-python/fireemu_runner/bodies.py +96 -0
- package/bin/runner-python/fireemu_runner/diagnostics.py +36 -0
- package/bin/runner-python/fireemu_runner/dispatch.py +63 -26
- package/bin/runner-python/fireemu_runner/environment.py +119 -0
- package/bin/runner-python/fireemu_runner/events.py +118 -28
- package/bin/runner-python/fireemu_runner/framing.py +51 -1
- package/bin/runner-python/fireemu_runner/main.py +20 -10
- package/bin/runner-python/fireemu_runner/manifest.py +91 -8
- package/bin/runner-python/fireemu_runner/request_view.py +65 -0
- package/bin/runner-python/fireemu_runner/scheduler.py +87 -0
- package/bin/runner-python/fireemu_runner/serve.py +83 -11
- package/package.json +1 -1
|
@@ -10,11 +10,25 @@ dict (`firestore_fn.py:108-131`). The adapter therefore builds the production fo
|
|
|
10
10
|
import json
|
|
11
11
|
import os
|
|
12
12
|
import re
|
|
13
|
+
from typing import NamedTuple
|
|
13
14
|
|
|
14
15
|
from cloudevents.http import from_http
|
|
15
16
|
from google.events.cloud import firestore as firestore_events
|
|
16
17
|
|
|
18
|
+
from . import bodies
|
|
19
|
+
|
|
17
20
|
_NOT_AN_ATTRIBUTE = frozenset({"data", "datacontenttype"})
|
|
21
|
+
# The daemon's strict 2nd gen frame names a `traceparent` for the Node runner's event. Production sends the trace context as an
|
|
22
|
+
# ordinary HTTP header (Cloud Run's), never as a `ce-` attribute, so the strict Python event has none (sandbox run
|
|
23
|
+
# PYCHECK-EVENTS-20261104-01, the raw twins of the Pub/Sub, Storage, Eventarc and Firestore events). The emulator profile
|
|
24
|
+
# stamps no such member, so a `traceparent` there is the publisher's own tracing extension and stays an attribute.
|
|
25
|
+
_HTTP_HEADER_ONLY = frozenset({"traceparent"})
|
|
26
|
+
# Eventarc sends `ce-dataschema` with every Firestore event. Recorded for `created` only; for `updated`, `deleted`, `written`
|
|
27
|
+
# and any other type routed here it is UNRECORDED (the schema is one document for the family in google-cloudevents). The strict
|
|
28
|
+
# profile delivers it; the emulator profile keeps the official emulator's attributes, which have none.
|
|
29
|
+
_FIRESTORE_DATASCHEMA = (
|
|
30
|
+
"https://github.com/googleapis/google-cloudevents/blob/main/proto/google/events/cloud/firestore/v1/data.proto"
|
|
31
|
+
)
|
|
18
32
|
|
|
19
33
|
|
|
20
34
|
_WHOLE_SECOND_TIME = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(Z|[+-]\d{2}:\d{2})$")
|
|
@@ -25,10 +39,11 @@ def _attribute_headers(event: dict) -> dict:
|
|
|
25
39
|
if not isinstance(event.get("specversion"), str):
|
|
26
40
|
# Without it `from_http` would read the body as a structured JSON event and fail on the body.
|
|
27
41
|
raise ValueError("the event has no specversion")
|
|
42
|
+
header_only = _HTTP_HEADER_ONLY if is_strict() else frozenset()
|
|
28
43
|
return {
|
|
29
44
|
f"ce-{name}": value
|
|
30
45
|
for name, value in event.items()
|
|
31
|
-
if name not in _NOT_AN_ATTRIBUTE and isinstance(value, str)
|
|
46
|
+
if name not in _NOT_AN_ATTRIBUTE and name not in header_only and isinstance(value, str)
|
|
32
47
|
}
|
|
33
48
|
|
|
34
49
|
|
|
@@ -37,24 +52,52 @@ def _is_json(content_type: str) -> bool:
|
|
|
37
52
|
return media == "application/json" or media.endswith("+json")
|
|
38
53
|
|
|
39
54
|
|
|
40
|
-
|
|
41
|
-
"""
|
|
55
|
+
class Delivery(NamedTuple):
|
|
56
|
+
"""One delivery: the headers, the body the event is built from, and the body the request carries.
|
|
57
|
+
|
|
58
|
+
The two bodies are the same bytes for a Firestore event. For the JSON families the event is built from the encoding of
|
|
59
|
+
the data the adapters have always used (`event_body`), and the request shows the layout the recorded delivery had
|
|
60
|
+
(`request_body`); the function is given the same event whichever layout the request has.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
headers: dict
|
|
64
|
+
event_body: bytes
|
|
65
|
+
request_body: bytes
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _json_delivery(event: dict, request_text: str | None = None) -> Delivery:
|
|
69
|
+
"""The delivery of an event whose data is JSON (or text, with a text content type).
|
|
42
70
|
|
|
43
71
|
Eventarc delivers these in binary content mode: attributes as `ce-*` headers and the data as the body,
|
|
44
72
|
which functions-framework hands to `cloudevents.http.from_http` (`functions_framework/__init__.py:174-188`).
|
|
45
|
-
A delivery without a content type is JSON.
|
|
73
|
+
A delivery without a content type is JSON. `request_text` is the body text of the request when it differs from the
|
|
74
|
+
compact encoding of the data (see `bodies.py`).
|
|
46
75
|
"""
|
|
47
76
|
headers = _attribute_headers(event)
|
|
48
77
|
declared = event.get("datacontenttype")
|
|
49
78
|
content_type = declared if isinstance(declared, str) and declared else "application/json"
|
|
50
79
|
headers["content-type"] = content_type
|
|
51
80
|
if "data" not in event:
|
|
52
|
-
|
|
53
|
-
|
|
81
|
+
return Delivery(headers, b"", b"")
|
|
82
|
+
if isinstance(event["data"], str) and not _is_json(content_type):
|
|
54
83
|
body = event["data"].encode()
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
return
|
|
84
|
+
return Delivery(headers, body, body)
|
|
85
|
+
request = request_text if request_text is not None else bodies.compact(event["data"])
|
|
86
|
+
return Delivery(headers, json.dumps(event["data"]).encode(), request.encode())
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _delivered_value(event: dict, raw_data: str | None):
|
|
90
|
+
"""The data with its numbers as the daemon wrote them, or the event's own data when there is no text or it does not parse."""
|
|
91
|
+
if raw_data is not None:
|
|
92
|
+
try:
|
|
93
|
+
return bodies.parse_keeping_numbers(raw_data)
|
|
94
|
+
except ValueError:
|
|
95
|
+
pass
|
|
96
|
+
return event.get("data")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def is_strict() -> bool:
|
|
100
|
+
return os.environ.get("FIREEMU_HTTP_PROFILE") == "strict"
|
|
58
101
|
|
|
59
102
|
|
|
60
103
|
def _lenient_time(event: dict) -> dict:
|
|
@@ -65,49 +108,96 @@ def _lenient_time(event: dict) -> dict:
|
|
|
65
108
|
emulator profile must serve; the strict profile delivers the time as production's event has it.
|
|
66
109
|
"""
|
|
67
110
|
time = event.get("time")
|
|
68
|
-
if
|
|
111
|
+
if is_strict() or not isinstance(time, str):
|
|
69
112
|
return event
|
|
70
113
|
match = _WHOLE_SECOND_TIME.match(time)
|
|
71
114
|
return event if match is None else {**event, "time": f"{match.group(1)}.000{match.group(2)}"}
|
|
72
115
|
|
|
73
116
|
|
|
74
|
-
def
|
|
75
|
-
"""
|
|
76
|
-
|
|
77
|
-
The SDK reads `message_id` and `publish_time` (`pubsub_fn.py:98-160`), the snake-case spellings Pub/Sub's
|
|
78
|
-
push body carries next to `messageId` and `publishTime`. `orderingKey` has no alternative: the SDK pops
|
|
79
|
-
it and passes `ordering_key=` itself, so a copied key would be passed twice (`TypeError`).
|
|
80
|
-
"""
|
|
81
|
-
data = event.get("data")
|
|
117
|
+
def _with_snake_case_keys(data):
|
|
118
|
+
"""A Pub/Sub push body with `message_id` and `publish_time` next to `messageId` and `publishTime` (`pubsub_fn.py:98-160`)."""
|
|
82
119
|
message = data.get("message") if isinstance(data, dict) else None
|
|
83
120
|
if not isinstance(message, dict):
|
|
84
|
-
return
|
|
121
|
+
return data
|
|
85
122
|
message = dict(message)
|
|
86
123
|
for camel, snake in (("messageId", "message_id"), ("publishTime", "publish_time")):
|
|
87
124
|
if snake not in message and camel in message and message[camel] is not None:
|
|
88
125
|
message[snake] = message[camel]
|
|
89
|
-
return
|
|
126
|
+
return {**data, "message": message}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def pubsub_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
130
|
+
"""The delivery of a Pub/Sub message event.
|
|
131
|
+
|
|
132
|
+
The SDK reads `message_id` and `publish_time` (`pubsub_fn.py:98-160`), the snake-case spellings Pub/Sub's
|
|
133
|
+
push body carries next to `messageId` and `publishTime`. `orderingKey` has no alternative: the SDK pops
|
|
134
|
+
it and passes `ordering_key=` itself, so a copied key would be passed twice (`TypeError`).
|
|
135
|
+
The strict request carries the recorded layout: sorted compact JSON and a newline.
|
|
136
|
+
"""
|
|
137
|
+
data = event.get("data")
|
|
138
|
+
if not isinstance(data, dict) or not isinstance(data.get("message"), dict):
|
|
139
|
+
return _json_delivery(event)
|
|
140
|
+
event = {**event, "data": _with_snake_case_keys(data)}
|
|
141
|
+
value = _with_snake_case_keys(_delivered_value(event, raw_data))
|
|
142
|
+
text = bodies.pubsub_layout(value) if is_strict() else bodies.compact(value)
|
|
143
|
+
return _json_delivery(event, text)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def storage_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
147
|
+
"""The delivery of a Storage object event: the text the daemon sent, and in the strict profile the recorded layout."""
|
|
148
|
+
event = _lenient_time(event)
|
|
149
|
+
if is_strict() and "data" in event:
|
|
150
|
+
return _json_delivery(event, bodies.storage_layout(_delivered_value(event, raw_data)))
|
|
151
|
+
return _json_delivery(event, raw_data)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def eventarc_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
155
|
+
"""The delivery of a custom Eventarc event: the text of the data as the daemon framed it (the publisher's own in the strict profile)."""
|
|
156
|
+
return _json_delivery(_lenient_time(event), raw_data)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def database_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
160
|
+
"""The delivery of a Realtime Database event: the text the daemon sent (no raw delivery was recorded; UNRECORDED)."""
|
|
161
|
+
return _json_delivery(event, raw_data)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def firestore_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
165
|
+
"""The delivery of a Firestore event frame (`created`, `updated`, `deleted`, `written`): a protobuf body."""
|
|
166
|
+
headers = _attribute_headers(event)
|
|
167
|
+
if is_strict():
|
|
168
|
+
headers.setdefault("ce-dataschema", _FIRESTORE_DATASCHEMA)
|
|
169
|
+
data = event.get("data")
|
|
170
|
+
message = firestore_events.DocumentEventData.from_json(json.dumps(data if isinstance(data, dict) else {}))
|
|
171
|
+
headers["content-type"] = "application/protobuf"
|
|
172
|
+
body = firestore_events.DocumentEventData.serialize(message)
|
|
173
|
+
return Delivery(headers, body, body)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def pubsub_cloud_event(event: dict):
|
|
177
|
+
"""The `CloudEvent` for a Pub/Sub message event (see `pubsub_request`)."""
|
|
178
|
+
delivery = pubsub_request(event)
|
|
179
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
90
180
|
|
|
91
181
|
|
|
92
182
|
def storage_cloud_event(event: dict):
|
|
93
183
|
"""The `CloudEvent` for a Storage object event."""
|
|
94
|
-
|
|
184
|
+
delivery = storage_request(event)
|
|
185
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
95
186
|
|
|
96
187
|
|
|
97
188
|
def eventarc_cloud_event(event: dict):
|
|
98
189
|
"""The `CloudEvent` for a custom Eventarc event."""
|
|
99
|
-
|
|
190
|
+
delivery = eventarc_request(event)
|
|
191
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
100
192
|
|
|
101
193
|
|
|
102
194
|
def database_cloud_event(event: dict):
|
|
103
195
|
"""The `CloudEvent` for a Realtime Database event."""
|
|
104
|
-
|
|
196
|
+
delivery = database_request(event)
|
|
197
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
105
198
|
|
|
106
199
|
|
|
107
200
|
def firestore_cloud_event(event: dict):
|
|
108
201
|
"""The `CloudEvent` for a Firestore event frame (`created`, `updated`, `deleted`, `written`)."""
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
message = firestore_events.DocumentEventData.from_json(json.dumps(data if isinstance(data, dict) else {}))
|
|
112
|
-
headers["content-type"] = "application/protobuf"
|
|
113
|
-
return from_http(headers, firestore_events.DocumentEventData.serialize(message))
|
|
202
|
+
delivery = firestore_request(event)
|
|
203
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
@@ -27,13 +27,59 @@ def encode_frame(message: dict) -> bytes:
|
|
|
27
27
|
return str(len(payload)).encode("ascii") + b"\n" + payload
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
_DECODER = json.JSONDecoder()
|
|
31
|
+
_WHITESPACE = " \t\n\r"
|
|
32
|
+
# The trigger types whose request body is the event's JSON data as the daemon framed it.
|
|
33
|
+
_RAW_DATA_TRIGGERS = frozenset(("eventarc", "pubsub", "storage", "database"))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _after_whitespace(text: str, index: int) -> int:
|
|
37
|
+
while index < len(text) and text[index] in _WHITESPACE:
|
|
38
|
+
index += 1
|
|
39
|
+
return index
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def raw_event_data(text: str) -> str | None:
|
|
43
|
+
"""The text of `event.data` exactly as it stands in a frame, or None when the frame has none or is malformed.
|
|
44
|
+
|
|
45
|
+
The decoder turns the data into Python values and forgets its spelling (`1e-7`, `\\u00e9`, member order, white space).
|
|
46
|
+
For a custom Eventarc event in the strict profile the daemon frames the publisher's own JSON text, and that text is the
|
|
47
|
+
request body production delivered, so it is read here from the frame text, member by member, without parsing the data.
|
|
48
|
+
"""
|
|
49
|
+
try:
|
|
50
|
+
index = _after_whitespace(text, 0)
|
|
51
|
+
for wanted in ("event", "data"):
|
|
52
|
+
if text[index] != "{":
|
|
53
|
+
return None
|
|
54
|
+
index += 1
|
|
55
|
+
while True:
|
|
56
|
+
index = _after_whitespace(text, index)
|
|
57
|
+
key, index = _DECODER.raw_decode(text, index)
|
|
58
|
+
index = _after_whitespace(text, index)
|
|
59
|
+
if text[index] != ":" or not isinstance(key, str):
|
|
60
|
+
return None
|
|
61
|
+
index = _after_whitespace(text, index + 1)
|
|
62
|
+
if key == wanted:
|
|
63
|
+
break
|
|
64
|
+
index = _after_whitespace(text, _DECODER.raw_decode(text, index)[1])
|
|
65
|
+
if text[index] == ",":
|
|
66
|
+
index += 1
|
|
67
|
+
start = index
|
|
68
|
+
end = _DECODER.raw_decode(text, start)[1]
|
|
69
|
+
# A value must be followed by what ends it, or the frame is cut short.
|
|
70
|
+
return text[start:end] if text[_after_whitespace(text, end)] in ",}" else None
|
|
71
|
+
except (IndexError, ValueError):
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
30
75
|
def _text(value) -> bool:
|
|
31
76
|
return isinstance(value, str) and len(value) > 0
|
|
32
77
|
|
|
33
78
|
|
|
34
79
|
def _decode_message(payload: bytes) -> dict:
|
|
35
80
|
try:
|
|
36
|
-
|
|
81
|
+
text = payload.decode("utf-8")
|
|
82
|
+
message = json.loads(text)
|
|
37
83
|
except (UnicodeDecodeError, ValueError):
|
|
38
84
|
raise ProtocolError("UTF-8 or JSON") from None
|
|
39
85
|
if not isinstance(message, dict):
|
|
@@ -49,6 +95,10 @@ def _decode_message(payload: bytes) -> dict:
|
|
|
49
95
|
or ("entryPoint" in message and not _text(message["entryPoint"]))
|
|
50
96
|
):
|
|
51
97
|
raise ProtocolError("message fields")
|
|
98
|
+
if message["trigger"] in _RAW_DATA_TRIGGERS and "data" in message["event"]:
|
|
99
|
+
raw = raw_event_data(text)
|
|
100
|
+
if raw is not None:
|
|
101
|
+
message["rawEventData"] = raw
|
|
52
102
|
return message
|
|
53
103
|
|
|
54
104
|
|
|
@@ -6,10 +6,12 @@ import platform
|
|
|
6
6
|
import sys
|
|
7
7
|
import traceback
|
|
8
8
|
|
|
9
|
+
from . import appcheck
|
|
9
10
|
from .dispatch import Dispatcher
|
|
11
|
+
from .environment import EnvironmentGate, load_local_secrets
|
|
10
12
|
from .framing import FrameWriter, ProtocolError, read_frame
|
|
11
13
|
from .manifest import build_manifest
|
|
12
|
-
from .serve import start_server
|
|
14
|
+
from .serve import HTTP_TRIGGERS, start_server
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
def _parse(argv):
|
|
@@ -27,13 +29,15 @@ def load_codebase(source: str) -> dict:
|
|
|
27
29
|
return serving.get_functions()
|
|
28
30
|
|
|
29
31
|
|
|
30
|
-
def serve(
|
|
32
|
+
def serve(
|
|
33
|
+
functions: dict, codebase: str, secret: str, stdin, writer: FrameWriter, gate: EnvironmentGate | None = None, app_check_verified: bool = False
|
|
34
|
+
) -> int:
|
|
31
35
|
"""Says hello, then serves frames from `stdin` until `shutdown` or the end of the stream."""
|
|
32
|
-
manifest = build_manifest(functions)
|
|
36
|
+
manifest = build_manifest(functions, app_check_verified)
|
|
33
37
|
server = None
|
|
34
38
|
port = None
|
|
35
|
-
if any(spec["trigger"]["type"]
|
|
36
|
-
server, port = start_server(functions, manifest, secret)
|
|
39
|
+
if any(spec["trigger"]["type"] in HTTP_TRIGGERS for spec in manifest["functions"]):
|
|
40
|
+
server, port = start_server(functions, manifest, secret, gate)
|
|
37
41
|
writer.write(
|
|
38
42
|
{
|
|
39
43
|
"type": "hello",
|
|
@@ -44,7 +48,7 @@ def serve(functions: dict, codebase: str, secret: str, stdin, writer: FrameWrite
|
|
|
44
48
|
"manifest": manifest,
|
|
45
49
|
}
|
|
46
50
|
)
|
|
47
|
-
dispatcher = Dispatcher(functions, manifest, writer)
|
|
51
|
+
dispatcher = Dispatcher(functions, manifest, writer, gate)
|
|
48
52
|
try:
|
|
49
53
|
while True:
|
|
50
54
|
message = read_frame(stdin)
|
|
@@ -70,13 +74,19 @@ def run(argv) -> int:
|
|
|
70
74
|
os.dup2(2, 1)
|
|
71
75
|
writer = FrameWriter(protocol)
|
|
72
76
|
secret = os.environ.get("FIREEMU_RUNNER_SECRET", "")
|
|
73
|
-
# The daemon's local secrets are
|
|
74
|
-
# that carried them, nor the key to this runner's HTTP port.
|
|
75
|
-
os.environ.pop("FIREEMU_LOCAL_SECRETS_JSON", None)
|
|
77
|
+
# The daemon's local secrets are put back into the environment, a function at a time, by the gate; user code
|
|
78
|
+
# does not see the channel that carried them, nor the key to this runner's HTTP port.
|
|
76
79
|
os.environ.pop("FIREEMU_RUNNER_SECRET", None)
|
|
80
|
+
try:
|
|
81
|
+
gate = EnvironmentGate(load_local_secrets(os.environ))
|
|
82
|
+
except ValueError as error:
|
|
83
|
+
sys.stderr.write(f"[functions] {error}\n")
|
|
84
|
+
return 1
|
|
85
|
+
# Before any user code runs, so the Admin SDK's App Check service is built against the emulator's keys.
|
|
86
|
+
app_check_verified = appcheck.install_from_environment(os.environ)
|
|
77
87
|
try:
|
|
78
88
|
functions = load_codebase(args.source)
|
|
79
89
|
except BaseException: # a failing import in user code is reported with its traceback
|
|
80
90
|
traceback.print_exc()
|
|
81
91
|
return 1
|
|
82
|
-
return serve(functions, args.codebase, secret, sys.stdin.buffer, writer)
|
|
92
|
+
return serve(functions, args.codebase, secret, sys.stdin.buffer, writer, gate, app_check_verified)
|
|
@@ -10,6 +10,7 @@ under `ignored[]`, never dropped.
|
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
12
|
import enum
|
|
13
|
+
import math
|
|
13
14
|
from typing import Any
|
|
14
15
|
|
|
15
16
|
from firebase_functions.params import Expression
|
|
@@ -32,7 +33,10 @@ _UNSERVED_PRODUCTS = (
|
|
|
32
33
|
("analytics", "analytics", "notPlanned", "not planned: Google Analytics triggers have no local emulator"),
|
|
33
34
|
("testing", "testLab", "notPlanned", "not planned: Test Lab triggers have no local emulator"),
|
|
34
35
|
)
|
|
35
|
-
|
|
36
|
+
_SCHEDULE_RETRY_FIELDS = frozenset(("retryCount", "maxRetrySeconds", "minBackoffSeconds", "maxBackoffSeconds", "maxDoublings"))
|
|
37
|
+
_TASK_RETRY_FIELDS = frozenset(("maxAttempts", "maxRetrySeconds", "minBackoffSeconds", "maxBackoffSeconds", "maxDoublings"))
|
|
38
|
+
_TASK_RATE_FIELDS = frozenset(("maxConcurrentDispatches", "maxDispatchesPerSecond"))
|
|
39
|
+
_BLOCKING_AUTH_EVENTS = ("beforeCreate", "beforeSignIn", "beforeSendEmail", "beforeSendSms")
|
|
36
40
|
|
|
37
41
|
|
|
38
42
|
class Unserved(Exception):
|
|
@@ -224,7 +228,79 @@ def _event_trigger(event_trigger: dict) -> dict:
|
|
|
224
228
|
raise Unserved(family, "unsupported", f"event type {event_type} is not recognised")
|
|
225
229
|
|
|
226
230
|
|
|
227
|
-
def
|
|
231
|
+
def _record(value, field: str, family: str) -> dict:
|
|
232
|
+
"""An options record: absent is empty, an object is itself, anything else is an error."""
|
|
233
|
+
if _unset(value):
|
|
234
|
+
return {}
|
|
235
|
+
if not isinstance(value, dict):
|
|
236
|
+
raise Unserved(family, "unsupported", f"{field} must be an object")
|
|
237
|
+
return value
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _numbers(value, field: str, allowed: frozenset, family: str) -> dict:
|
|
241
|
+
"""The numbers of a retry or rate record. A member the SDK left unset is null on the wire."""
|
|
242
|
+
record = _record(value, field, family)
|
|
243
|
+
numbers: dict[str, Any] = {}
|
|
244
|
+
for key, original in record.items():
|
|
245
|
+
if key not in allowed:
|
|
246
|
+
raise Unserved(family, "unsupported", f"{field}.{key} is not supported")
|
|
247
|
+
resolved = _resolved(original, f"{field}.{key}")
|
|
248
|
+
if resolved is None:
|
|
249
|
+
numbers[key] = None
|
|
250
|
+
continue
|
|
251
|
+
if isinstance(resolved, bool) or not isinstance(resolved, (int, float)) or not math.isfinite(resolved):
|
|
252
|
+
raise Unserved(family, "unsupported", f"{field}.{key} did not resolve to a finite number")
|
|
253
|
+
numbers[key] = resolved
|
|
254
|
+
return numbers
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _schedule_trigger(trigger) -> tuple[dict, bool]:
|
|
258
|
+
"""The canonical trigger of a `scheduleTrigger` and whether the function retries."""
|
|
259
|
+
record = _record(trigger, "scheduleTrigger", "schedule")
|
|
260
|
+
schedule = _text(record, "schedule", "scheduleTrigger.schedule", "schedule", required=True, allow_empty=False)
|
|
261
|
+
time_zone = _text(record, "timeZone", "scheduleTrigger.timeZone", "schedule")
|
|
262
|
+
# The SDK keeps a deployment expression as braced CEL; this runner resolves values, it does not evaluate CEL.
|
|
263
|
+
if "{{" in schedule or (time_zone is not None and "{{" in time_zone):
|
|
264
|
+
raise Unserved("schedule", "unsupported", "scheduleTrigger contains an unresolved deployment expression")
|
|
265
|
+
retry_config = _numbers(record.get("retryConfig"), "scheduleTrigger.retryConfig", _SCHEDULE_RETRY_FIELDS, "schedule")
|
|
266
|
+
# A Gen2 job with a retry window and no count is retried until the window ends (run 156715222b86ea44), so a
|
|
267
|
+
# window alone declares a retry too.
|
|
268
|
+
retry = (retry_config.get("retryCount") or 0) > 0 or (retry_config.get("maxRetrySeconds") or 0) > 0
|
|
269
|
+
described: dict[str, Any] = {"type": "schedule", "schedule": schedule}
|
|
270
|
+
if time_zone is not None:
|
|
271
|
+
described["timeZone"] = time_zone
|
|
272
|
+
described["retryConfig"] = retry_config
|
|
273
|
+
return described, retry
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _task_queue_trigger(queue) -> dict:
|
|
277
|
+
record = _record(queue, "taskQueueTrigger", "tasks")
|
|
278
|
+
return {
|
|
279
|
+
"type": "tasks",
|
|
280
|
+
"retryConfig": _numbers(record.get("retryConfig"), "taskQueueTrigger.retryConfig", _TASK_RETRY_FIELDS, "tasks"),
|
|
281
|
+
"rateLimits": _numbers(record.get("rateLimits"), "taskQueueTrigger.rateLimits", _TASK_RATE_FIELDS, "tasks"),
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _blocking_trigger(blocking) -> dict:
|
|
286
|
+
record = _record(blocking, "blockingTrigger", "blocking")
|
|
287
|
+
event_type = str(_resolved(record.get("eventType"), "blockingTrigger.eventType") or "")
|
|
288
|
+
if not any(event_type.endswith(event) for event in _BLOCKING_AUTH_EVENTS):
|
|
289
|
+
for fragment, trigger_type, scope, reason in _UNSERVED_PRODUCTS:
|
|
290
|
+
if fragment in event_type:
|
|
291
|
+
raise Unserved(trigger_type, scope, reason)
|
|
292
|
+
raise Unserved("blocking", "unsupported", f"blocking identity event {event_type} is not served")
|
|
293
|
+
options = record.get("options") if isinstance(record.get("options"), dict) else {}
|
|
294
|
+
return {
|
|
295
|
+
"type": "blockingAuth",
|
|
296
|
+
"eventType": event_type,
|
|
297
|
+
"accessToken": options.get("accessToken") is True,
|
|
298
|
+
"idToken": options.get("idToken") is True,
|
|
299
|
+
"refreshToken": options.get("refreshToken") is True,
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def describe_endpoint(name: str, endpoint, app_check_verified: bool = False) -> dict:
|
|
228
304
|
"""The canonical `functions[]` entry of one endpoint, or `Unserved` when it cannot be served."""
|
|
229
305
|
if endpoint.platform != "gcfv2":
|
|
230
306
|
raise Unserved("unknown", "unsupported", "endpoint platform is not recognised")
|
|
@@ -242,15 +318,19 @@ def describe_endpoint(name: str, endpoint) -> dict:
|
|
|
242
318
|
if options:
|
|
243
319
|
described["platformOptions"] = options
|
|
244
320
|
if endpoint.taskQueueTrigger is not None:
|
|
245
|
-
|
|
321
|
+
described["trigger"] = _task_queue_trigger(endpoint.taskQueueTrigger)
|
|
322
|
+
return described
|
|
246
323
|
if endpoint.httpsTrigger is not None:
|
|
247
324
|
described["trigger"] = {"type": "http", "callable": False}
|
|
248
325
|
elif endpoint.callableTrigger is not None:
|
|
249
326
|
described["trigger"] = {"type": "http", "callable": True}
|
|
327
|
+
if app_check_verified:
|
|
328
|
+
# The SDK verifies the token itself, against the App Check emulator's keys, and never consumes it.
|
|
329
|
+
described["trigger"]["consumeAppCheckToken"] = "disabled"
|
|
250
330
|
elif endpoint.scheduleTrigger is not None:
|
|
251
|
-
|
|
331
|
+
described["trigger"], described["retry"] = _schedule_trigger(endpoint.scheduleTrigger)
|
|
252
332
|
elif endpoint.blockingTrigger is not None:
|
|
253
|
-
|
|
333
|
+
described["trigger"] = _blocking_trigger(endpoint.blockingTrigger)
|
|
254
334
|
elif endpoint.eventTrigger is not None:
|
|
255
335
|
described["trigger"] = _event_trigger(endpoint.eventTrigger)
|
|
256
336
|
retry = _resolved(endpoint.eventTrigger.get("retry"), "eventTrigger.retry")
|
|
@@ -262,14 +342,17 @@ def describe_endpoint(name: str, endpoint) -> dict:
|
|
|
262
342
|
return described
|
|
263
343
|
|
|
264
344
|
|
|
265
|
-
def build_manifest(functions: dict) -> dict:
|
|
266
|
-
"""`{"functions": [...], "ignored": [...]}` for the decorated functions of a codebase.
|
|
345
|
+
def build_manifest(functions: dict, app_check_verified: bool = False) -> dict:
|
|
346
|
+
"""`{"functions": [...], "ignored": [...]}` for the decorated functions of a codebase.
|
|
347
|
+
|
|
348
|
+
`app_check_verified` says the Admin SDK verifies App Check tokens against the emulator's keys (`appcheck.py`).
|
|
349
|
+
"""
|
|
267
350
|
served: list[dict] = []
|
|
268
351
|
ignored: list[dict] = []
|
|
269
352
|
for name, function in functions.items():
|
|
270
353
|
endpoint = function.__firebase_endpoint__
|
|
271
354
|
try:
|
|
272
|
-
served.append(describe_endpoint(name, endpoint))
|
|
355
|
+
served.append(describe_endpoint(name, endpoint, app_check_verified))
|
|
273
356
|
except Unserved as unserved:
|
|
274
357
|
try:
|
|
275
358
|
region = _first_region(endpoint) or DEFAULT_REGION
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""The HTTP request a CloudEvent function runs inside.
|
|
2
|
+
|
|
3
|
+
functions-framework serves a CloudEvent function from a Flask view and calls the function inside the request context, so
|
|
4
|
+
`flask.request` shows the delivered request. Production's raw twins were recorded on 2026-11-04 (sandbox run
|
|
5
|
+
PYCHECK-EVENTS-20261104-01): `POST /` with the binary-mode `ce-*` headers, `content-type`, `content-length` and, in the strict
|
|
6
|
+
profile, constants every recorded frame carries (the sender's user agent, the encodings, the protocol) plus Cloud Run's plain
|
|
7
|
+
`traceparent` header. The body is the text the delivery had, not a re-encoding of the event: the layouts recorded per family are in
|
|
8
|
+
`bodies.py` (strict Pub/Sub and Storage; Eventarc keeps the publisher's text), and the emulator and unset profiles carry the framed JSON value text
|
|
9
|
+
of the data as the daemon sent it (a publisher's surrounding white space is trimmed upstream; a Pub/Sub body is the exception, it gets the SDK's snake-case aliases and is rewritten compactly). A Realtime Database body was not recorded. Header policy: a name that is not an HTTP token (or has an underscore) is
|
|
10
|
+
left out of the request, a control character other than TAB in a value becomes a space, and of two names that differ only in case the first one
|
|
11
|
+
is the header; the event the function receives is never changed by the view. The OIDC `Authorization`, `Forwarded`, `X-Forwarded-For`, `Function-Execution-Id`, `X-Cloud-Trace-Context`,
|
|
12
|
+
`From`, `Accept` and the Cloud Run host have no local source and are not invented.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import contextlib
|
|
16
|
+
import re
|
|
17
|
+
|
|
18
|
+
import flask
|
|
19
|
+
from werkzeug.test import EnvironBuilder
|
|
20
|
+
|
|
21
|
+
from .events import is_strict
|
|
22
|
+
|
|
23
|
+
_APP = flask.Flask("fireemu-cloudevent")
|
|
24
|
+
# What an HTTP header may be called: a token without an underscore. Header names with an underscore are dropped by the usual
|
|
25
|
+
# servers, and Werkzeug folds `foo_bar` into `foo-bar`; both are UNRECORDED, so such an attribute stays out of the request
|
|
26
|
+
# (the event keeps it).
|
|
27
|
+
_HEADER_NAME = re.compile(r"[!#$%&'*+.^`|~0-9A-Za-z-]+")
|
|
28
|
+
# A control character other than the tab cannot be in a header value; it becomes a space, as a line break does.
|
|
29
|
+
_CONTROL = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]")
|
|
30
|
+
# Constant in every recorded Eventarc delivery (the four raw twins of run 57ab27296dedaedc).
|
|
31
|
+
_STRICT_CONSTANTS = {
|
|
32
|
+
"user-agent": "APIs-Google; (+https://developers.google.com/webmasters/APIs-Google.html)",
|
|
33
|
+
"accept-encoding": "gzip, deflate, br",
|
|
34
|
+
"x-forwarded-proto": "https",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def request_headers(headers: dict, event: dict) -> dict:
|
|
39
|
+
"""The headers of the request: the event's binary-mode headers, and in the strict profile the recorded constants and the
|
|
40
|
+
trace context as the plain `traceparent` header.
|
|
41
|
+
|
|
42
|
+
The request is what a server would have received, so a name that is not a header name is left out, and when two names
|
|
43
|
+
differ only in case the first one stays. A control character in a value other than the tab, a line break included, is
|
|
44
|
+
replaced by a space. The event the function receives keeps every attribute and its text.
|
|
45
|
+
"""
|
|
46
|
+
view = {}
|
|
47
|
+
taken = set()
|
|
48
|
+
for name, value in headers.items():
|
|
49
|
+
if _HEADER_NAME.fullmatch(name) and name.lower() not in taken:
|
|
50
|
+
taken.add(name.lower())
|
|
51
|
+
view[name] = _CONTROL.sub(" ", value)
|
|
52
|
+
if is_strict():
|
|
53
|
+
view.update(_STRICT_CONSTANTS)
|
|
54
|
+
traceparent = event.get("traceparent")
|
|
55
|
+
if isinstance(traceparent, str):
|
|
56
|
+
view["traceparent"] = _CONTROL.sub(" ", traceparent)
|
|
57
|
+
return view
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@contextlib.contextmanager
|
|
61
|
+
def delivery_request(headers: dict, body: bytes, event: dict):
|
|
62
|
+
"""Runs the block inside the Flask request context of the delivery described by `headers` and `body`."""
|
|
63
|
+
environ = EnvironBuilder(method="POST", path="/", headers=request_headers(headers, event), data=body).get_environ()
|
|
64
|
+
with _APP.request_context(environ):
|
|
65
|
+
yield
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Scheduled runs: the request Cloud Scheduler sends a Gen2 function, built in process.
|
|
2
|
+
|
|
3
|
+
A Gen2 `on_schedule` function is an HTTP function. Production's Scheduler calls it with `POST /`, the job id in
|
|
4
|
+
`X-CloudScheduler-JobName` and the schedule time in `X-CloudScheduler-ScheduleTime`, and judges the attempt by the
|
|
5
|
+
status the SDK wrapper answers: 200, or 500 when the handler raised (the wrapper logs the error and answers its
|
|
6
|
+
message, `scheduler_fn.py:99-137`). The Node runner does the same in the strict profile
|
|
7
|
+
(`tools/runner-node/schedule-delivery.mjs`, recorded in run 156715222b86ea44).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import datetime
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
from zoneinfo import ZoneInfo
|
|
14
|
+
|
|
15
|
+
import flask
|
|
16
|
+
from werkzeug.test import EnvironBuilder
|
|
17
|
+
|
|
18
|
+
_INSTANT = re.compile(r"^(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)(?:\.(\d{1,9}))?Z$")
|
|
19
|
+
_LOS_ANGELES = ZoneInfo("America/Los_Angeles")
|
|
20
|
+
_JOB_NAME = re.compile(r"projects/([^/]*)/locations/([^/]*)/jobs/")
|
|
21
|
+
_NAME = re.compile(r"[A-Za-z0-9-]+")
|
|
22
|
+
_MAX_ANSWER_CHARS = 256
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SchedulerFailure(Exception):
|
|
26
|
+
"""A scheduled attempt that failed: the run was malformed or the function answered 400 or more."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _schedule_time(instant) -> str:
|
|
30
|
+
"""The schedule time as the header carries it: Los Angeles with its offset in the strict profile (whatever the
|
|
31
|
+
job's own zone, as recorded), the daemon's UTC instant otherwise."""
|
|
32
|
+
match = _INSTANT.match(instant) if isinstance(instant, str) else None
|
|
33
|
+
if match is None:
|
|
34
|
+
raise SchedulerFailure(f"the scheduled time is not an RFC 3339 UTC instant: {instant!r}")
|
|
35
|
+
year, month, day, hour, minute, second = (int(part) for part in match.groups()[:6])
|
|
36
|
+
try:
|
|
37
|
+
moment = datetime.datetime(year, month, day, hour, minute, second, tzinfo=datetime.timezone.utc)
|
|
38
|
+
except ValueError:
|
|
39
|
+
raise SchedulerFailure(f"the scheduled time is not an RFC 3339 UTC instant: {instant!r}") from None
|
|
40
|
+
if os.environ.get("FIREEMU_HTTP_PROFILE", "emulator") != "strict":
|
|
41
|
+
return instant
|
|
42
|
+
local = moment.astimezone(_LOS_ANGELES)
|
|
43
|
+
fraction = (match.group(7) or "").rstrip("0")
|
|
44
|
+
offset = local.strftime("%z")
|
|
45
|
+
return local.strftime("%Y-%m-%dT%H:%M:%S") + (f".{fraction}" if fraction else "") + f"{offset[:3]}:{offset[3:]}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _function_host(job: str) -> str | None:
|
|
49
|
+
"""The public host of a function, `<region>-<project>.cloudfunctions.net`, from a job's resource name.
|
|
50
|
+
|
|
51
|
+
Recorded for us-central1 only (all 76 distinct Gen2 frames of run 156715222b86ea44); another region follows the
|
|
52
|
+
same rule, unrecorded. A name without a project and a location in this form invents no host (the Node runner's
|
|
53
|
+
`functionHost`).
|
|
54
|
+
"""
|
|
55
|
+
match = _JOB_NAME.match(job)
|
|
56
|
+
if match is None or not _NAME.fullmatch(match.group(1)) or not _NAME.fullmatch(match.group(2)):
|
|
57
|
+
return None
|
|
58
|
+
return f"{match.group(2)}-{match.group(1)}.cloudfunctions.net"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def scheduler_headers(data) -> dict:
|
|
62
|
+
"""The headers of the request for one scheduled run (`{jobName, scheduleTime}`)."""
|
|
63
|
+
job = data.get("jobName") if isinstance(data, dict) else None
|
|
64
|
+
if not isinstance(job, str) or not job:
|
|
65
|
+
raise SchedulerFailure("the scheduled run names no job name")
|
|
66
|
+
host = _function_host(job)
|
|
67
|
+
return {
|
|
68
|
+
"x-cloudscheduler": "true",
|
|
69
|
+
"x-cloudscheduler-jobname": job.rsplit("/", 1)[-1],
|
|
70
|
+
"x-cloudscheduler-scheduletime": _schedule_time(data.get("scheduleTime")),
|
|
71
|
+
"user-agent": "Google-Cloud-Scheduler",
|
|
72
|
+
"content-length": "0",
|
|
73
|
+
# Constant in every recorded frame: the front end's encodings and protocol.
|
|
74
|
+
"accept-encoding": "gzip, deflate, br",
|
|
75
|
+
"x-forwarded-proto": "https",
|
|
76
|
+
**({"host": host} if host else {}),
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def deliver_schedule(function, data) -> None:
|
|
81
|
+
"""Calls the SDK wrapper with the scheduler's request. Returns when it answers below 400; raises otherwise."""
|
|
82
|
+
environ = EnvironBuilder(method="POST", path="/", headers=scheduler_headers(data)).get_environ()
|
|
83
|
+
with flask.Flask("fireemu-scheduler").request_context(environ):
|
|
84
|
+
response = function(flask.request)
|
|
85
|
+
if response.status_code >= 400:
|
|
86
|
+
body = response.get_data(as_text=True)[:_MAX_ANSWER_CHARS]
|
|
87
|
+
raise SchedulerFailure(f"the scheduled function answered {response.status_code}" + (f": {body}" if body else ""))
|