@fireemu/darwin-x64 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/fireemu +2 -2
- package/bin/runner-node/index.mjs +25 -3
- package/bin/runner-python/fireemu_runner/appcheck.py +255 -0
- package/bin/runner-python/fireemu_runner/bodies.py +96 -0
- package/bin/runner-python/fireemu_runner/diagnostics.py +36 -0
- package/bin/runner-python/fireemu_runner/dispatch.py +63 -26
- package/bin/runner-python/fireemu_runner/environment.py +119 -0
- package/bin/runner-python/fireemu_runner/events.py +126 -28
- package/bin/runner-python/fireemu_runner/framing.py +51 -1
- package/bin/runner-python/fireemu_runner/main.py +20 -10
- package/bin/runner-python/fireemu_runner/manifest.py +91 -8
- package/bin/runner-python/fireemu_runner/request_view.py +65 -0
- package/bin/runner-python/fireemu_runner/scheduler.py +87 -0
- package/bin/runner-python/fireemu_runner/serve.py +83 -11
- package/package.json +1 -1
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Local secrets: which invocation sees which secret in the one process environment.
|
|
2
|
+
|
|
3
|
+
Production gives every function its own instance, so a secret a function declares is in that instance's environment
|
|
4
|
+
only. One runner serves a whole codebase here and has one `os.environ`, so the runner takes the daemon's local
|
|
5
|
+
secrets (`FIREEMU_LOCAL_SECRETS_JSON`, from `.secret.local`) out of the environment at start and puts them back for
|
|
6
|
+
the duration of an invocation, as the Node runner's `withFunctionEnvironment` does:
|
|
7
|
+
|
|
8
|
+
- a plain invocation (one that declares no secret with a local value) runs with every local secret hidden, beside
|
|
9
|
+
other plain invocations;
|
|
10
|
+
- an invocation that declares a secret with a local value runs alone, with only the secrets it declares;
|
|
11
|
+
- invocations start in arrival order, so a plain one that arrives behind a queued secret invocation waits.
|
|
12
|
+
|
|
13
|
+
The model is `EnvGate.tla` (checked with TLC): an exclusive invocation never overlaps any other, a plain one never
|
|
14
|
+
sees a secret, and nothing is exposed while nothing runs.
|
|
15
|
+
|
|
16
|
+
What this does not isolate, as in the Node runner (one process serves the whole codebase):
|
|
17
|
+
- work a function leaves running after its invocation ends (a thread it started, a child process it launched) runs in
|
|
18
|
+
the shared process and sees whatever the environment holds at that moment;
|
|
19
|
+
- a function that holds a secret and waits, synchronously, for another function of the same codebase (a request to its
|
|
20
|
+
own URL) deadlocks: the other invocation queues behind the gate the caller holds.
|
|
21
|
+
Both are stated on the Functions page.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import collections
|
|
25
|
+
import contextlib
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import threading
|
|
29
|
+
|
|
30
|
+
SECRETS_VARIABLE = "FIREEMU_LOCAL_SECRETS_JSON"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def load_local_secrets(environ) -> dict:
|
|
34
|
+
"""Takes the daemon's local secrets out of `environ`: name to value, string values only."""
|
|
35
|
+
text = environ.pop(SECRETS_VARIABLE, None)
|
|
36
|
+
if not text:
|
|
37
|
+
return {}
|
|
38
|
+
try:
|
|
39
|
+
parsed = json.loads(text)
|
|
40
|
+
except ValueError:
|
|
41
|
+
raise ValueError("the local secrets are not valid JSON") from None
|
|
42
|
+
if not isinstance(parsed, dict):
|
|
43
|
+
raise ValueError("the local secrets are not a JSON object")
|
|
44
|
+
return {name: value for name, value in parsed.items() if isinstance(value, str)}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class EnvironmentGate:
|
|
48
|
+
"""One queue of invocations and the environment each may see."""
|
|
49
|
+
|
|
50
|
+
def __init__(self, secrets: dict):
|
|
51
|
+
self._secrets = dict(secrets)
|
|
52
|
+
self._condition = threading.Condition()
|
|
53
|
+
self._queue = collections.deque() # tickets waiting, oldest first
|
|
54
|
+
self._running = 0
|
|
55
|
+
self._exclusive_running = False
|
|
56
|
+
self._saved = {}
|
|
57
|
+
|
|
58
|
+
def _hide(self) -> None:
|
|
59
|
+
for name in self._secrets:
|
|
60
|
+
self._saved[name] = os.environ.pop(name, None)
|
|
61
|
+
|
|
62
|
+
def _restore(self) -> None:
|
|
63
|
+
for name, value in self._saved.items():
|
|
64
|
+
if value is None:
|
|
65
|
+
os.environ.pop(name, None)
|
|
66
|
+
else:
|
|
67
|
+
os.environ[name] = value
|
|
68
|
+
self._saved = {}
|
|
69
|
+
|
|
70
|
+
def _may_start(self, ticket) -> bool:
|
|
71
|
+
if self._queue[0] is not ticket:
|
|
72
|
+
return False
|
|
73
|
+
return self._running == 0 if ticket.exclusive else not self._exclusive_running
|
|
74
|
+
|
|
75
|
+
def _enter(self, ticket) -> None:
|
|
76
|
+
with self._condition:
|
|
77
|
+
self._queue.append(ticket)
|
|
78
|
+
self._condition.wait_for(lambda: self._may_start(ticket))
|
|
79
|
+
self._queue.popleft()
|
|
80
|
+
if self._running == 0:
|
|
81
|
+
self._hide()
|
|
82
|
+
self._running += 1
|
|
83
|
+
if ticket.exclusive:
|
|
84
|
+
self._exclusive_running = True
|
|
85
|
+
for name in ticket.names:
|
|
86
|
+
os.environ[name] = self._secrets[name]
|
|
87
|
+
# The next in line may be able to start beside this one.
|
|
88
|
+
self._condition.notify_all()
|
|
89
|
+
|
|
90
|
+
def _leave(self, ticket) -> None:
|
|
91
|
+
with self._condition:
|
|
92
|
+
if ticket.exclusive:
|
|
93
|
+
self._exclusive_running = False
|
|
94
|
+
for name in ticket.names:
|
|
95
|
+
os.environ.pop(name, None)
|
|
96
|
+
self._running -= 1
|
|
97
|
+
if self._running == 0:
|
|
98
|
+
self._restore()
|
|
99
|
+
self._condition.notify_all()
|
|
100
|
+
|
|
101
|
+
@contextlib.contextmanager
|
|
102
|
+
def hold(self, declared):
|
|
103
|
+
"""Runs the body with the environment `declared` (the function's secret names) is entitled to."""
|
|
104
|
+
if not self._secrets:
|
|
105
|
+
yield
|
|
106
|
+
return
|
|
107
|
+
names = [name for name in dict.fromkeys(declared or ()) if name in self._secrets]
|
|
108
|
+
ticket = _Ticket(names)
|
|
109
|
+
self._enter(ticket)
|
|
110
|
+
try:
|
|
111
|
+
yield
|
|
112
|
+
finally:
|
|
113
|
+
self._leave(ticket)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _Ticket:
|
|
117
|
+
def __init__(self, names):
|
|
118
|
+
self.names = names
|
|
119
|
+
self.exclusive = bool(names)
|
|
@@ -7,14 +7,29 @@ dict (`firestore_fn.py:108-131`). The adapter therefore builds the production fo
|
|
|
7
7
|
(`functions_framework/__init__.py:174-188`).
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
+
import base64
|
|
10
11
|
import json
|
|
11
12
|
import os
|
|
12
13
|
import re
|
|
14
|
+
from typing import NamedTuple
|
|
13
15
|
|
|
14
16
|
from cloudevents.http import from_http
|
|
15
17
|
from google.events.cloud import firestore as firestore_events
|
|
16
18
|
|
|
19
|
+
from . import bodies
|
|
20
|
+
|
|
17
21
|
_NOT_AN_ATTRIBUTE = frozenset({"data", "datacontenttype"})
|
|
22
|
+
# The daemon's strict 2nd gen frame names a `traceparent` for the Node runner's event. Production sends the trace context as an
|
|
23
|
+
# ordinary HTTP header (Cloud Run's), never as a `ce-` attribute, so the strict Python event has none (sandbox run
|
|
24
|
+
# PYCHECK-EVENTS-20261104-01, the raw twins of the Pub/Sub, Storage, Eventarc and Firestore events). The emulator profile
|
|
25
|
+
# stamps no such member, so a `traceparent` there is the publisher's own tracing extension and stays an attribute.
|
|
26
|
+
_HTTP_HEADER_ONLY = frozenset({"traceparent"})
|
|
27
|
+
# Eventarc sends `ce-dataschema` with every Firestore event. Recorded for `created` only; for `updated`, `deleted`, `written`
|
|
28
|
+
# and any other type routed here it is UNRECORDED (the schema is one document for the family in google-cloudevents). The strict
|
|
29
|
+
# profile delivers it; the emulator profile keeps the official emulator's attributes, which have none.
|
|
30
|
+
_FIRESTORE_DATASCHEMA = (
|
|
31
|
+
"https://github.com/googleapis/google-cloudevents/blob/main/proto/google/events/cloud/firestore/v1/data.proto"
|
|
32
|
+
)
|
|
18
33
|
|
|
19
34
|
|
|
20
35
|
_WHOLE_SECOND_TIME = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(Z|[+-]\d{2}:\d{2})$")
|
|
@@ -25,10 +40,11 @@ def _attribute_headers(event: dict) -> dict:
|
|
|
25
40
|
if not isinstance(event.get("specversion"), str):
|
|
26
41
|
# Without it `from_http` would read the body as a structured JSON event and fail on the body.
|
|
27
42
|
raise ValueError("the event has no specversion")
|
|
43
|
+
header_only = _HTTP_HEADER_ONLY if is_strict() else frozenset()
|
|
28
44
|
return {
|
|
29
45
|
f"ce-{name}": value
|
|
30
46
|
for name, value in event.items()
|
|
31
|
-
if name not in _NOT_AN_ATTRIBUTE and isinstance(value, str)
|
|
47
|
+
if name not in _NOT_AN_ATTRIBUTE and name not in header_only and isinstance(value, str)
|
|
32
48
|
}
|
|
33
49
|
|
|
34
50
|
|
|
@@ -37,24 +53,52 @@ def _is_json(content_type: str) -> bool:
|
|
|
37
53
|
return media == "application/json" or media.endswith("+json")
|
|
38
54
|
|
|
39
55
|
|
|
40
|
-
|
|
41
|
-
"""
|
|
56
|
+
class Delivery(NamedTuple):
|
|
57
|
+
"""One delivery: the headers, the body the event is built from, and the body the request carries.
|
|
58
|
+
|
|
59
|
+
The two bodies are the same bytes for a Firestore event. For the JSON families the event is built from the encoding of
|
|
60
|
+
the data the adapters have always used (`event_body`), and the request shows the layout the recorded delivery had
|
|
61
|
+
(`request_body`); the function is given the same event whichever layout the request has.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
headers: dict
|
|
65
|
+
event_body: bytes
|
|
66
|
+
request_body: bytes
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _json_delivery(event: dict, request_text: str | None = None) -> Delivery:
|
|
70
|
+
"""The delivery of an event whose data is JSON (or text, with a text content type).
|
|
42
71
|
|
|
43
72
|
Eventarc delivers these in binary content mode: attributes as `ce-*` headers and the data as the body,
|
|
44
73
|
which functions-framework hands to `cloudevents.http.from_http` (`functions_framework/__init__.py:174-188`).
|
|
45
|
-
A delivery without a content type is JSON.
|
|
74
|
+
A delivery without a content type is JSON. `request_text` is the body text of the request when it differs from the
|
|
75
|
+
compact encoding of the data (see `bodies.py`).
|
|
46
76
|
"""
|
|
47
77
|
headers = _attribute_headers(event)
|
|
48
78
|
declared = event.get("datacontenttype")
|
|
49
79
|
content_type = declared if isinstance(declared, str) and declared else "application/json"
|
|
50
80
|
headers["content-type"] = content_type
|
|
51
81
|
if "data" not in event:
|
|
52
|
-
|
|
53
|
-
|
|
82
|
+
return Delivery(headers, b"", b"")
|
|
83
|
+
if isinstance(event["data"], str) and not _is_json(content_type):
|
|
54
84
|
body = event["data"].encode()
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
return
|
|
85
|
+
return Delivery(headers, body, body)
|
|
86
|
+
request = request_text if request_text is not None else bodies.compact(event["data"])
|
|
87
|
+
return Delivery(headers, json.dumps(event["data"]).encode(), request.encode())
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _delivered_value(event: dict, raw_data: str | None):
|
|
91
|
+
"""The data with its numbers as the daemon wrote them, or the event's own data when there is no text or it does not parse."""
|
|
92
|
+
if raw_data is not None:
|
|
93
|
+
try:
|
|
94
|
+
return bodies.parse_keeping_numbers(raw_data)
|
|
95
|
+
except ValueError:
|
|
96
|
+
pass
|
|
97
|
+
return event.get("data")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def is_strict() -> bool:
|
|
101
|
+
return os.environ.get("FIREEMU_HTTP_PROFILE") == "strict"
|
|
58
102
|
|
|
59
103
|
|
|
60
104
|
def _lenient_time(event: dict) -> dict:
|
|
@@ -65,49 +109,103 @@ def _lenient_time(event: dict) -> dict:
|
|
|
65
109
|
emulator profile must serve; the strict profile delivers the time as production's event has it.
|
|
66
110
|
"""
|
|
67
111
|
time = event.get("time")
|
|
68
|
-
if
|
|
112
|
+
if is_strict() or not isinstance(time, str):
|
|
69
113
|
return event
|
|
70
114
|
match = _WHOLE_SECOND_TIME.match(time)
|
|
71
115
|
return event if match is None else {**event, "time": f"{match.group(1)}.000{match.group(2)}"}
|
|
72
116
|
|
|
73
117
|
|
|
74
|
-
def
|
|
75
|
-
"""
|
|
76
|
-
|
|
77
|
-
The SDK reads `message_id` and `publish_time` (`pubsub_fn.py:98-160`), the snake-case spellings Pub/Sub's
|
|
78
|
-
push body carries next to `messageId` and `publishTime`. `orderingKey` has no alternative: the SDK pops
|
|
79
|
-
it and passes `ordering_key=` itself, so a copied key would be passed twice (`TypeError`).
|
|
80
|
-
"""
|
|
81
|
-
data = event.get("data")
|
|
118
|
+
def _with_snake_case_keys(data):
|
|
119
|
+
"""A Pub/Sub push body with `message_id` and `publish_time` next to `messageId` and `publishTime` (`pubsub_fn.py:98-160`)."""
|
|
82
120
|
message = data.get("message") if isinstance(data, dict) else None
|
|
83
121
|
if not isinstance(message, dict):
|
|
84
|
-
return
|
|
122
|
+
return data
|
|
85
123
|
message = dict(message)
|
|
86
124
|
for camel, snake in (("messageId", "message_id"), ("publishTime", "publish_time")):
|
|
87
125
|
if snake not in message and camel in message and message[camel] is not None:
|
|
88
126
|
message[snake] = message[camel]
|
|
89
|
-
return
|
|
127
|
+
return {**data, "message": message}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def pubsub_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
131
|
+
"""The delivery of a Pub/Sub message event.
|
|
132
|
+
|
|
133
|
+
The SDK reads `message_id` and `publish_time` (`pubsub_fn.py:98-160`), the snake-case spellings Pub/Sub's
|
|
134
|
+
push body carries next to `messageId` and `publishTime`. `orderingKey` has no alternative: the SDK pops
|
|
135
|
+
it and passes `ordering_key=` itself, so a copied key would be passed twice (`TypeError`).
|
|
136
|
+
The strict request carries the recorded layout: sorted compact JSON and a newline.
|
|
137
|
+
"""
|
|
138
|
+
data = event.get("data")
|
|
139
|
+
if not isinstance(data, dict) or not isinstance(data.get("message"), dict):
|
|
140
|
+
return _json_delivery(event)
|
|
141
|
+
event = {**event, "data": _with_snake_case_keys(data)}
|
|
142
|
+
value = _with_snake_case_keys(_delivered_value(event, raw_data))
|
|
143
|
+
text = bodies.pubsub_layout(value) if is_strict() else bodies.compact(value)
|
|
144
|
+
return _json_delivery(event, text)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def storage_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
148
|
+
"""The delivery of a Storage object event: the text the daemon sent, and in the strict profile the recorded layout."""
|
|
149
|
+
event = _lenient_time(event)
|
|
150
|
+
if is_strict() and "data" in event:
|
|
151
|
+
return _json_delivery(event, bodies.storage_layout(_delivered_value(event, raw_data)))
|
|
152
|
+
return _json_delivery(event, raw_data)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def eventarc_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
156
|
+
"""The delivery of a custom Eventarc event: the text of the data as the daemon framed it (the publisher's own in the strict profile)."""
|
|
157
|
+
return _json_delivery(_lenient_time(event), raw_data)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def database_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
161
|
+
"""The delivery of a Realtime Database event: the text the daemon sent (no raw delivery was recorded; UNRECORDED)."""
|
|
162
|
+
return _json_delivery(event, raw_data)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def firestore_request(event: dict, raw_data: str | None = None) -> Delivery:
|
|
166
|
+
"""The delivery of a Firestore event frame (`created`, `updated`, `deleted`, `written`): a protobuf body."""
|
|
167
|
+
headers = _attribute_headers(event)
|
|
168
|
+
if is_strict():
|
|
169
|
+
headers.setdefault("ce-dataschema", _FIRESTORE_DATASCHEMA)
|
|
170
|
+
data = event.get("data")
|
|
171
|
+
if event.get("datacontenttype") == "application/protobuf":
|
|
172
|
+
# Binary IPC data is base64; preserve Rust's protobuf bytes for the real SDK.
|
|
173
|
+
if not isinstance(data, str):
|
|
174
|
+
raise ValueError("Firestore protobuf IPC data must be base64 text")
|
|
175
|
+
body = base64.b64decode(data, validate=True)
|
|
176
|
+
headers["content-type"] = "application/protobuf"
|
|
177
|
+
return Delivery(headers, body, body)
|
|
178
|
+
message = firestore_events.DocumentEventData.from_json(json.dumps(data if isinstance(data, dict) else {}))
|
|
179
|
+
headers["content-type"] = "application/protobuf"
|
|
180
|
+
body = firestore_events.DocumentEventData.serialize(message)
|
|
181
|
+
return Delivery(headers, body, body)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def pubsub_cloud_event(event: dict):
|
|
185
|
+
"""The `CloudEvent` for a Pub/Sub message event (see `pubsub_request`)."""
|
|
186
|
+
delivery = pubsub_request(event)
|
|
187
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
90
188
|
|
|
91
189
|
|
|
92
190
|
def storage_cloud_event(event: dict):
|
|
93
191
|
"""The `CloudEvent` for a Storage object event."""
|
|
94
|
-
|
|
192
|
+
delivery = storage_request(event)
|
|
193
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
95
194
|
|
|
96
195
|
|
|
97
196
|
def eventarc_cloud_event(event: dict):
|
|
98
197
|
"""The `CloudEvent` for a custom Eventarc event."""
|
|
99
|
-
|
|
198
|
+
delivery = eventarc_request(event)
|
|
199
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
100
200
|
|
|
101
201
|
|
|
102
202
|
def database_cloud_event(event: dict):
|
|
103
203
|
"""The `CloudEvent` for a Realtime Database event."""
|
|
104
|
-
|
|
204
|
+
delivery = database_request(event)
|
|
205
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
105
206
|
|
|
106
207
|
|
|
107
208
|
def firestore_cloud_event(event: dict):
|
|
108
209
|
"""The `CloudEvent` for a Firestore event frame (`created`, `updated`, `deleted`, `written`)."""
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
message = firestore_events.DocumentEventData.from_json(json.dumps(data if isinstance(data, dict) else {}))
|
|
112
|
-
headers["content-type"] = "application/protobuf"
|
|
113
|
-
return from_http(headers, firestore_events.DocumentEventData.serialize(message))
|
|
210
|
+
delivery = firestore_request(event)
|
|
211
|
+
return from_http(delivery.headers, delivery.event_body)
|
|
@@ -27,13 +27,59 @@ def encode_frame(message: dict) -> bytes:
|
|
|
27
27
|
return str(len(payload)).encode("ascii") + b"\n" + payload
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
_DECODER = json.JSONDecoder()
|
|
31
|
+
_WHITESPACE = " \t\n\r"
|
|
32
|
+
# The trigger types whose request body is the event's JSON data as the daemon framed it.
|
|
33
|
+
_RAW_DATA_TRIGGERS = frozenset(("eventarc", "pubsub", "storage", "database"))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _after_whitespace(text: str, index: int) -> int:
|
|
37
|
+
while index < len(text) and text[index] in _WHITESPACE:
|
|
38
|
+
index += 1
|
|
39
|
+
return index
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def raw_event_data(text: str) -> str | None:
|
|
43
|
+
"""The text of `event.data` exactly as it stands in a frame, or None when the frame has none or is malformed.
|
|
44
|
+
|
|
45
|
+
The decoder turns the data into Python values and forgets its spelling (`1e-7`, `\\u00e9`, member order, white space).
|
|
46
|
+
For a custom Eventarc event in the strict profile the daemon frames the publisher's own JSON text, and that text is the
|
|
47
|
+
request body production delivered, so it is read here from the frame text, member by member, without parsing the data.
|
|
48
|
+
"""
|
|
49
|
+
try:
|
|
50
|
+
index = _after_whitespace(text, 0)
|
|
51
|
+
for wanted in ("event", "data"):
|
|
52
|
+
if text[index] != "{":
|
|
53
|
+
return None
|
|
54
|
+
index += 1
|
|
55
|
+
while True:
|
|
56
|
+
index = _after_whitespace(text, index)
|
|
57
|
+
key, index = _DECODER.raw_decode(text, index)
|
|
58
|
+
index = _after_whitespace(text, index)
|
|
59
|
+
if text[index] != ":" or not isinstance(key, str):
|
|
60
|
+
return None
|
|
61
|
+
index = _after_whitespace(text, index + 1)
|
|
62
|
+
if key == wanted:
|
|
63
|
+
break
|
|
64
|
+
index = _after_whitespace(text, _DECODER.raw_decode(text, index)[1])
|
|
65
|
+
if text[index] == ",":
|
|
66
|
+
index += 1
|
|
67
|
+
start = index
|
|
68
|
+
end = _DECODER.raw_decode(text, start)[1]
|
|
69
|
+
# A value must be followed by what ends it, or the frame is cut short.
|
|
70
|
+
return text[start:end] if text[_after_whitespace(text, end)] in ",}" else None
|
|
71
|
+
except (IndexError, ValueError):
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
30
75
|
def _text(value) -> bool:
|
|
31
76
|
return isinstance(value, str) and len(value) > 0
|
|
32
77
|
|
|
33
78
|
|
|
34
79
|
def _decode_message(payload: bytes) -> dict:
|
|
35
80
|
try:
|
|
36
|
-
|
|
81
|
+
text = payload.decode("utf-8")
|
|
82
|
+
message = json.loads(text)
|
|
37
83
|
except (UnicodeDecodeError, ValueError):
|
|
38
84
|
raise ProtocolError("UTF-8 or JSON") from None
|
|
39
85
|
if not isinstance(message, dict):
|
|
@@ -49,6 +95,10 @@ def _decode_message(payload: bytes) -> dict:
|
|
|
49
95
|
or ("entryPoint" in message and not _text(message["entryPoint"]))
|
|
50
96
|
):
|
|
51
97
|
raise ProtocolError("message fields")
|
|
98
|
+
if message["trigger"] in _RAW_DATA_TRIGGERS and "data" in message["event"]:
|
|
99
|
+
raw = raw_event_data(text)
|
|
100
|
+
if raw is not None:
|
|
101
|
+
message["rawEventData"] = raw
|
|
52
102
|
return message
|
|
53
103
|
|
|
54
104
|
|
|
@@ -6,10 +6,12 @@ import platform
|
|
|
6
6
|
import sys
|
|
7
7
|
import traceback
|
|
8
8
|
|
|
9
|
+
from . import appcheck
|
|
9
10
|
from .dispatch import Dispatcher
|
|
11
|
+
from .environment import EnvironmentGate, load_local_secrets
|
|
10
12
|
from .framing import FrameWriter, ProtocolError, read_frame
|
|
11
13
|
from .manifest import build_manifest
|
|
12
|
-
from .serve import start_server
|
|
14
|
+
from .serve import HTTP_TRIGGERS, start_server
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
def _parse(argv):
|
|
@@ -27,13 +29,15 @@ def load_codebase(source: str) -> dict:
|
|
|
27
29
|
return serving.get_functions()
|
|
28
30
|
|
|
29
31
|
|
|
30
|
-
def serve(
|
|
32
|
+
def serve(
|
|
33
|
+
functions: dict, codebase: str, secret: str, stdin, writer: FrameWriter, gate: EnvironmentGate | None = None, app_check_verified: bool = False
|
|
34
|
+
) -> int:
|
|
31
35
|
"""Says hello, then serves frames from `stdin` until `shutdown` or the end of the stream."""
|
|
32
|
-
manifest = build_manifest(functions)
|
|
36
|
+
manifest = build_manifest(functions, app_check_verified)
|
|
33
37
|
server = None
|
|
34
38
|
port = None
|
|
35
|
-
if any(spec["trigger"]["type"]
|
|
36
|
-
server, port = start_server(functions, manifest, secret)
|
|
39
|
+
if any(spec["trigger"]["type"] in HTTP_TRIGGERS for spec in manifest["functions"]):
|
|
40
|
+
server, port = start_server(functions, manifest, secret, gate)
|
|
37
41
|
writer.write(
|
|
38
42
|
{
|
|
39
43
|
"type": "hello",
|
|
@@ -44,7 +48,7 @@ def serve(functions: dict, codebase: str, secret: str, stdin, writer: FrameWrite
|
|
|
44
48
|
"manifest": manifest,
|
|
45
49
|
}
|
|
46
50
|
)
|
|
47
|
-
dispatcher = Dispatcher(functions, manifest, writer)
|
|
51
|
+
dispatcher = Dispatcher(functions, manifest, writer, gate)
|
|
48
52
|
try:
|
|
49
53
|
while True:
|
|
50
54
|
message = read_frame(stdin)
|
|
@@ -70,13 +74,19 @@ def run(argv) -> int:
|
|
|
70
74
|
os.dup2(2, 1)
|
|
71
75
|
writer = FrameWriter(protocol)
|
|
72
76
|
secret = os.environ.get("FIREEMU_RUNNER_SECRET", "")
|
|
73
|
-
# The daemon's local secrets are
|
|
74
|
-
# that carried them, nor the key to this runner's HTTP port.
|
|
75
|
-
os.environ.pop("FIREEMU_LOCAL_SECRETS_JSON", None)
|
|
77
|
+
# The daemon's local secrets are put back into the environment, a function at a time, by the gate; user code
|
|
78
|
+
# does not see the channel that carried them, nor the key to this runner's HTTP port.
|
|
76
79
|
os.environ.pop("FIREEMU_RUNNER_SECRET", None)
|
|
80
|
+
try:
|
|
81
|
+
gate = EnvironmentGate(load_local_secrets(os.environ))
|
|
82
|
+
except ValueError as error:
|
|
83
|
+
sys.stderr.write(f"[functions] {error}\n")
|
|
84
|
+
return 1
|
|
85
|
+
# Before any user code runs, so the Admin SDK's App Check service is built against the emulator's keys.
|
|
86
|
+
app_check_verified = appcheck.install_from_environment(os.environ)
|
|
77
87
|
try:
|
|
78
88
|
functions = load_codebase(args.source)
|
|
79
89
|
except BaseException: # a failing import in user code is reported with its traceback
|
|
80
90
|
traceback.print_exc()
|
|
81
91
|
return 1
|
|
82
|
-
return serve(functions, args.codebase, secret, sys.stdin.buffer, writer)
|
|
92
|
+
return serve(functions, args.codebase, secret, sys.stdin.buffer, writer, gate, app_check_verified)
|
|
@@ -10,6 +10,7 @@ under `ignored[]`, never dropped.
|
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
12
|
import enum
|
|
13
|
+
import math
|
|
13
14
|
from typing import Any
|
|
14
15
|
|
|
15
16
|
from firebase_functions.params import Expression
|
|
@@ -32,7 +33,10 @@ _UNSERVED_PRODUCTS = (
|
|
|
32
33
|
("analytics", "analytics", "notPlanned", "not planned: Google Analytics triggers have no local emulator"),
|
|
33
34
|
("testing", "testLab", "notPlanned", "not planned: Test Lab triggers have no local emulator"),
|
|
34
35
|
)
|
|
35
|
-
|
|
36
|
+
_SCHEDULE_RETRY_FIELDS = frozenset(("retryCount", "maxRetrySeconds", "minBackoffSeconds", "maxBackoffSeconds", "maxDoublings"))
|
|
37
|
+
_TASK_RETRY_FIELDS = frozenset(("maxAttempts", "maxRetrySeconds", "minBackoffSeconds", "maxBackoffSeconds", "maxDoublings"))
|
|
38
|
+
_TASK_RATE_FIELDS = frozenset(("maxConcurrentDispatches", "maxDispatchesPerSecond"))
|
|
39
|
+
_BLOCKING_AUTH_EVENTS = ("beforeCreate", "beforeSignIn", "beforeSendEmail", "beforeSendSms")
|
|
36
40
|
|
|
37
41
|
|
|
38
42
|
class Unserved(Exception):
|
|
@@ -224,7 +228,79 @@ def _event_trigger(event_trigger: dict) -> dict:
|
|
|
224
228
|
raise Unserved(family, "unsupported", f"event type {event_type} is not recognised")
|
|
225
229
|
|
|
226
230
|
|
|
227
|
-
def
|
|
231
|
+
def _record(value, field: str, family: str) -> dict:
|
|
232
|
+
"""An options record: absent is empty, an object is itself, anything else is an error."""
|
|
233
|
+
if _unset(value):
|
|
234
|
+
return {}
|
|
235
|
+
if not isinstance(value, dict):
|
|
236
|
+
raise Unserved(family, "unsupported", f"{field} must be an object")
|
|
237
|
+
return value
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _numbers(value, field: str, allowed: frozenset, family: str) -> dict:
|
|
241
|
+
"""The numbers of a retry or rate record. A member the SDK left unset is null on the wire."""
|
|
242
|
+
record = _record(value, field, family)
|
|
243
|
+
numbers: dict[str, Any] = {}
|
|
244
|
+
for key, original in record.items():
|
|
245
|
+
if key not in allowed:
|
|
246
|
+
raise Unserved(family, "unsupported", f"{field}.{key} is not supported")
|
|
247
|
+
resolved = _resolved(original, f"{field}.{key}")
|
|
248
|
+
if resolved is None:
|
|
249
|
+
numbers[key] = None
|
|
250
|
+
continue
|
|
251
|
+
if isinstance(resolved, bool) or not isinstance(resolved, (int, float)) or not math.isfinite(resolved):
|
|
252
|
+
raise Unserved(family, "unsupported", f"{field}.{key} did not resolve to a finite number")
|
|
253
|
+
numbers[key] = resolved
|
|
254
|
+
return numbers
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _schedule_trigger(trigger) -> tuple[dict, bool]:
|
|
258
|
+
"""The canonical trigger of a `scheduleTrigger` and whether the function retries."""
|
|
259
|
+
record = _record(trigger, "scheduleTrigger", "schedule")
|
|
260
|
+
schedule = _text(record, "schedule", "scheduleTrigger.schedule", "schedule", required=True, allow_empty=False)
|
|
261
|
+
time_zone = _text(record, "timeZone", "scheduleTrigger.timeZone", "schedule")
|
|
262
|
+
# The SDK keeps a deployment expression as braced CEL; this runner resolves values, it does not evaluate CEL.
|
|
263
|
+
if "{{" in schedule or (time_zone is not None and "{{" in time_zone):
|
|
264
|
+
raise Unserved("schedule", "unsupported", "scheduleTrigger contains an unresolved deployment expression")
|
|
265
|
+
retry_config = _numbers(record.get("retryConfig"), "scheduleTrigger.retryConfig", _SCHEDULE_RETRY_FIELDS, "schedule")
|
|
266
|
+
# A Gen2 job with a retry window and no count is retried until the window ends (run 156715222b86ea44), so a
|
|
267
|
+
# window alone declares a retry too.
|
|
268
|
+
retry = (retry_config.get("retryCount") or 0) > 0 or (retry_config.get("maxRetrySeconds") or 0) > 0
|
|
269
|
+
described: dict[str, Any] = {"type": "schedule", "schedule": schedule}
|
|
270
|
+
if time_zone is not None:
|
|
271
|
+
described["timeZone"] = time_zone
|
|
272
|
+
described["retryConfig"] = retry_config
|
|
273
|
+
return described, retry
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _task_queue_trigger(queue) -> dict:
|
|
277
|
+
record = _record(queue, "taskQueueTrigger", "tasks")
|
|
278
|
+
return {
|
|
279
|
+
"type": "tasks",
|
|
280
|
+
"retryConfig": _numbers(record.get("retryConfig"), "taskQueueTrigger.retryConfig", _TASK_RETRY_FIELDS, "tasks"),
|
|
281
|
+
"rateLimits": _numbers(record.get("rateLimits"), "taskQueueTrigger.rateLimits", _TASK_RATE_FIELDS, "tasks"),
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _blocking_trigger(blocking) -> dict:
|
|
286
|
+
record = _record(blocking, "blockingTrigger", "blocking")
|
|
287
|
+
event_type = str(_resolved(record.get("eventType"), "blockingTrigger.eventType") or "")
|
|
288
|
+
if not any(event_type.endswith(event) for event in _BLOCKING_AUTH_EVENTS):
|
|
289
|
+
for fragment, trigger_type, scope, reason in _UNSERVED_PRODUCTS:
|
|
290
|
+
if fragment in event_type:
|
|
291
|
+
raise Unserved(trigger_type, scope, reason)
|
|
292
|
+
raise Unserved("blocking", "unsupported", f"blocking identity event {event_type} is not served")
|
|
293
|
+
options = record.get("options") if isinstance(record.get("options"), dict) else {}
|
|
294
|
+
return {
|
|
295
|
+
"type": "blockingAuth",
|
|
296
|
+
"eventType": event_type,
|
|
297
|
+
"accessToken": options.get("accessToken") is True,
|
|
298
|
+
"idToken": options.get("idToken") is True,
|
|
299
|
+
"refreshToken": options.get("refreshToken") is True,
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def describe_endpoint(name: str, endpoint, app_check_verified: bool = False) -> dict:
|
|
228
304
|
"""The canonical `functions[]` entry of one endpoint, or `Unserved` when it cannot be served."""
|
|
229
305
|
if endpoint.platform != "gcfv2":
|
|
230
306
|
raise Unserved("unknown", "unsupported", "endpoint platform is not recognised")
|
|
@@ -242,15 +318,19 @@ def describe_endpoint(name: str, endpoint) -> dict:
|
|
|
242
318
|
if options:
|
|
243
319
|
described["platformOptions"] = options
|
|
244
320
|
if endpoint.taskQueueTrigger is not None:
|
|
245
|
-
|
|
321
|
+
described["trigger"] = _task_queue_trigger(endpoint.taskQueueTrigger)
|
|
322
|
+
return described
|
|
246
323
|
if endpoint.httpsTrigger is not None:
|
|
247
324
|
described["trigger"] = {"type": "http", "callable": False}
|
|
248
325
|
elif endpoint.callableTrigger is not None:
|
|
249
326
|
described["trigger"] = {"type": "http", "callable": True}
|
|
327
|
+
if app_check_verified:
|
|
328
|
+
# The SDK verifies the token itself, against the App Check emulator's keys, and never consumes it.
|
|
329
|
+
described["trigger"]["consumeAppCheckToken"] = "disabled"
|
|
250
330
|
elif endpoint.scheduleTrigger is not None:
|
|
251
|
-
|
|
331
|
+
described["trigger"], described["retry"] = _schedule_trigger(endpoint.scheduleTrigger)
|
|
252
332
|
elif endpoint.blockingTrigger is not None:
|
|
253
|
-
|
|
333
|
+
described["trigger"] = _blocking_trigger(endpoint.blockingTrigger)
|
|
254
334
|
elif endpoint.eventTrigger is not None:
|
|
255
335
|
described["trigger"] = _event_trigger(endpoint.eventTrigger)
|
|
256
336
|
retry = _resolved(endpoint.eventTrigger.get("retry"), "eventTrigger.retry")
|
|
@@ -262,14 +342,17 @@ def describe_endpoint(name: str, endpoint) -> dict:
|
|
|
262
342
|
return described
|
|
263
343
|
|
|
264
344
|
|
|
265
|
-
def build_manifest(functions: dict) -> dict:
|
|
266
|
-
"""`{"functions": [...], "ignored": [...]}` for the decorated functions of a codebase.
|
|
345
|
+
def build_manifest(functions: dict, app_check_verified: bool = False) -> dict:
|
|
346
|
+
"""`{"functions": [...], "ignored": [...]}` for the decorated functions of a codebase.
|
|
347
|
+
|
|
348
|
+
`app_check_verified` says the Admin SDK verifies App Check tokens against the emulator's keys (`appcheck.py`).
|
|
349
|
+
"""
|
|
267
350
|
served: list[dict] = []
|
|
268
351
|
ignored: list[dict] = []
|
|
269
352
|
for name, function in functions.items():
|
|
270
353
|
endpoint = function.__firebase_endpoint__
|
|
271
354
|
try:
|
|
272
|
-
served.append(describe_endpoint(name, endpoint))
|
|
355
|
+
served.append(describe_endpoint(name, endpoint, app_check_verified))
|
|
273
356
|
except Unserved as unserved:
|
|
274
357
|
try:
|
|
275
358
|
region = _first_region(endpoint) or DEFAULT_REGION
|