@fireemu/darwin-x64 0.42.0 → 0.44.0

package/bin/runner-python/fireemu_runner/environment.py

@@ -0,0 +1,119 @@
+"""Local secrets: which invocation sees which secret in the one process environment.
+
+Production gives every function its own instance, so a secret a function declares is in that instance's environment
+only. One runner serves a whole codebase here and has one `os.environ`, so the runner takes the daemon's local
+secrets (`FIREEMU_LOCAL_SECRETS_JSON`, from `.secret.local`) out of the environment at start and puts them back for
+the duration of an invocation, as the Node runner's `withFunctionEnvironment` does:
+
+- a plain invocation (one that declares no secret with a local value) runs with every local secret hidden, beside
+  other plain invocations;
+- an invocation that declares a secret with a local value runs alone, with only the secrets it declares;
+- invocations start in arrival order, so a plain one that arrives behind a queued secret invocation waits.
+
+The model is `EnvGate.tla` (checked with TLC): an exclusive invocation never overlaps any other, a plain one never
+sees a secret, and nothing is exposed while nothing runs.
+
+What this does not isolate, as in the Node runner (one process serves the whole codebase):
+- work a function leaves running after its invocation ends (a thread it started, a child process it launched) runs in
+  the shared process and sees whatever the environment holds at that moment;
+- a function that holds a secret and waits, synchronously, for another function of the same codebase (a request to its
+  own URL) deadlocks: the other invocation queues behind the gate the caller holds.
+Both are stated on the Functions page.
+"""
+
+import collections
+import contextlib
+import json
+import os
+import threading
+
+SECRETS_VARIABLE = "FIREEMU_LOCAL_SECRETS_JSON"
+
+
+def load_local_secrets(environ) -> dict:
+    """Takes the daemon's local secrets out of `environ`: name to value, string values only."""
+    text = environ.pop(SECRETS_VARIABLE, None)
+    if not text:
+        return {}
+    try:
+        parsed = json.loads(text)
+    except ValueError:
+        raise ValueError("the local secrets are not valid JSON") from None
+    if not isinstance(parsed, dict):
+        raise ValueError("the local secrets are not a JSON object")
+    return {name: value for name, value in parsed.items() if isinstance(value, str)}
+
+
+class EnvironmentGate:
+    """One queue of invocations and the environment each may see."""
+
+    def __init__(self, secrets: dict):
+        self._secrets = dict(secrets)
+        self._condition = threading.Condition()
+        self._queue = collections.deque()  # tickets waiting, oldest first
+        self._running = 0
+        self._exclusive_running = False
+        self._saved = {}
+
+    def _hide(self) -> None:
+        for name in self._secrets:
+            self._saved[name] = os.environ.pop(name, None)
+
+    def _restore(self) -> None:
+        for name, value in self._saved.items():
+            if value is None:
+                os.environ.pop(name, None)
+            else:
+                os.environ[name] = value
+        self._saved = {}
+
+    def _may_start(self, ticket) -> bool:
+        if self._queue[0] is not ticket:
+            return False
+        return self._running == 0 if ticket.exclusive else not self._exclusive_running
+
+    def _enter(self, ticket) -> None:
+        with self._condition:
+            self._queue.append(ticket)
+            self._condition.wait_for(lambda: self._may_start(ticket))
+            self._queue.popleft()
+            if self._running == 0:
+                self._hide()
+            self._running += 1
+            if ticket.exclusive:
+                self._exclusive_running = True
+                for name in ticket.names:
+                    os.environ[name] = self._secrets[name]
+            # The next in line may be able to start beside this one.
+            self._condition.notify_all()
+
+    def _leave(self, ticket) -> None:
+        with self._condition:
+            if ticket.exclusive:
+                self._exclusive_running = False
+                for name in ticket.names:
+                    os.environ.pop(name, None)
+            self._running -= 1
+            if self._running == 0:
+                self._restore()
+            self._condition.notify_all()
+
+    @contextlib.contextmanager
+    def hold(self, declared):
+        """Runs the body with the environment `declared` (the function's secret names) is entitled to."""
+        if not self._secrets:
+            yield
+            return
+        names = [name for name in dict.fromkeys(declared or ()) if name in self._secrets]
+        ticket = _Ticket(names)
+        self._enter(ticket)
+        try:
+            yield
+        finally:
+            self._leave(ticket)
+
+
+class _Ticket:
+    def __init__(self, names):
+        self.names = names
+        self.exclusive = bool(names)

package/bin/runner-python/fireemu_runner/events.py

@@ -0,0 +1,203 @@
+"""Event adapters: the daemon's structured event as the CloudEvent the Python SDK expects.
+
+Production delivers a Firestore event as a binary-mode CloudEvent (`ce-*` headers) whose body is a
+protobuf `DocumentEventData`; the SDK's JSON branch needs a JSON *string* and fails on a decoded
+dict (`firestore_fn.py:108-131`). The adapter therefore builds the production form and lets
+`cloudevents.http.from_http` produce the object, exactly the path functions-framework takes
+(`functions_framework/__init__.py:174-188`).
+"""
+
+import json
+import os
+import re
+from typing import NamedTuple
+
+from cloudevents.http import from_http
+from google.events.cloud import firestore as firestore_events
+
+from . import bodies
+
+_NOT_AN_ATTRIBUTE = frozenset({"data", "datacontenttype"})
+# The daemon's strict 2nd gen frame names a `traceparent` for the Node runner's event. Production sends the trace context as an
+# ordinary HTTP header (Cloud Run's), never as a `ce-` attribute, so the strict Python event has none (sandbox run
+# PYCHECK-EVENTS-20261104-01, the raw twins of the Pub/Sub, Storage, Eventarc and Firestore events). The emulator profile
+# stamps no such member, so a `traceparent` there is the publisher's own tracing extension and stays an attribute.
+_HTTP_HEADER_ONLY = frozenset({"traceparent"})
+# Eventarc sends `ce-dataschema` with every Firestore event. Recorded for `created` only; for `updated`, `deleted`, `written`
+# and any other type routed here it is UNRECORDED (the schema is one document for the family in google-cloudevents). The strict
+# profile delivers it; the emulator profile keeps the official emulator's attributes, which have none.
+_FIRESTORE_DATASCHEMA = (
+    "https://github.com/googleapis/google-cloudevents/blob/main/proto/google/events/cloud/firestore/v1/data.proto"
+)
+
+
+_WHOLE_SECOND_TIME = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(Z|[+-]\d{2}:\d{2})$")
+
+
+def _attribute_headers(event: dict) -> dict:
+    """The binary-mode `ce-*` headers of an event: its string members, except the data and the content type."""
+    if not isinstance(event.get("specversion"), str):
+        # Without it `from_http` would read the body as a structured JSON event and fail on the body.
+        raise ValueError("the event has no specversion")
+    header_only = _HTTP_HEADER_ONLY if is_strict() else frozenset()
+    return {
+        f"ce-{name}": value
+        for name, value in event.items()
+        if name not in _NOT_AN_ATTRIBUTE and name not in header_only and isinstance(value, str)
+    }
+
+
+def _is_json(content_type: str) -> bool:
+    media = content_type.split(";", 1)[0].strip().lower()
+    return media == "application/json" or media.endswith("+json")
+
+
+class Delivery(NamedTuple):
+    """One delivery: the headers, the body the event is built from, and the body the request carries.
+
+    The two bodies are the same bytes for a Firestore event. For the JSON families the event is built from the encoding of
+    the data the adapters have always used (`event_body`), and the request shows the layout the recorded delivery had
+    (`request_body`); the function is given the same event whichever layout the request has.
+    """
+
+    headers: dict
+    event_body: bytes
+    request_body: bytes
+
+
+def _json_delivery(event: dict, request_text: str | None = None) -> Delivery:
+    """The delivery of an event whose data is JSON (or text, with a text content type).
+
+    Eventarc delivers these in binary content mode: attributes as `ce-*` headers and the data as the body,
+    which functions-framework hands to `cloudevents.http.from_http` (`functions_framework/__init__.py:174-188`).
+    A delivery without a content type is JSON. `request_text` is the body text of the request when it differs from the
+    compact encoding of the data (see `bodies.py`).
+    """
+    headers = _attribute_headers(event)
+    declared = event.get("datacontenttype")
+    content_type = declared if isinstance(declared, str) and declared else "application/json"
+    headers["content-type"] = content_type
+    if "data" not in event:
+        return Delivery(headers, b"", b"")
+    if isinstance(event["data"], str) and not _is_json(content_type):
+        body = event["data"].encode()
+        return Delivery(headers, body, body)
+    request = request_text if request_text is not None else bodies.compact(event["data"])
+    return Delivery(headers, json.dumps(event["data"]).encode(), request.encode())
+
+
+def _delivered_value(event: dict, raw_data: str | None):
+    """The data with its numbers as the daemon wrote them, or the event's own data when there is no text or it does not parse."""
+    if raw_data is not None:
+        try:
+            return bodies.parse_keeping_numbers(raw_data)
+        except ValueError:
+            pass
+    return event.get("data")
+
+
+def is_strict() -> bool:
+    return os.environ.get("FIREEMU_HTTP_PROFILE") == "strict"
+
+
+def _lenient_time(event: dict) -> dict:
+    """The emulator profile gives a whole-second `time` a fraction.
+
+    `storage_fn` and `eventarc_fn` parse `time` with `%f` (`storage_fn.py:249-252`, `eventarc_fn.py:65-68`) and
+    fail on `2026-01-01T00:00:02Z`. The official emulator always prints milliseconds, so what it served the
+    emulator profile must serve; the strict profile delivers the time as production's event has it.
+    """
+    time = event.get("time")
+    if is_strict() or not isinstance(time, str):
+        return event
+    match = _WHOLE_SECOND_TIME.match(time)
+    return event if match is None else {**event, "time": f"{match.group(1)}.000{match.group(2)}"}
+
+
+def _with_snake_case_keys(data):
+    """A Pub/Sub push body with `message_id` and `publish_time` next to `messageId` and `publishTime` (`pubsub_fn.py:98-160`)."""
+    message = data.get("message") if isinstance(data, dict) else None
+    if not isinstance(message, dict):
+        return data
+    message = dict(message)
+    for camel, snake in (("messageId", "message_id"), ("publishTime", "publish_time")):
+        if snake not in message and camel in message and message[camel] is not None:
+            message[snake] = message[camel]
+    return {**data, "message": message}
+
+
+def pubsub_request(event: dict, raw_data: str | None = None) -> Delivery:
+    """The delivery of a Pub/Sub message event.
+
+    The SDK reads `message_id` and `publish_time` (`pubsub_fn.py:98-160`), the snake-case spellings Pub/Sub's
+    push body carries next to `messageId` and `publishTime`. `orderingKey` has no alternative: the SDK pops
+    it and passes `ordering_key=` itself, so a copied key would be passed twice (`TypeError`).
+    The strict request carries the recorded layout: sorted compact JSON and a newline.
+    """
+    data = event.get("data")
+    if not isinstance(data, dict) or not isinstance(data.get("message"), dict):
+        return _json_delivery(event)
+    event = {**event, "data": _with_snake_case_keys(data)}
+    value = _with_snake_case_keys(_delivered_value(event, raw_data))
+    text = bodies.pubsub_layout(value) if is_strict() else bodies.compact(value)
+    return _json_delivery(event, text)
+
+
+def storage_request(event: dict, raw_data: str | None = None) -> Delivery:
+    """The delivery of a Storage object event: the text the daemon sent, and in the strict profile the recorded layout."""
+    event = _lenient_time(event)
+    if is_strict() and "data" in event:
+        return _json_delivery(event, bodies.storage_layout(_delivered_value(event, raw_data)))
+    return _json_delivery(event, raw_data)
+
+
+def eventarc_request(event: dict, raw_data: str | None = None) -> Delivery:
+    """The delivery of a custom Eventarc event: the text of the data as the daemon framed it (the publisher's own in the strict profile)."""
+    return _json_delivery(_lenient_time(event), raw_data)
+
+
+def database_request(event: dict, raw_data: str | None = None) -> Delivery:
+    """The delivery of a Realtime Database event: the text the daemon sent (no raw delivery was recorded; UNRECORDED)."""
+    return _json_delivery(event, raw_data)
+
+
+def firestore_request(event: dict, raw_data: str | None = None) -> Delivery:
+    """The delivery of a Firestore event frame (`created`, `updated`, `deleted`, `written`): a protobuf body."""
+    headers = _attribute_headers(event)
+    if is_strict():
+        headers.setdefault("ce-dataschema", _FIRESTORE_DATASCHEMA)
+    data = event.get("data")
+    message = firestore_events.DocumentEventData.from_json(json.dumps(data if isinstance(data, dict) else {}))
+    headers["content-type"] = "application/protobuf"
+    body = firestore_events.DocumentEventData.serialize(message)
+    return Delivery(headers, body, body)
+
+
+def pubsub_cloud_event(event: dict):
+    """The `CloudEvent` for a Pub/Sub message event (see `pubsub_request`)."""
+    delivery = pubsub_request(event)
+    return from_http(delivery.headers, delivery.event_body)
+
+
+def storage_cloud_event(event: dict):
+    """The `CloudEvent` for a Storage object event."""
+    delivery = storage_request(event)
+    return from_http(delivery.headers, delivery.event_body)
+
+
+def eventarc_cloud_event(event: dict):
+    """The `CloudEvent` for a custom Eventarc event."""
+    delivery = eventarc_request(event)
+    return from_http(delivery.headers, delivery.event_body)
+
+
+def database_cloud_event(event: dict):
+    """The `CloudEvent` for a Realtime Database event."""
+    delivery = database_request(event)
+    return from_http(delivery.headers, delivery.event_body)
+
+
+def firestore_cloud_event(event: dict):
+    """The `CloudEvent` for a Firestore event frame (`created`, `updated`, `deleted`, `written`)."""
+    delivery = firestore_request(event)
+    return from_http(delivery.headers, delivery.event_body)

package/bin/runner-python/fireemu_runner/framing.py

@@ -0,0 +1,145 @@
+"""The frame codec: `<decimal byte length>\\n<UTF-8 JSON object>`.
+
+It matches `crates/fireemu-adapter-functions/src/protocol.rs` and `tools/runner-node/protocol.mjs`:
+a canonical decimal length (no sign, no leading zero, at most 8 digits), a cap of 16 MiB, and only
+the daemon messages this runner serves (`invoke` and `shutdown`). Diagnostics are fixed strings:
+a JSON parser error can quote the payload.
+"""
+
+import json
+import threading
+
+MAX_FRAME_BYTES = 16 * 1024 * 1024
+_MAX_HEADER_DIGITS = len(str(MAX_FRAME_BYTES))
+
+
+class ProtocolError(Exception):
+    """A frame the runner cannot accept; the reason is a fixed string."""
+
+    def __init__(self, reason: str):
+        super().__init__(f"invalid runner frame ({reason})")
+
+
+def encode_frame(message: dict) -> bytes:
+    payload = json.dumps(message, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+    if len(payload) > MAX_FRAME_BYTES:
+        raise ProtocolError("frame too large")
+    return str(len(payload)).encode("ascii") + b"\n" + payload
+
+
+_DECODER = json.JSONDecoder()
+_WHITESPACE = " \t\n\r"
+# The trigger types whose request body is the event's JSON data as the daemon framed it.
+_RAW_DATA_TRIGGERS = frozenset(("eventarc", "pubsub", "storage", "database"))
+
+
+def _after_whitespace(text: str, index: int) -> int:
+    while index < len(text) and text[index] in _WHITESPACE:
+        index += 1
+    return index
+
+
+def raw_event_data(text: str) -> str | None:
+    """The text of `event.data` exactly as it stands in a frame, or None when the frame has none or is malformed.
+
+    The decoder turns the data into Python values and forgets its spelling (`1e-7`, `\\u00e9`, member order, white space).
+    For a custom Eventarc event in the strict profile the daemon frames the publisher's own JSON text, and that text is the
+    request body production delivered, so it is read here from the frame text, member by member, without parsing the data.
+    """
+    try:
+        index = _after_whitespace(text, 0)
+        for wanted in ("event", "data"):
+            if text[index] != "{":
+                return None
+            index += 1
+            while True:
+                index = _after_whitespace(text, index)
+                key, index = _DECODER.raw_decode(text, index)
+                index = _after_whitespace(text, index)
+                if text[index] != ":" or not isinstance(key, str):
+                    return None
+                index = _after_whitespace(text, index + 1)
+                if key == wanted:
+                    break
+                index = _after_whitespace(text, _DECODER.raw_decode(text, index)[1])
+                if text[index] == ",":
+                    index += 1
+        start = index
+        end = _DECODER.raw_decode(text, start)[1]
+        # A value must be followed by what ends it, or the frame is cut short.
+        return text[start:end] if text[_after_whitespace(text, end)] in ",}" else None
+    except (IndexError, ValueError):
+        return None
+
+
+def _text(value) -> bool:
+    return isinstance(value, str) and len(value) > 0
+
+
+def _decode_message(payload: bytes) -> dict:
+    try:
+        text = payload.decode("utf-8")
+        message = json.loads(text)
+    except (UnicodeDecodeError, ValueError):
+        raise ProtocolError("UTF-8 or JSON") from None
+    if not isinstance(message, dict):
+        raise ProtocolError("message object")
+    if message.get("type") == "shutdown":
+        return message
+    if (
+        message.get("type") != "invoke"
+        or not _text(message.get("invocationId"))
+        or not _text(message.get("function"))
+        or not _text(message.get("trigger"))
+        or not isinstance(message.get("event"), dict)
+        or ("entryPoint" in message and not _text(message["entryPoint"]))
+    ):
+        raise ProtocolError("message fields")
+    if message["trigger"] in _RAW_DATA_TRIGGERS and "data" in message["event"]:
+        raw = raw_event_data(text)
+        if raw is not None:
+            message["rawEventData"] = raw
+    return message
+
+
+def read_frame(stream) -> dict | None:
+    """Reads one message from a binary stream; `None` at a clean end between frames."""
+    length = 0
+    digits = 0
+    while True:
+        byte = stream.read(1)
+        if not byte:
+            if digits == 0:
+                return None
+            raise ProtocolError("truncated")
+        value = byte[0]
+        if value == 10:
+            if digits == 0:
+                raise ProtocolError("empty length")
+            break
+        if value < 48 or value > 57 or (digits == 0 and value == 48):
+            raise ProtocolError("decimal length")
+        digits += 1
+        length = length * 10 + value - 48
+        if digits > _MAX_HEADER_DIGITS or length > MAX_FRAME_BYTES:
+            raise ProtocolError("frame too large")
+    payload = stream.read(length)
+    if len(payload) != length:
+        raise ProtocolError("truncated")
+    return _decode_message(payload)
+
+
+class FrameWriter:
+    """Writes frames to the protocol channel, one whole frame at a time."""
+
+    def __init__(self, stream):
+        self._stream = stream
+        self._lock = threading.Lock()
+
+    def write(self, message: dict) -> None:
+        data = encode_frame(message)
+        with self._lock:
+            self._stream.write(data)
+            flush = getattr(self._stream, "flush", None)
+            if flush is not None:
+                flush()

package/bin/runner-python/fireemu_runner/main.py

@@ -0,0 +1,92 @@
+"""The runner process: load the codebase, say hello, serve frames until told to stop."""
+
+import argparse
+import os
+import platform
+import sys
+import traceback
+
+from . import appcheck
+from .dispatch import Dispatcher
+from .environment import EnvironmentGate, load_local_secrets
+from .framing import FrameWriter, ProtocolError, read_frame
+from .manifest import build_manifest
+from .serve import HTTP_TRIGGERS, start_server
+
+
+def _parse(argv):
+    parser = argparse.ArgumentParser(prog="runner.py")
+    parser.add_argument("--source", required=True)
+    parser.add_argument("--codebase", default="default")
+    return parser.parse_args(argv)
+
+
+def load_codebase(source: str) -> dict:
+    """Imports the codebase's `main.py` the way the SDK's own admin server does and returns its functions."""
+    os.chdir(source)
+    from firebase_functions.private import serving
+
+    return serving.get_functions()
+
+
+def serve(
+    functions: dict, codebase: str, secret: str, stdin, writer: FrameWriter, gate: EnvironmentGate | None = None, app_check_verified: bool = False
+) -> int:
+    """Says hello, then serves frames from `stdin` until `shutdown` or the end of the stream."""
+    manifest = build_manifest(functions, app_check_verified)
+    server = None
+    port = None
+    if any(spec["trigger"]["type"] in HTTP_TRIGGERS for spec in manifest["functions"]):
+        server, port = start_server(functions, manifest, secret, gate)
+    writer.write(
+        {
+            "type": "hello",
+            "runner": "python",
+            "codebase": codebase,
+            "version": platform.python_version(),
+            "httpPort": port,
+            "manifest": manifest,
+        }
+    )
+    dispatcher = Dispatcher(functions, manifest, writer, gate)
+    try:
+        while True:
+            message = read_frame(stdin)
+            if message is None or message["type"] == "shutdown":
+                return 0
+            if not dispatcher.reserve():
+                sys.stderr.write("[functions] too many active invocations\n")
+                return 2
+            dispatcher.start(message)
+    except ProtocolError as error:
+        sys.stderr.write(f"[functions] {error}\n")
+        return 2
+    finally:
+        if server is not None:
+            server.shutdown()
+
+
+def run(argv) -> int:
+    args = _parse(argv)
+    # The protocol owns the original stdout. Everything that writes to file descriptor 1 afterwards
+    # (print, C extensions, child processes, the SDK logger) goes to stderr instead.
+    protocol = os.fdopen(os.dup(1), "wb", buffering=0)
+    os.dup2(2, 1)
+    writer = FrameWriter(protocol)
+    secret = os.environ.get("FIREEMU_RUNNER_SECRET", "")
+    # The daemon's local secrets are put back into the environment, a function at a time, by the gate; user code
+    # does not see the channel that carried them, nor the key to this runner's HTTP port.
+    os.environ.pop("FIREEMU_RUNNER_SECRET", None)
+    try:
+        gate = EnvironmentGate(load_local_secrets(os.environ))
+    except ValueError as error:
+        sys.stderr.write(f"[functions] {error}\n")
+        return 1
+    # Before any user code runs, so the Admin SDK's App Check service is built against the emulator's keys.
+    app_check_verified = appcheck.install_from_environment(os.environ)
+    try:
+        functions = load_codebase(args.source)
+    except BaseException:  # a failing import in user code is reported with its traceback
+        traceback.print_exc()
+        return 1
+    return serve(functions, args.codebase, secret, sys.stdin.buffer, writer, gate, app_check_verified)