@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-python/src/hkp/services/gather.py

@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+# Service Documentation
+# Service ID: gather
+# Service Name: Gather
+# Runtime: hkp-python
+# Modes: none — the items as an array, or what the nested pipeline makes of them
+# Key Config: pipeline, maxBytes, maxGroups, timeoutMs, bypass
+# IO: in=one part a Spread sent on -> out=the whole group, once every part is in; None until then
+# Arrays: what a group becomes, in the order it was spread
+# Binary: an item may be bytes; joining bytes is the nested pipeline's to do
+# MixedData: not native in runtime
+
+"""The other half of Spread: waits until it has seen every part of a group and
+turns them back into one value.
+
+Spread sends each item as ``{item, group, index, total}``. Gather collects the
+parts of one ``group``, puts each at its ``index``, and once it holds ``total``
+of them the group is complete. Parts may arrive in any order; a part sent twice
+counts once. Until then it passes nothing on. Anything that is not a part
+passes through, so the same Gather sits on a board that sometimes sends a value
+whole.
+
+**What the group becomes is a pipeline, not a list of strategies**, the same
+answer Tracks gives its reducer. With no pipeline the items travel on as a list;
+with one, the pipeline is given ``{"group", "items"}`` and its answer is what
+goes on.
+
+**Collecting a group holds it in memory.** ``maxBytes`` bounds one group,
+``maxGroups`` how many may be incomplete at once — so the most held is the two
+multiplied — and a group still incomplete after ``timeoutMs`` is let go — each
+said in the log, because a group that never arrives looks, from outside, exactly
+like nothing having been sent.
+
+Matches ``gather`` in hkp-node, hkp-rt and the browser runtime.
+"""
+
+import json
+import time
+from typing import Any
+
+from ..runtime import child_run
+from ..types import (
+    JsonRecord,
+    NotifyCallback,
+    ServiceConfiguration,
+    ServiceCreator,
+    ServiceRegistryEntry,
+)
+from .sub_service import SubService
+
+GATHER_DESCRIPTOR = ServiceRegistryEntry(
+    service_id="gather",
+    service_name="Gather",
+    version="v1",
+    capabilities=["subservices"],
+)
+
+#: As much as the coordinator lets through in one frame by default.
+DEFAULT_MAX_BYTES = 64 * 1024 * 1024
+DEFAULT_TIMEOUT_MS = 60_000
+#: Groups incomplete at once. A new group beyond it is refused rather than
+#: making room: what is already arriving is not given up for what arrives next.
+DEFAULT_MAX_GROUPS = 16
+#: Refusals remembered at once, the oldest forgotten first. They expire anyway;
+#: this keeps a stream of new groups, each refused, from growing the list
+#: instead of the groups.
+MAX_REFUSED = 1024
+#: The longest group a part may name. A group is held as long as its parts are,
+#: and is not counted against ``maxBytes``; Spread names its groups by UUID.
+MAX_GROUP_LENGTH = 256
+
+
+def is_part(value: Any) -> bool:
+    """Whether a value is a part a Spread sent on, rather than one to pass through."""
+    if not isinstance(value, dict) or "item" not in value:
+        return False
+    group, index, total = value.get("group"), value.get("index"), value.get("total")
+    return (
+        isinstance(group, str)
+        and group != ""
+        and len(group) <= MAX_GROUP_LENGTH
+        and _is_int(total)
+        and total > 0
+        and _is_int(index)
+        and 0 <= index < total
+    )
+
+
+def _is_int(value: Any) -> bool:
+    if isinstance(value, bool):
+        return False
+    if isinstance(value, float):
+        return value.is_integer()
+    return isinstance(value, int)
+
+
+def _size_of(item: Any) -> int:
+    """What a part costs to hold, near enough to bound memory by."""
+    if isinstance(item, str):
+        return len(item.encode("utf-8"))
+    if isinstance(item, (bytes, bytearray)):
+        return len(item)
+    if item is None:
+        return 0
+    try:
+        return len(json.dumps(item).encode("utf-8"))
+    except (TypeError, ValueError):
+        return 0
+
+
+class _Collecting:
+    def __init__(self, total: int, started_at: float) -> None:
+        self.items: dict[int, Any] = {}
+        self.total = total
+        self.bytes = 0
+        self.started_at = started_at
+
+
+class GatherService(SubService):
+    service_id = GATHER_DESCRIPTOR.service_id
+    service_name = GATHER_DESCRIPTOR.service_name
+    version = GATHER_DESCRIPTOR.version
+    capabilities = GATHER_DESCRIPTOR.capabilities
+
+    def __init__(self, config: ServiceConfiguration, create_service: ServiceCreator) -> None:
+        # Before the base constructor, which configures from the board's state
+        # and so reaches the override below.
+        self._max_bytes = DEFAULT_MAX_BYTES
+        self._max_groups = DEFAULT_MAX_GROUPS
+        self._timeout_ms = DEFAULT_TIMEOUT_MS
+        self._last_error = ""
+        self._collecting: dict[str, _Collecting] = {}
+        #: Groups given up on, and when. The parts of a group still on their way
+        #: when it was refused would otherwise start it again.
+        self._refused: dict[str, float] = {}
+        super().__init__(config, create_service)
+
+    def configure(self, config: JsonRecord) -> JsonRecord:
+        max_bytes = config.get("maxBytes")
+        if isinstance(max_bytes, (int, float)) and not isinstance(max_bytes, bool) and max_bytes > 0:
+            self._max_bytes = int(max_bytes)
+        max_groups = config.get("maxGroups")
+        if isinstance(max_groups, (int, float)) and not isinstance(max_groups, bool) and max_groups >= 1:
+            self._max_groups = int(max_groups)
+        timeout_ms = config.get("timeoutMs")
+        if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) and timeout_ms > 0:
+            self._timeout_ms = int(timeout_ms)
+        # The pipeline, bypass and the editing commands belong to the base.
+        super().configure(config)
+        return self.get_state()
+
+    def get_state(self) -> JsonRecord:
+        return {
+            **super().get_state(),
+            "maxBytes": self._max_bytes,
+            "maxGroups": self._max_groups,
+            "timeoutMs": self._timeout_ms,
+            "pending": len(self._collecting),
+            "error": self._last_error,
+        }
+
+    def process(self, input: Any, notify: NotifyCallback) -> Any:
+        if self._bypass or not is_part(input):
+            return input
+
+        now = time.monotonic()
+        self._let_go_of_stale(now, notify)
+
+        group = input["group"]
+        index = int(input["index"])
+        total = int(input["total"])
+        item = input["item"]
+        if group in self._refused:
+            return None
+
+        collecting = self._collecting.get(group)
+        if collecting is None:
+            if len(self._collecting) >= self._max_groups:
+                self._refuse(
+                    group,
+                    now,
+                    f"group {group} refused: {self._max_groups} groups are already incomplete",
+                    notify,
+                )
+                return None
+            collecting = _Collecting(total, now)
+            self._collecting[group] = collecting
+
+        if collecting.total != total:
+            self._refuse(group, now, f"group {group} said {collecting.total} parts, then {total}", notify)
+            return None
+        if index in collecting.items:
+            # Sent twice; the first one counts.
+            return None
+
+        collecting.bytes += _size_of(item)
+        if collecting.bytes > self._max_bytes:
+            self._refuse(group, now, f"group {group} is larger than {self._max_bytes} bytes", notify)
+            return None
+
+        collecting.items[index] = item
+        notify(
+            {
+                "group": group,
+                "received": len(collecting.items),
+                "total": total,
+                "pending": len(self._collecting),
+            },
+            None,
+        )
+
+        if len(collecting.items) < total:
+            return None
+
+        del self._collecting[group]
+        items = [collecting.items[i] for i in range(total)]
+        if not self._pipeline or not self._pipeline.list_services():
+            return None if self._stop_propagation else items
+        # The run that brought the last part is the one the whole arrives in;
+        # the pipeline making it runs as a run of its own beneath it.
+        result = self._pipeline.process(
+            {"group": group, "items": items},
+            lambda _n: None,
+            child_run(self._host.current_context() if self._host else None),
+        )
+        return None if self._stop_propagation else result
+
+    def destroy(self) -> None:
+        self._collecting.clear()
+        self._refused.clear()
+        super().destroy()
+
+    # ── Private ────────────────────────────────────────────────────────────────
+
+    def _refuse(self, group: str, now: float, message: str, notify: NotifyCallback) -> None:
+        self._collecting.pop(group, None)
+        self._refused[group] = now
+        # A dict keeps insertion order, so the first key is the oldest refusal.
+        while len(self._refused) > MAX_REFUSED:
+            del self._refused[next(iter(self._refused))]
+        self._last_error = message
+        if self._host:
+            self._host.log("error", "gather.refused", {"group": group, "message": message})
+        notify({"error": message, "pending": len(self._collecting)}, None)
+
+    def _let_go_of_stale(self, now: float, notify: NotifyCallback) -> None:
+        """Lets go of groups that stopped arriving, and of refusals that are old."""
+        timeout = self._timeout_ms / 1000
+        for group, collecting in list(self._collecting.items()):
+            if now - collecting.started_at > timeout:
+                del self._collecting[group]
+                received = len(collecting.items)
+                self._last_error = (
+                    f"group {group} timed out with {received} of {collecting.total} parts"
+                )
+                if self._host:
+                    self._host.log(
+                        "warn",
+                        "gather.timed-out",
+                        {"group": group, "received": received, "total": collecting.total},
+                    )
+                notify({"error": self._last_error, "pending": len(self._collecting)}, None)
+        for group, at in list(self._refused.items()):
+            if now - at > timeout:
+                del self._refused[group]

package/bundle/hkp-python/src/hkp/services/hold.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+# Service Documentation
+# Service ID: hold
+# Service Name: Hold
+# Runtime: hkp-python
+# Modes: none — either a slot with a declared role, or a property that discriminates
+# Key Config: slot + op, or property
+# IO: in=anything -> out=the held value, or None while nothing is held
+# Arrays: a list carries no property, so it reads
+# Binary: holdable in a slot; a property cannot discriminate on one
+#
+# Sample-and-hold: a pipeline entered from two sides — a producer that runs on
+# its own schedule and a consumer that arrives whenever it arrives — needs the
+# producer's latest value to survive between runs. Hold keeps it.
+#
+# **Which side is calling can be said two ways**, and a board picks one.
+#
+# With a `slot`, the board says outright: `op` is `write` or `read`, and two
+# Holds naming one slot are the two ends of it. Nothing inspects the value, so
+# anything can be held — bytes, a document, None — and the two ends may sit in
+# pipelines that never meet, which is what an endpoint's separate entry points
+# are. Where the cells live is the host's to decide (RuntimeHost.slots): the
+# service owning both pipelines, or failing that the runtime.
+#
+# With a `property` and no slot, the input says: an input carrying that property
+# is the producer, its value replaces what is held, and every call — that one
+# included — emits the held value under the same property name, so the services
+# after Hold cannot tell the two sides apart. That is the older arrangement, and
+# it is the only one available where the two sides share one pipeline, since
+# there is nothing but the value to tell them apart. A None held value is an
+# empty one, so a producer cannot hold None: an input carrying the property as
+# None reads like any other.
+#
+# Mirrors hkp-node's and hkp-rt's `hold`.
+
+import json
+from typing import Any
+
+from ..types import JsonRecord, NotifyCallback, RuntimeHost, ServiceConfiguration, ServiceRegistryEntry
+
+HOLD_DESCRIPTOR = ServiceRegistryEntry(
+    service_id="hold",
+    service_name="Hold",
+    version="v1",
+    capabilities=[],
+)
+
+_MISSING = object()
+
+
+class HoldService:
+    service_id = HOLD_DESCRIPTOR.service_id
+    service_name = HOLD_DESCRIPTOR.service_name
+    version = HOLD_DESCRIPTOR.version
+    capabilities = HOLD_DESCRIPTOR.capabilities
+
+    def __init__(self, config: ServiceConfiguration, _create_service: Any = None) -> None:
+        self.uuid = config.uuid
+        self._host: RuntimeHost | None = None
+
+        self._property = ""
+        #: See the header: named, this is a cell the host owns rather than this one.
+        self._slot = ""
+        self._op = "read"
+        #: What is held when no slot names somewhere else to hold it.
+        self._own: Any = None
+        self._read_count = 0
+        self._write_count = 0
+
+        if config.state:
+            self.configure(config.state)
+
+    def set_host(self, host: RuntimeHost) -> None:
+        self._host = host
+
+    def get_state(self) -> JsonRecord:
+        # Only the arrangement in use is reported. A state property a service
+        # does not act on is one a board keeps and a reader has to discount, and
+        # an omitted one is erased from the board the next time it is saved —
+        # which is what should happen to the half of this service a board is not
+        # using.
+        which = {"slot": self._slot, "op": self._op} if self._slot else {"property": self._property}
+        return {
+            **which,
+            "held": _reportable(self._read()),
+            "readCount": self._read_count,
+            "writeCount": self._write_count,
+        }
+
+    def configure(self, config: JsonRecord) -> JsonRecord:
+        prop = config.get("property")
+        if isinstance(prop, str):
+            if prop != self._property:
+                # What is held belongs to the property it was written for.
+                self._forget()
+            self._property = prop
+
+        slot = config.get("slot")
+        if isinstance(slot, str):
+            if slot != self._slot:
+                # A slot is an address, and what was held belongs to the old one
+                # — but it belongs to whoever else is still reading it, so only
+                # this service's own cell is cleared, never the host's.
+                self._own = None
+                self._read_count = 0
+                self._write_count = 0
+            self._slot = slot
+
+        op = config.get("op")
+        if op in ("read", "write"):
+            self._op = op
+
+        if config.get("action") == "clear":
+            self._forget()
+
+        state = self.get_state()
+        self._notify(state)
+        return state
+
+    def process(self, input: Any, _notify: NotifyCallback) -> Any:
+        if self._slot:
+            return self._use_slot(input)
+
+        # Nothing named is nothing to hold: an unconfigured Hold is a wire.
+        if not self._property:
+            return input
+
+        incoming = _carried_value(input, self._property)
+        if incoming is not _MISSING and incoming is not None:
+            self._write(incoming)
+            self._write_count += 1
+        else:
+            self._read_count += 1
+
+        self._notify(self.get_state())
+
+        held = self._read()
+        if held is None:
+            return None
+
+        return {self._property: held}
+
+    def destroy(self) -> None:
+        # Only what this service holds itself. A slot belongs to the host, and
+        # the other end of it outlives this one — a pipeline rebuilt while a
+        # board is running destroys the services in it, and that must not empty
+        # a cell the service on the other side is still answering from.
+        self._own = None
+        self._read_count = 0
+        self._write_count = 0
+
+    # ── Private ──────────────────────────────────────────────────────────────
+
+    def _use_slot(self, input: Any) -> Any:
+        """A call on a Hold whose role is declared rather than inferred.
+
+        A write emits **its input unchanged**, so the pass it belongs to carries
+        on as though the Hold were not there; a read emits what is held, **raw**,
+        so it can be the whole of what a pipeline answers with. Neither looks at
+        the value, which is what lets a slot hold what a property never could.
+        """
+        if self._op == "write":
+            self._write(input)
+            self._write_count += 1
+            self._notify(self.get_state())
+            return input
+
+        self._read_count += 1
+        self._notify(self.get_state())
+        # Nothing held is nothing to pass on, the same as everywhere else — a
+        # consumer that arrives before the producer has run stops here.
+        return self._read()
+
+    def _read(self) -> Any:
+        """What is held, from wherever this Hold holds it."""
+        if not self._slot:
+            return self._own
+        store = self._store()
+        return store.get(self._slot) if store else self._own
+
+    def _write(self, value: Any) -> None:
+        store = self._store() if self._slot else None
+        if store is not None:
+            store.set(self._slot, value)
+            return
+        # No store to share through — a Hold outside any host that provides one
+        # still holds, for itself alone, rather than dropping what it was given.
+        self._own = value
+
+    def _store(self) -> Any:
+        # Optional on the host, as mounts are: a host that lends no cells is a
+        # host this Hold keeps its value to itself in.
+        slots = getattr(self._host, "slots", None)
+        return slots() if callable(slots) else None
+
+    def _forget(self) -> None:
+        """Back to how the service started.
+
+        The counts go with the value: they say how often each side has called
+        for what is held now, and left running across a clear they would
+        describe a value that is gone.
+        """
+        self._own = None
+        if self._slot:
+            store = self._store()
+            if store is not None:
+                store.set(self._slot, None)
+        self._read_count = 0
+        self._write_count = 0
+
+    def _notify(self, payload: JsonRecord) -> None:
+        if self._host:
+            self._host.notify(payload, self.uuid)
+
+
+def _carried_value(input: Any, prop: str) -> Any:
+    """The value an input carries for the held property, if it carries one at
+    all — anything else makes the call a read rather than a write."""
+    if not isinstance(input, dict):
+        return _MISSING
+    return input.get(prop, _MISSING)
+
+
+#: Beyond this, what is held is described rather than sent.
+_REPORTABLE_LIMIT = 2048
+
+
+def _reportable(value: Any) -> Any:
+    """What is held, as something safe to put in state.
+
+    State is read back into the board and sent to everyone watching, so what a
+    Hold reports has to be worth carrying. A value that is bytes, or simply
+    large — a rendered document, an audio buffer — is described instead of
+    copied: the size is what a reader is looking for at that point, and the
+    value itself is on its way to whatever asked for it regardless.
+    """
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return f"[{len(bytes(value))} bytes]"
+    try:
+        encoded = json.dumps(value)
+    except (TypeError, ValueError):
+        return f"[{type(value).__name__}]"
+    if len(encoded) > _REPORTABLE_LIMIT:
+        return f"[{type(value).__name__}, {len(encoded)} characters]"
+    return value