@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-python/src/hkp/services/tracks.py

@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+# Service Documentation
+# Service ID: tracks
+# Service Name: Tracks
+# Runtime: hkp-python
+# Modes: serial | parallel
+# Key Config: tracks, reduce, run, bypass
+# IO: in=any -> out=what the reducer made of the tracks' answers
+# Arrays: the answers are an array, one element per track, in declaration order
+# Binary: handed to every track untouched
+# MixedData: not native in runtime
+
+"""Several pipelines over one input, and one answer out.
+
+``iterator`` runs one pipeline over many items. This is the other half of that
+pair: many pipelines over one item. A board that has to do two unrelated things
+with the same value — write it to a table *and* tell somebody about it, ask
+three services and compare — has otherwise to fake it by putting those things in
+a row and teaching each of them to pass its input through. That works and says
+nothing: nothing in the board records that they are siblings rather than a
+sequence, in which order they may run, or which of their answers matters.
+
+    input ──┬── track ── answer ──┬── reduce ── output
+            ├── track ── answer ──┤
+            └── track ── answer ──┘
+
+**Every track is given the same input.** They do not feed each other, which is
+the whole point: a track can be read on its own.
+
+**The answers are a list with one element per track, nulls included.** A track
+that stopped — declined, found nothing, failed — leaves a hole rather than being
+left out, so position still names the track that produced it.
+
+**``reduce`` is a pipeline, not a list of strategies.** It is given
+``{"input": …, "results": [...]}``: the value the tracks were run on, and what
+they answered. Carrying on as though the tracks were side effects is then
+``{"=": "params.input"}``. With no reducer the answers travel on as they are,
+and a reducer returning ``None`` stops the pipeline as any service does.
+
+**``run`` is recorded and honoured where a runtime can.** This one cannot: the
+python runtime processes synchronously, so tracks here always run one at a time.
+A board says the same thing in either runtime and gets the same answers; only
+the wall clock differs.
+"""
+
+from typing import Any, Callable
+
+from ..types import (
+    JsonRecord,
+    NotifyCallback,
+    RuntimeHost,
+    ServiceConfiguration,
+    ServiceCreator,
+    ServiceRegistryEntry,
+    SlotStore,
+)
+from .nested_pipeline import NestedPipeline
+from .sub_service import _is_json_record
+
+TRACKS_DESCRIPTOR = ServiceRegistryEntry(
+    service_id="tracks",
+    service_name="Tracks",
+    version="v1",
+    capabilities=["subservices"],
+)
+
+RUN_MODES = ("serial", "parallel")
+
+#: What the reducer answers to when an edit names a branch. A track may not take it.
+REDUCE = "reduce"
+
+
+class _Track:
+    def __init__(self, name: str, bypass: bool, pipeline: NestedPipeline) -> None:
+        self.name = name
+        self.bypass = bypass
+        self.pipeline = pipeline
+
+
+class TracksService:
+    service_id = TRACKS_DESCRIPTOR.service_id
+    service_name = TRACKS_DESCRIPTOR.service_name
+    version = TRACKS_DESCRIPTOR.version
+    capabilities = TRACKS_DESCRIPTOR.capabilities
+
+    def __init__(self, config: ServiceConfiguration, create_service: ServiceCreator) -> None:
+        self.uuid = config.uuid
+        self._create_service = create_service
+        self._host: RuntimeHost | None = None
+        self._bypass = False
+        self._run = "serial"
+        self._tracks: list[_Track] = []
+        #: The cells this service's pipelines hold values in.
+        #:
+        #: Owned here, like an endpoint's: the branches are pipelines of one
+        #: arrangement, so a value one leaves for another belongs to this
+        #: service rather than to the runtime around it — and two of these on a
+        #: runtime may both use a name without meeting.
+        self._slot_store = SlotStore()
+        self._reduce = NestedPipeline(
+            f"{self.uuid}:{REDUCE}", create_service, "Tracks", self.uuid
+        )
+        self._reduce.share_slots(self._slot_store)
+        self._last_error = ""
+
+        if config.state:
+            self.configure(config.state)
+
+    def set_host(self, host: RuntimeHost) -> None:
+        self._host = host
+        self._reduce.attach(host)
+        for track in self._tracks:
+            track.pipeline.attach(host)
+
+    def find_nested(self, instance_id: str) -> Any | None:
+        """The nested service a scoped address names, tracks first and in
+        declaration order, then the reducer. A name used in two tracks resolves
+        to the earlier, which is the cost of addressing a branch by what is in
+        it rather than by the branch's own name.
+        """
+        for track in self._tracks:
+            found = track.pipeline.find(instance_id)
+            if found is not None:
+                return found
+        return self._reduce.find(instance_id)
+
+    def get_state(self) -> JsonRecord:
+        return {
+            "bypass": self._bypass,
+            "run": self._run,
+            "tracks": [
+                {"name": t.name, "bypass": t.bypass, "pipeline": t.pipeline.state()}
+                for t in self._tracks
+            ],
+            "reduce": self._reduce.state(),
+            "error": self._last_error,
+        }
+
+    def configure(self, config: JsonRecord) -> JsonRecord:
+        # An edit naming a track is about one pipeline under this service rather
+        # than about the service; without it there is no way to say which.
+        if isinstance(config.get("track"), str):
+            self._configure_track(config["track"], config)
+            return self.get_state()
+
+        if isinstance(config.get("bypass"), bool):
+            self._bypass = config["bypass"]
+        if isinstance(config.get("run"), str) and config["run"] in RUN_MODES:
+            self._run = config["run"]
+        if isinstance(config.get("tracks"), list):
+            self._set_tracks(config["tracks"])
+        if isinstance(config.get("reduce"), list):
+            self._reduce.set_pipeline(config["reduce"])
+
+        return self.get_state()
+
+    def process(self, input: Any, _notify: NotifyCallback) -> Any:
+        if self._bypass or not self._tracks:
+            return input
+
+        parent = self._host.current_context() if self._host else None
+        results = [self._run_track(track, input, parent) for track in self._tracks]
+
+        if self._reduce.is_empty():
+            return results
+        return self._reduce.process({"input": input, "results": results}, parent)
+
+    def destroy(self) -> None:
+        self._reduce.destroy()
+        for track in self._tracks:
+            track.pipeline.destroy()
+        self._tracks = []
+        self._host = None
+
+    # ── Private ────────────────────────────────────────────────────────────────
+
+    def _run_track(self, track: _Track, input: Any, parent: Any) -> Any:
+        """One track's answer, or None where it has none.
+
+        A track that raises is reported and leaves a hole rather than taking the
+        others down with it: they were given the same input and have nothing to
+        do with each other, which is exactly why they are tracks.
+        """
+        if track.bypass or track.pipeline.is_empty():
+            return None
+        try:
+            return track.pipeline.process(input, parent)
+        except Exception as error:  # noqa: BLE001 - reported, not swallowed
+            self._last_error = f"track '{track.name}' failed: {error}"
+            if self._host:
+                self._host.log("error", "service.failed", {"message": self._last_error})
+            return None
+
+    def _set_tracks(self, value: list[Any]) -> None:
+        previous = {track.name: track for track in self._tracks}
+        nxt: list[_Track] = []
+
+        for index, entry in enumerate(value):
+            if not _is_json_record(entry):
+                raise ValueError("every track is an object with a name and a pipeline")
+            # A name is what a reducer, a log and a panel call this track.
+            # Unnamed, it is called by the position it was declared in.
+            name = entry["name"] if isinstance(entry.get("name"), str) and entry.get("name") else str(index)
+            if name == REDUCE:
+                raise ValueError(f"'{REDUCE}' is the reducer's name and cannot be a track's")
+            if any(track.name == name for track in nxt):
+                raise ValueError(f"two tracks are both called '{name}'")
+
+            # A track already here keeps its pipeline: rebuilding it would
+            # destroy what is running inside it because a track beside it was
+            # edited.
+            existing = previous.pop(name, None)
+            pipeline = (
+                existing.pipeline
+                if existing
+                else NestedPipeline(
+                    f"{self.uuid}:{name}", self._create_service, "Tracks", self.uuid
+                )
+            )
+            pipeline.share_slots(self._slot_store)
+            if isinstance(entry.get("pipeline"), list):
+                pipeline.set_pipeline(entry["pipeline"])
+            if existing is None and self._host:
+                pipeline.attach(self._host)
+
+            nxt.append(_Track(name, entry.get("bypass") is True, pipeline))
+
+        # Whatever is left was removed, and holds services nothing will reach.
+        for dropped in previous.values():
+            dropped.pipeline.destroy()
+        self._tracks = nxt
+
+    def _configure_track(self, name: str, config: JsonRecord) -> None:
+        track = next((t for t in self._tracks if t.name == name), None)
+        pipeline = self._reduce if name == REDUCE else (track.pipeline if track else None)
+        if pipeline is None:
+            raise ValueError(f"no track called '{name}'")
+
+        if isinstance(config.get("bypass"), bool) and track:
+            track.bypass = config["bypass"]
+        if isinstance(config.get("pipeline"), list):
+            pipeline.set_pipeline(config["pipeline"])
+        if _is_json_record(config.get("appendService")):
+            pipeline.append(config["appendService"])
+        if isinstance(config.get("removeService"), str):
+            pipeline.remove(config["removeService"])
+        if _is_json_record(config.get("configureService")):
+            payload = config["configureService"]
+            if isinstance(payload.get("instanceId"), str) and _is_json_record(payload.get("state")):
+                pipeline.configure_service(payload["instanceId"], payload["state"])

package/bundle/hkp-python/src/hkp/types.py

@@ -0,0 +1,412 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Literal, Protocol, runtime_checkable
+
+from .secrets import SecretEntry, SecretVault
+
+# Generic JSON object type
+JsonRecord = dict[str, Any]
+
+# Callback signatures
+NotifyCallback = Callable[[Any, str | None], None]
+NotificationCallback = Callable[["RuntimeNotification"], None]
+ServiceCreator = Callable[["ServiceConfiguration"], "HostedService"]
+
+
+@dataclass
+class ServiceRegistryEntry:
+    service_id: str
+    service_name: str
+    version: str | None = None
+    capabilities: list[str] | None = None
+
+
+@dataclass
+class ServiceConfiguration:
+    service_id: str
+    uuid: str
+    name: str | None = None
+    service_name: str | None = None
+    state: JsonRecord | None = None
+
+
+@dataclass
+class RuntimeConfiguration:
+    id: str
+    name: str
+    board_name: str = ""
+    #: Whether this runtime should be torn down once the last client that was
+    #: connected to it disconnects.
+    #:
+    #: Declared by whoever creates it, because only they know: a browser
+    #: running a board says ``True`` — it is the controller, and its runtimes
+    #: should not outlive it — while a coordinator, a config file or a script
+    #: says nothing and gets a runtime that lives until it is deleted.
+    #:
+    #: Absent means persist. Cleanup is opted into, so nothing that exists
+    #: today starts disappearing, and a runtime is never reaped because of who
+    #: happened to connect to it.
+    garbage_collected: bool = False
+    #: Whether this runtime records anything at all.
+    #:
+    #: Off unless the board turns it on. A board that is not being looked into
+    #: has no reason to be writing a line per call to somebody's disk, and a log
+    #: kept by default is one nobody decided to keep — including for the data it
+    #: holds.
+    logging: bool = False
+    #: The least severe level this runtime records.
+    #:
+    #: The flow itself — every service call and return — is recorded at
+    #: ``debug``, so this is what decides whether a board keeps a trace of what
+    #: ran or only what its services chose to say.
+    log_level: str = "info"
+
+    #: Whether an entry may carry the ``data`` a service passed with it.
+    #:
+    #: Only service-authored entries have one: the flow this runtime records for
+    #: itself never carries the values passing through, so nothing reaches a log
+    #: unless a service was configured to put it there. That per-service opt-in
+    #: is the real gate, which is why this defaults to allowed. Setting it False
+    #: is a board-wide override for a deployment that must never write payloads.
+    log_data: bool = True
+    #: Values for the ``{{secret.<alias>}}`` references this runtime's services
+    #: carry, by alias.
+    #:
+    #: They ride with the create payload because provisioning is one call: the
+    #: services in it are constructed *and* configured before it returns, and a
+    #: service that opens a connection while being configured needs its
+    #: credential by then. Sending them later would be too late for exactly the
+    #: services that have one.
+    #:
+    #: They are unpacked into the runtime's vault and go no further — never into
+    #: a service's state, never into a serialized runtime, never back out.
+    secrets: dict[str, SecretEntry] = field(default_factory=dict)
+    #: Descriptors for the ``hkp-asset://<id>`` references this runtime's
+    #: services carry, by id — with the create payload for the reason secrets
+    #: are, and only what the services reference. They go into the runtime's
+    #: asset store and never into service state. See ``assets.py``.
+    assets: dict[str, dict[str, Any]] = field(default_factory=dict)
+    services: list[ServiceConfiguration] = field(default_factory=list)
+
+
+@dataclass
+class ServiceDescriptor:
+    service_id: str
+    service_name: str
+    uuid: str
+    state: JsonRecord
+    version: str | None = None
+    capabilities: list[str] | None = None
+
+
+@dataclass
+class RuntimeDescriptor:
+    id: str
+    name: str
+    board_name: str
+    services: list[ServiceDescriptor]
+    inputs: list[dict[str, Any]] = field(default_factory=list)
+    output_url: str | None = None
+
+
+@dataclass
+class RuntimeNotification:
+    instance_id: str
+    payload: Any
+    context: "ProcessContext | None" = None
+
+
+CallerRole = Literal["owner", "member"]
+
+
+@dataclass
+class Caller:
+    """Who took the action a run began with.
+
+    Stated by the server that verified their token, never read from what they
+    sent: a payload saying who its sender is proves nothing about them. A run
+    nobody began — a timer tick, a request arriving at a mount — has none, and
+    neither does anything on a server running without authentication, where
+    there is nobody to tell apart.
+    """
+
+    #: The token's ``sub``.
+    sub: str
+    #: Present only when the token carried a verified one; normalised.
+    email: str | None = None
+    #: What the board's member list calls that email. Set by a coordinator
+    #: that keeps such a list, and absent everywhere else — a name from the
+    #: token would be the person's own choice.
+    name: str | None = None
+    #: What they are to the board: its ``owner``, in whose tenant it runs, or a
+    #: ``member`` its list admitted. Stated by whoever admitted them.
+    role: CallerRole | None = None
+
+    def to_wire(self) -> dict[str, Any]:
+        wire: dict[str, Any] = {"sub": self.sub}
+        if self.email:
+            wire["email"] = self.email
+        if self.name:
+            wire["name"] = self.name
+        if self.role:
+            wire["role"] = self.role
+        return wire
+
+
+@dataclass(frozen=True)
+class PersonRunActor:
+    """The verified person acting in a run and their authority deadline."""
+
+    kind: Literal["person"]
+    sub: str
+    expires_at: int
+    email: str | None = None
+    name: str | None = None
+    role: CallerRole | None = None
+
+    def to_wire(self) -> dict[str, Any]:
+        wire: dict[str, Any] = {
+            "kind": self.kind,
+            "sub": self.sub,
+            "expiresAt": self.expires_at,
+        }
+        if self.email:
+            wire["email"] = self.email
+        if self.name:
+            wire["name"] = self.name
+        if self.role:
+            wire["role"] = self.role
+        return wire
+
+
+@dataclass(frozen=True)
+class SourceRunActor:
+    """A run created without delegated person authority."""
+
+    kind: Literal["board", "mount", "local"]
+
+    def to_wire(self) -> dict[str, Any]:
+        return {"kind": self.kind}
+
+
+RunActor = PersonRunActor | SourceRunActor
+
+
+@dataclass
+class ProcessContext:
+    """What travels with a process call rather than with the data it carries.
+
+    The ordered service list says what runs; this says which invocation it is
+    running as. The distinction matters as soon as anything has to attribute
+    work after the fact — which run produced this, and what invoked that run —
+    because the payload cannot answer it: the same data can flow through the
+    same services for entirely unrelated reasons.
+
+    Kept deliberately separate from ``request_id``, which the browser and hkp-rt
+    runtimes carry for a different purpose: that is a *reply address*, exists
+    only while someone awaits a response, and is consumed on resolution. A run
+    outlives any number of those, so the two are not interchangeable — see
+    plans/TODO-CONSOLIDATION.md section 4.
+    """
+
+    #: Identifies one invocation of a board — one webhook, one timer tick, one
+    #: user action — across every service and runtime it reaches.
+    run_id: str
+    #: What is acting in this run. Only the person variant carries identity
+    #: and an authority deadline.
+    actor: RunActor
+    #: The run this one was invoked from, for a nested pipeline. Absent on a run
+    #: triggered from outside rather than from inside another run, which is what
+    #: makes a trace reconstructable as a tree rather than a list.
+    parent_run_id: str | None = None
+    #: Where to send a result somebody is waiting for. Absent for the fire-and-
+    #: forget calls that make up most traffic. Unused by this runtime today;
+    #: named here so the shape matches the runtimes that do carry one.
+    request_id: str | None = None
+
+
+LogLevel = Literal["debug", "info", "warn", "error"]
+
+
+@dataclass
+class LogEntry:
+    """One thing worth recording about a run.
+
+    A board's log is assembled by its coordinator from every runtime it spans,
+    because only the coordinator can see the whole board — a log held per
+    runtime would have to be stitched back together by timestamp to answer the
+    first question anyone asks it, which is what one run did.
+
+    ``data`` is the only free-form field and therefore the only one that can
+    carry something a service did not mean to record. It is dropped unless a
+    board asks for it, so a service that forgets to redact can only leak
+    through a channel somebody deliberately opened.
+    """
+
+    run_id: str
+    #: ISO 8601, set by the runtime that produced the entry.
+    ts: str
+    runtime_id: str
+    service_uuid: str
+    level: LogLevel
+    #: What happened, as a short stable name a reader can group by.
+    event: str
+    parent_run_id: str | None = None
+    data: Any = None
+    duration_ms: float | None = None
+    #: The ``sub`` of whoever began the run, when somebody did. Enough to
+    #: answer "who did this" from a board's log without it collecting addresses.
+    caller: str | None = None
+
+    def to_wire(self) -> dict[str, Any]:
+        """The entry as the other runtimes spell it.
+
+        A board's log is one file assembled from every runtime it spans, so an
+        entry leaving this process has to be indistinguishable from one written
+        by hkp-node or hkp-rt — camelCase, and the optional fields absent rather
+        than null, since a reader that filters on ``data`` being present would
+        otherwise see every entry as carrying one.
+        """
+        wire: dict[str, Any] = {
+            "runId": self.run_id,
+            "ts": self.ts,
+            "runtimeId": self.runtime_id,
+            "serviceUuid": self.service_uuid,
+            "level": self.level,
+            "event": self.event,
+        }
+        if self.parent_run_id:
+            wire["parentRunId"] = self.parent_run_id
+        if self.caller:
+            wire["caller"] = self.caller
+        if self.data is not None:
+            wire["data"] = self.data
+        if self.duration_ms is not None:
+            wire["durationMs"] = self.duration_ms
+        return wire
+
+
+class RuntimeHost(Protocol):
+    def process_from(
+        self,
+        start_after_uuid: str,
+        data: Any,
+        on_notification: NotificationCallback,
+        context: "ProcessContext | None" = None,
+    ) -> Any: ...
+
+    def notify(
+        self,
+        payload: Any,
+        instance_id: str,
+        context: "ProcessContext | None" = None,
+    ) -> None:
+        """Report something to whoever is watching.
+
+        Said from inside a run, it is reported as part of it. ``context`` names
+        the run instead where the report is being carried out of a nested
+        pipeline, which knows the run it was made in when the runtime around it
+        does not.
+        """
+        ...
+
+    def emit_result(self, output: Any) -> None: ...
+
+    def spawn(self, coro: Any) -> bool:
+        """Run a coroutine on the server's loop, from wherever this is called.
+
+        A service that starts work during ``process`` is running on a worker
+        thread, where there is no loop to schedule on. False means there was
+        none to schedule on at all, so the caller can say so in its own words.
+        """
+        ...
+
+    def log(self, level: LogLevel, event: str, data: Any = None) -> None:
+        """Record something about the run in progress.
+
+        Unlike ``notify``, which exists for whoever is watching and may be
+        dropped when nobody is, an entry has to survive with nobody attached —
+        a board running unwatched is exactly the case a log is for. The run and
+        the service are taken from the call in progress, so a service says only
+        what happened.
+        """
+        ...
+
+    def log_settings(self) -> dict[str, bool]:
+        """What the runtime around a nested pipeline records.
+
+        A nested runtime is built from a service's own configuration, which says
+        nothing about logging — so without asking, a sub-pipeline would sit
+        silent inside a board that is being looked into.
+        """
+        ...
+
+    def forward_log(self, entry: "LogEntry") -> None:
+        """Pass an entry a nested pipeline produced outward, unchanged.
+
+        Distinct from ``log`` because the entry already names its own run and
+        service: re-deriving those from the call in progress would relabel work
+        done inside a sub-pipeline as the work of the service hosting it, which
+        is the nesting the entry exists to record.
+        """
+        ...
+
+    def current_context(self) -> "ProcessContext | None":
+        """The context of the call being processed now, or None outside one.
+
+        A service that finishes its work after ``process`` returns — an HTTP
+        response arriving, a socket pushing — has left the call it belongs to by
+        the time it has something to pass on. Capturing this while still inside
+        ``process`` and handing it back to ``process_from`` is what keeps the
+        two halves recognisable as one run.
+        """
+        ...
+
+    def slots(self) -> "SlotStore | None":
+        """The cells this pipeline's values may be held in, or None.
+
+        A nested runtime answers with the store the service hosting it gave it,
+        and otherwise with its parent's — so a slot named inside a sub-pipeline
+        reaches the nearest owner that declared one, and a service that declared
+        none does not silently isolate what is nested in it.
+        """
+        ...
+
+
+class SlotStore:
+    """Named cells two pipelines can share a value through.
+
+    A pipeline pass carries one value and ends; anything that has to survive
+    until a *different* pipeline runs has nowhere to live. A store gives it a
+    name, and whoever owns the pipelines that must share decides which store
+    they see — which is what keeps the sharing scoped to the arrangement that
+    needs it rather than being ambient across a runtime.
+
+    Deliberately not a cache: nothing expires, nothing is computed on a miss.
+    It is a cell, and the services that read and write it say what it means.
+    """
+
+    def __init__(self) -> None:
+        self._cells: dict[str, Any] = {}
+
+    def get(self, name: str) -> Any:
+        return self._cells.get(name)
+
+    def set(self, name: str, value: Any) -> None:
+        self._cells[name] = value
+
+
+@runtime_checkable
+class HostedService(Protocol):
+    service_id: str
+    service_name: str
+    uuid: str
+    version: str | None
+    capabilities: list[str] | None
+
+    def configure(self, config: JsonRecord) -> JsonRecord: ...
+    def get_state(self) -> JsonRecord: ...
+    def process(self, input: Any, notify: NotifyCallback) -> Any: ...
+    def set_host(self, host: RuntimeHost) -> None: ...
+    def destroy(self) -> None: ...