@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-node/src/types.ts

@@ -0,0 +1,465 @@
+import { MountHandle, MountHandlers } from "./mounts";
+import { SecretVault } from "./secrets";
+import { AssetDescriptor, AssetStore } from "./assets";
+
+export type JsonRecord = Record<string, unknown>;
+
+export type ServiceRegistryEntry = {
+  serviceId: string;
+  serviceName: string;
+  version?: string;
+  capabilities?: string[];
+};
+
+export type ServiceConfiguration = {
+  serviceId: string;
+  uuid: string;
+  name?: string;
+  serviceName?: string;
+  state?: JsonRecord;
+};
+
+export type RuntimeConfiguration = {
+  id: string;
+  name: string;
+  boardName?: string;
+  /**
+   * Whether this runtime should be torn down once the last client that was
+   * connected to it disconnects.
+   *
+   * Declared by whoever creates it, because only they know: a browser
+   * provisioning a board it is running says `true` — it is the controller, and
+   * its runtimes should not outlive it — while a coordinator, a config file or
+   * a script says nothing and gets a runtime that lives until it is deleted.
+   *
+   * Absent means persist. Cleanup is opted into, so nothing that exists today
+   * starts disappearing, and a runtime is never reaped because of who happened
+   * to connect to it.
+   */
+  garbageCollected?: boolean;
+  /**
+   * Values for the `{{secret.<alias>}}` references this runtime's services
+   * carry, by alias.
+   *
+   * They ride with the create payload because provisioning is one call: the
+   * services in it are constructed *and* configured before it returns, and a
+   * service that opens a connection on configure needs its credential by then.
+   * Sending them later would be too late for exactly the services that have
+   * one.
+   *
+   * They are unpacked into the runtime's vault and go no further — never into
+   * a service's state, never into a serialized runtime, never back out over
+   * the wire.
+   */
+  secrets?: Record<string, { value: string; audience?: string[] }>;
+  /**
+   * Descriptors for the `hkp-asset://<id>` references this runtime's services
+   * carry, by id.
+   *
+   * With the create payload for the reason secrets are: provisioning is one
+   * call, and a service that loads its content while being configured needs
+   * the descriptor by then. Only what this runtime's services reference — a
+   * board's inline assets can be large, and a runtime holds none it has no use
+   * for. They go into the runtime's asset store and never into service state.
+   */
+  assets?: Record<string, AssetDescriptor>;
+  /**
+   * Whether this runtime records anything at all.
+   *
+   * Off unless the board turns it on. A board that is not being looked into has
+   * no reason to be writing a line per call to somebody's disk, and a log kept
+   * by default is one nobody decided to keep — including for the data it holds.
+   * Turning it on is the act that makes the rest of this meaningful.
+   */
+  logging?: boolean;
+  /**
+   * The least severe level this runtime records. Absent means `info`.
+   *
+   * The flow itself — every service call and return — is recorded at `debug`,
+   * so this is what decides whether a board keeps a trace of what ran or only
+   * what its services chose to say. That is the difference between a log that
+   * answers "where did this stop" and one that costs almost nothing to keep.
+   */
+  logLevel?: LogLevel;
+  /**
+   * Whether this runtime's log entries may carry their `data` payload.
+   *
+   * Declared by the board, in the runtime's state, because it is a decision
+   * about what the board is willing to record rather than one a service can
+   * make for itself: `data` is the one free-form field, so it is the only place
+   * a service can put something it did not mean to keep. Absent means off, so
+   * the quiet default is the safe one and switching it on is deliberate.
+   */
+  logData?: boolean;
+  services: ServiceConfiguration[];
+  inputs?: Array<Record<string, unknown>>;
+};
+
+export type RuntimeDescriptor = {
+  id: string;
+  name: string;
+  /** How this runtime is cleaned up; see RuntimeConfiguration. Reported so a
+   *  client can see whether it outlives them. */
+  garbageCollected?: boolean;
+  boardName: string;
+  services: ServiceDescriptor[];
+  inputs: Array<Record<string, unknown>>;
+  outputUrl?: string;
+};
+
+export type ServiceDescriptor = {
+  serviceId: string;
+  serviceName: string;
+  version?: string;
+  capabilities?: string[];
+  uuid: string;
+  state: JsonRecord;
+};
+
+export interface HostedService {
+  readonly serviceId: string;
+  readonly serviceName: string;
+  readonly version?: string;
+  readonly capabilities?: string[];
+  readonly uuid: string;
+  configure(config: JsonRecord): JsonRecord;
+  getState(): JsonRecord;
+  process(
+    input: unknown,
+    notify: (payload: unknown, instanceId?: string) => void,
+  ): unknown;
+  setHost?(host: RuntimeHost): void;
+  /**
+   * The tenant and board this service now answers to.
+   *
+   * Only a service holding a pipeline of its own needs this: its nested runtime
+   * was built before anything knew the scope, and being told once at `setHost`
+   * only reaches one level down. A scope that stops propagating is not a
+   * cosmetic problem — a nested runtime's default owner is the anonymous one,
+   * so a service that never heard the real scope reads and writes what every
+   * tenant shares.
+   */
+  setScope?(scope: RuntimeScope): void;
+  /**
+   * The service one segment of a scoped address names inside this one.
+   *
+   * Only a service holding a pipeline answers: it is what lets an address
+   * reach past the runtime's flat list into what a scope contains. A service
+   * with several pipelines searches them in declaration order. See
+   * `address.ts`.
+   */
+  /**
+   * Try again to claim an endpoint, now that the runtime can serve one.
+   *
+   * A mount is claimed once, eagerly, when a service is given its host — and a
+   * service inside a pipeline is built before the pipeline has a host to pass
+   * on, so its first attempt reaches nothing. Secrets and slots do not need
+   * this because they are looked up on each use; a mount is held, so the
+   * service that gave up on one has to be told to ask again.
+   *
+   * Implemented by a service that claims a mount, and by one holding a
+   * pipeline, which passes it down.
+   */
+  remount?(): void;
+  findNested?(instanceId: string): HostedService | undefined;
+  /**
+   * Runs from a service inside this one to the end of the pipeline holding it,
+   * the way `processAt` does at the top level.
+   *
+   * Only a service whose pipeline is a chain answers. An endpoint's entries and
+   * a Tracks branch are driven by the thing that owns them — a request
+   * arriving, a fan-out — so entering one from outside would run half of an
+   * arrangement whose other half never happened. Those stay readable and
+   * configurable by address without being enterable by one.
+   */
+  processNested?(
+    address: string,
+    input: unknown,
+    context?: ProcessContext,
+  ): Promise<unknown>;
+  destroy?(): void;
+}
+
+export type ServiceCreator = (config: ServiceConfiguration) => HostedService;
+
+export type HostedServiceFactory = {
+  descriptor: ServiceRegistryEntry;
+  create: (
+    config: ServiceConfiguration,
+    createService: ServiceCreator,
+  ) => HostedService;
+};
+
+export type RuntimeNotification = {
+  instanceId: string;
+  payload: unknown;
+  /** The run in which it was raised, captured at the service call boundary. */
+  context?: ProcessContext;
+};
+
+/**
+ * The tenant and board a runtime belongs to.
+ *
+ * Board rather than runtime, because a board is the unit a person thinks in:
+ * two runtimes of one board are two halves of one app, and something one half
+ * stored is something the other half should find.
+ */
+export type RuntimeScope = {
+  /** The authenticated `sub`, or ANONYMOUS_SUB where auth is off. */
+  owner: string;
+  boardName: string;
+};
+
+/**
+ * What travels with a process call rather than with the data it carries.
+ *
+ * The ordered service list says what runs; this says which invocation it is
+ * running as. The distinction matters as soon as anything has to attribute work
+ * after the fact — which run produced this, and what invoked that run — because
+ * the payload cannot answer it: the same data can flow through the same
+ * services for entirely unrelated reasons.
+ *
+ * Kept deliberately separate from `requestId`, which the browser and hkp-rt
+ * runtimes carry for a different purpose: `requestId` is a *reply address*,
+ * exists only while someone awaits a response, and is consumed on resolution.
+ * A run outlives any number of those, so the two are not interchangeable — see
+ * plans/TODO-CONSOLIDATION.md section 4.
+ */
+/**
+ * Who took the action a run began with.
+ *
+ * Stated by the server that verified their token, never read from what they
+ * sent: a payload saying who its sender is proves nothing about them. A run
+ * nobody began — a timer tick, a request arriving at a mount — has none, and
+ * neither does anything on a server running without authentication, where
+ * there is nobody to tell apart.
+ */
+export type Caller = {
+  /** The token's `sub`. */
+  sub: string;
+  /** Present only when the token carried a verified one; normalised. */
+  email?: string;
+  /**
+   * What the board's member list calls that email. Set by a coordinator that
+   * keeps such a list, and absent everywhere else — a name from the token
+   * would be the person's own choice.
+   */
+  name?: string;
+  /**
+   * What they are to the board: its `owner`, in whose tenant it runs, or a
+   * `member` its list admitted. Stated by whoever admitted them — the
+   * coordinator for a bridge, the runtime server for a client acting in its
+   * own tenant — so a statement can tell who may set things up from who may
+   * only take part.
+   */
+  role?: CallerRole;
+};
+
+export type CallerRole = "owner" | "member";
+
+/** What is acting in a run. Person identity is present only in that case. */
+export type RunActor =
+  | ({ kind: "person"; expiresAt: number } & Caller)
+  | { kind: "board" }
+  | { kind: "mount" }
+  | { kind: "local" };
+
+export type ProcessContext = {
+  /**
+   * Identifies one invocation of a board — one webhook, one timer tick, one
+   * user action — across every service and runtime it reaches.
+   */
+  runId: string;
+  /**
+   * The run this one was invoked from, for a nested pipeline. Absent on a run
+   * that was triggered from outside rather than from inside another run, which
+   * is what makes a trace reconstructable as a tree rather than a list.
+   */
+  parentRunId?: string;
+  /**
+   * Where to send a result somebody is waiting for. Absent for the fire-and-
+   * forget calls that make up most traffic. Unused by this runtime today; named
+   * here so the shape matches the runtimes that do carry one.
+   */
+  requestId?: string;
+  /**
+   * What is acting in this run. A person actor is stated only by the server
+   * that verified them and carries the deadline of that delegated authority.
+   */
+  actor: RunActor;
+};
+
+export type LogLevel = "debug" | "info" | "warn" | "error";
+
+/**
+ * One thing worth recording about a run.
+ *
+ * A board's log is assembled by its coordinator from every runtime it spans,
+ * because only the coordinator can see the whole board — a log held per runtime
+ * would have to be stitched back together by timestamp to answer the first
+ * question anyone asks it, which is what one run did.
+ *
+ * `data` is the only free-form field and therefore the only one that can carry
+ * something a service did not mean to record. It is dropped unless a board asks
+ * for it, so a service that forgets to redact can only leak through a channel
+ * somebody deliberately opened.
+ */
+export type LogEntry = {
+  runId: string;
+  parentRunId?: string;
+  /** ISO 8601, set by the runtime that produced the entry. */
+  ts: string;
+  runtimeId: string;
+  serviceUuid: string;
+  level: LogLevel;
+  /** What happened, as a short stable name a reader can group by. */
+  event: string;
+  data?: unknown;
+  durationMs?: number;
+  /**
+   * The `sub` of whoever began the run, when somebody did. Enough to answer
+   * "who did this" from a board's log without the log collecting addresses.
+   */
+  caller?: string;
+};
+
+/**
+ * Named cells two pipelines can share a value through.
+ *
+ * A pipeline pass carries one value and ends; anything that has to survive
+ * until a *different* pipeline runs has nowhere to live. A store gives it a
+ * name, and whoever owns the pipelines that must share decides which store they
+ * see — which is what keeps the sharing scoped to the arrangement that needs it
+ * rather than being ambient across a runtime.
+ *
+ * Deliberately not a cache: nothing expires, nothing is computed on a miss. It
+ * is a cell, and the services that read and write it say what it means.
+ */
+export interface SlotStore {
+  get(name: string): unknown;
+  set(name: string, value: unknown): void;
+}
+
+export interface RuntimeHost {
+  /**
+   * Runs the services after `startAfterUuid` and answers with what they
+   * produced.
+   *
+   * Asynchronous because a pass is: the runtime awaits each service, so a
+   * service that pulls the ones behind it has to await them too.
+   */
+  processFrom(
+    startAfterUuid: string,
+    data: unknown,
+    onNotification: (notification: RuntimeNotification) => void,
+    context?: ProcessContext,
+  ): Promise<unknown>;
+  /**
+   * Reports something to whoever is watching. Said from inside a run, it is
+   * reported as part of it; `context` names the run instead where the report
+   * is being carried out of a nested pipeline, which knows the run it was made
+   * in when the runtime around it does not.
+   */
+  notify(
+    payload: unknown,
+    instanceId: string,
+    context?: ProcessContext | null,
+  ): void;
+  /**
+   * Hands a value to whatever follows this runtime. Called from inside a run,
+   * the value goes on as part of it — the next runtime of a deployed board is
+   * told the same run and the same caller. `context` names the run where the
+   * value is handed on from outside the call that produced it.
+   */
+  emitResult(output: unknown, context?: ProcessContext | null): void;
+  /**
+   * The context of the call currently being processed, or null outside one.
+   *
+   * A service that finishes its work after its `process` returns — an HTTP
+   * response arriving, a socket pushing — has left the call it belongs to by
+   * the time it has something to pass on. Capturing this while still inside
+   * `process` and handing it back to `processFrom` is what keeps the two halves
+   * recognisable as one run.
+   */
+  currentContext(): ProcessContext | null;
+  /**
+   * Record something about the run in progress.
+   *
+   * Unlike `notify`, which exists for whoever is watching and may be dropped
+   * when nobody is, an entry has to survive with nobody attached — a board
+   * running unwatched is exactly the case a log is for. The run and the service
+   * are taken from the call in progress, so a service says only what happened.
+   */
+  log(level: LogLevel, event: string, data?: unknown): void;
+  /**
+   * Pass an entry a nested pipeline produced outward, unchanged.
+   *
+   * Distinct from `log` because the entry already names its own run and
+   * service: re-deriving those from the call in progress would relabel work
+   * done inside a sub-pipeline as the work of the service hosting it, which is
+   * the nesting the entry exists to record.
+   */
+  forwardLog(entry: LogEntry): void;
+  /**
+   * What the runtime around a nested pipeline records, so the pipeline can
+   * record the same.
+   *
+   * A nested runtime is built from a service's own configuration, which says
+   * nothing about logging — so without asking, a sub-pipeline would sit silent
+   * inside a board that is being looked into, which is where its entries are
+   * most wanted.
+   */
+  logSettings(): { logging: boolean; logData: boolean; logLevel: LogLevel };
+  /**
+   * Who this runtime belongs to and which board it is part of.
+   *
+   * For a service that keeps something between calls: this server is
+   * multi-tenant and runtimes are namespaced by the authenticated `sub`, so
+   * anything durable has to be namespaced the same way or one tenant's data
+   * becomes another's. A service cannot work that out for itself — it is told
+   * its own configuration and nothing about who asked for it — which is why it
+   * comes from the host rather than from state.
+   */
+  scope(): RuntimeScope;
+  /**
+   * Claim a publicly reachable endpoint served by the shared server, for a
+   * service that needs to be called from outside (an HTTP endpoint, a
+   * signalling server). Returns null when the host cannot serve mounts — an
+   * inner sub-service pipeline, or a server that is not listening yet — in
+   * which case the service has no public endpoint and should say so in its
+   * state rather than falling back to a port of its own.
+   */
+  mount?(
+    serviceUuid: string,
+    handlers: MountHandlers,
+    options?: { mountName?: string },
+  ): MountHandle | null;
+  /**
+   * The runtime's secrets, for a service that has a credential to send.
+   *
+   * A service holds the reference it was configured with and asks here for the
+   * value, naming where it is about to send it. What comes back is used and
+   * dropped: assigning it to state would put it back on the path a board is
+   * saved from, which is the whole thing this arrangement exists to prevent.
+   */
+  secrets(): SecretVault;
+  /**
+   * The runtime's assets, for a service that consumes content by reference.
+   *
+   * A service holds `hkp-asset://<id>` and resolves it here at the moment it
+   * uses it, so an asset edited while the board runs is what the next use
+   * gets. The content is used and dropped, never put into state. Optional
+   * because a host that was not provisioned with any has nothing to answer.
+   */
+  assets?(): AssetStore;
+  /**
+   * The slots this pipeline's values may be held in, or nothing where no store
+   * has been provided.
+   *
+   * A nested runtime answers with the store the service hosting it gave it, and
+   * otherwise with its parent's — so a slot named inside a sub-pipeline reaches
+   * the nearest owner that declared one, and a service that declared none does
+   * not silently isolate what is nested in it.
+   */
+  slots?(): SlotStore | null;
+}

package/bundle/hkp-node/src/wsTickets.ts

@@ -0,0 +1,77 @@
+import { randomBytes } from "node:crypto";
+
+import { AuthenticatedUser } from "./auth";
+
+/**
+ * Single-use tickets that open a WebSocket in place of a token.
+ *
+ * A browser cannot set a header on a WebSocket handshake, so a token it
+ * authenticated one with would have to ride in the address — and an address
+ * is what proxies and access logs keep. So a browser first asks for a ticket
+ * (`POST /ws-tickets`, with its token in the `Authorization` header like any
+ * request) and opens the socket with `?ticket=` instead. A ticket opens one
+ * socket, within seconds, as whoever asked for it, and is worth nothing once
+ * written down.
+ *
+ * What the ticket remembers is what checking the token found: who the caller
+ * is, and whether this server's allowlist lets them own. The socket is then
+ * judged exactly as the token would have been, without the token.
+ *
+ * The same in hkp-python (`ws_tickets.py`) and hkp-rt (`http/server.cpp`).
+ */
+
+export type TicketHolder = {
+  /** Who the token named; null for a token that names nobody. */
+  identity: AuthenticatedUser | null;
+  /** The same person, when they may own here; null otherwise. */
+  owner: AuthenticatedUser | null;
+};
+
+/** Long enough for a browser to open the socket it asked for, and no longer. */
+export const TICKET_TTL_MS = 30_000;
+
+/** Outstanding tickets kept; past this the oldest go first. */
+const MAX_OUTSTANDING = 10_000;
+
+export class WsTickets {
+  private readonly tickets = new Map<string, TicketHolder & { expires: number }>();
+
+  constructor(private readonly now: () => number = Date.now) {}
+
+  /** A ticket for `holder`, good for one socket within TICKET_TTL_MS. */
+  mint(holder: TicketHolder): string {
+    this.prune();
+    while (this.tickets.size >= MAX_OUTSTANDING) {
+      const oldest = this.tickets.keys().next().value;
+      if (oldest === undefined) {
+        break;
+      }
+      this.tickets.delete(oldest);
+    }
+    const ticket = randomBytes(32).toString("base64url");
+    this.tickets.set(ticket, { ...holder, expires: this.now() + TICKET_TTL_MS });
+    return ticket;
+  }
+
+  /** Who a ticket was minted for — once. Unknown, used or expired: null. */
+  redeem(ticket: string): TicketHolder | null {
+    const held = this.tickets.get(ticket);
+    if (!held) {
+      return null;
+    }
+    this.tickets.delete(ticket);
+    if (held.expires < this.now()) {
+      return null;
+    }
+    return { identity: held.identity, owner: held.owner };
+  }
+
+  private prune(): void {
+    const now = this.now();
+    for (const [ticket, held] of this.tickets) {
+      if (held.expires < now) {
+        this.tickets.delete(ticket);
+      }
+    }
+  }
+}

package/bundle/hkp-node/src/yas.ts

@@ -0,0 +1,95 @@
+/**
+ * The YAS frames a runtime socket carries when what flows is not JSON.
+ *
+ * Reference implementations:
+ *   hkp-frontend/src/runtime/rest/Message.ts   (TypeScript, the peer here)
+ *   hkp-rt/lib/src/types/message.cpp           (C++, yas library)
+ *   hkp-python/src/hkp/yas.py                  (Python)
+ *
+ * Frame layout, integers little-endian:
+ *
+ *   7 bytes   YAS header, b"yas" + 4 flag/version bytes (only "yas" is checked)
+ *   uint16    message purpose
+ *   uint16    data type id
+ *   uint64    sender length
+ *   bytes     sender (ascii)
+ *   ...       payload — for BinaryData the raw bytes to the end of the frame,
+ *             for Null a single ignored byte
+ *
+ * Only those two types are read and written: a Node service has nothing to do
+ * with a FloatRingBuffer, and JSON and text travel as JSON frames.
+ */
+
+export enum MessagePurpose {
+  NOTIFICATION = 0,
+  RESULT = 1,
+  RESULT_AWAITING_RESPONSE = 2,
+  RESULT_WITH_REQUEST_ID = 3,
+}
+
+/** In sync with hkp-rt/lib/include/types/data.h. */
+export enum DataTypeId {
+  Undefined = 0,
+  FloatRingBuffer = 1,
+  JSON = 2,
+  BinaryData = 3,
+  String = 4,
+  Null = 5,
+}
+
+export type YasMessage = {
+  purpose: number;
+  dataType: number;
+  sender: string;
+  /** A Buffer for BinaryData, null for Null; undefined for a type not read. */
+  data: Buffer | null | undefined;
+};
+
+const HEADER = Buffer.from("yas0017", "latin1");
+const FIXED = HEADER.length + 2 + 2 + 8;
+
+/** The frame, or null when the bytes are not one. */
+export function decodeYasMessage(frame: Buffer): YasMessage | null {
+  if (frame.length < FIXED || frame.toString("latin1", 0, 3) !== "yas") {
+    return null;
+  }
+  let offset = HEADER.length;
+  const purpose = frame.readUInt16LE(offset);
+  offset += 2;
+  const dataType = frame.readUInt16LE(offset);
+  offset += 2;
+  const senderLength = Number(frame.readBigUInt64LE(offset));
+  offset += 8;
+  if (offset + senderLength > frame.length) {
+    return null;
+  }
+  const sender = frame.toString("latin1", offset, offset + senderLength);
+  offset += senderLength;
+
+  let data: Buffer | null | undefined;
+  if (dataType === DataTypeId.BinaryData) {
+    data = frame.subarray(offset);
+  } else if (dataType === DataTypeId.Null) {
+    data = null;
+  }
+  return { purpose, dataType, sender, data };
+}
+
+/** The bytes as a BinaryData frame. */
+export function encodeYasBinary(
+  bytes: Uint8Array,
+  purpose: MessagePurpose,
+  sender = "",
+): Buffer {
+  const head = Buffer.alloc(FIXED + sender.length);
+  HEADER.copy(head, 0);
+  let offset = HEADER.length;
+  head.writeUInt16LE(purpose, offset);
+  offset += 2;
+  head.writeUInt16LE(DataTypeId.BinaryData, offset);
+  offset += 2;
+  head.writeBigUInt64LE(BigInt(sender.length), offset);
+  offset += 8;
+  head.write(sender, offset, "latin1");
+  return Buffer.concat([head, bytes]);
+}

package/bundle/hkp-python/src/hkp/__init__.py

@@ -0,0 +1 @@
+"""hkp-python — asyncio port of hkp-node runtime engine."""