@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-node/src/services/gather.ts

@@ -0,0 +1,335 @@
+/**
+ * Service Documentation
+ * Service ID: gather
+ * Service Name: Gather
+ * Runtime: hkp-node
+ * Modes: none — the items as an array, or what the nested pipeline makes of them
+ * Key Config: pipeline, maxBytes, maxGroups, timeoutMs, bypass
+ * IO: in=one part a Spread sent on -> out=the whole group, once every part is in; null until then
+ * Arrays: what a group becomes, in the order it was spread
+ * Binary: an item may be bytes; joining bytes is the nested pipeline's to do
+ * MixedData: not native in runtime
+ *
+ * The other half of Spread. Spread turns one array into as many values as it
+ * has items, each sent on by itself; Gather waits until it has seen them all
+ * and turns them back into one.
+ *
+ *     ui:   file-source (chunkSize) → spread     one value per chunk
+ *     node: gather → measure                      one value per file
+ *
+ * What makes that worth doing is the space between the two. A value leaving a
+ * runtime travels as one message, and a coordinator drops a message above its
+ * frame limit (`HKP_COORDINATOR_MAX_FRAME_BYTES`) — so a file too large to send
+ * at once can still be sent as chunks, each under the limit, and arrive whole.
+ *
+ * **A part says which whole it belongs to.** Spread sends each item as
+ * `{ item, group, index, total }`: the item itself, untouched under `item`, and
+ * where it sits in the array it came from. The index is what puts the group
+ * back in order, so parts may arrive in any order; the total is what says it is
+ * complete. Anything that is not a part passes through, so the same Gather sits
+ * on a board that sometimes sends a file whole.
+ *
+ * **What the group becomes is a pipeline, not a list of strategies** — the same
+ * answer Tracks gives its reducer. With no pipeline the items travel on as an
+ * array. With one, the pipeline is given `{ group, items }` and its answer is
+ * what goes on: joining text is one `map` term (`join(params.items, '')`), and
+ * a board that needs something stranger writes more services rather than
+ * waiting for a strategy to be added here. A pipeline that stops stops this
+ * service too.
+ *
+ * **Collecting a group holds it in memory**, which a frame limit set to bound
+ * memory would otherwise have prevented. `maxBytes` bounds one group,
+ * `maxGroups` how many may be incomplete at once — so the most held is the two
+ * multiplied — and a group still incomplete after `timeoutMs` is let go; each is
+ * said in the log rather than kept quiet, because a group that never arrives
+ * looks, from outside, exactly like nothing having been sent.
+ */
+import { Buffer } from "node:buffer";
+
+import { childRun } from "../runtime";
+import {
+  JsonRecord,
+  ServiceConfiguration,
+  ServiceCreator,
+  ServiceRegistryEntry,
+} from "../types";
+import { SubService } from "./sub-service";
+
+export const gatherDescriptor: ServiceRegistryEntry = {
+  serviceId: "gather",
+  serviceName: "Gather",
+  version: "v1",
+  // Like every service holding a pipeline: the board's UI needs to know there
+  // is something to look inside.
+  capabilities: ["subservices"],
+};
+
+/** As much as the coordinator lets through in one frame by default. */
+const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
+const DEFAULT_TIMEOUT_MS = 60_000;
+/**
+ * Groups incomplete at once. A new group beyond it is refused rather than
+ * making room: what is already arriving is not given up for what arrives next.
+ */
+const DEFAULT_MAX_GROUPS = 16;
+/**
+ * Refusals remembered at once, the oldest forgotten first. They expire anyway;
+ * this keeps a stream of new groups, each refused, from growing the list
+ * instead of the groups.
+ */
+export const MAX_REFUSED = 1024;
+/**
+ * The longest group a part may name. A group is held as long as its parts are,
+ * and is not counted against `maxBytes`; Spread names its groups by UUID.
+ */
+export const MAX_GROUP_LENGTH = 256;
+
+/** One item of an array a Spread sent on, and where it sits in it. */
+export type Part = {
+  item: unknown;
+  group: string;
+  index: number;
+  total: number;
+};
+
+/** Whether a value is a part, rather than a value to pass through. */
+export function isPart(value: unknown): value is Part {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  const part = value as JsonRecord;
+  return (
+    "item" in part &&
+    typeof part.group === "string" &&
+    part.group !== "" &&
+    part.group.length <= MAX_GROUP_LENGTH &&
+    Number.isInteger(part.total) &&
+    (part.total as number) > 0 &&
+    Number.isInteger(part.index) &&
+    (part.index as number) >= 0 &&
+    (part.index as number) < (part.total as number)
+  );
+}
+
+type Collecting = {
+  /**
+   * By index rather than an array of `total`: a part names its own total, and
+   * an array that size would be allocated — or refused outright — before a
+   * second part had arrived.
+   */
+  items: Map<number, unknown>;
+  total: number;
+  bytes: number;
+  startedAt: number;
+};
+
+/** What a part costs to hold, near enough to bound memory by. */
+function sizeOf(item: unknown): number {
+  if (typeof item === "string") {
+    return Buffer.byteLength(item, "utf8");
+  }
+  if (item instanceof Uint8Array) {
+    return item.byteLength;
+  }
+  if (item === null || item === undefined) {
+    return 0;
+  }
+  try {
+    return Buffer.byteLength(JSON.stringify(item) ?? "", "utf8");
+  } catch {
+    return 0;
+  }
+}
+
+export class GatherService extends SubService {
+  readonly serviceId = gatherDescriptor.serviceId;
+  readonly serviceName = gatherDescriptor.serviceName;
+  readonly version = gatherDescriptor.version;
+  readonly capabilities = gatherDescriptor.capabilities;
+
+  private maxBytes = DEFAULT_MAX_BYTES;
+  private maxGroups = DEFAULT_MAX_GROUPS;
+  private timeoutMs = DEFAULT_TIMEOUT_MS;
+  private lastError = "";
+  private collecting = new Map<string, Collecting>();
+  /**
+   * Groups given up on, and when. The parts of a group still on their way when
+   * it was refused would otherwise start it again, and wait out the timeout
+   * for parts that went with the refusal.
+   */
+  private refused = new Map<string, number>();
+
+  constructor(config: ServiceConfiguration, createService: ServiceCreator) {
+    super(config, createService);
+    // Again, for the reason Iterator gives: the base constructor's configure
+    // ran before this class's fields were defined, and they overwrote it.
+    if (config.state) {
+      this.settle(config.state);
+    }
+  }
+
+  configure(config: JsonRecord): JsonRecord {
+    this.settle(config);
+    // The pipeline, bypass and the editing commands belong to the base.
+    super.configure(config);
+    return this.getState();
+  }
+
+  getState(): JsonRecord {
+    return {
+      ...super.getState(),
+      maxBytes: this.maxBytes,
+      maxGroups: this.maxGroups,
+      timeoutMs: this.timeoutMs,
+      pending: this.collecting?.size ?? 0,
+      error: this.lastError ?? "",
+    };
+  }
+
+  /** The part of a configuration this service owns rather than its base. */
+  private settle(config: JsonRecord): void {
+    if (typeof config.maxBytes === "number" && config.maxBytes > 0) {
+      this.maxBytes = Math.floor(config.maxBytes);
+    }
+    if (typeof config.maxGroups === "number" && config.maxGroups >= 1) {
+      this.maxGroups = Math.floor(config.maxGroups);
+    }
+    if (typeof config.timeoutMs === "number" && config.timeoutMs > 0) {
+      this.timeoutMs = Math.floor(config.timeoutMs);
+    }
+  }
+
+  async process(
+    input: unknown,
+    notify: (payload: unknown, instanceId?: string) => void,
+  ): Promise<unknown> {
+    if (this.bypass || !isPart(input)) {
+      return input;
+    }
+
+    const now = Date.now();
+    this.letGoOfStale(now, notify);
+
+    const { group, index, total, item } = input;
+    if (this.refused.has(group)) {
+      return null;
+    }
+
+    let collecting = this.collecting.get(group);
+    if (!collecting) {
+      if (this.collecting.size >= this.maxGroups) {
+        this.refuse(
+          group,
+          now,
+          `group ${group} refused: ${this.maxGroups} groups are already incomplete`,
+          notify,
+        );
+        return null;
+      }
+      collecting = {
+        items: new Map(),
+        total,
+        bytes: 0,
+        startedAt: now,
+      };
+      this.collecting.set(group, collecting);
+    }
+
+    if (collecting.total !== total) {
+      this.refuse(
+        group,
+        now,
+        `group ${group} said ${collecting.total} parts, then ${total}`,
+        notify,
+      );
+      return null;
+    }
+    if (collecting.items.has(index)) {
+      // Sent twice; the first one counts.
+      return null;
+    }
+
+    collecting.bytes += sizeOf(item);
+    if (collecting.bytes > this.maxBytes) {
+      this.refuse(
+        group,
+        now,
+        `group ${group} is larger than ${this.maxBytes} bytes`,
+        notify,
+      );
+      return null;
+    }
+
+    collecting.items.set(index, item);
+    notify({
+      group,
+      received: collecting.items.size,
+      total,
+      pending: this.collecting.size,
+    });
+
+    if (collecting.items.size < total) {
+      return null;
+    }
+
+    this.collecting.delete(group);
+    const items = Array.from({ length: total }, (_, i) => collecting.items.get(i));
+    if (!this.pipeline || this.pipeline.listServices().length === 0) {
+      return this.stopPropagation ? null : items;
+    }
+    // The run that brought the last part is the one the whole arrives in; the
+    // pipeline making it runs as a run of its own beneath it, like a scope's.
+    const result = await this.pipeline.process(
+      { group, items },
+      () => {},
+      childRun(this.host?.currentContext() ?? null),
+    );
+    return this.stopPropagation ? null : result;
+  }
+
+  destroy(): void {
+    this.collecting.clear();
+    this.refused.clear();
+    super.destroy();
+  }
+
+  // ── Private ───────────────────────────────────────────────────────────────
+
+  private refuse(
+    group: string,
+    now: number,
+    message: string,
+    notify: (payload: unknown) => void,
+  ): void {
+    this.collecting.delete(group);
+    this.refused.set(group, now);
+    // A Map keeps insertion order, so the first key is the oldest refusal.
+    while (this.refused.size > MAX_REFUSED) {
+      this.refused.delete(this.refused.keys().next().value as string);
+    }
+    this.lastError = message;
+    this.host?.log("error", "gather.refused", { group, message });
+    notify({ error: message, pending: this.collecting.size });
+  }
+
+  /** Lets go of groups that stopped arriving, and of refusals that are old. */
+  private letGoOfStale(now: number, notify: (payload: unknown) => void): void {
+    for (const [group, collecting] of this.collecting) {
+      if (now - collecting.startedAt > this.timeoutMs) {
+        this.collecting.delete(group);
+        this.lastError = `group ${group} timed out with ${collecting.items.size} of ${collecting.total} parts`;
+        this.host?.log("warn", "gather.timed-out", {
+          group,
+          received: collecting.items.size,
+          total: collecting.total,
+        });
+        notify({ error: this.lastError, pending: this.collecting.size });
+      }
+    }
+    for (const [group, at] of this.refused) {
+      if (now - at > this.timeoutMs) {
+        this.refused.delete(group);
+      }
+    }
+  }
+}

package/bundle/hkp-node/src/services/hold.ts

@@ -0,0 +1,275 @@
+/**
+ * Service Documentation
+ * Service ID: hold
+ * Service Name: Hold
+ * Runtime: hkp-node
+ * Modes: none — either a slot with a declared role, or a property that discriminates
+ * Key Config: slot + op, or property
+ * IO: in=any -> out=the held value, or null while nothing is held
+ * Arrays: an array input carries no property, so it reads
+ * Binary: holdable in a slot; a property cannot discriminate on one
+ * MixedData: not native in runtime
+ *
+ * Sample-and-hold: a pipeline entered from two sides — a producer that runs on
+ * its own schedule and a consumer that arrives whenever it arrives — needs the
+ * producer's latest value to survive between runs. Hold keeps it.
+ *
+ * **Which side is calling can be said two ways**, and a board picks one.
+ *
+ * With a `slot`, the board says outright: `op` is `write` or `read`, and two
+ * Holds naming one slot are the two ends of it. Nothing inspects the value, so
+ * anything can be held — bytes, a document, null — and the two ends may sit in
+ * pipelines that never meet, which is what an endpoint's separate entry points
+ * are. Where the cells live is the host's to decide (`RuntimeHost.slots`): the
+ * service owning both pipelines, or failing that the runtime.
+ *
+ * With a `property` and no slot, the input says: an input carrying that
+ * property is the producer, its value replaces what is held, and every call —
+ * that one included — emits the held value under the same property name, so
+ * the services after Hold cannot tell the two sides apart. That is the older
+ * arrangement, and it is the only one available where the two sides share one
+ * pipeline, since there is nothing but the value to tell them apart. A null
+ * held value is an empty one, so a producer cannot hold null: an input carrying
+ * the property as null reads like any other.
+ */
+import {
+  HostedService,
+  JsonRecord,
+  RuntimeHost,
+  ServiceConfiguration,
+  ServiceRegistryEntry,
+} from "../types";
+
+export const holdDescriptor: ServiceRegistryEntry = {
+  serviceId: "hold",
+  serviceName: "Hold",
+};
+
+export class HoldService implements HostedService {
+  readonly serviceId = holdDescriptor.serviceId;
+  readonly serviceName = holdDescriptor.serviceName;
+  readonly uuid: string;
+
+  private host: RuntimeHost | null = null;
+
+  private property = "";
+  /** See the header: named, this is a cell the host owns rather than this one. */
+  private slot = "";
+  private op: "read" | "write" = "read";
+  /** What is held when no slot names somewhere else to hold it. */
+  private own: unknown = null;
+  private readCount = 0;
+  private writeCount = 0;
+
+  constructor(config: ServiceConfiguration) {
+    this.uuid = config.uuid;
+    if (config.state) {
+      this.configure(config.state);
+    }
+  }
+
+  setHost(host: RuntimeHost): void {
+    this.host = host;
+  }
+
+  getState(): JsonRecord {
+    // Only the arrangement in use is reported. A state property a service does
+    // not act on is one a board keeps and a reader has to discount, and an
+    // omitted one is erased from the board the next time it is saved — which is
+    // what should happen to the half of this service a board is not using.
+    const which: JsonRecord = this.slot
+      ? { slot: this.slot, op: this.op }
+      : { property: this.property };
+    return {
+      ...which,
+      held: reportable(this.read()),
+      readCount: this.readCount,
+      writeCount: this.writeCount,
+    };
+  }
+
+  configure(config: JsonRecord): JsonRecord {
+    if (typeof config.property === "string") {
+      if (config.property !== this.property) {
+        // What is held belongs to the property it was written for.
+        this.forget();
+      }
+      this.property = config.property;
+    }
+
+    if (typeof config.slot === "string") {
+      if (config.slot !== this.slot) {
+        // A slot is an address, and what was held belongs to the old one — but
+        // it belongs to whoever else is still reading it, so only this
+        // service's own cell is cleared, never the host's.
+        this.own = null;
+        this.readCount = 0;
+        this.writeCount = 0;
+      }
+      this.slot = config.slot;
+    }
+
+    if (config.op === "read" || config.op === "write") {
+      this.op = config.op;
+    }
+
+    if (config.action === "clear") {
+      this.forget();
+    }
+
+    const state = this.getState();
+    this.notify(state);
+    return state;
+  }
+
+  process(
+    input: unknown,
+    _notify: (payload: unknown, instanceId?: string) => void,
+  ): unknown {
+    if (this.slot) {
+      return this.useSlot(input);
+    }
+
+    // Nothing named is nothing to hold: an unconfigured Hold is a wire.
+    if (!this.property) {
+      return input;
+    }
+
+    const incoming = carriedValue(input, this.property);
+    if (incoming !== null && incoming !== undefined) {
+      this.write(incoming);
+      this.writeCount += 1;
+    } else {
+      this.readCount += 1;
+    }
+
+    this.notify(this.getState());
+
+    const held = this.read();
+    return held === null ? null : { [this.property]: held };
+  }
+
+  destroy(): void {
+    // Only what this service holds itself. A slot belongs to the host, and the
+    // other end of it outlives this one — a pipeline rebuilt while a board is
+    // running destroys the services in it, and that must not empty a cell the
+    // service on the other side is still answering from.
+    this.own = null;
+    this.readCount = 0;
+    this.writeCount = 0;
+  }
+
+  // ── Private ────────────────────────────────────────────────────────────────
+
+  /**
+   * A call on a Hold whose role is declared rather than inferred.
+   *
+   * A write emits **its input unchanged**, so the pass it belongs to carries on
+   * as though the Hold were not there; a read emits what is held, **raw**, so
+   * it can be the whole of what a pipeline answers with. Neither looks at the
+   * value, which is what lets a slot hold what a property never could.
+   */
+  private useSlot(input: unknown): unknown {
+    if (this.op === "write") {
+      this.write(input);
+      this.writeCount += 1;
+      this.notify(this.getState());
+      return input;
+    }
+
+    this.readCount += 1;
+    this.notify(this.getState());
+    // Nothing held is nothing to pass on, the same as everywhere else — a
+    // consumer that arrives before the producer has run stops here.
+    return this.read() ?? null;
+  }
+
+  /** What is held, from wherever this Hold holds it. */
+  private read(): unknown {
+    if (!this.slot) {
+      return this.own;
+    }
+    const store = this.host?.slots?.();
+    return (store ? store.get(this.slot) : this.own) ?? null;
+  }
+
+  private write(value: unknown): void {
+    const store = this.slot ? this.host?.slots?.() : null;
+    if (store) {
+      store.set(this.slot, value);
+      return;
+    }
+    // No store to share through — a Hold outside any host that provides one
+    // still holds, for itself alone, rather than dropping what it was given.
+    this.own = value;
+  }
+
+  /**
+   * Back to how the service started. The counts go with the value: they say how
+   * often each side has called for what is held now, and left running across a
+   * clear they would describe a value that is gone.
+   */
+  private forget(): void {
+    this.own = null;
+    if (this.slot) {
+      this.host?.slots?.()?.set(this.slot, null);
+    }
+    this.readCount = 0;
+    this.writeCount = 0;
+  }
+
+  private notify(payload: JsonRecord): void {
+    this.host?.notify(payload, this.uuid);
+  }
+}
+
+/**
+ * The value an input carries for the held property, if it carries one at all —
+ * anything else makes the call a read rather than a write.
+ */
+function carriedValue(input: unknown, property: string): unknown {
+  if (typeof input !== "object" || input === null || Array.isArray(input)) {
+    return undefined;
+  }
+  return (input as Record<string, unknown>)[property];
+}
+
+/**
+ * The held value as it can be reported over REST. State travels as JSON, so a
+ * value that does not survive the trip is described rather than sent.
+ */
+/**
+ * What is held, as something safe to put in state.
+ *
+ * State is read back into the board and sent to everyone watching, so what a
+ * Hold reports has to be worth carrying. A value that is bytes, or simply large
+ * — a rendered document, an audio buffer — is described instead of copied: the
+ * size is what a reader is looking for at that point, and the value itself is
+ * on its way to whatever asked for it regardless.
+ */
+const REPORTABLE_LIMIT = 2048;
+
+function reportable(value: unknown): unknown {
+  if (value instanceof Uint8Array) {
+    return `[${value.byteLength} bytes]`;
+  }
+  try {
+    const json = JSON.stringify(value);
+    if (json === undefined) {
+      return null;
+    }
+    if (json.length > REPORTABLE_LIMIT) {
+      return `[${describe(value)}, ${json.length} characters]`;
+    }
+    return JSON.parse(json);
+  } catch {
+    return `[${describe(value)}]`;
+  }
+}
+
+function describe(value: unknown): string {
+  if (typeof value === "object" && value !== null) {
+    return value.constructor?.name ?? "object";
+  }
+  return typeof value;
+}