@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-node/src/services/peer-server.ts

@@ -0,0 +1,268 @@
+/**
+ * Service Documentation
+ * Service ID: peer-server
+ * Service Name: PeerServer
+ * Runtime: hkp-node
+ * Modes: PeerJS signaling server on a runtime-assigned mount
+ * Key Config: bypass, emitEvents (the endpoint is assigned, not configured)
+ * IO: passthrough — the service's value is the signaling side-effect
+ *     When emitEvents=true, peer connect/disconnect events are pushed to
+ *     the next service in the pipeline (e.g. a Monitor).
+ * Compatible with the peerjs browser client.
+ */
+import http from "node:http";
+
+import express, { Express } from "express";
+import { ExpressPeerServer, IClient } from "peer";
+import { WebSocketServer } from "ws";
+
+import { MountHandle } from "../mounts";
+import {
+  HostedService,
+  JsonRecord,
+  RuntimeHost,
+  RuntimeNotification,
+  ServiceConfiguration,
+  ServiceRegistryEntry,
+} from "../types";
+
+export const peerServerDescriptor: ServiceRegistryEntry = {
+  serviceId: "peer-server",
+  serviceName: "PeerServer",
+};
+
+export class PeerServerService implements HostedService {
+  readonly serviceId = peerServerDescriptor.serviceId;
+  readonly serviceName = peerServerDescriptor.serviceName;
+  readonly uuid: string;
+
+  private bypass = true;
+  private emitEvents = false;
+  private mount: MountHandle | null = null;
+  private app: Express | null = null;
+  private socketServer: WebSocketServer | null = null;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private peerServer: any = null;
+  private host: RuntimeHost | null = null;
+  private connectedPeers: string[] = [];
+
+  constructor(config: ServiceConfiguration) {
+    this.uuid = config.uuid;
+    if (config.state) {
+      this.configure(config.state);
+    }
+  }
+
+  setHost(host: RuntimeHost): void {
+    this.host = host;
+    // State is applied in the constructor, before the host exists, so a service
+    // configured as already-active has nothing to claim its mount from until now.
+    if (!this.bypass && !this.mount) {
+      this.claimMount();
+    }
+  }
+
+  configure(config: JsonRecord): JsonRecord {
+    // `port` and `path` are accepted and ignored: the signalling endpoint is
+    // served by the shared runtime server at an assigned path, so the service
+    // no longer picks either. Older boards still carry the fields, and
+    // rejecting them would fail those boards on load.
+
+    if (typeof config.emitEvents === "boolean") {
+      this.emitEvents = config.emitEvents;
+    }
+
+    if (typeof config.bypass === "boolean" && config.bypass !== this.bypass) {
+      this.bypass = config.bypass;
+      if (this.bypass) {
+        this.releaseMount();
+      } else {
+        this.claimMount();
+      }
+    }
+
+    if (!this.bypass && !this.mount) {
+      this.claimMount();
+    }
+
+    return this.getState();
+  }
+
+  getState(): JsonRecord {
+    return {
+      bypass: this.bypass,
+      // Public signalling endpoint, assigned by the runtime. Reserved name: a
+      // client points at this service and resolves the address from here (see
+      // the frontend's runtime/board/mount).
+      __hkpMount: this.mount?.url ?? "",
+      emitEvents: this.emitEvents,
+      connectedPeers: this.connectedPeers,
+    };
+  }
+
+  process(
+    input: unknown,
+    _notify: (payload: unknown, instanceId?: string) => void,
+  ): unknown {
+    return input;
+  }
+
+  destroy(): void {
+    this.releaseMount();
+  }
+
+  // ── Private ───────────────────────────────────────────────────────────────
+
+  private onPeerConnected = (client: IClient): void => {
+    if (!this.mount) return;
+    const peerId = client.getId();
+    this.connectedPeers = [...this.connectedPeers, peerId];
+    this.host?.notify({ connectedPeers: this.connectedPeers }, this.uuid);
+    void this.reportFailure(
+      this.emitEvent({
+        event: "peer-connected",
+        peerId,
+        connectedPeers: this.connectedPeers,
+      }),
+    );
+  };
+
+  private onPeerDisconnected = (client: IClient): void => {
+    if (!this.mount) return;
+    const peerId = client.getId();
+    this.connectedPeers = this.connectedPeers.filter((id) => id !== peerId);
+    this.host?.notify({ connectedPeers: this.connectedPeers }, this.uuid);
+    void this.reportFailure(
+      this.emitEvent({
+        event: "peer-disconnected",
+        peerId,
+        connectedPeers: this.connectedPeers,
+      }),
+    );
+  };
+
+  /**
+   * A push nothing is waiting on.
+   *
+   * A peer connecting is not a call anyone made, so there is no caller for a
+   * failure to reach. Reported rather than left to surface as an unhandled
+   * rejection, which would take the runtime down over one bad event.
+   */
+  private async reportFailure(work: Promise<void>): Promise<void> {
+    try {
+      await work;
+    } catch (err) {
+      this.host?.log("error", "service.failed", {
+        message: `peer event failed: ${err instanceof Error ? err.message : String(err)}`,
+      });
+    }
+  }
+
+  /**
+   * Push a peer event through the rest of this runtime and on to the next one.
+   *
+   * A service that produces data without being asked has to emit the runtime's
+   * result itself; nothing else will. Running the remaining services alone only
+   * updates this runtime, which looks right in a Monitor sitting behind the
+   * service but leaves the chain dead from there on.
+   */
+  private async emitEvent(payload: JsonRecord): Promise<void> {
+    if (!this.emitEvents || !this.host) {
+      return;
+    }
+    const output = await this.host.processFrom(
+      this.uuid,
+      payload,
+      (_n: RuntimeNotification) => {},
+    );
+    // A downstream service returning null means "stop" — honour it rather than
+    // forwarding a dead result to the next runtime.
+    if (output !== null && output !== undefined) {
+      this.host.emitResult(output);
+    }
+  }
+
+  /** See HostedService.remount: the runtime can serve one now. */
+  remount(): void {
+    this.claimMount();
+  }
+
+  private claimMount(): void {
+    if (this.mount || !this.host?.mount) {
+      return;
+    }
+
+    // Claim the public path first: PeerJS derives its WebSocket route from the
+    // Express mountpath, so the sub-app can only be built once the path is known.
+    const mount = this.host.mount(this.uuid, {
+      request: (req, res) => {
+        this.app?.(req, res);
+      },
+      upgrade: (req, socket, head) => {
+        const socketServer = this.socketServer;
+        if (!socketServer) {
+          socket.destroy();
+          return;
+        }
+        socketServer.handleUpgrade(req, socket, head, (ws) => {
+          socketServer.emit("connection", ws, req);
+        });
+      },
+    });
+    if (!mount) {
+      return;
+    }
+    this.mount = mount;
+
+    const app = express();
+    // A detached http.Server that is never listened on. PeerJS insists on a
+    // server to attach to, and giving it this one rather than the shared runtime
+    // server is what keeps upgrade routing ours: attaching to a live server
+    // installs a `ws` upgrade listener that answers 400 on every path it does
+    // not own, which would take out the runtime notification sockets.
+    const carrier = http.createServer();
+    const peerServer = ExpressPeerServer(carrier, {
+      allow_discovery: true,
+      // The carrier receives nothing, so without a reference to the socket
+      // server PeerJS builds there is no way to hand it an upgrade. This hook is
+      // that reference; `noServer` only makes the detachment explicit.
+      createWebSocketServer: (options) => {
+        this.socketServer = new WebSocketServer({
+          ...options,
+          server: undefined,
+          noServer: true,
+        });
+        return this.socketServer;
+      },
+    });
+    app.use(mount.path, peerServer);
+
+    peerServer.on("connection", this.onPeerConnected);
+    peerServer.on("disconnect", this.onPeerDisconnected);
+
+    this.peerServer = peerServer;
+    this.app = app;
+
+    this.host?.notify({ __hkpMount: mount.url }, this.uuid);
+  }
+
+  private releaseMount(): void {
+    if (this.peerServer) {
+      this.peerServer.removeListener("connection", this.onPeerConnected);
+      this.peerServer.removeListener("disconnect", this.onPeerDisconnected);
+      this.peerServer = null;
+    }
+
+    this.socketServer?.close();
+    this.socketServer = null;
+    this.app = null;
+
+    this.mount?.release();
+    this.mount = null;
+
+    if (this.connectedPeers.length > 0) {
+      this.connectedPeers = [];
+      this.host?.notify({ connectedPeers: this.connectedPeers }, this.uuid);
+    }
+  }
+}

package/bundle/hkp-node/src/services/queue-store.ts

@@ -0,0 +1,301 @@
+/**
+ * The queue itself: one table, and the five things that can be done to it.
+ *
+ * A module rather than part of the service, for the same reason `database.ts`
+ * is one. Two callers need this and they are not both services: the `queue`
+ * service running in a pipeline, and the runtime's own `/queue` route, which
+ * is how a board on *another* server reaches these rows. One implementation
+ * means a remote claim and a local claim are the same claim — the guarantee
+ * cannot drift between the two paths, because there is only one path.
+ *
+ * Every function takes the database it works on. Which database that is —
+ * whose, and therefore which messages exist at all — is decided by the caller
+ * from something it was told rather than something it was asked: a service
+ * reads its runtime's scope, the route reads the authenticated user.
+ */
+import { randomUUID } from "node:crypto";
+
+import { JsonRecord } from "../types";
+import { Database } from "./database";
+
+/** What a message is doing, as far as the queue is concerned. */
+export type Status = "pending" | "claimed" | "done" | "dead";
+
+export const DEFAULT_LIMIT = 10;
+/**
+ * How long a claim holds before the message is handed out again.
+ *
+ * Long enough for a pipeline that calls a model and sends mail — the work
+ * between a consume and its ack is I/O with other people's servers in it —
+ * and short enough that a runtime killed mid-flight does not strand the
+ * message for the rest of the afternoon.
+ */
+export const DEFAULT_VISIBILITY_SECONDS = 300;
+/**
+ * Attempts before a message is left alone.
+ *
+ * A message that five separate runs could not finish is not going to be
+ * finished by a sixth, and a queue that keeps trying turns one bad payload
+ * into an unbounded bill.
+ */
+export const DEFAULT_MAX_ATTEMPTS = 5;
+
+/**
+ * One table, and the index the two hot queries need.
+ *
+ * `consume` asks for the oldest claimable message on a topic and `list` reads
+ * a topic in order; both are served by (topic, status, availableAt). Nothing
+ * here is indexed on a guess.
+ */
+const SCHEMA = `
+  CREATE TABLE IF NOT EXISTS message (
+    id TEXT PRIMARY KEY,
+    topic TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    status TEXT NOT NULL,
+    publishedBy TEXT NOT NULL DEFAULT '',
+    publishedAt TEXT NOT NULL,
+    availableAt TEXT NOT NULL,
+    claimedAt TEXT,
+    attempts INTEGER NOT NULL DEFAULT 0,
+    updatedAt TEXT NOT NULL,
+    error TEXT NOT NULL DEFAULT ''
+  );
+  CREATE INDEX IF NOT EXISTS message_by_topic
+    ON message (topic, status, availableAt);
+`;
+
+/** Creates the table if this database has not seen it yet. */
+export function prepare(db: Database): void {
+  db.exec(SCHEMA);
+}
+
+/** Reads a JSON column, tolerating one that was never written by us. */
+function parsed(value: unknown, fallback: unknown): unknown {
+  if (typeof value !== "string") {
+    return fallback;
+  }
+  try {
+    return JSON.parse(value);
+  } catch {
+    return fallback;
+  }
+}
+
+/** A row as a board sees it: the payload opened, the bookkeeping visible. */
+export function describedMessage(row: JsonRecord): JsonRecord {
+  return {
+    id: row.id,
+    topic: row.topic,
+    payload: parsed(row.payload, row.payload),
+    status: row.status,
+    publishedBy: row.publishedBy,
+    publishedAt: row.publishedAt,
+    availableAt: row.availableAt,
+    claimedAt: row.claimedAt ?? null,
+    attempts: row.attempts,
+    updatedAt: row.updatedAt,
+    error: row.error,
+  };
+}
+
+/**
+ * Puts one message on the topic.
+ *
+ * `id` is the caller's when it has one. A publish that crossed a network may
+ * arrive twice — the sender timed out on a request that in fact succeeded —
+ * and a message identified by the sender lands once however many times it is
+ * delivered.
+ */
+export function publish(
+  db: Database,
+  params: {
+    topic: string;
+    payload: unknown;
+    publishedBy: string;
+    id?: string;
+  },
+): JsonRecord {
+  const now = new Date().toISOString();
+  const id = params.id || randomUUID();
+  db.run(
+    `INSERT INTO message
+       (id, topic, payload, status, publishedBy, publishedAt, availableAt,
+        claimedAt, attempts, updatedAt, error)
+     VALUES ($id, $topic, $payload, 'pending', $board, $now, $now, NULL, 0, $now, '')
+     ON CONFLICT (id) DO NOTHING`,
+    {
+      $id: id,
+      $topic: params.topic,
+      $payload: JSON.stringify(params.payload),
+      $board: params.publishedBy,
+      $now: now,
+    },
+  );
+  const row = db.query("SELECT * FROM message WHERE id = $id", { $id: id })[0];
+  return describedMessage(row ?? {});
+}
+
+/**
+ * Hands out the messages that may be worked on now.
+ *
+ * "May be worked on" is one condition, not two: a message is claimable when
+ * its `availableAt` has passed, which covers both one nobody has taken and one
+ * whose claim ran out. A claim is therefore not a lock to be released — it is
+ * a deadline, and a runtime that dies holds nothing.
+ */
+export function consume(
+  db: Database,
+  params: {
+    topic: string;
+    limit?: number;
+    visibilitySeconds?: number;
+    maxAttempts?: number;
+  },
+): { messages: JsonRecord[]; count: number } {
+  const limit = params.limit ?? DEFAULT_LIMIT;
+  const visibilitySeconds = params.visibilitySeconds ?? DEFAULT_VISIBILITY_SECONDS;
+  const maxAttempts = params.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+
+  const now = new Date();
+  const nowText = now.toISOString();
+  const rows = db.query(
+    `SELECT * FROM message
+      WHERE topic = $topic
+        AND status IN ('pending', 'claimed')
+        AND availableAt <= $now
+      ORDER BY publishedAt ASC, id ASC
+      LIMIT $limit`,
+    { $topic: params.topic, $now: nowText, $limit: limit },
+  );
+
+  const claimed: JsonRecord[] = [];
+  for (const row of rows) {
+    const attempts = Number(row.attempts) + 1;
+    if (attempts > maxAttempts) {
+      // Handed out its share of times and never acked. Left where it is, with
+      // the count that says why, rather than deleted.
+      db.run(
+        `UPDATE message
+            SET status = 'dead', updatedAt = $now, error = $error
+          WHERE id = $id AND attempts = $previousAttempts`,
+        {
+          $id: String(row.id),
+          $previousAttempts: Number(row.attempts),
+          $now: nowText,
+          $error: `not acknowledged after ${maxAttempts} attempts`,
+        },
+      );
+      continue;
+    }
+
+    const availableAt = new Date(
+      now.getTime() + visibilitySeconds * 1000,
+    ).toISOString();
+    // The claim re-states the conditions the row was selected under, and
+    // `attempts` doubles as its version. Between the select and this update
+    // another consumer may have taken the same row — another process on the
+    // same file, or a board on another server claiming over the route — and
+    // the one whose update changes no rows simply does not deliver it.
+    // Without the guard both would hand out the same message and one of the
+    // two acks would land on work the other did.
+    const { changes } = db.run(
+      `UPDATE message
+          SET status = 'claimed', attempts = $attempts, claimedAt = $now,
+              availableAt = $availableAt, updatedAt = $now
+        WHERE id = $id
+          AND attempts = $previousAttempts
+          AND status IN ('pending', 'claimed')
+          AND availableAt <= $now`,
+      {
+        $id: String(row.id),
+        $attempts: attempts,
+        $previousAttempts: Number(row.attempts),
+        $now: nowText,
+        $availableAt: availableAt,
+      },
+    );
+    if (changes === 0) {
+      continue;
+    }
+    claimed.push(describedMessage({ ...row, status: "claimed", attempts }));
+  }
+
+  return { messages: claimed, count: claimed.length };
+}
+
+/**
+ * Closes a claimed message: acknowledged, or handed back.
+ *
+ * Handing back returns it immediately rather than waiting out the claim,
+ * because a pipeline that knows it failed knows it now. The attempt has
+ * already been counted, so a payload nothing can process still reaches `dead`
+ * rather than cycling for ever.
+ */
+export function close(
+  db: Database,
+  params: {
+    id: string;
+    outcome: "done" | "pending";
+    error?: string;
+    maxAttempts?: number;
+  },
+): JsonRecord | null {
+  const maxAttempts = params.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const row = db.query("SELECT * FROM message WHERE id = $id", {
+    $id: params.id,
+  })[0];
+  if (!row) {
+    return null;
+  }
+
+  const now = new Date().toISOString();
+  const error = params.outcome === "pending" ? (params.error ?? "") : "";
+  // A failure that has used up its attempts is finished here rather than being
+  // handed out once more only to be buried on the next consume.
+  const status: Status =
+    params.outcome === "done"
+      ? "done"
+      : Number(row.attempts) >= maxAttempts
+        ? "dead"
+        : "pending";
+
+  db.run(
+    `UPDATE message
+        SET status = $status, availableAt = $now, claimedAt = NULL,
+            updatedAt = $now, error = $error
+      WHERE id = $id`,
+    { $id: params.id, $status: status, $now: now, $error: error },
+  );
+
+  return { id: params.id, status, attempts: Number(row.attempts) };
+}
+
+/** Reads the log without touching it. For a facade table, and for looking. */
+export function list(
+  db: Database,
+  params: { topic?: string; status?: string; limit?: number },
+): { messages: JsonRecord[]; count: number } {
+  const values: Record<string, string | number> = {
+    $limit: params.limit ?? DEFAULT_LIMIT,
+  };
+  const conditions: string[] = [];
+  if (params.topic) {
+    conditions.push("topic = $topic");
+    values.$topic = params.topic;
+  }
+  if (params.status) {
+    conditions.push("status = $status");
+    values.$status = params.status;
+  }
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+
+  const rows = db.query(
+    `SELECT * FROM message ${where}
+      ORDER BY publishedAt DESC, id DESC
+      LIMIT $limit`,
+    values,
+  );
+  const messages = rows.map(describedMessage);
+  return { messages, count: messages.length };
+}