@scp-dev/hub 0.4.0 → 0.5.0

package/dist/index.d.ts

@@ -583,10 +583,14 @@ type GraphEdge = {
     count: number;
     last_at: string;
 };
+/** a machine as the graph draws it: `actor` is the member who bound it, null for the personal account's owner or a revoked name */
+interface GraphMachine extends MachineRow {
+    actor: string | null;
+}
 interface HubGraph {
     /** the window message edges count over */
     minutes: number;
-    machines: MachineRow[];
+    machines: GraphMachine[];
     nodes: GraphNode[];
     edges: GraphEdge[];
 }
@@ -606,6 +610,7 @@ type DashboardEvent = {
     kind: string;
     from: string;
     to: string[];
+    local?: true;
 } | {
     type: "endpoints";
     at: string;
@@ -668,6 +673,14 @@ declare const HUB_LIMITS: {
 type HubLimits = {
     [K in keyof typeof HUB_LIMITS]: number;
 };
+/** kinds a bridge reports for a delivery its broker made on its own machine */
+declare const LOCAL_EVENT_KINDS: readonly ["message", "request", "reply", "status", "handback", "spawn"];
+/** events per `local_events` frame, recipients per event, and how far back an event's `at` may lie */
+declare const LOCAL_EVENTS: {
+    readonly perFrame: 200;
+    readonly recipients: 64;
+    readonly maxAgeMs: 300000;
+};
 /** the hosted relay of one account: machines, bridges, direct clients, push and replay over a FederatedCore; pure, sockets are injected */
 declare class RelayHub {
     private readonly deps;
@@ -799,6 +812,11 @@ declare class RelayHub {
     private deliverOn;
     private readonly deliverJudged;
     private delivered;
+    /**
+     * a bridge's metadata for deliveries its broker made on its own machine: counted for the graph and streamed to watchers and the
+     * dashboard as `local`; unknown endpoints and malformed events are skipped
+     */
+    private localEvents;
     /** the body's bytes when the account can hold them; refuses a send past the `storedBytes` limit */
     private room;
     /** bytes of message bodies held, and of the messages from other accounts */
@@ -865,4 +883,4 @@ declare class RelayHub {
     xnotice(endpoint: string, text: string): void;
 }
 
-export { type DashboardEvent, type Deliver, type Delivery, FederatedCore, type FederatedOptions, GRAPH_MINUTES, GRAPH_MINUTES_MAX, type GraphEdge, type GraphNode, HUB_LIMITS, type HoldView, type HubAuth, type HubConn, type HubDeps, type HubGrant, type HubGraph, type HubJudgeOptions, type HubLimits, type HubRole, type HubSocket, JUDGE_LIMITS, type JudgeClientFactory, JudgeGate, type JudgeGateOptions, type JudgeLimits, type Origin as JudgeOrigin, MACHINE_NAME, type MachineRow, type MirrorCaps, type MirrorChange, type MirrorRules, type MirrorScope, type MirrorView, type RelayAuth, RelayHub, type RemoteView, type XEndpoint, dashboardEvent };
+export { type DashboardEvent, type Deliver, type Delivery, FederatedCore, type FederatedOptions, GRAPH_MINUTES, GRAPH_MINUTES_MAX, type GraphEdge, type GraphMachine, type GraphNode, HUB_LIMITS, type HoldView, type HubAuth, type HubConn, type HubDeps, type HubGrant, type HubGraph, type HubJudgeOptions, type HubLimits, type HubRole, type HubSocket, JUDGE_LIMITS, type JudgeClientFactory, JudgeGate, type JudgeGateOptions, type JudgeLimits, type Origin as JudgeOrigin, LOCAL_EVENTS, LOCAL_EVENT_KINDS, MACHINE_NAME, type MachineRow, type MirrorCaps, type MirrorChange, type MirrorRules, type MirrorScope, type MirrorView, type RelayAuth, RelayHub, type RemoteView, type XEndpoint, dashboardEvent };

package/dist/index.js

@@ -20,6 +20,8 @@ import {
   SYSTEM_HANDLE,
   seqList,
   unreachable,
+  harnessFits,
+  keptHarness,
   validHandle,
   validHarness,
   validSelector,
@@ -497,9 +499,9 @@ var FederatedCore = class extends RelayCore {
           this.checkCap();
           row = this.byId(this.insertRemote(`${machine}/${handle}`, "agent", harness, auth, now));
           created = true;
-        } else if (held.kind === "agent" && held.connections === 0 && held.harness === harness) {
+        } else if (held.kind === "agent" && held.connections === 0 && harnessFits(held.harness, harness)) {
           row = held;
-        } else if (exact && held.kind === "agent" && held.token_hash !== null && held.process === null && held.session === null && held.parent_ep === null && held.harness === harness) {
+        } else if (exact && held.kind === "agent" && held.token_hash !== null && held.process === null && held.session === null && held.parent_ep === null && harnessFits(held.harness, harness)) {
           row = held;
           joined = true;
         } else if (held.kind === "agent" && held.connections === 0 && (held.last_seen ?? held.created_at) < now - LIMITS.handleBoundS * 1e3) {
@@ -519,7 +521,7 @@ var FederatedCore = class extends RelayCore {
       const token = joined ? void 0 : `kre_${randomHex(32)}`;
       this.sql.all(
         `UPDATE endpoints SET harness = ?, model = ?, cwd = ?, parent = ?, labels = ?, token_hash = ${token ? "?" : "token_hash"}, auth = ?, connections = connections + 1, last_seen = ? WHERE id = ?`,
-        harness,
+        keptHarness(row.harness, harness),
         fields.model,
         fields.cwd,
         fields.parent,
@@ -2017,7 +2019,8 @@ function dashboardEvent(event, at) {
   if (event.type === "message") {
     const from = str(event.from);
     const to = Array.isArray(event.to) ? event.to.filter((a) => typeof a === "string") : [];
-    return from && to.length ? { type: "message", at, kind: str(event.kind) ?? "message", from, to } : null;
+    if (!from || !to.length) return null;
+    return { type: "message", at, kind: str(event.kind) ?? "message", from, to, ...event.local === true ? { local: true } : {} };
   }
   if (event.type === "presence") {
     const endpoint = str(event.endpoint);
@@ -2055,8 +2058,14 @@ var TOUCH_MS = 1e4;
 var HUB_MIGRATION = [
   `CREATE TABLE IF NOT EXISTS machines (name TEXT PRIMARY KEY, token_id TEXT NOT NULL, kind TEXT NOT NULL, created_at INTEGER NOT NULL, last_seen INTEGER)`,
   `CREATE TABLE IF NOT EXISTS hub_state (key TEXT PRIMARY KEY, value INTEGER NOT NULL)`,
-  `CREATE TABLE IF NOT EXISTS machine_tombstones (name TEXT PRIMARY KEY, actor_id TEXT, user_id TEXT, at INTEGER NOT NULL)`
+  `CREATE TABLE IF NOT EXISTS machine_tombstones (name TEXT PRIMARY KEY, actor_id TEXT, user_id TEXT, at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS local_events (from_ep TEXT NOT NULL, to_ep TEXT NOT NULL, at INTEGER NOT NULL)`,
+  `CREATE INDEX IF NOT EXISTS local_events_at ON local_events (at)`
 ];
+var LOCAL_EVENT_KINDS = ["message", "request", "reply", "status", "handback", "spawn"];
+var LOCAL_EVENTS = { perFrame: 200, recipients: 64, maxAgeMs: 3e5 };
+var LOCAL_KEEP_MS = 36e5;
+var HASH_ID = /^[0-9a-f]{8,64}$/;
 var OBSERVER_METHODS = /* @__PURE__ */ new Set(["list", "snapshot", "watch", "log", "machines", "machines_rm"]);
 var CLIENT_METHODS = /* @__PURE__ */ new Set(["whoami", "send", "ask", "ask_wait", "inbox", "ack", "status", "event", ...OBSERVER_METHODS]);
 var iso2 = (ms) => ms === null ? null : new Date(ms).toISOString();
@@ -2335,6 +2344,7 @@ var RelayHub = class _RelayHub {
     this.setState("last_validated", now);
     if (now - (this.state("last_sweep") ?? 0) >= SWEEP_MS) {
       this.relay.sweep();
+      this.deps.sql.all(`DELETE FROM local_events WHERE at <= ?`, now - LOCAL_KEEP_MS);
       this.expireMachines(now);
       this.forgetTombstones(now);
       const denied = this.deniedGrants();
@@ -2790,6 +2800,8 @@ var RelayHub = class _RelayHub {
         this.fleet({ type: "activity", machine, ...row });
         return row;
       }
+      case "local_events":
+        return { recorded: this.localEvents(machine, params.events) };
       case "list":
         if (params.contacts === true) return { contacts: await this.contactList() };
         return { endpoints: core.list(listFilters(params)) };
@@ -3137,6 +3149,47 @@ var RelayHub = class _RelayHub {
       hop: result.hop
     });
   }
+  /**
+   * a bridge's metadata for deliveries its broker made on its own machine: counted for the graph and streamed to watchers and the
+   * dashboard as `local`; unknown endpoints and malformed events are skipped
+   */
+  localEvents(machine, value) {
+    if (!Array.isArray(value) || value.length > LOCAL_EVENTS.perFrame)
+      throw new RelayError3("invalid_params", `events is a list of at most ${LOCAL_EVENTS.perFrame}`);
+    const core = this.relay;
+    const now = this.now();
+    const resolve = (handle) => {
+      try {
+        return core.remoteEndpoint(machine, handle);
+      } catch {
+        return null;
+      }
+    };
+    let recorded = 0;
+    for (const raw of value) {
+      if (typeof raw !== "object" || raw === null) continue;
+      const event = raw;
+      const kind = LOCAL_EVENT_KINDS.includes(event.kind) ? event.kind : null;
+      const at = typeof event.at === "number" && Number.isFinite(event.at) ? Math.min(Math.max(event.at, now - LOCAL_EVENTS.maxAgeMs), now) : null;
+      const from = resolve(event.from);
+      if (!kind || at === null || !from || !Array.isArray(event.to) || event.to.length > LOCAL_EVENTS.recipients) continue;
+      const to = [...new Set(event.to.map(resolve).filter((id) => id !== null && id !== from))];
+      if (!to.length) continue;
+      for (const ep of to) this.deps.sql.all(`INSERT INTO local_events (from_ep, to_ep, at) VALUES (?, ?, ?)`, from, ep, at);
+      this.fleet({
+        type: "message",
+        machine,
+        local: true,
+        ...typeof event.id === "string" && HASH_ID.test(event.id) ? { id: event.id } : {},
+        ...typeof event.thread === "string" && HASH_ID.test(event.thread) ? { thread: event.thread } : {},
+        kind,
+        from: core.addressOf(from),
+        to: to.map((ep) => core.addressOf(ep))
+      });
+      recorded++;
+    }
+    return recorded;
+  }
   /** the body's bytes when the account can hold them; refuses a send past the `storedBytes` limit */
   room(sender, body) {
     if (typeof body !== "string") return 0;
@@ -3354,9 +3407,13 @@ var RelayHub = class _RelayHub {
     const edges = nodes.flatMap(
       (n) => n.parent && n.parent !== n.address && listed.has(n.parent) ? [{ kind: "spawn", from: n.parent, to: n.address }] : []
     );
+    const since = this.now() - window * 6e4;
     const pairs = this.deps.sql.all(
-      `SELECT from_ep, to_ep, COUNT(*) AS n, MAX(at) AS last FROM pair_events WHERE at > ? GROUP BY from_ep, to_ep`,
-      this.now() - window * 6e4
+      `SELECT from_ep, to_ep, COUNT(*) AS n, MAX(at) AS last FROM (
+         SELECT from_ep, to_ep, at FROM pair_events WHERE at > ? UNION ALL SELECT from_ep, to_ep, at FROM local_events WHERE at > ?
+       ) GROUP BY from_ep, to_ep`,
+      since,
+      since
     );
     const traffic = pairs.flatMap((row) => {
       const from = byId.get(row.from_ep);
@@ -3364,7 +3421,9 @@ var RelayHub = class _RelayHub {
       return from && to && from !== to ? [{ kind: "message", from, to, count: row.n, last_at: iso2(row.last) }] : [];
     });
     edges.push(...traffic.sort((a, b) => a.from.localeCompare(b.from) || a.to.localeCompare(b.to)));
-    return { minutes: window, machines: this.machineList(), nodes, edges };
+    const actors = new Map(this.deps.sql.all(`SELECT name, actor_id FROM machines`).map((r) => [r.name, r.actor_id]));
+    const machines = this.machineList().map((m) => ({ ...m, actor: actors.get(m.name) ?? null }));
+    return { minutes: window, machines, nodes, edges };
   }
   machineList() {
     const sockets = this.deps.sockets().filter((s) => !this.gone.has(s.conn.id));
@@ -3522,6 +3581,8 @@ export {
   HUB_LIMITS,
   JUDGE_LIMITS,
   JudgeGate,
+  LOCAL_EVENTS,
+  LOCAL_EVENT_KINDS,
   MACHINE_NAME,
   RelayHub,
   dashboardEvent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scp-dev/hub",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Swarm Communication Protocol federation and account hub: mirrored endpoints across machines, bridges, direct clients and cross-account delivery.",
   "license": "Apache-2.0",
   "homepage": "https://scp.dev",
@@ -29,7 +29,7 @@
     "typecheck": "tsc -p ."
   },
   "dependencies": {
-    "@scp-dev/core": "0.1.4"
+    "@scp-dev/core": "0.1.5"
   },
   "devDependencies": {
     "tsup": "^8.5.1"

package/src/federation.ts

@@ -40,6 +40,8 @@ import {
   SYSTEM_HANDLE,
   seqList,
   unreachable,
+  harnessFits,
+  keptHarness,
   validHandle,
   validHarness,
   validSelector,
@@ -734,7 +736,7 @@ export class FederatedCore extends RelayCore {
           this.checkCap()
           row = this.byId(this.insertRemote(`${machine}/${handle}`, "agent", harness, auth, now)) as FederatedRow
           created = true
-        } else if (held.kind === "agent" && held.connections === 0 && held.harness === harness) {
+        } else if (held.kind === "agent" && held.connections === 0 && harnessFits(held.harness, harness)) {
           row = held
         } else if (
           exact &&
@@ -743,7 +745,7 @@ export class FederatedCore extends RelayCore {
           held.process === null &&
           held.session === null &&
           held.parent_ep === null &&
-          held.harness === harness
+          harnessFits(held.harness, harness)
         ) {
           row = held
           joined = true
@@ -764,7 +766,7 @@ export class FederatedCore extends RelayCore {
       const token = joined ? undefined : `kre_${randomHex(32)}`
       this.sql.all(
         `UPDATE endpoints SET harness = ?, model = ?, cwd = ?, parent = ?, labels = ?, token_hash = ${token ? "?" : "token_hash"}, auth = ?, connections = connections + 1, last_seen = ? WHERE id = ?`,
-        harness,
+        keptHarness(row.harness, harness),
         fields.model,
         fields.cwd,
         fields.parent,

package/src/hub.ts

@@ -111,10 +111,15 @@ export interface GraphNode {
 /** `spawn`: parent to child; `message`: sender to recipient, with the count and newest send in the graph's window */
 export type GraphEdge = { kind: "spawn"; from: string; to: string } | { kind: "message"; from: string; to: string; count: number; last_at: string }
 
+/** a machine as the graph draws it: `actor` is the member who bound it, null for the personal account's owner or a revoked name */
+export interface GraphMachine extends MachineRow {
+  actor: string | null
+}
+
 export interface HubGraph {
   /** the window message edges count over */
   minutes: number
-  machines: MachineRow[]
+  machines: GraphMachine[]
   nodes: GraphNode[]
   edges: GraphEdge[]
 }
@@ -126,7 +131,7 @@ export const GRAPH_MINUTES_MAX = 60
 /** what a read-only dashboard stream sees as it happens: addresses, kinds and presence, never a body, note, summary or label */
 export type DashboardEvent =
   | { type: "presence"; at: string; machine: string | null; endpoint: string; presence: Presence }
-  | { type: "message"; at: string; kind: string; from: string; to: string[] }
+  | { type: "message"; at: string; kind: string; from: string; to: string[]; local?: true }
   | { type: "endpoints"; at: string; machine: string }
 
 const str = (v: unknown): string | null => (typeof v === "string" ? v : null)
@@ -136,7 +141,8 @@ export function dashboardEvent(event: Record<string, unknown>, at: string): Dash
   if (event.type === "message") {
     const from = str(event.from)
     const to = Array.isArray(event.to) ? event.to.filter((a): a is string => typeof a === "string") : []
-    return from && to.length ? { type: "message", at, kind: str(event.kind) ?? "message", from, to } : null
+    if (!from || !to.length) return null
+    return { type: "message", at, kind: str(event.kind) ?? "message", from, to, ...(event.local === true ? { local: true as const } : {}) }
   }
   if (event.type === "presence") {
     const endpoint = str(event.endpoint)
@@ -218,8 +224,17 @@ const HUB_MIGRATION = [
   `CREATE TABLE IF NOT EXISTS machines (name TEXT PRIMARY KEY, token_id TEXT NOT NULL, kind TEXT NOT NULL, created_at INTEGER NOT NULL, last_seen INTEGER)`,
   `CREATE TABLE IF NOT EXISTS hub_state (key TEXT PRIMARY KEY, value INTEGER NOT NULL)`,
   `CREATE TABLE IF NOT EXISTS machine_tombstones (name TEXT PRIMARY KEY, actor_id TEXT, user_id TEXT, at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS local_events (from_ep TEXT NOT NULL, to_ep TEXT NOT NULL, at INTEGER NOT NULL)`,
+  `CREATE INDEX IF NOT EXISTS local_events_at ON local_events (at)`,
 ]
 
+/** kinds a bridge reports for a delivery its broker made on its own machine */
+export const LOCAL_EVENT_KINDS = ["message", "request", "reply", "status", "handback", "spawn"] as const
+/** events per `local_events` frame, recipients per event, and how far back an event's `at` may lie */
+export const LOCAL_EVENTS = { perFrame: 200, recipients: 64, maxAgeMs: 300_000 } as const
+const LOCAL_KEEP_MS = 3_600_000
+const HASH_ID = /^[0-9a-f]{8,64}$/
+
 const OBSERVER_METHODS = new Set(["list", "snapshot", "watch", "log", "machines", "machines_rm"])
 const CLIENT_METHODS = new Set(["whoami", "send", "ask", "ask_wait", "inbox", "ack", "status", "event", ...OBSERVER_METHODS])
 
@@ -542,6 +557,7 @@ export class RelayHub {
     this.setState("last_validated", now)
     if (now - (this.state("last_sweep") ?? 0) >= SWEEP_MS) {
       this.relay.sweep()
+      this.deps.sql.all(`DELETE FROM local_events WHERE at <= ?`, now - LOCAL_KEEP_MS)
       this.expireMachines(now)
       this.forgetTombstones(now)
       const denied = this.deniedGrants()
@@ -1024,6 +1040,8 @@ export class RelayHub {
         this.fleet({ type: "activity", machine, ...row })
         return row
       }
+      case "local_events":
+        return { recorded: this.localEvents(machine, params.events) }
       case "list":
         if (params.contacts === true) return { contacts: await this.contactList() }
         return { endpoints: core.list(listFilters(params)) }
@@ -1405,6 +1423,48 @@ export class RelayHub {
     })
   }
 
+  /**
+   * a bridge's metadata for deliveries its broker made on its own machine: counted for the graph and streamed to watchers and the
+   * dashboard as `local`; unknown endpoints and malformed events are skipped
+   */
+  private localEvents(machine: string, value: unknown): number {
+    if (!Array.isArray(value) || value.length > LOCAL_EVENTS.perFrame)
+      throw new RelayError("invalid_params", `events is a list of at most ${LOCAL_EVENTS.perFrame}`)
+    const core = this.relay
+    const now = this.now()
+    const resolve = (handle: unknown): string | null => {
+      try {
+        return core.remoteEndpoint(machine, handle)
+      } catch {
+        return null
+      }
+    }
+    let recorded = 0
+    for (const raw of value) {
+      if (typeof raw !== "object" || raw === null) continue
+      const event = raw as Params
+      const kind = (LOCAL_EVENT_KINDS as readonly unknown[]).includes(event.kind) ? (event.kind as string) : null
+      const at = typeof event.at === "number" && Number.isFinite(event.at) ? Math.min(Math.max(event.at, now - LOCAL_EVENTS.maxAgeMs), now) : null
+      const from = resolve(event.from)
+      if (!kind || at === null || !from || !Array.isArray(event.to) || event.to.length > LOCAL_EVENTS.recipients) continue
+      const to = [...new Set(event.to.map(resolve).filter((id): id is string => id !== null && id !== from))]
+      if (!to.length) continue
+      for (const ep of to) this.deps.sql.all(`INSERT INTO local_events (from_ep, to_ep, at) VALUES (?, ?, ?)`, from, ep, at)
+      this.fleet({
+        type: "message",
+        machine,
+        local: true,
+        ...(typeof event.id === "string" && HASH_ID.test(event.id) ? { id: event.id } : {}),
+        ...(typeof event.thread === "string" && HASH_ID.test(event.thread) ? { thread: event.thread } : {}),
+        kind,
+        from: core.addressOf(from),
+        to: to.map((ep) => core.addressOf(ep)),
+      })
+      recorded++
+    }
+    return recorded
+  }
+
   /** the body's bytes when the account can hold them; refuses a send past the `storedBytes` limit */
   private room(sender: string, body: unknown): number {
     if (typeof body !== "string") return 0
@@ -1647,9 +1707,13 @@ export class RelayHub {
     const edges: GraphEdge[] = nodes.flatMap((n) =>
       n.parent && n.parent !== n.address && listed.has(n.parent) ? [{ kind: "spawn" as const, from: n.parent, to: n.address }] : [],
     )
+    const since = this.now() - window * 60_000
     const pairs = this.deps.sql.all<{ from_ep: string; to_ep: string; n: number; last: number }>(
-      `SELECT from_ep, to_ep, COUNT(*) AS n, MAX(at) AS last FROM pair_events WHERE at > ? GROUP BY from_ep, to_ep`,
-      this.now() - window * 60_000,
+      `SELECT from_ep, to_ep, COUNT(*) AS n, MAX(at) AS last FROM (
+         SELECT from_ep, to_ep, at FROM pair_events WHERE at > ? UNION ALL SELECT from_ep, to_ep, at FROM local_events WHERE at > ?
+       ) GROUP BY from_ep, to_ep`,
+      since,
+      since,
     )
     const traffic: GraphEdge[] = pairs.flatMap((row) => {
       const from = byId.get(row.from_ep)
@@ -1657,7 +1721,9 @@ export class RelayHub {
       return from && to && from !== to ? [{ kind: "message" as const, from, to, count: row.n, last_at: iso(row.last)! }] : []
     })
     edges.push(...traffic.sort((a, b) => a.from.localeCompare(b.from) || a.to.localeCompare(b.to)))
-    return { minutes: window, machines: this.machineList(), nodes, edges }
+    const actors = new Map(this.deps.sql.all<{ name: string; actor_id: string | null }>(`SELECT name, actor_id FROM machines`).map((r) => [r.name, r.actor_id]))
+    const machines = this.machineList().map((m) => ({ ...m, actor: actors.get(m.name) ?? null }))
+    return { minutes: window, machines, nodes, edges }
   }
 
   private machineList(): MachineRow[] {

package/src/index.ts

@@ -18,9 +18,12 @@ export {
   GRAPH_MINUTES,
   GRAPH_MINUTES_MAX,
   type GraphEdge,
+  type GraphMachine,
   type GraphNode,
   type HubGraph,
   HUB_LIMITS,
+  LOCAL_EVENT_KINDS,
+  LOCAL_EVENTS,
   RelayHub,
   type HubAuth,
   type HubConn,