@scp-dev/broker 0.1.5 → 0.2.0

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { ScopePreset, Claim, EndpointView, ScopeView, ScopeChange, Envelope, StopWaitResult, PeerView, WhyResult, SendParams, SendResult, ActivityRow, XListed, Selector } from '@scp-dev/core';
-import { JudgeGate as JudgeGate$1, JudgeGateOptions, FederatedCore, Deliver, JudgeClientFactory } from '@scp-dev/hub';
+import { LOCAL_EVENT_KINDS, JudgeGate as JudgeGate$1, JudgeGateOptions, FederatedCore, Deliver, JudgeClientFactory } from '@scp-dev/hub';
 export { Deliver } from '@scp-dev/hub';
 
 interface Hello {
@@ -203,6 +203,10 @@ interface BridgeConfig {
     redact?: boolean | undefined;
     /** let messages marked private leave this machine */
     privateOk?: boolean | undefined;
+    /** report metadata of deliveries made on this machine to the hub, never bodies (default true) */
+    mirrorLocal?: boolean | undefined;
+    /** how often reported local deliveries are sent, batched (default 1 s) */
+    localFlushMs?: number | undefined;
     /** a WebSocket constructor (tests) */
     webSocket?: WebSocketWithHeaders | undefined;
     /** reconnect backoff bounds (default 1 s to 60 s) */
@@ -236,6 +240,8 @@ interface Bridge {
     renamed(endpoint: string, was: string): void;
     /** a local endpoint's activity, posted to the hub */
     activity(endpoint: string, row: ActivityRow): void;
+    /** a delivery between endpoints of this machine: its metadata is reported to the hub in the next batch */
+    local?(event: LocalDelivery): void;
     /** the endpoints of other accounts this account's contacts let it reach */
     contacts(): Promise<XListed[]>;
     status(): BridgeStatus;
@@ -243,6 +249,15 @@ interface Bridge {
     relink(): void;
     stop(): Promise<void>;
 }
+/** a delivery the broker made on its own machine, by local endpoint ids */
+interface LocalDelivery {
+    id: string;
+    thread: string;
+    kind: (typeof LOCAL_EVENT_KINDS)[number];
+    from: string;
+    to: string[];
+    at: number;
+}
 /** a WebSocket constructor that takes request headers, as Bun's and Node's do */
 type WebSocketWithHeaders = new (url: string, options: {
     headers: Record<string, string>;

package/dist/index.js

@@ -524,6 +524,7 @@ async function connectBroker(dir, start, timeoutMs = 5e3) {
 }
 
 // src/bridge.ts
+import { createHash as createHash2 } from "crypto";
 import {
   findSecret,
   matches,
@@ -532,7 +533,9 @@ import {
   RelayError as RelayError2,
   XLIMITS
 } from "@scp-dev/core";
-import { HUB_LIMITS } from "@scp-dev/hub";
+import { HUB_LIMITS, LOCAL_EVENTS } from "@scp-dev/hub";
+var LOCAL_BUFFER = 1e3;
+var idHash = (id) => createHash2("sha256").update(id).digest("hex").slice(0, 32);
 var CONNECT_TIMEOUT_MS = 15e3;
 var MIRROR_FRAME_BYTES = HUB_LIMITS.frameBytes - 1024;
 var DEAD_LETTER_ATTEMPTS = 3;
@@ -578,6 +581,10 @@ function startBridge(options) {
   const inflight = /* @__PURE__ */ new Set();
   const injectFailures = /* @__PURE__ */ new Map();
   const fromHub = /* @__PURE__ */ new WeakSet();
+  const mirrorLocal = options.mirrorLocal !== false;
+  const localFlushMs = options.localFlushMs ?? 1e3;
+  let localQueue = [];
+  let localTimer;
   const setState = (up, error) => {
     if (up !== connected || error !== lastError) since = (/* @__PURE__ */ new Date()).toISOString();
     connected = up;
@@ -887,21 +894,21 @@ function startBridge(options) {
     if (event.machine === machine) return;
     if (event.type === "message" && Array.isArray(event.to) && account && event.to.some((to) => typeof to === "string" && to.startsWith(`${account}/${machine}/`)))
       return;
-    const local = {
+    const local2 = {
       ...event,
       ...event.from !== void 0 ? { from: localAddress(event.from) } : {},
       ...event.endpoint !== void 0 ? { endpoint: localAddress(event.endpoint) } : {}
     };
-    if (Array.isArray(event.to)) local.to = event.to.map(localAddress);
+    if (Array.isArray(event.to)) local2.to = event.to.map(localAddress);
     if (event.type === "activity" && typeof event.machine === "string" && typeof event.endpoint === "string") {
       try {
         const proxy = core.remoteEndpoint(event.machine, event.endpoint.slice(event.endpoint.lastIndexOf("/") + 1));
         const row = core.event(proxy, { kind: event.kind, tool: event.tool, summary: event.summary });
-        local.id = row.id;
+        local2.id = row.id;
       } catch {
       }
     }
-    hooks.fleet(local);
+    hooks.fleet(local2);
   }
   function linkUp(ms) {
     if (connected) return Promise.resolve(true);
@@ -1017,6 +1024,30 @@ function startBridge(options) {
     if (handle === null) return;
     fireAndForget("event", { from: handle, kind: row.kind, tool: row.tool ?? void 0, summary: options.redact ? void 0 : row.summary ?? void 0 });
   }
+  function local(event) {
+    if (!mirrorLocal || !ready || localQueue.length >= LOCAL_BUFFER) return;
+    let from;
+    let to;
+    try {
+      from = handleOf(event.from);
+      to = event.to.flatMap((endpoint) => {
+        const handle = handleOf(endpoint);
+        return handle === null ? [] : [handle];
+      });
+    } catch {
+      return;
+    }
+    if (from === null || !to.length) return;
+    localQueue.push({ from, to: to.slice(0, LOCAL_EVENTS.recipients), kind: event.kind, at: event.at, id: idHash(event.id), thread: idHash(event.thread) });
+    localTimer ??= setTimeout(flushLocal, localFlushMs);
+  }
+  function flushLocal() {
+    localTimer = void 0;
+    const batch = localQueue;
+    localQueue = [];
+    if (!ready) return;
+    for (let i = 0; i < batch.length; i += LOCAL_EVENTS.perFrame) fireAndForget("local_events", { events: batch.slice(i, i + LOCAL_EVENTS.perFrame) });
+  }
   async function contacts() {
     if (!connected) throw unavailable();
     return (await request("list", { contacts: true })).contacts;
@@ -1038,6 +1069,8 @@ function startBridge(options) {
     stopped = true;
     shutDown = true;
     clearTimeout(retryTimer);
+    clearTimeout(localTimer);
+    localQueue = [];
     for (const fn of waitingForLink.splice(0)) fn();
     const socket = ws;
     ws = null;
@@ -1048,11 +1081,11 @@ function startBridge(options) {
     if (socket && socket.readyState <= 1) socket.close(1e3, "the broker stopped");
   }
   connect();
-  return { forward, changed, resync, renamed, activity, contacts, status, relink, stop };
+  return { forward, changed, resync, renamed, activity, local, contacts, status, relink, stop };
 }
 
 // src/server.ts
-import { createHash as createHash3 } from "crypto";
+import { createHash as createHash4 } from "crypto";
 import { chmodSync as chmodSync2, existsSync as existsSync2, lstatSync as lstatSync2, readFileSync as readFileSync6, realpathSync, rmdirSync, unlinkSync as unlinkSync2 } from "fs";
 import { createServer } from "net";
 import { homedir as homedir2 } from "os";
@@ -1089,11 +1122,11 @@ var JudgeGate = class extends HubJudgeGate {
 };
 
 // src/sessions.ts
-import { createHash as createHash2, randomUUID } from "crypto";
+import { createHash as createHash3, randomUUID } from "crypto";
 import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, renameSync as renameSync2, rmSync, writeFileSync } from "fs";
 import { join as join3 } from "path";
 function markerPath(dir, session) {
-  return join3(dir, "sessions", `${createHash2("sha256").update(session).digest("hex").slice(0, 32)}.json`);
+  return join3(dir, "sessions", `${createHash3("sha256").update(session).digest("hex").slice(0, 32)}.json`);
 }
 function readSessionMarker(dir, session) {
   try {
@@ -1282,7 +1315,7 @@ async function startBroker(options) {
     core = new FederatedCore(sql, {
       ...compact({ user: options.user, machine, now: options.now }),
       hostname: shortHostname(),
-      hash: (value) => createHash3("sha256").update(value).digest("hex"),
+      hash: (value) => createHash4("sha256").update(value).digest("hex"),
       transaction: (fn) => db.transaction(fn),
       alive: running
     });
@@ -1981,13 +2014,23 @@ async function startBroker(options) {
   }
   const deliver = async (from, send, origin) => {
     let result;
-    if (remoteSend(core, send.to)) {
+    const remote = remoteSend(core, send.to);
+    if (remote) {
       if (!bridge)
         throw new RelayError3("not_found", `no endpoint ${[send.to].flat().join(", ")}: this broker is not linked to a hub (${CLI_NAME} cloud enable)`);
       result = await bridge.forward(from, send, origin);
     } else {
       result = core.send(from, send, origin);
     }
+    if (!result.duplicate && !remote)
+      bridge?.local?.({
+        id: result.id,
+        thread: result.thread,
+        kind: send.kind ?? "message",
+        from,
+        to: result.deliveries.map((d) => d.endpoint),
+        at: (options.now ?? Date.now)()
+      });
     if (!result.duplicate) {
       for (const delivery of result.deliveries) wake(delivery.endpoint);
       notifyFleet({
@@ -2113,6 +2156,14 @@ async function startBroker(options) {
         if (result.handback) {
           const message = core.delivered(spawner, result.handback);
           wake(spawner);
+          bridge?.local?.({
+            id: message.id,
+            thread: message.thread,
+            kind: "handback",
+            from: message.from.endpoint,
+            to: [spawner],
+            at: (options.now ?? Date.now)()
+          });
           notifyFleet({
             type: "message",
             id: message.id,

package/dist/testing.d.ts

@@ -1,4 +1,4 @@
-import { RelayHub, HubSocket } from '@scp-dev/hub';
+import { RelayHub, HubSocket, DashboardEvent } from '@scp-dev/hub';
 
 interface FakeToken {
     tokenId: string;
@@ -27,6 +27,8 @@ declare function startFakeHubServer(options: {
     tokens: Record<string, FakeToken>;
     now?: (() => number) | undefined;
     contacts?: [string, string][] | undefined;
+    /** receives every account's dashboard events */
+    dashboard?: ((account: string, event: DashboardEvent) => void) | undefined;
 }): FakeHubServer;
 
 export { type FakeHubServer, type FakeToken, startFakeHubServer };

package/dist/testing.js

@@ -37,7 +37,8 @@ function startFakeHubServer(options) {
       sockets: () => [...sockets.values()],
       save: () => {
       },
-      ...options.contacts ? { crossAccount: linkOf(name) } : {}
+      ...options.contacts ? { crossAccount: linkOf(name) } : {},
+      ...options.dashboard ? { dashboard: (event) => options.dashboard(name, event) } : {}
     });
     hub.init({ user: name });
     found = { hub, sockets, traps: /* @__PURE__ */ new Map() };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scp-dev/broker",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "description": "Swarm Communication Protocol local broker: a unix socket JSON-RPC daemon over the relay core, its client and the bridge to an account hub.",
   "license": "Apache-2.0",
   "homepage": "https://scp.dev",
@@ -37,8 +37,8 @@
     "typecheck": "tsc -p ."
   },
   "dependencies": {
-    "@scp-dev/core": "0.1.4",
-    "@scp-dev/hub": "0.3.1"
+    "@scp-dev/core": "0.1.5",
+    "@scp-dev/hub": "0.5.0"
   },
   "devDependencies": {
     "tsup": "^8.5.1"

package/src/bridge.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto"
 import {
   type ActivityRow,
   type EndpointView,
@@ -13,7 +14,7 @@ import {
   XLIMITS,
   type XListed,
 } from "@scp-dev/core"
-import { type FederatedCore, HUB_LIMITS, type RemoteView } from "@scp-dev/hub"
+import { type FederatedCore, HUB_LIMITS, type LOCAL_EVENT_KINDS, LOCAL_EVENTS, type RemoteView } from "@scp-dev/hub"
 
 /** what the broker gives the bridge: waking local waiters, telling local watchers, the local policy's state */
 export interface BridgeHooks {
@@ -42,6 +43,10 @@ export interface BridgeConfig {
   redact?: boolean | undefined
   /** let messages marked private leave this machine */
   privateOk?: boolean | undefined
+  /** report metadata of deliveries made on this machine to the hub, never bodies (default true) */
+  mirrorLocal?: boolean | undefined
+  /** how often reported local deliveries are sent, batched (default 1 s) */
+  localFlushMs?: number | undefined
   /** a WebSocket constructor (tests) */
   webSocket?: WebSocketWithHeaders | undefined
   /** reconnect backoff bounds (default 1 s to 60 s) */
@@ -77,6 +82,8 @@ export interface Bridge {
   renamed(endpoint: string, was: string): void
   /** a local endpoint's activity, posted to the hub */
   activity(endpoint: string, row: ActivityRow): void
+  /** a delivery between endpoints of this machine: its metadata is reported to the hub in the next batch */
+  local?(event: LocalDelivery): void
   /** the endpoints of other accounts this account's contacts let it reach */
   contacts(): Promise<XListed[]>
   status(): BridgeStatus
@@ -85,6 +92,21 @@ export interface Bridge {
   stop(): Promise<void>
 }
 
+/** a delivery the broker made on its own machine, by local endpoint ids */
+export interface LocalDelivery {
+  id: string
+  thread: string
+  kind: (typeof LOCAL_EVENT_KINDS)[number]
+  from: string
+  to: string[]
+  at: number
+}
+
+/** local deliveries waiting for the next batch; past this, new ones are dropped until it is sent */
+const LOCAL_BUFFER = 1000
+
+const idHash = (id: string): string => createHash("sha256").update(id).digest("hex").slice(0, 32)
+
 /** a WebSocket constructor that takes request headers, as Bun's and Node's do */
 export type WebSocketWithHeaders = new (url: string, options: { headers: Record<string, string> }) => WebSocket
 
@@ -153,6 +175,10 @@ export function startBridge(options: BridgeOptions): Bridge {
   const inflight = new Set<number>()
   const injectFailures = new Map<number, number>()
   const fromHub = new WeakSet<Error>()
+  const mirrorLocal = options.mirrorLocal !== false
+  const localFlushMs = options.localFlushMs ?? 1000
+  let localQueue: object[] = []
+  let localTimer: ReturnType<typeof setTimeout> | undefined
 
   const setState = (up: boolean, error: string | null) => {
     if (up !== connected || error !== lastError) since = new Date().toISOString()
@@ -636,6 +662,32 @@ export function startBridge(options: BridgeOptions): Bridge {
     fireAndForget("event", { from: handle, kind: row.kind, tool: row.tool ?? undefined, summary: options.redact ? undefined : (row.summary ?? undefined) })
   }
 
+  function local(event: LocalDelivery) {
+    if (!mirrorLocal || !ready || localQueue.length >= LOCAL_BUFFER) return
+    let from: string | null
+    let to: string[]
+    try {
+      from = handleOf(event.from)
+      to = event.to.flatMap((endpoint) => {
+        const handle = handleOf(endpoint)
+        return handle === null ? [] : [handle]
+      })
+    } catch {
+      return
+    }
+    if (from === null || !to.length) return
+    localQueue.push({ from, to: to.slice(0, LOCAL_EVENTS.recipients), kind: event.kind, at: event.at, id: idHash(event.id), thread: idHash(event.thread) })
+    localTimer ??= setTimeout(flushLocal, localFlushMs)
+  }
+
+  function flushLocal() {
+    localTimer = undefined
+    const batch = localQueue
+    localQueue = []
+    if (!ready) return
+    for (let i = 0; i < batch.length; i += LOCAL_EVENTS.perFrame) fireAndForget("local_events", { events: batch.slice(i, i + LOCAL_EVENTS.perFrame) })
+  }
+
   async function contacts(): Promise<XListed[]> {
     if (!connected) throw unavailable()
     return (await request<{ contacts: XListed[] }>("list", { contacts: true })).contacts
@@ -660,6 +712,8 @@ export function startBridge(options: BridgeOptions): Bridge {
     stopped = true
     shutDown = true
     clearTimeout(retryTimer)
+    clearTimeout(localTimer)
+    localQueue = []
     for (const fn of waitingForLink.splice(0)) fn()
     const socket = ws
     ws = null
@@ -671,5 +725,5 @@ export function startBridge(options: BridgeOptions): Bridge {
   }
 
   connect()
-  return { forward, changed, resync, renamed, activity, contacts, status, relink, stop }
+  return { forward, changed, resync, renamed, activity, local, contacts, status, relink, stop }
 }

package/src/server.ts

@@ -996,13 +996,23 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
   /** a send as `from` to local endpoints or, through the bridge, other machines; wakes the recipients and tells the watchers */
   const deliver: Deliver = async (from, send, origin) => {
     let result: SendResult
-    if (remoteSend(core, send.to)) {
+    const remote = remoteSend(core, send.to)
+    if (remote) {
       if (!bridge)
         throw new RelayError("not_found", `no endpoint ${[send.to].flat().join(", ")}: this broker is not linked to a hub (${CLI_NAME} cloud enable)`)
       result = await bridge.forward(from, send, origin)
     } else {
       result = core.send(from, send, origin)
     }
+    if (!result.duplicate && !remote)
+      bridge?.local?.({
+        id: result.id,
+        thread: result.thread,
+        kind: send.kind ?? "message",
+        from,
+        to: result.deliveries.map((d) => d.endpoint),
+        at: (options.now ?? Date.now)(),
+      })
     if (!result.duplicate) {
       for (const delivery of result.deliveries) wake(delivery.endpoint)
       notifyFleet({
@@ -1138,6 +1148,14 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
         if (result.handback) {
           const message = core.delivered(spawner, result.handback)!
           wake(spawner)
+          bridge?.local?.({
+            id: message.id,
+            thread: message.thread,
+            kind: "handback",
+            from: message.from.endpoint,
+            to: [spawner],
+            at: (options.now ?? Date.now)(),
+          })
           notifyFleet({
             type: "message",
             id: message.id,

package/src/testing.ts

@@ -1,7 +1,7 @@
 import { Database } from "bun:sqlite"
 import type { Server, ServerWebSocket } from "bun"
 import { type CrossAccountLink, PASS_GATE, sha256Hex, type Sql, type SqlValue, type XContact } from "@scp-dev/core"
-import { type HubGrant, type HubSocket, RelayHub } from "@scp-dev/hub"
+import { type DashboardEvent, type HubGrant, type HubSocket, RelayHub } from "@scp-dev/hub"
 
 export interface FakeToken {
   tokenId: string
@@ -40,6 +40,8 @@ export function startFakeHubServer(options: {
   tokens: Record<string, FakeToken>
   now?: (() => number) | undefined
   contacts?: [string, string][] | undefined
+  /** receives every account's dashboard events */
+  dashboard?: ((account: string, event: DashboardEvent) => void) | undefined
 }): FakeHubServer {
   const accounts = new Map<string, Account>()
 
@@ -77,6 +79,7 @@ export function startFakeHubServer(options: {
       sockets: () => [...sockets.values()],
       save: () => {},
       ...(options.contacts ? { crossAccount: linkOf(name) } : {}),
+      ...(options.dashboard ? { dashboard: (event: DashboardEvent) => options.dashboard!(name, event) } : {}),
     })
     hub.init({ user: name })
     found = { hub, sockets, traps: new Map() }