@semiont/gateway 0.5.29 → 0.5.30

package/dist/index.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 import winston from 'winston';
-import { recordSubscriberConnect, recordSubscriberDisconnect, withTraceparent, withSpan, recordBusEmit, SpanKind, injectTraceparent, getLogTraceContext } from '@semiont/observability';
+import { registerCorrelationRegistryProvider, recordSubscriberConnect, withTraceparent, withSpan, recordBusEmit, recordUnanswerableRequest, SpanKind, recordSubscriberDisconnect, recordReplySuppressed, recordResumeGap, injectTraceparent, getLogTraceContext } from '@semiont/observability';
 import { z } from 'zod';
 import jwt from 'jsonwebtoken';
-import { email, userId, googleCredential, agentToDid, kbDid, BUS_OPERATIONS, EventBus, accessToken, userToDid, resourceId, CHANNEL_SCHEMAS, busLog, baseMediaType, isSupportedMediaType } from '@semiont/core';
+import { email, userId, googleCredential, agentToDid, kbDid, replyChannelsFor, BUS_OPERATIONS, EventBus, accessToken, userToDid, resourceId, CHANNEL_SCHEMAS, busLog, baseMediaType, isSupportedMediaType } from '@semiont/core';
 import { cors } from 'hono/cors';
 import { serve } from '@hono/node-server';
 import { Hono } from 'hono';
@@ -10346,7 +10346,7 @@ healthRouter.get("/api/health", async (c) => {
   const response = {
     status: "operational",
     message: "Semiont API is running",
-    version: "0.5.29",
+    version: "0.5.30",
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     database: dbStatus ? "connected" : "disconnected",
     environment: nodeEnv
@@ -11011,10 +11011,14 @@ authRouter.get("/api/cookies/export", authMiddleware, async (c) => {
 
 // src/lib/archivist.ts
 init_logger();
+var archivistSpan = (op, run) => withSpan(`archivist.${op}`, run, {
+  kind: SpanKind.CLIENT,
+  attrs: { "peer.service": "archivist" }
+});
 async function kbBranch(config2) {
   try {
     const { base, headers } = archivistEndpoint(config2);
-    const res = await fetch(`${base}/kb/branch`, { headers });
+    const res = await archivistSpan("kb.branch", () => fetch(`${base}/kb/branch`, { headers }));
     if (!res.ok) return void 0;
     const { branch } = await res.json();
     return branch ?? void 0;
@@ -11027,7 +11031,7 @@ async function putContent(config2, storageUri, body) {
   const url = `${base}/content/${encodeURIComponent(storageUri)}`;
   let res;
   try {
-    res = await fetch(url, { method: "PUT", headers, body });
+    res = await archivistSpan("content.put", () => fetch(url, { method: "PUT", headers, body }));
   } catch (error) {
     getLogger().error("Archivist content write unreachable", {
       component: "archivist-client",
@@ -11052,7 +11056,7 @@ async function getContent(config2, resourceId2) {
   const url = `${base}/resources/${encodeURIComponent(resourceId2)}/content`;
   let res;
   try {
-    res = await fetch(url, { headers });
+    res = await archivistSpan("content.get", () => fetch(url, { headers }));
   } catch (error) {
     getLogger().error("Archivist content read unreachable", {
       component: "archivist-client",
@@ -11096,7 +11100,7 @@ statusRouter.get("/api/status", async (c) => {
   const did = kbDid(siteDomain);
   const response = {
     status: "operational",
-    version: "0.5.29",
+    version: "0.5.30",
     features: {
       semanticContent: "planned",
       collaboration: "planned",
@@ -11494,9 +11498,13 @@ init_logger();
 var getBusLogger = () => getLogger().child({ component: "bus" });
 async function fetchArchivistReplay(config2, resourceId2, fromSequence) {
   const { base, headers } = archivistEndpoint(config2);
-  const res = await fetch(
-    `${base}/events/${encodeURIComponent(resourceId2)}?fromSequence=${fromSequence}`,
-    { headers }
+  const res = await withSpan(
+    "archivist.events.replay",
+    () => fetch(
+      `${base}/events/${encodeURIComponent(resourceId2)}?fromSequence=${fromSequence}`,
+      { headers }
+    ),
+    { kind: SpanKind.CLIENT, attrs: { "peer.service": "archivist" } }
   );
   if (!res.ok) {
     throw new Error(`Archivist replay read failed: ${res.status} ${res.statusText}`);
@@ -11528,43 +11536,138 @@ function extractSequence(payload) {
 }
 var MAX_SCOPES = 512;
 var SCOPE_WARN_THRESHOLD = 128;
-var REPLY_CHANNELS = [
-  ...new Set(Object.values(BUS_OPERATIONS).flatMap((op) => [op.result, op.failure]))
-];
+var MAX_PENDING_WRITE_BYTES = 16 * 1024 * 1024;
+var MAX_REPLAY_BUFFER_EVENTS = 1e3;
+var CORRELATED_CHANNELS = new Set(replyChannelsFor(Object.keys(BUS_OPERATIONS)));
+var PROGRESS_CHANNELS = new Set(
+  Object.values(BUS_OPERATIONS).flatMap(
+    (op) => "progress" in op && op.progress ? [op.progress] : []
+  )
+);
 var REPLY_RETENTION_TTL_MS = 6e4;
 var REPLY_RETENTION_MAX = 1024;
 var PENDING_REPLIES_MAX = 256;
-function createReplyRetention(eventBus2, opts = {}) {
+var CLAIM_TTL_MS = 15 * 6e4;
+var CLAIM_MAX_GLOBAL = 4096;
+var correlationIdOf = (payload) => {
+  const cid = payload?.correlationId;
+  return typeof cid === "string" && cid.length > 0 ? cid : void 0;
+};
+function createCorrelationRegistry(eventBus2, opts = {}) {
   const ttlMs = opts.ttlMs ?? REPLY_RETENTION_TTL_MS;
   const max = opts.max ?? REPLY_RETENTION_MAX;
+  const claimTtlMs = opts.claimTtlMs ?? CLAIM_TTL_MS;
   const now = opts.now ?? Date.now;
-  const buffer = /* @__PURE__ */ new Map();
-  const subs = REPLY_CHANNELS.map(
-    (channel) => eventBus2.get(channel).subscribe((payload) => {
-      const cid = payload?.correlationId;
-      if (typeof cid !== "string" || cid.length === 0) return;
-      buffer.delete(cid);
-      buffer.set(cid, { channel, payload, retainedAt: now() });
-      while (buffer.size > max) {
-        const oldest = buffer.keys().next().value;
-        if (oldest === void 0) break;
-        buffer.delete(oldest);
+  const claims = /* @__PURE__ */ new Map();
+  const perClient = /* @__PURE__ */ new Map();
+  const release = (clientId) => {
+    const n = (perClient.get(clientId) ?? 1) - 1;
+    if (n <= 0) perClient.delete(clientId);
+    else perClient.set(clientId, n);
+  };
+  const forget = (cid) => {
+    const claim = claims.get(cid);
+    if (!claim) return;
+    claims.delete(cid);
+    if (!claim.answered) release(claim.clientId);
+  };
+  const sweepClaims = () => {
+    const cutoff = now() - claimTtlMs;
+    for (const [cid, claim] of claims) {
+      if (claim.claimedAt > cutoff) break;
+      if (!claim.reply) {
+        getBusLogger().warn("[bus CLAIM-EXPIRED] claim swept with no reply", {
+          correlationId: cid,
+          clientId: claim.clientId,
+          ageMs: now() - claim.claimedAt
+        });
       }
+      forget(cid);
+    }
+  };
+  const sweepReplies = () => {
+    const cutoff = now() - ttlMs;
+    let retained = 0;
+    for (const claim of claims.values()) {
+      if (!claim.reply) continue;
+      if (claim.reply.retainedAt <= cutoff) delete claim.reply;
+      else retained++;
+    }
+    if (retained <= max) return;
+    let excess = retained - max;
+    for (const claim of claims.values()) {
+      if (excess === 0) break;
+      if (claim.reply) {
+        delete claim.reply;
+        excess--;
+      }
+    }
+  };
+  const subs = [...CORRELATED_CHANNELS].map(
+    (channel) => eventBus2.get(channel).subscribe((payload) => {
+      const cid = correlationIdOf(payload);
+      if (!cid) return;
+      const claim = claims.get(cid);
+      if (!claim) return;
+      claim.claimedAt = now();
+      if (PROGRESS_CHANNELS.has(channel)) return;
+      if (!claim.answered) {
+        claim.answered = true;
+        release(claim.clientId);
+      }
+      claim.reply = { channel, payload, retainedAt: now() };
+      sweepClaims();
+      sweepReplies();
     })
   );
   return {
-    lookup(correlationId) {
-      const entry = buffer.get(correlationId);
-      if (!entry) return void 0;
-      if (now() - entry.retainedAt > ttlMs) {
-        buffer.delete(correlationId);
+    claim(cid, clientId, principalDid) {
+      sweepClaims();
+      const existing = claims.get(cid);
+      if (existing) return "conflict";
+      if ((perClient.get(clientId) ?? 0) >= PENDING_REPLIES_MAX) return "at-capacity";
+      if (claims.size >= CLAIM_MAX_GLOBAL) {
+        const oldest = claims.keys().next().value;
+        if (oldest !== void 0) {
+          getBusLogger().warn("[bus CLAIM-EVICTED] global claim cap reached", {
+            correlationId: oldest,
+            cap: CLAIM_MAX_GLOBAL
+          });
+          forget(oldest);
+        }
+      }
+      claims.set(cid, { clientId, principalDid, claimedAt: now() });
+      perClient.set(clientId, (perClient.get(clientId) ?? 0) + 1);
+      return "ok";
+    },
+    owner(cid) {
+      const claim = claims.get(cid);
+      if (!claim) return void 0;
+      if (now() - claim.claimedAt > claimTtlMs) return void 0;
+      return { clientId: claim.clientId, principalDid: claim.principalDid };
+    },
+    lookupReply(cid, clientId, principalDid) {
+      const claim = claims.get(cid);
+      if (!claim?.reply) return void 0;
+      if (claim.clientId !== clientId || claim.principalDid !== principalDid) return void 0;
+      if (now() - claim.reply.retainedAt > ttlMs) {
+        delete claim.reply;
         return void 0;
       }
-      return entry;
+      return claim.reply;
+    },
+    size() {
+      return claims.size;
+    },
+    occupancy() {
+      let retainedReplies = 0;
+      for (const claim of claims.values()) if (claim.reply) retainedReplies++;
+      return { claims: claims.size, retainedReplies };
     },
     dispose() {
-      for (const s of subs) s.unsubscribe();
-      buffer.clear();
+      for (const sub of subs) sub.unsubscribe();
+      claims.clear();
+      perClient.clear();
     }
   };
 }
@@ -11573,7 +11676,10 @@ function parseSubscribeBody(raw) {
   if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
     return { error: "body must be a JSON object" };
   }
-  const { global: rawGlobal, scoped: rawScoped, pendingReplies: rawPending } = raw;
+  const { global: rawGlobal, scoped: rawScoped, pendingReplies: rawPending, clientId } = raw;
+  if (typeof clientId !== "string" || clientId === "") {
+    return { error: "`clientId` is required (BusSubscribeRequest)" };
+  }
   const global2 = rawGlobal === void 0 ? [] : rawGlobal;
   if (!isStringArray(global2)) return { error: "`global` must be an array of channel names" };
   const pendingReplies = rawPending === void 0 ? [] : rawPending;
@@ -11601,27 +11707,28 @@ function parseSubscribeBody(raw) {
   if (scoped.length > MAX_SCOPES) {
     return { error: `scope count ${scoped.length} exceeds the per-connection cap of ${MAX_SCOPES}` };
   }
-  return { global: global2, scoped, pendingReplies };
+  return { global: global2, scoped, pendingReplies, clientId };
 }
 function createBusRouter(authMiddleware2) {
   const busRouter2 = new Hono();
   busRouter2.use("/bus/*", authMiddleware2);
-  const retentionByBus = /* @__PURE__ */ new WeakMap();
+  const registryByBus = /* @__PURE__ */ new WeakMap();
   busRouter2.post("/bus/subscribe", async (c) => {
     const raw = await c.req.json().catch(() => null);
     const parsed = parseSubscribeBody(raw);
     if ("error" in parsed) {
       throw new HTTPException(400, { message: parsed.error });
     }
-    const { global: channels, scoped, pendingReplies } = parsed;
+    const { global: channels, scoped, pendingReplies, clientId } = parsed;
     const eventBus2 = c.get("eventBus");
     const subscriberDid = c.get("principalDid");
-    let retention = retentionByBus.get(eventBus2);
-    if (!retention) {
-      retention = createReplyRetention(eventBus2);
-      retentionByBus.set(eventBus2, retention);
+    let registry = registryByBus.get(eventBus2);
+    if (!registry) {
+      registry = createCorrelationRegistry(eventBus2);
+      registryByBus.set(eventBus2, registry);
+      registerCorrelationRegistryProvider(() => registry.occupancy());
     }
-    const replyRetention = retention;
+    const correlations = registry;
     if (scoped.length >= SCOPE_WARN_THRESHOLD) {
       getBusLogger().warn("large scope matrix", { scopeCount: scoped.length, cap: MAX_SCOPES });
     }
@@ -11641,11 +11748,45 @@ function createBusRouter(authMiddleware2) {
       const presence = { participant: subscriberDid ?? "", connectionId };
       recordSubscriberConnect();
       eventBus2.get("session:joined").next(presence);
-      stream.onAbort(() => {
+      const subs = [];
+      let pendingBytes = 0;
+      let tornDown = false;
+      const { outgoing } = c.env ?? {};
+      const teardown = (reason) => {
+        if (tornDown) return;
+        tornDown = true;
+        for (const s of subs) s.unsubscribe();
         recordSubscriberDisconnect();
         eventBus2.get("session:left").next(presence);
-        getBusLogger().info("SSE disconnect", { connectionId });
-      });
+        getBusLogger().info("SSE disconnect", { connectionId, reason, pendingBytes });
+        stream.abort();
+        outgoing?.destroy();
+      };
+      stream.onAbort(() => teardown("stream-abort"));
+      c.req.raw.signal.addEventListener("abort", () => stream.abort(), { once: true });
+      if (c.req.raw.signal.aborted) {
+        teardown("pre-aborted");
+        return;
+      }
+      const boundedWrite = async (frame) => {
+        if (tornDown) return;
+        const cost = frame.data.length;
+        pendingBytes += cost;
+        if (pendingBytes > MAX_PENDING_WRITE_BYTES) {
+          getBusLogger().warn("SSE pending-write overflow \u2014 disconnecting dead or stalled subscriber", {
+            connectionId,
+            pendingBytes,
+            cap: MAX_PENDING_WRITE_BYTES
+          });
+          teardown("pending-write-overflow");
+          return;
+        }
+        try {
+          await stream.writeSSE(frame);
+        } finally {
+          pendingBytes -= cost;
+        }
+      };
       const lastDeliveredSeq = /* @__PURE__ */ new Map();
       const writeBusEvent = async (channel, payload, eventScope) => {
         const seq = extractSequence(payload);
@@ -11666,8 +11807,7 @@ function createBusRouter(authMiddleware2) {
           }
           const data = eventScope ? JSON.stringify({ channel, payload, scope: eventScope }) : JSON.stringify({ channel, payload });
           busLog("SSE", channel, payload, eventScope);
-          await stream.writeSSE({ event: "bus-event", data, id }).catch(() => {
-          });
+          await boundedWrite({ event: "bus-event", data, id });
         };
         if (typeof cid === "string" && cid.length > 0) {
           await withSpan(`sse.deliver:${channel}`, doWrite, {
@@ -11683,32 +11823,58 @@ function createBusRouter(authMiddleware2) {
         }
       };
       const emitResumeGap = async (reason, gapScope, lastSeenId) => {
+        recordResumeGap(reason);
         const payload = { reason };
         if (gapScope !== void 0) payload.scope = gapScope;
         if (lastSeenId !== void 0) payload.lastSeenId = lastSeenId;
-        await stream.writeSSE({
+        await boundedWrite({
           event: "bus-event",
           data: JSON.stringify({ channel: "bus:resume-gap", payload }),
           id: nextEphemeralId()
-        }).catch(() => {
         });
       };
       const liveBuffer = [];
       let mode = "live";
       const emitOrBuffer = (channel, payload, eventScope) => {
         if (mode === "buffering") {
+          if (liveBuffer.length >= MAX_REPLAY_BUFFER_EVENTS) {
+            getBusLogger().warn("SSE replay-buffer overflow \u2014 disconnecting stalled subscriber", {
+              connectionId,
+              cap: MAX_REPLAY_BUFFER_EVENTS
+            });
+            teardown("replay-buffer-overflow");
+            return;
+          }
           liveBuffer.push({ channel, payload, scope: eventScope });
         } else {
           void writeBusEvent(channel, payload, eventScope);
         }
       };
+      const mayDeliver = (channel, payload) => {
+        const cid = correlationIdOf(payload);
+        if (!cid) {
+          if (!PROGRESS_CHANNELS.has(channel)) {
+            getBusLogger().warn("[bus REPLY-NO-CID] correlated frame without a correlationId", { channel });
+          }
+          return false;
+        }
+        const owner = correlations.owner(cid);
+        if (!owner) return false;
+        if (owner.clientId === clientId && owner.principalDid === subscriberDid) return true;
+        recordReplySuppressed(channel);
+        return false;
+      };
       const willReplay = scoped.some((entry) => entry.lastEventId !== void 0);
       if (willReplay) mode = "buffering";
-      const subs = channels.map(
-        (channel) => eventBus2.get(channel).subscribe((payload) => {
-          emitOrBuffer(channel, payload, void 0);
-        })
-      );
+      for (const channel of channels) {
+        const correlated = CORRELATED_CHANNELS.has(channel);
+        subs.push(
+          eventBus2.get(channel).subscribe((payload) => {
+            if (correlated && !mayDeliver(channel, payload)) return;
+            emitOrBuffer(channel, payload, void 0);
+          })
+        );
+      }
       for (const entry of scoped) {
         const scopedBus = eventBus2.scope(entry.scope);
         for (const channel of entry.channels) {
@@ -11719,8 +11885,8 @@ function createBusRouter(authMiddleware2) {
           );
         }
       }
-      stream.onAbort(() => subs.forEach((s) => s.unsubscribe()));
       for (const entry of scoped) {
+        if (tornDown) break;
         if (entry.lastEventId === void 0) continue;
         const parsed2 = parsePersistedId(entry.lastEventId);
         if (!parsed2) {
@@ -11750,18 +11916,19 @@ function createBusRouter(authMiddleware2) {
         }
       }
       for (const cid of pendingReplies) {
-        const retained = replyRetention.lookup(cid);
+        if (tornDown) break;
+        const retained = correlations.lookupReply(cid, clientId, subscriberDid);
         if (retained) {
           await writeBusEvent(retained.channel, retained.payload, void 0);
         }
       }
-      while (liveBuffer.length > 0) {
+      while (liveBuffer.length > 0 && !tornDown) {
         const next = liveBuffer.shift();
         await writeBusEvent(next.channel, next.payload, next.scope);
       }
       mode = "live";
-      while (true) {
-        await stream.writeSSE({ event: "ping", data: "" });
+      while (!tornDown && !stream.aborted && !stream.closed) {
+        await boundedWrite({ event: "ping", data: "" });
         await stream.sleep(15e3);
       }
     });
@@ -11770,6 +11937,7 @@ function createBusRouter(authMiddleware2) {
     const eventBus2 = c.get("eventBus");
     const body = await c.req.json();
     const { channel, payload, scope } = body;
+    const emitterClientId = typeof body.clientId === "string" && body.clientId !== "" ? body.clientId : void 0;
     if (!channel || typeof channel !== "string") {
       throw new HTTPException(400, { message: "channel is required" });
     }
@@ -11796,6 +11964,30 @@ function createBusRouter(authMiddleware2) {
     if (principalDid) {
       payload._userId = principalDid;
     }
+    const claimCid = channel in BUS_OPERATIONS ? correlationIdOf(payload) : void 0;
+    if (claimCid) {
+      const clientId = emitterClientId;
+      if (clientId === void 0) {
+        throw new HTTPException(400, {
+          message: `clientId is required to emit ${channel} with a correlationId`
+        });
+      }
+      let registry = registryByBus.get(eventBus2);
+      if (!registry) {
+        registry = createCorrelationRegistry(eventBus2);
+        registryByBus.set(eventBus2, registry);
+      }
+      const outcome = registry.claim(claimCid, clientId, principalDid);
+      if (outcome === "conflict") {
+        getBusLogger().warn("[bus CLAIM-CONFLICT] correlationId already claimed", { channel, correlationId: claimCid });
+        throw new HTTPException(409, { message: `correlationId ${claimCid} is already claimed` });
+      }
+      if (outcome === "at-capacity") {
+        throw new HTTPException(429, {
+          message: `client has ${PENDING_REPLIES_MAX} unanswered requests; retry when one settles`
+        });
+      }
+    }
     const traceparent = c.req.header("traceparent");
     const tracestate = c.req.header("tracestate");
     const carrier = traceparent ? tracestate ? { traceparent, tracestate } : { traceparent } : void 0;
@@ -11811,13 +12003,35 @@ function createBusRouter(authMiddleware2) {
           subject.next(payload);
           busLog("EMIT", channel, payload, scope);
           recordBusEmit(channel, scope);
-          getBusLogger().info("emit", { channel, scope, subscribers, correlationId: payload.correlationId });
+          getBusLogger().info("emit", {
+            channel,
+            scope,
+            subscribers,
+            clientId: emitterClientId,
+            correlationId: payload.correlationId
+          });
           if (subscribers === 0) {
             getBusLogger().warn("emit reached no subscribers", {
               channel,
               scope,
               hint: "Nothing on this gateway subscribes to that channel. For a UI signal meant to cross to a participant, check that the channel is in BRIDGED_BROADCASTS and that a client subscribed to it."
             });
+            const operation = BUS_OPERATIONS[channel];
+            const failureCid = correlationIdOf(payload);
+            if (operation?.failure && failureCid) {
+              const { _userId: _injected, ...echo } = payload;
+              const failure = {
+                ...echo,
+                message: `No subscriber for ${channel}: the service that answers it is not connected`
+              };
+              recordUnanswerableRequest(channel);
+              getBusLogger().warn("[bus UNANSWERABLE] synthesizing failure for an unsubscribed request", {
+                channel,
+                failureChannel: operation.failure,
+                correlationId: failureCid
+              });
+              eventBus2.get(operation.failure).next(failure);
+            }
           }
         },
         {
@@ -12032,7 +12246,7 @@ app.get("/api/openapi.json", (c) => {
   const openApiPath = fs.existsSync(distPath) ? distPath : path.join(__dirname$1, "../../../specs/openapi.json");
   const openApiContent = fs.readFileSync(openApiPath, "utf-8");
   const openApiSpec = JSON.parse(openApiContent);
-  openApiSpec.info = { ...openApiSpec.info, version: "0.5.29" };
+  openApiSpec.info = { ...openApiSpec.info, version: "0.5.30" };
   const port2 = gatewayService.port || 4e3;
   const apiUrl = gatewayService.publicURL || `http://localhost:${port2}`;
   if (apiUrl) {