@sema-agent/server 1.187.0 → 1.189.0

package/dist/http/server.js

@@ -17,7 +17,7 @@ import { manifestToIndexEntry } from "../images/manifest.js";
 import { runInBackground, evictIfConflict, stripCheckpointToken, resumeAtHttpStatus, TurnAnchorCapture, HEARTBEAT_MS, markChildrenStoppedByUserOnAbort } from "../runs.js";
 import { looksLikeJwt } from "../auth-bridge.js";
 import { redactSteerIn } from "../orchestration/workflow-agent-steer.js";
-import { emitPendingWorkflowCompletions, taskNotificationInboxEntry, taskNotificationFoldKey } from "../orchestration/workflow-completion-inbox.js";
+import { emitPendingWorkflowCompletions, taskNotificationInboxEntry, taskNotificationFoldKey, taskNotificationStreamKey, NotifiedKeys } from "../orchestration/workflow-completion-inbox.js";
 import { fleetRunPublisher, fleetRunLabels } from "../fleet/fleet-bus.js";
 import { RateLimiter } from "../observability/rate-limit.js";
 import { withPrincipal } from "../observability/principal-context.js";
@@ -181,6 +181,14 @@ export function createHttpServer(deps) {
     let uncountedBillableInflight = 0;
     if (deps.drainState)
         deps.drainState.inflight = () => new Set([...inflightRuns.keys(), ...steerableRuns.keys()]).size + uncountedBillableInflight + (deps.leaderEndpoint?.inflight?.() ?? 0);
+    let lastBillableActivityAt = Date.now();
+    const noteActivity = (url) => {
+        if (url === "/health" || url.startsWith("/metrics"))
+            return;
+        lastBillableActivityAt = Date.now();
+    };
+    if (deps.drainState)
+        deps.drainState.lastActivityAt = () => lastBillableActivityAt;
     if (deps.hookWakeBus) {
         deps.hookWakeBus.deliver = async (sessionId, text) => {
             try {
@@ -249,6 +257,7 @@ export function createHttpServer(deps) {
         const startedAt = Date.now();
         const method = req.method ?? "GET";
         const url = (req.url ?? "/").split("?")[0];
+        noteActivity(url);
         let logged = false;
         const record = (viaClose) => {
             if (logged)
@@ -674,6 +683,7 @@ export function createHttpServer(deps) {
                 }, 15_000);
                 let ranLive = false;
                 let resp;
+                uncountedBillableInflight++;
                 try {
                     resp = await idemCache.run(idemKey, async () => {
                         ranLive = true;
@@ -725,7 +735,7 @@ export function createHttpServer(deps) {
                         let liveStreamRef;
                         const subagentHandleEvictions = [];
                         let syncLegLive = true;
-                        const syncNotifiedKeys = new Set();
+                        const syncNotifiedKeys = new NotifiedKeys();
                         fleetPub?.onStart();
                         try {
                             const streamBody = async () => {
@@ -767,9 +777,9 @@ export function createHttpServer(deps) {
                                                 void deps.workflowCompletionInbox.enqueue(taskNotificationInboxEntry(prepared.spec.sessionId, askOwner, n, Date.now(), durableTaskId)).catch((err) => deps.logger?.warn?.("park_enqueue_failed", { route: "sync-stream", taskId: n.task_id, err: err instanceof Error ? err.message : String(err) }));
                                             }
                                             else if (deliverable) {
-                                                const key = `${n.task_id}:${n.status}`;
-                                                if (!syncNotifiedKeys.has(key)) {
-                                                    syncNotifiedKeys.add(key);
+                                                const key = taskNotificationStreamKey(n);
+                                                if (syncNotifiedKeys.get(key) === undefined) {
+                                                    syncNotifiedKeys.set(key, Promise.resolve(true));
                                                     res.write(`data: ${JSON.stringify({ type: "task_notification", ...taskNotificationEventData({ notification: n }) })}\n\n`);
                                                 }
                                             }
@@ -814,9 +824,13 @@ export function createHttpServer(deps) {
                                         }
                                         else if (ev.type === "task_notification") {
                                             const tnv = ev.notification;
-                                            if (tnv?.task_id)
-                                                syncNotifiedKeys.add(`${tnv.task_id}:${tnv.status}`);
-                                            res.write(`data: ${JSON.stringify({ type: "task_notification", ...taskNotificationEventData(ev) })}\n\n`);
+                                            const tnvKey = tnv?.task_id ? taskNotificationStreamKey({ task_type: tnv.task_type, task_id: tnv.task_id, status: String(tnv.status), seq: tnv.seq }) : undefined;
+                                            const tnvPrior = tnvKey !== undefined ? syncNotifiedKeys.get(tnvKey) : undefined;
+                                            if (tnvPrior === undefined || !(await tnvPrior)) {
+                                                if (tnvKey !== undefined)
+                                                    syncNotifiedKeys.set(tnvKey, Promise.resolve(true));
+                                                res.write(`data: ${JSON.stringify({ type: "task_notification", ...taskNotificationEventData(ev) })}\n\n`);
+                                            }
                                         }
                                         else if (ev.type === "reasoning_delta" && typeof ev.delta === "string") {
                                             res.write(`data: ${JSON.stringify({ ...ev, delta: redactSecrets(ev.delta) })}\n\n`);
@@ -845,6 +859,9 @@ export function createHttpServer(deps) {
                             await emitPendingWorkflowCompletions(deps.workflowCompletionInbox, prepared.spec.sessionId, askOwner, (frame) => {
                                 if (closed || res.writableEnded || res.destroyed)
                                     throw new Error("stream ended before the completion frame was written");
+                                const f = frame;
+                                if (f.type === "task_notification" && f.task_id)
+                                    syncNotifiedKeys.set(taskNotificationStreamKey({ task_type: f.task_type, task_id: f.task_id, status: String(f.status), seq: f.seq }), Promise.resolve(true));
                                 res.write(`data: ${JSON.stringify(frame)}\n\n`);
                             }, { route: "sync-stream-open", connection: "live-sse", log: (m, x) => deps.logger?.info?.(m, x) });
                             const withElicit = deps.elicitation
@@ -896,6 +913,7 @@ export function createHttpServer(deps) {
                         throw e;
                 }
                 finally {
+                    uncountedBillableInflight--;
                     clearInterval(hb);
                 }
                 if (!ranLive && resp && !res.writableEnded) {
@@ -3780,11 +3798,15 @@ export function createHttpServer(deps) {
         }
         try {
             result = await withPrincipal(principal, async () => {
+                const resumeNotifiedKeys = new NotifiedKeys();
                 if (deps.runStore && taskId) {
                     const rs0 = deps.runStore;
                     let drainSeq = (await rs0.maxSeq(taskId)) + 1;
                     await emitPendingWorkflowCompletions(deps.workflowCompletionInbox, sessionId, principal ?? null, (frame) => {
                         const { type, ...rest } = frame;
+                        const f = frame;
+                        if (f.type === "task_notification" && f.task_id)
+                            resumeNotifiedKeys.set(taskNotificationStreamKey({ task_type: f.task_type, task_id: f.task_id, status: String(f.status), seq: f.seq }), Promise.resolve(true));
                         return rs0.appendEvent(taskId, drainSeq++, type, rest);
                     }, { route: "resume-open", connection: "durable-append", log: (m, x) => deps.logger?.info?.(m, x) });
                 }
@@ -3817,7 +3839,6 @@ export function createHttpServer(deps) {
                 let reasoning = "";
                 const persistThinking = deps.config.traceThinking;
                 const append = (type, data) => (rs && taskId ? rs.appendEvent(taskId, ++seq, type, data) : Promise.resolve());
-                const resumeNotifiedKeys = new Set();
                 const anchorOwner = principal ?? null;
                 const getLeafId = deps.sessionStorage?.getLeafId?.bind(deps.sessionStorage);
                 const captureTurnAnchor = deps.resumeAnchorStore && getLeafId
@@ -3866,10 +3887,9 @@ export function createHttpServer(deps) {
                                 void deps.workflowCompletionInbox.enqueue(taskNotificationInboxEntry(sessionId, principal ?? null, n, Date.now(), taskId)).catch((err) => deps.logger?.warn?.("park_enqueue_failed", { route: "resume", taskId: n.task_id, err: err instanceof Error ? err.message : String(err) }));
                             }
                             else if (resumeLegLive) {
-                                const key = `${n.task_id}:${n.status}`;
-                                if (!resumeNotifiedKeys.has(key)) {
-                                    resumeNotifiedKeys.add(key);
-                                    void append("task_notification", taskNotificationEventData({ notification: n })).catch(() => undefined);
+                                const key = taskNotificationStreamKey(n);
+                                if (resumeNotifiedKeys.get(key) === undefined) {
+                                    resumeNotifiedKeys.set(key, append("task_notification", taskNotificationEventData({ notification: n })).then(() => true, () => false));
                                 }
                             }
                         },
@@ -3930,9 +3950,14 @@ export function createHttpServer(deps) {
                             case "task_notification": {
                                 await flush();
                                 const tnr = ev.notification;
-                                if (tnr?.task_id)
-                                    resumeNotifiedKeys.add(`${tnr.task_id}:${tnr.status}`);
-                                await append("task_notification", taskNotificationEventData(ev));
+                                const tnrKey = tnr?.task_id ? taskNotificationStreamKey({ task_type: tnr.task_type, task_id: tnr.task_id, status: String(tnr.status), seq: tnr.seq }) : undefined;
+                                const tnrPrior = tnrKey !== undefined ? resumeNotifiedKeys.get(tnrKey) : undefined;
+                                if (tnrPrior === undefined || !(await tnrPrior)) {
+                                    const write = append("task_notification", taskNotificationEventData(ev));
+                                    if (tnrKey !== undefined)
+                                        resumeNotifiedKeys.set(tnrKey, write.then(() => true, () => false));
+                                    await write;
+                                }
                                 break;
                             }
                             case "turn_end": {