@sema-agent/server 7.63.0 → 7.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +8 -0
- package/USAGE.md +17 -15
- package/deploy/sema-up/smoke.sh +2 -2
- package/dist/approval-ask-audit-store.d.ts +149 -3
- package/dist/approval-ask-audit-store.js +11 -4
- package/dist/bench/s1/arms.d.ts +2 -2
- package/dist/bench/s1/arms.js +20 -15
- package/dist/bench/s1/live-deps.js +3 -1
- package/dist/bench/s1/run-firm.js +2 -2
- package/dist/degenerate-instrument.d.ts +0 -33
- package/dist/degenerate-instrument.js +2 -1
- package/dist/fleet/fleet-bus.d.ts +9 -4
- package/dist/fleet/fleet-bus.js +2 -1
- package/dist/host-decision.d.ts +39 -0
- package/dist/host-decision.js +6 -0
- package/dist/http/route-ctx.d.ts +8 -5
- package/dist/http/routes/a2a-serve.js +4 -2
- package/dist/http/routes/approvals-assistant.d.ts +0 -13
- package/dist/http/routes/approvals-assistant.js +2 -1
- package/dist/http/routes/runs.js +3 -2
- package/dist/http/routes/tasks.js +33 -29
- package/dist/http/routes/workflows.d.ts +0 -11
- package/dist/http/routes/workflows.js +18 -14
- package/dist/http/server.js +50 -46
- package/dist/leader/fanout.d.ts +6 -23
- package/dist/leader/fanout.js +8 -19
- package/dist/leader/leader.d.ts +13 -10
- package/dist/leader/leader.js +30 -22
- package/dist/leader/repair-wire.d.ts +4 -1
- package/dist/leader/repair-wire.js +7 -5
- package/dist/leader/wire.js +5 -4
- package/dist/main.js +5 -3
- package/dist/observability/fail-open.d.ts +8 -0
- package/dist/observability/fail-open.js +8 -0
- package/dist/observability/metrics.js +1 -1
- package/dist/observability/run-terminal-log.d.ts +25 -7
- package/dist/observability/run-terminal-log.js +15 -4
- package/dist/observability/tool-trace.d.ts +13 -12
- package/dist/observability/tool-trace.js +8 -13
- package/dist/parked-decide.d.ts +8 -4
- package/dist/parked-decide.js +1 -1
- package/dist/plugins/file-run-store.js +12 -21
- package/dist/plugins/memory-run-store.js +3 -2
- package/dist/plugins/run-store-sql.js +3 -2
- package/dist/run-local.js +6 -4
- package/dist/runs.d.ts +4 -19
- package/dist/runs.js +49 -47
- package/dist/terminal.d.ts +100 -0
- package/dist/terminal.js +55 -0
- package/dist/trace/core-keyset-guard.d.ts +20 -4
- package/dist/trace/ledger-sink.d.ts +2 -1
- package/dist/trace/ledger-sink.js +14 -7
- package/dist/trace/project.d.ts +9 -12
- package/dist/trace/project.js +54 -9
- package/package.json +4 -4
|
@@ -3,14 +3,11 @@ import { projectUsageStats, USAGE_SCAN_LIMIT } from "../usage-analytics.js";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";
|
|
5
5
|
import { randomBytes } from "node:crypto";
|
|
6
|
-
import {
|
|
6
|
+
import { terminalProjection } from "@sema-agent/core";
|
|
7
|
+
import { AppendLog, atomicWriteFile, ensureDir, isProcessLive, processFingerprint, readJsonlRecords, sanitizePathComponent, writeThenLink } from "@sema-agent/core";
|
|
7
8
|
import { recordFailOpen } from "../observability/fail-open.js";
|
|
8
9
|
import { notifyRunTerminal } from "../observability/run-terminal-log.js";
|
|
9
10
|
const ownerEq = (a, b) => (a ?? null) === (b ?? null);
|
|
10
|
-
const PROCESS_FACTS = {
|
|
11
|
-
pid: process.pid,
|
|
12
|
-
startedAtMs: Date.now() - Math.round(process.uptime() * 1000),
|
|
13
|
-
};
|
|
14
11
|
const CLAIM_TAKEOVER_ATTEMPTS = 3;
|
|
15
12
|
const SAFE_PATH_COMPONENT = /^[A-Za-z0-9_.-]+$/;
|
|
16
13
|
const isSafePathComponent = (s) => s !== "" && s !== "." && s !== ".." && SAFE_PATH_COMPONENT.test(s);
|
|
@@ -22,17 +19,6 @@ function readClaimRecord(recs, sessionId) {
|
|
|
22
19
|
return undefined;
|
|
23
20
|
return c;
|
|
24
21
|
}
|
|
25
|
-
function isPidAlive(pid) {
|
|
26
|
-
if (!Number.isInteger(pid) || pid <= 0)
|
|
27
|
-
return true;
|
|
28
|
-
try {
|
|
29
|
-
process.kill(pid, 0);
|
|
30
|
-
return true;
|
|
31
|
-
}
|
|
32
|
-
catch (e) {
|
|
33
|
-
return e.code !== "ESRCH";
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
22
|
const PARK_ORPHAN_SETTLE_MS = 30_000;
|
|
37
23
|
export class RunStoreStrictHydrateError extends Error {
|
|
38
24
|
name = "RunStoreStrictHydrateError";
|
|
@@ -46,7 +32,7 @@ export class FileRunStore {
|
|
|
46
32
|
events = new Map();
|
|
47
33
|
eventLogs = new Map();
|
|
48
34
|
checkpointProbe;
|
|
49
|
-
claimOwner = { nonce: randomBytes(12).toString("hex"), ...
|
|
35
|
+
claimOwner = { nonce: randomBytes(12).toString("hex"), ...processFingerprint() };
|
|
50
36
|
claimLocks = new Map();
|
|
51
37
|
logger;
|
|
52
38
|
strictHydrate;
|
|
@@ -382,7 +368,7 @@ export class FileRunStore {
|
|
|
382
368
|
r.status = status;
|
|
383
369
|
r.result = result;
|
|
384
370
|
r.error = error;
|
|
385
|
-
r.errorCode = result
|
|
371
|
+
r.errorCode = result === null ? null : (terminalProjection(result.terminal).errorCode ?? null);
|
|
386
372
|
r.updatedAt = new Date();
|
|
387
373
|
this.persist(r);
|
|
388
374
|
pending = {
|
|
@@ -393,7 +379,7 @@ export class FileRunStore {
|
|
|
393
379
|
resultTaskId: result?.taskId,
|
|
394
380
|
model: result?.model,
|
|
395
381
|
errorCode: r.errorCode ?? undefined,
|
|
396
|
-
errorMessage: result
|
|
382
|
+
errorMessage: (result === null ? undefined : terminalProjection(result.terminal).errorMessage) ?? error ?? undefined,
|
|
397
383
|
elapsedMs: Math.max(0, Date.now() - r.createdAt.getTime()),
|
|
398
384
|
};
|
|
399
385
|
}
|
|
@@ -728,7 +714,7 @@ export class FileRunStore {
|
|
|
728
714
|
async judgeClaim(sessionId, claim, holderTaskId) {
|
|
729
715
|
const owner = claim?.owner;
|
|
730
716
|
const who = owner
|
|
731
|
-
? `pid ${owner.pid} started ${new Date(owner.startedAtMs).toISOString()}`
|
|
717
|
+
? `pid ${owner.pid} on ${owner.host ?? "an unnamed host"} started ${owner.startedAtMs === null || owner.startedAtMs === undefined ? "(start time unobtainable)" : new Date(owner.startedAtMs).toISOString()}`
|
|
732
718
|
: "an owner-less claim (written before the liveness stamp existed)";
|
|
733
719
|
const torn = claim === undefined;
|
|
734
720
|
let row = !torn && holderTaskId ? this.authoritativeRow(holderTaskId) : undefined;
|
|
@@ -770,7 +756,7 @@ export class FileRunStore {
|
|
|
770
756
|
return { kind: "stale", reason: `parked run (${row.status}) has no resumable checkpoint — claim reclaimed`, snapshot };
|
|
771
757
|
}
|
|
772
758
|
const mine = owner?.nonce === this.claimOwner.nonce;
|
|
773
|
-
const ownerGone = owner !== undefined && !
|
|
759
|
+
const ownerGone = owner !== undefined && !isProcessLive(owner);
|
|
774
760
|
if (!mine && !ownerGone)
|
|
775
761
|
return live();
|
|
776
762
|
if (!row)
|
|
@@ -782,6 +768,11 @@ export class FileRunStore {
|
|
|
782
768
|
const probe = this.checkpointProbe;
|
|
783
769
|
if (probe && (await probe.hasPending(row.sessionId)))
|
|
784
770
|
return live();
|
|
771
|
+
this.logger?.error?.("file_run_store_claim_taken_over", {
|
|
772
|
+
sessionId, holder: holder || null, owner: owner ?? null,
|
|
773
|
+
pidStillRunning: owner !== undefined && isProcessLive({ pid: owner.pid }),
|
|
774
|
+
note: "the claim's owner process is provably not the one that wrote it (fingerprint falsified, or the pid is gone) and the row has nothing left to resume — taking the claim over",
|
|
775
|
+
});
|
|
785
776
|
return { kind: "stale", reason: `run orphaned by engine restart (claim held by ${who}, which is gone)`, snapshot };
|
|
786
777
|
}
|
|
787
778
|
takeOverClaim(sessionId, reason, snapshot) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { redactErrorMessage, redactLedgerEventData, redactTerminalResult } from "../observability/run-terminal-log.js";
|
|
2
2
|
import { projectUsageStats, USAGE_SCAN_LIMIT } from "../usage-analytics.js";
|
|
3
|
+
import { terminalProjection } from "@sema-agent/core";
|
|
3
4
|
import { notifyRunTerminal } from "../observability/run-terminal-log.js";
|
|
4
5
|
const ownerEq = (a, b) => (a ?? null) === (b ?? null);
|
|
5
6
|
export class MemoryRunStore {
|
|
@@ -119,7 +120,7 @@ export class MemoryRunStore {
|
|
|
119
120
|
r.status = status;
|
|
120
121
|
r.result = result;
|
|
121
122
|
r.error = error;
|
|
122
|
-
r.errorCode = result
|
|
123
|
+
r.errorCode = result === null ? null : (terminalProjection(result.terminal).errorCode ?? null);
|
|
123
124
|
r.updatedAt = new Date();
|
|
124
125
|
pending = {
|
|
125
126
|
runId: taskId,
|
|
@@ -129,7 +130,7 @@ export class MemoryRunStore {
|
|
|
129
130
|
resultTaskId: result?.taskId,
|
|
130
131
|
model: result?.model,
|
|
131
132
|
errorCode: r.errorCode ?? undefined,
|
|
132
|
-
errorMessage: result
|
|
133
|
+
errorMessage: (result === null ? undefined : terminalProjection(result.terminal).errorMessage) ?? error ?? undefined,
|
|
133
134
|
elapsedMs: Math.max(0, Date.now() - r.createdAt.getTime()),
|
|
134
135
|
};
|
|
135
136
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { redactErrorMessage, redactLedgerEventData, redactTerminalResult } from "../observability/run-terminal-log.js";
|
|
2
|
+
import { terminalProjection } from "@sema-agent/core";
|
|
2
3
|
import { projectUsageStats, USAGE_SCAN_LIMIT } from "../usage-analytics.js";
|
|
3
4
|
import { escapeLike } from "./sql-escape.js";
|
|
4
5
|
import { pgSanitizeText } from "./pg-safe-json.js";
|
|
@@ -118,7 +119,7 @@ export class SqlRunStore {
|
|
|
118
119
|
async setTerminal(taskId, status, result, error) {
|
|
119
120
|
result = redactTerminalResult(result);
|
|
120
121
|
error = error === null ? null : (redactErrorMessage(error) ?? null);
|
|
121
|
-
const errorCode = result
|
|
122
|
+
const errorCode = result === null ? null : (terminalProjection(result.terminal).errorCode ?? null);
|
|
122
123
|
let observed;
|
|
123
124
|
await this.tx(async (conn) => {
|
|
124
125
|
const w = await conn.query(this.q("UPDATE task_run SET status = ?, result = ?, error = ?, error_code = ?, updated_at = ? WHERE task_id = ? AND status IN ('running','suspended','needs_review')", "UPDATE task_run SET status = $1, result = $2::jsonb, error = $3, error_code = $4, updated_at = $5 WHERE task_id = $6 AND status IN ('running','suspended','needs_review')"), [
|
|
@@ -150,7 +151,7 @@ export class SqlRunStore {
|
|
|
150
151
|
resultTaskId: result?.taskId,
|
|
151
152
|
model: result?.model,
|
|
152
153
|
errorCode: errorCode ?? undefined,
|
|
153
|
-
errorMessage: result
|
|
154
|
+
errorMessage: (result === null ? undefined : terminalProjection(result.terminal).errorMessage) ?? error ?? undefined,
|
|
154
155
|
elapsedMs: observed.createdAt === undefined ? undefined : Math.max(0, Date.now() - observed.createdAt),
|
|
155
156
|
});
|
|
156
157
|
}
|
package/dist/run-local.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { terminalProjection } from "@sema-agent/core";
|
|
2
3
|
import { homedir } from "node:os";
|
|
3
4
|
import { join } from "node:path";
|
|
4
5
|
import { realpathSync } from "node:fs";
|
|
@@ -513,16 +514,17 @@ export async function runLocal(argv, deps = {}) {
|
|
|
513
514
|
if (args.json) {
|
|
514
515
|
print(JSON.stringify(result, null, 2));
|
|
515
516
|
}
|
|
516
|
-
else if (result.
|
|
517
|
+
else if (result.terminal.kind === "completed") {
|
|
517
518
|
print(result.result);
|
|
518
519
|
}
|
|
519
520
|
else {
|
|
520
|
-
const
|
|
521
|
+
const plane = terminalProjection(result.terminal);
|
|
522
|
+
const salvage = result.result || result.salvagedOutput || plane.blockedReason || "";
|
|
521
523
|
if (salvage)
|
|
522
524
|
print(salvage);
|
|
523
|
-
printErr(`task ${
|
|
525
|
+
printErr(`task ${plane.status}${plane.errorCode ? ` (${plane.errorCode})` : ""}${plane.errorMessage ? `: ${plane.errorMessage}` : ""}`);
|
|
524
526
|
}
|
|
525
|
-
if (result.
|
|
527
|
+
if (result.terminal.kind !== "completed")
|
|
526
528
|
exitCode = 1;
|
|
527
529
|
if (memorySync) {
|
|
528
530
|
try {
|
package/dist/runs.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { defaultTaskRegistry, type CascadeConfig, type CheckpointToken, type Runner, type SubagentSteerHandle, type TaskResult, type TaskSpec, type TaskStream } from "@sema-agent/core";
|
|
1
|
+
import { defaultTaskRegistry, type CascadeConfig, type CheckpointToken, type Runner, type SubagentSteerHandle, type TaskResult, type TaskSpec, type TaskStream, type TerminalCause } from "@sema-agent/core";
|
|
2
2
|
import type { SendUserFileEmitter } from "./capabilities/send-user-file-tool.js";
|
|
3
3
|
import type { RunStore } from "./plugins/store-backend.js";
|
|
4
4
|
import type { Metrics } from "./observability/metrics.js";
|
|
@@ -91,13 +91,11 @@ export declare function taskHandleStop(registry: typeof defaultTaskRegistry, han
|
|
|
91
91
|
export declare function markChildrenStoppedByUserOnAbort(signal: AbortSignal, taskId: string, principal: string | null | undefined): void;
|
|
92
92
|
/**
|
|
93
93
|
* A session CAS conflict is swallowed by core into a failed `TaskResult` (it is not thrown from
|
|
94
|
-
* `runTaskStream`), so detect it from the result. Prefer the structured `
|
|
94
|
+
* `runTaskStream`), so detect it from the result. Prefer the structured `code` (core ≥1.8); fall
|
|
95
95
|
* back to the conflict wording only for older results without one.
|
|
96
96
|
*/
|
|
97
97
|
export declare function isSessionConflictResult(result: {
|
|
98
|
-
|
|
99
|
-
errorMessage?: string;
|
|
100
|
-
errorCode?: string;
|
|
98
|
+
terminal: TerminalCause;
|
|
101
99
|
}): boolean;
|
|
102
100
|
/**
|
|
103
101
|
* E18 resume-at CALLER error → an HTTP 4xx. A bad/stale resume-at target (the resolved entryId is gone or is not a
|
|
@@ -151,8 +149,7 @@ export declare function isSessionConflictResult(result: {
|
|
|
151
149
|
* 缺口(不是本批回归),已在上述对表格里逐码显式登记为债务,并由那道门看住不许陈腐。
|
|
152
150
|
*/
|
|
153
151
|
export declare function resumeAtHttpStatus(result: {
|
|
154
|
-
|
|
155
|
-
errorCode?: string;
|
|
152
|
+
terminal: TerminalCause;
|
|
156
153
|
}): number | undefined;
|
|
157
154
|
/**
|
|
158
155
|
* E18 resume-at — per-turn anchor capture, shared by runInBackground AND the resume-leg driveResume so the (single,
|
|
@@ -207,18 +204,6 @@ export declare class TurnAnchorCapture {
|
|
|
207
204
|
/** turn_end — capture the anchor iff this was a settled assistant-TEXT turn, then reset for the next turn. */
|
|
208
205
|
onTurnEnd(): Promise<void>;
|
|
209
206
|
}
|
|
210
|
-
/**
|
|
211
|
-
* Strip the durable-resume capability token from a result before it is persisted to the replayable event
|
|
212
|
-
* log / run row or returned to the submitter. The token IS the resume credential (token-as-auth, search
|
|
213
|
-
* [18] Q1b) and must never leave the service. Safe on any result (no-op when absent) — notably core maps a
|
|
214
|
-
* NESTED suspend under verify/cascade to status:"failed" + errorCode "unexpected.suspended"(inner status
|
|
215
|
-
* "needs_review" ⇒ "unexpected.needs_review";core `agents/suspend-guard.js` `mapNestedSuspend`,由 verify.js
|
|
216
|
-
* 191/201/208 与 cascade.js:125 调用)while still
|
|
217
|
-
* spreading `checkpointToken`, so that "failed" path (unlike the stream path's explicit suspend handling)
|
|
218
|
-
* would otherwise leak it into task_event / task_run.result / the sync 200 response. `checkpointGate` is
|
|
219
|
-
* non-secret and intentionally kept.
|
|
220
|
-
*/
|
|
221
|
-
export declare function stripCheckpointToken<T>(result: T): T;
|
|
222
207
|
/** [ref] 投影腿的最小 checkpoint 读口(结构型——真店 `CheckpointStore.get` 直接对上)。 */
|
|
223
208
|
type ParkCheckpointReader = {
|
|
224
209
|
get(token: CheckpointToken): Promise<{
|
package/dist/runs.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { runWithVerification, runCascade, defaultTaskRegistry, DURABLE_AGENT_HANDLE_RE } from "@sema-agent/core";
|
|
1
|
+
import { runWithVerification, runCascade, defaultTaskRegistry, terminalProjection, DURABLE_AGENT_HANDLE_RE } from "@sema-agent/core";
|
|
2
|
+
import { failureCodeOf, stripResumeToken } from "./terminal.js";
|
|
2
3
|
import { redactErrorMessage } from "./observability/run-terminal-log.js";
|
|
3
4
|
import { withPrincipal } from "./observability/principal-context.js";
|
|
4
5
|
import { redactSecrets } from "./trace/redact.js";
|
|
@@ -59,36 +60,39 @@ export function markChildrenStoppedByUserOnAbort(signal, taskId, principal) {
|
|
|
59
60
|
signal.addEventListener("abort", mark, { once: true });
|
|
60
61
|
}
|
|
61
62
|
export function isSessionConflictResult(result) {
|
|
62
|
-
if (result.
|
|
63
|
+
if (result.terminal.kind !== "failed")
|
|
63
64
|
return false;
|
|
64
|
-
if (result.
|
|
65
|
-
return result.
|
|
66
|
-
return /stale write|concurrent write|conflict/i.test(result.
|
|
65
|
+
if (result.terminal.code)
|
|
66
|
+
return result.terminal.code === "conflict";
|
|
67
|
+
return /stale write|concurrent write|conflict/i.test(result.terminal.message ?? "");
|
|
67
68
|
}
|
|
68
69
|
export function resumeAtHttpStatus(result) {
|
|
69
|
-
|
|
70
|
+
const code = failureCodeOf(result.terminal);
|
|
71
|
+
if (code === undefined)
|
|
70
72
|
return undefined;
|
|
71
|
-
if (
|
|
73
|
+
if (code === "resume.session_not_found")
|
|
72
74
|
return 404;
|
|
73
|
-
if (
|
|
75
|
+
if (code === "rewind_snapshot.unresolvable")
|
|
74
76
|
return 422;
|
|
75
|
-
if (
|
|
77
|
+
if (code === "rewind.child_scope_unsupported")
|
|
76
78
|
return 422;
|
|
77
|
-
if (
|
|
79
|
+
if (code === "memory.admission_denied")
|
|
78
80
|
return 403;
|
|
79
|
-
if (
|
|
81
|
+
if (code === "memory.admission_required")
|
|
80
82
|
return 503;
|
|
81
|
-
if (
|
|
83
|
+
if (code === "memory.capture_optout_denied")
|
|
82
84
|
return 403;
|
|
83
|
-
if (
|
|
85
|
+
if (code === "memory.capture_optout_unpersisted")
|
|
84
86
|
return 503;
|
|
85
|
-
if (
|
|
87
|
+
if (code === "config.memory_capture_spelling")
|
|
86
88
|
return 400;
|
|
87
|
-
if (
|
|
89
|
+
if (code === "config.invalid_checkpoint_store")
|
|
90
|
+
return 400;
|
|
91
|
+
if (code === "config.memory_capture_unsupported")
|
|
88
92
|
return 409;
|
|
89
|
-
if (!
|
|
93
|
+
if (!code.startsWith("resume_at."))
|
|
90
94
|
return undefined;
|
|
91
|
-
return
|
|
95
|
+
return code === "resume_at.not_found" ? 404 : 422;
|
|
92
96
|
}
|
|
93
97
|
export class TurnAnchorCapture {
|
|
94
98
|
capture;
|
|
@@ -123,21 +127,14 @@ export class TurnAnchorCapture {
|
|
|
123
127
|
this.turnHadTool = false;
|
|
124
128
|
}
|
|
125
129
|
}
|
|
126
|
-
export function stripCheckpointToken(result) {
|
|
127
|
-
if (result !== null && typeof result === "object" && !Array.isArray(result) && Object.hasOwn(result, "checkpointToken")) {
|
|
128
|
-
const { checkpointToken: _drop, ...rest } = result;
|
|
129
|
-
return rest;
|
|
130
|
-
}
|
|
131
|
-
return result;
|
|
132
|
-
}
|
|
133
130
|
export async function parkToolCallId(result, cs) {
|
|
134
131
|
if (!cs || result === null || typeof result !== "object")
|
|
135
132
|
return undefined;
|
|
136
|
-
const
|
|
137
|
-
if (
|
|
133
|
+
const t = result.terminal;
|
|
134
|
+
if (t === undefined || t.kind !== "paused" || typeof t.token !== "string")
|
|
138
135
|
return undefined;
|
|
139
136
|
try {
|
|
140
|
-
const pa = (await cs.get(
|
|
137
|
+
const pa = (await cs.get(t.token))?.pendingAction;
|
|
141
138
|
return pa?.kind === "tool_approval" && typeof pa.toolCallId === "string" ? pa.toolCallId : undefined;
|
|
142
139
|
}
|
|
143
140
|
catch (e) {
|
|
@@ -169,7 +166,7 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
169
166
|
preemptable?.set(taskId, preemptCtrl);
|
|
170
167
|
const cancelledResult = () => {
|
|
171
168
|
const ctx = buildCancelContext(taskId, startedAt);
|
|
172
|
-
return { taskId, sessionId: spec.sessionId,
|
|
169
|
+
return { taskId, sessionId: spec.sessionId, terminal: { kind: "failed", code: "cancelled", message: "cancelled by user" }, stats: { turns: 0, tokens: 0 }, ...(ctx ? { cancelContext: ctx } : {}) };
|
|
173
170
|
};
|
|
174
171
|
const recordDone = (result, status) => {
|
|
175
172
|
metrics?.addGauge("runs_active", -1);
|
|
@@ -318,11 +315,12 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
318
315
|
await flush();
|
|
319
316
|
evictIfConflict(runner, spec.sessionId, result);
|
|
320
317
|
await flushPromptManifest();
|
|
321
|
-
const safe = await withModelUsage(
|
|
318
|
+
const safe = await withModelUsage(stripResumeToken(result));
|
|
322
319
|
await append("done", { result: safe });
|
|
323
320
|
reached = { kind: "done", result: safe };
|
|
324
|
-
|
|
325
|
-
|
|
321
|
+
const plane = terminalProjection(safe.terminal);
|
|
322
|
+
await runStore.setTerminal(taskId, plane.status, safe, plane.errorMessage ?? null);
|
|
323
|
+
recordDone(safe, plane.status);
|
|
326
324
|
return;
|
|
327
325
|
}
|
|
328
326
|
if (cascade !== undefined) {
|
|
@@ -330,11 +328,12 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
330
328
|
await flush();
|
|
331
329
|
evictIfConflict(runner, spec.sessionId, result);
|
|
332
330
|
await flushPromptManifest();
|
|
333
|
-
const safe = await withModelUsage(
|
|
331
|
+
const safe = await withModelUsage(stripResumeToken(result));
|
|
334
332
|
await append("done", { result: safe });
|
|
335
333
|
reached = { kind: "done", result: safe };
|
|
336
|
-
|
|
337
|
-
|
|
334
|
+
const plane = terminalProjection(safe.terminal);
|
|
335
|
+
await runStore.setTerminal(taskId, plane.status, safe, plane.errorMessage ?? null);
|
|
336
|
+
recordDone(safe, plane.status);
|
|
338
337
|
return;
|
|
339
338
|
}
|
|
340
339
|
const stream = runner.runTaskStream({ ...spec, taskId, signal: cancelCtrl.signal, preemptSignal: preemptCtrl.signal }, undefined, fwdInternals);
|
|
@@ -359,7 +358,8 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
359
358
|
}
|
|
360
359
|
if (ev.type === "done") {
|
|
361
360
|
await flush();
|
|
362
|
-
|
|
361
|
+
const parkStatus = ev.result.terminal.kind === "paused" ? terminalProjection(ev.result.terminal).status : undefined;
|
|
362
|
+
if (parkStatus === "suspended") {
|
|
363
363
|
await sink.appendParked("suspended", await graftParkToolCallId(ev.result, parkCheckpointReader));
|
|
364
364
|
await flushModelUsage();
|
|
365
365
|
reached = { kind: "suspended" };
|
|
@@ -368,17 +368,18 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
368
368
|
metrics?.inc("tasks_total", { status: "suspended" });
|
|
369
369
|
continue;
|
|
370
370
|
}
|
|
371
|
-
if (
|
|
371
|
+
if (parkStatus === "needs_review") {
|
|
372
372
|
await parkNeedsReview(ev.result);
|
|
373
373
|
continue;
|
|
374
374
|
}
|
|
375
375
|
evictIfConflict(runner, spec.sessionId, ev.result);
|
|
376
376
|
{
|
|
377
|
-
const safe = await withModelUsage(
|
|
377
|
+
const safe = await withModelUsage(stripResumeToken(ev.result));
|
|
378
378
|
await sink.appendDone(safe);
|
|
379
379
|
reached = { kind: "done", result: safe };
|
|
380
|
-
|
|
381
|
-
|
|
380
|
+
const plane = terminalProjection(safe.terminal);
|
|
381
|
+
await runStore.setTerminal(taskId, plane.status, safe, plane.errorMessage ?? null);
|
|
382
|
+
recordDone(safe, plane.status);
|
|
382
383
|
}
|
|
383
384
|
continue;
|
|
384
385
|
}
|
|
@@ -388,7 +389,7 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
388
389
|
await anchor.onTurnEnd();
|
|
389
390
|
}
|
|
390
391
|
}
|
|
391
|
-
if (spec.suggestNextPrompts && reached?.kind === "done" && reached.result.
|
|
392
|
+
if (spec.suggestNextPrompts && reached?.kind === "done" && reached.result.terminal.kind === "completed") {
|
|
392
393
|
const suggestions = await stream.suggestions().catch(() => []);
|
|
393
394
|
await append("suggestions", { suggestions: suggestions.map((s) => redactSecrets(s)) });
|
|
394
395
|
}
|
|
@@ -426,13 +427,13 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
426
427
|
}
|
|
427
428
|
const c = cancelledResult();
|
|
428
429
|
try {
|
|
429
|
-
await append("failed", { errorMessage:
|
|
430
|
+
await append("failed", { errorMessage: "cancelled by user", errorCode: "cancelled" });
|
|
430
431
|
}
|
|
431
432
|
catch (e) {
|
|
432
433
|
recordFailOpen("server.runs.terminal-ledger-write-failed", `arm=append-clean-cancel ${errText(e)}`);
|
|
433
434
|
}
|
|
434
435
|
reached = { kind: "done", result: c };
|
|
435
|
-
await runStore.setTerminal(taskId, "failed", c, c.errorMessage ?? null);
|
|
436
|
+
await runStore.setTerminal(taskId, "failed", c, terminalProjection(c.terminal).errorMessage ?? null);
|
|
436
437
|
recordDone(c, "cancelled");
|
|
437
438
|
}
|
|
438
439
|
}
|
|
@@ -446,7 +447,8 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
446
447
|
recordFailOpen("server.runs.terminal-ledger-write-failed", `arm=flush-catch ${errText(e)}`);
|
|
447
448
|
}
|
|
448
449
|
if (reached?.kind === "done") {
|
|
449
|
-
|
|
450
|
+
const plane = terminalProjection(reached.result.terminal);
|
|
451
|
+
await runStore.setTerminal(taskId, plane.status, reached.result, plane.errorMessage ?? null);
|
|
450
452
|
}
|
|
451
453
|
else if (reached?.kind === "suspended") {
|
|
452
454
|
await runStore.setSuspended(taskId);
|
|
@@ -457,12 +459,12 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
457
459
|
else if (cancelCtrl.signal.aborted) {
|
|
458
460
|
const c = cancelledResult();
|
|
459
461
|
try {
|
|
460
|
-
await append("failed", { errorMessage:
|
|
462
|
+
await append("failed", { errorMessage: "cancelled by user", errorCode: "cancelled" });
|
|
461
463
|
}
|
|
462
464
|
catch (e) {
|
|
463
465
|
recordFailOpen("server.runs.terminal-ledger-write-failed", `arm=append-thrown-cancel ${errText(e)}`);
|
|
464
466
|
}
|
|
465
|
-
await runStore.setTerminal(taskId, "failed", c, c.errorMessage ?? null);
|
|
467
|
+
await runStore.setTerminal(taskId, "failed", c, terminalProjection(c.terminal).errorMessage ?? null);
|
|
466
468
|
}
|
|
467
469
|
else {
|
|
468
470
|
const safeMsg = redactErrorMessage(message) ?? message;
|
|
@@ -478,7 +480,7 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
478
480
|
catch {
|
|
479
481
|
}
|
|
480
482
|
if (reached?.kind === "done")
|
|
481
|
-
recordDone(reached.result, reached.result.
|
|
483
|
+
recordDone(reached.result, failureCodeOf(reached.result.terminal) === "cancelled" ? "cancelled" : terminalProjection(reached.result.terminal).status);
|
|
482
484
|
else if (reached?.kind === "suspended") {
|
|
483
485
|
metrics?.addGauge("runs_active", -1);
|
|
484
486
|
metrics?.inc("tasks_total", { status: "suspended" });
|
|
@@ -495,7 +497,7 @@ export async function runInBackground(runner, spec, runStore, taskId, metrics, p
|
|
|
495
497
|
unregisterEngineNotice();
|
|
496
498
|
if (fleetPublisher) {
|
|
497
499
|
if (reached?.kind === "done")
|
|
498
|
-
fleetPublisher.onTerminal(reached.result.
|
|
500
|
+
fleetPublisher.onTerminal(failureCodeOf(reached.result.terminal) === "cancelled" ? "cancelled" : terminalProjection(reached.result.terminal).status, fleetRunResiduals(reached.result));
|
|
499
501
|
else if (reached?.kind === "suspended")
|
|
500
502
|
fleetPublisher.onTerminal("suspended");
|
|
501
503
|
else if (reached?.kind === "needs_review")
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 终局因由(`TaskResult.terminal`)的 server 侧**读**面 —— 一个叶子模块,只依赖 `@sema-agent/core`。
|
|
3
|
+
*
|
|
4
|
+
* core 7.6.0([ref] S6 D-8,BREAKING)把 `TaskResult` 上并列的八个平面键
|
|
5
|
+
* (`status` / `errorCode` / `errorMessage` / `blockedReason` / `checkpointToken` / `checkpointId` /
|
|
6
|
+
* `checkpointGate` / `workspaceRestoreMode`)删成**一条**带标因由 `terminal: TerminalCause`
|
|
7
|
+
* (`completed` | `failed{code?,message?,nestedPause?}` | `blocked{reason}` | `paused{gate,token,…}`)。
|
|
8
|
+
* 合法组合从此是**形状**而不是一串跨字段的口头约定(「`suspended` ⇔ 有 token」「review park 与 approval
|
|
9
|
+
* park 不同时置位」)。
|
|
10
|
+
*
|
|
11
|
+
* ## 本仓的两条纪律(与 core 的单铸律配套)
|
|
12
|
+
*
|
|
13
|
+
* 1. **判断读因由,不读平面词**。`terminal.kind === "paused"` 是判据;`terminalProjection(terminal).status
|
|
14
|
+
* === "suspended"` 不是 —— 后者把一次投影塞进读点,于是同一个事实有了两个真源。
|
|
15
|
+
* 2. **`terminalProjection` 只在明确的「投影出口」用一次**:必须继续说五词 `TaskStatus` 的那几张脸
|
|
16
|
+
* (run 行的 `status` / `error` 列、`tasks_total{status}` 计量标签、fleet 行的终态词)。出口之外不拼。
|
|
17
|
+
*
|
|
18
|
+
* wire 面(HTTP 任务体、SSE `done.result`)**就是结果本身**,所以它跟着 core 一起丢八键、带 `terminal`
|
|
19
|
+
* —— 那里没有投影,只有转发(见 {@link stripResumeToken} 的例外)。
|
|
20
|
+
*/
|
|
21
|
+
import { type PausedCause, type TaskStatus, type TerminalCause } from "@sema-agent/core";
|
|
22
|
+
/**
|
|
23
|
+
* 这次失败带的**机器码**(`terminal.failed.code`);非失败终局 ⇒ `undefined`。
|
|
24
|
+
*
|
|
25
|
+
* 旧形要两键合读(`status === "failed" && errorCode === X`),而「非 failed 的行上有 errorCode」在类型上
|
|
26
|
+
* 是可表达的 —— 那正是本键此前每个读点都要自己重述一遍 `status` 判断的原因。因由形下「码只属于失败」
|
|
27
|
+
* 是形状本身,所以判据收成这一个函数:**读点问码,不再各自复述状态**。
|
|
28
|
+
*
|
|
29
|
+
* 码是开集(core 的点名族 `limits.*` / `config.*` / `resume_at.*` / brain 扁平码 + 工具层自注册码),
|
|
30
|
+
* 消费点一律带 default 臂,禁按穷举写。
|
|
31
|
+
*/
|
|
32
|
+
export declare function failureCodeOf(terminal: TerminalCause): string | undefined;
|
|
33
|
+
/**
|
|
34
|
+
* 把**耐久恢复能力凭据**(`CheckpointToken`)从一条结果里摘掉,再让它进可重放账本 / run 行 / wire。
|
|
35
|
+
* token 就是 resume 凭据(token-as-auth,search [18] Q1b),**永不离开本服务**;非密的 `gate` /
|
|
36
|
+
* `checkpointId` / `restoreMode` 有意保留(对账位)。
|
|
37
|
+
*
|
|
38
|
+
* 🔴 core 7.6.0 起凭据换了住址:它不再是 `TaskResult` 顶层的 `checkpointToken`,而是因由里
|
|
39
|
+
* `paused` 那一臂的 `token`。而**一条结果上够得着 paused 因由的位置有两处**,这是类型的形状,不是特判:
|
|
40
|
+
* · `terminal`(kind `paused`)—— 本腿自己 park 了;
|
|
41
|
+
* · `terminal.nestedPause`(kind `failed`)—— 嵌套边界(verify / cascade)遇到自己驱动不了的耐久暂停,
|
|
42
|
+
* 在**它的**边界上如实报 `failed`,而把那次暂停的因由整只带在这里(顶层调用方据此仍能恢复)。
|
|
43
|
+
* 正是这条路径当年让 `checkpointToken` 漏进过账本 —— 摘除腿从来不是「理论上的对称」。
|
|
44
|
+
* 两处走**同一条**摘除规则({@link stripPausedToken}),不是两条。
|
|
45
|
+
*
|
|
46
|
+
* wire 上的形:paused 因由**缺席 `token`**(core 类型里它是必填座,那是**引擎内**的契约;本服务的 wire
|
|
47
|
+
* 契约逐字声明这一座不出服务边界 —— 见 `docs/ASSISTANT-WIRE-CONTRACT.md`)。消费端要动这次暂停,走
|
|
48
|
+
* `/v1/approvals` 决策端点,凭 `checkpointId` 对账,从来不是拿 token 直连引擎。
|
|
49
|
+
*
|
|
50
|
+
* 对任何输入安全(非普通对象 / 无 `terminal` / 因由不带 token ⇒ **同一引用**原样返回,不重塑)。
|
|
51
|
+
*/
|
|
52
|
+
export declare function stripResumeToken<T extends {
|
|
53
|
+
terminal: TerminalCause;
|
|
54
|
+
}>(result: T): T;
|
|
55
|
+
/**
|
|
56
|
+
* 这条腿的结果里**藏着的那次耐久暂停** —— 嵌套边界(verify / cascade)遇到自己驱动不了的耐久 park 时,
|
|
57
|
+
* 在**它的**边界上如实报 `failed`(码是 `unexpected.suspended` / `unexpected.needs_review`),而把那次
|
|
58
|
+
* park 的因由**整只**带在这里,顶层调用方据此仍能恢复。非嵌套 park ⇒ `undefined`。
|
|
59
|
+
*
|
|
60
|
+
* 🔴 core 7.6.0 之前这件事只能从**码的字面量**推:`status` 恒被改写成 `"failed"`(真 park 词挪进
|
|
61
|
+
* `errorCode`,token 顺着 `checkpointToken` 原样透传),于是本仓凡是按 `status` 判 park 的门在这条腿上
|
|
62
|
+
* **恒假**([ref] 二的真案:待批工具卡被谎报成 tool-less park),补救办法是「先把 status 换成真词再递进去」
|
|
63
|
+
* 这种再造事实的写法。因由形下 park 的门、门上的卡、恢复凭据都在同一个 `PausedCause` 上 —— 那些补救全部
|
|
64
|
+
* 消失,判据回到一条:**问因由**。
|
|
65
|
+
*/
|
|
66
|
+
export declare function nestedPauseOf(terminal: TerminalCause): PausedCause | undefined;
|
|
67
|
+
/**
|
|
68
|
+
* 这条结果**停在耐久门上**了吗 —— 若是,给出那次暂停的因由。两条路径合成一个判据:
|
|
69
|
+
* · 本腿自己 park(`terminal.kind === "paused"`);
|
|
70
|
+
* · 嵌套边界替它带回来的那次 park({@link nestedPauseOf})。
|
|
71
|
+
*
|
|
72
|
+
* 🔴 7.6.0 之前这两条路径要用**两个不同的判据**读:平面腿按 `status` 的五词穷举 switch
|
|
73
|
+
* (`suspended` / `needs_review` 是 park),嵌套腿按「还带着 `checkpointToken` 就是 park」——同一个问题
|
|
74
|
+
* 两套写法,而两套写法各漏过一次真事故(fanout 的手抄 `=== "suspended"` 漏 `needs_review`;verify 腿的
|
|
75
|
+
* status 恒 `"failed"`)。因由形下两条路径合成一句话,规则从二变一。
|
|
76
|
+
*/
|
|
77
|
+
export declare function pausedCauseOf(terminal: TerminalCause): PausedCause | undefined;
|
|
78
|
+
/**
|
|
79
|
+
* **持久 blob 的平面读面** —— 把一条**任何年代**的持久终局结果读成五词 `status` + 三个平面文本格。
|
|
80
|
+
*
|
|
81
|
+
* 🔴 为什么这不是「兼容读臂」(codex 对抗复审 r1 [high] ×2,亲读验真后修;首版把它当成兼容 shim 删掉了,
|
|
82
|
+
* 那是**错的**,理由如下):
|
|
83
|
+
* · 本仓 wire 面的纪律「不为老键留兼容读臂」管的是**同一条 wire 上两种形并存** —— 那会让两个写者共用一个
|
|
84
|
+
* 语义面。而这里是**读盘**:`task_run.result` / `task_event(done).result` / `workflow_journal` 里躺着
|
|
85
|
+
* 7.63.0 及以前写下的**真字节**,它们本来就是平面形。一个读盘面认识盘上真实存在的两种形,不是第二个写者
|
|
86
|
+
* ——**没有任何东西会再写出旧形**(写口只剩因由一条)。
|
|
87
|
+
* · 不认旧形的代价是**可复现的两条真回归**:① `terminalProjection(undefined)` 直接抛 TypeError ⇒ 历史 run 的
|
|
88
|
+
* `GET /v1/runs/:id` 整个 500;② 读面的二次脱敏(旧行纵深)对旧行**恒不生效** ⇒ 上线前落盘的凭据在
|
|
89
|
+
* `GET /v1/runs/:id/events` 回放时重新明文外发。
|
|
90
|
+
* ⇒ 规则仍然只有一条:**读盘认盘上真有的形,写盘只写一种形**。
|
|
91
|
+
*
|
|
92
|
+
* 形不合(非对象 / 两种形都不是)⇒ 全格缺席,不抛(读面永不因为一条坏行整只 500)。
|
|
93
|
+
*/
|
|
94
|
+
export declare function persistedPlaneOf(result: unknown): {
|
|
95
|
+
status?: TaskStatus;
|
|
96
|
+
errorCode?: string;
|
|
97
|
+
errorMessage?: string;
|
|
98
|
+
blockedReason?: string;
|
|
99
|
+
};
|
|
100
|
+
//# sourceMappingURL=terminal.d.ts.map
|
package/dist/terminal.js
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { terminalProjection } from "@sema-agent/core";
|
|
2
|
+
import { recordFailOpen } from "./observability/fail-open.js";
|
|
3
|
+
export function failureCodeOf(terminal) {
|
|
4
|
+
return terminal.kind === "failed" ? terminal.code : undefined;
|
|
5
|
+
}
|
|
6
|
+
export function stripResumeToken(result) {
|
|
7
|
+
const t = result?.terminal;
|
|
8
|
+
if (t === null || typeof t !== "object")
|
|
9
|
+
return result;
|
|
10
|
+
if (t.kind === "paused")
|
|
11
|
+
return { ...result, terminal: stripPausedToken(t) };
|
|
12
|
+
if (t.kind === "failed" && t.nestedPause !== undefined)
|
|
13
|
+
return { ...result, terminal: { ...t, nestedPause: stripPausedToken(t.nestedPause) } };
|
|
14
|
+
return result;
|
|
15
|
+
}
|
|
16
|
+
function stripPausedToken(paused) {
|
|
17
|
+
const { token: _drop, ...rest } = paused;
|
|
18
|
+
return rest;
|
|
19
|
+
}
|
|
20
|
+
export function nestedPauseOf(terminal) {
|
|
21
|
+
return terminal.kind === "failed" ? terminal.nestedPause : undefined;
|
|
22
|
+
}
|
|
23
|
+
export function pausedCauseOf(terminal) {
|
|
24
|
+
return terminal.kind === "paused" ? terminal : nestedPauseOf(terminal);
|
|
25
|
+
}
|
|
26
|
+
export function persistedPlaneOf(result) {
|
|
27
|
+
if (result === null || typeof result !== "object")
|
|
28
|
+
return {};
|
|
29
|
+
const r = result;
|
|
30
|
+
if (r.terminal !== null && typeof r.terminal === "object") {
|
|
31
|
+
let projected;
|
|
32
|
+
try {
|
|
33
|
+
projected = terminalProjection(r.terminal);
|
|
34
|
+
}
|
|
35
|
+
catch (e) {
|
|
36
|
+
recordFailOpen("server.terminal.persisted-plane-malformed", e instanceof Error ? e.message : String(e));
|
|
37
|
+
projected = undefined;
|
|
38
|
+
}
|
|
39
|
+
if (projected !== undefined) {
|
|
40
|
+
return {
|
|
41
|
+
status: projected.status,
|
|
42
|
+
...(projected.errorCode !== undefined ? { errorCode: projected.errorCode } : {}),
|
|
43
|
+
...(projected.errorMessage !== undefined ? { errorMessage: projected.errorMessage } : {}),
|
|
44
|
+
...(projected.blockedReason !== undefined ? { blockedReason: projected.blockedReason } : {}),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
...(typeof r.status === "string" ? { status: r.status } : {}),
|
|
50
|
+
...(typeof r.errorCode === "string" ? { errorCode: r.errorCode } : {}),
|
|
51
|
+
...(typeof r.errorMessage === "string" ? { errorMessage: r.errorMessage } : {}),
|
|
52
|
+
...(typeof r.blockedReason === "string" ? { blockedReason: r.blockedReason } : {}),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=terminal.js.map
|