@sema-agent/server 7.124.1 → 7.125.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +138 -2
- package/USAGE.md +46 -8
- package/dist/audit.d.ts +27 -5
- package/dist/audit.js +42 -8
- package/dist/boot/execution-env.d.ts +33 -1
- package/dist/boot/execution-env.js +31 -2
- package/dist/boot/resolve-spec.js +4 -3
- package/dist/boot/session-faces.d.ts +14 -1
- package/dist/boot/session-faces.js +6 -5
- package/dist/boot/stage-02-web-admin-pull.js +5 -1
- package/dist/boot/stage-05-execution-env.js +6 -2
- package/dist/boot/stage-09-leader.d.ts +1 -1
- package/dist/boot/stage-10-http-server.d.ts +1 -1
- package/dist/boot/web-admin.js +130 -140
- package/dist/capabilities/center-wiring.d.ts +2 -1
- package/dist/capabilities/composite-members.d.ts +1 -0
- package/dist/capabilities/composite-members.js +1 -0
- package/dist/capability-facts.d.ts +28 -3
- package/dist/capability-facts.js +20 -0
- package/dist/config-catalog.js +4 -1
- package/dist/config-d-migration.js +2 -1
- package/dist/config-lkg.d.ts +4 -1
- package/dist/config-lkg.js +16 -12
- package/dist/config-provider.d.ts +4 -1
- package/dist/config-provider.js +1 -0
- package/dist/config-types.d.ts +10 -3
- package/dist/config.js +44 -14
- package/dist/env-facts.d.ts +2 -2
- package/dist/env-facts.js +3 -2
- package/dist/fleet/fleet-bus.d.ts +24 -12
- package/dist/fleet/fleet-bus.js +4 -1
- package/dist/fleet/workspace-snapshot-residual.d.ts +10 -0
- package/dist/fleet/workspace-snapshot-residual.js +35 -0
- package/dist/hooks/hook-runner.d.ts +15 -27
- package/dist/hooks/hook-runner.js +19 -59
- package/dist/http/active-run-conflict.d.ts +3 -0
- package/dist/http/active-run-conflict.js +3 -0
- package/dist/http/admission.js +7 -7
- package/dist/http/approval-sweeps.js +2 -4
- package/dist/http/device-placement.js +1 -0
- package/dist/http/dispatch.js +1 -1
- package/dist/http/durable-run-leg.d.ts +20 -6
- package/dist/http/durable-run-leg.js +4 -4
- package/dist/http/idempotency.d.ts +83 -38
- package/dist/http/idempotency.js +38 -13
- package/dist/http/request-guards.d.ts +18 -0
- package/dist/http/request-guards.js +12 -3
- package/dist/http/resume-legs.js +7 -1
- package/dist/http/route-ctx.d.ts +3 -3
- package/dist/http/route-ctx.js +1 -1
- package/dist/http/routes/a2a-serve.js +27 -11
- package/dist/http/routes/capabilities.js +1 -0
- package/dist/http/routes/fleet.js +2 -0
- package/dist/http/routes/images.js +2 -2
- package/dist/http/routes/leader.d.ts +4 -0
- package/dist/http/routes/leader.js +13 -3
- package/dist/http/routes/observability.js +2 -0
- package/dist/http/routes/runs.d.ts +6 -2
- package/dist/http/routes/runs.js +71 -101
- package/dist/http/routes/session-sync.d.ts +3 -0
- package/dist/http/routes/session-sync.js +30 -1
- package/dist/http/routes/sessions-list.d.ts +11 -0
- package/dist/http/routes/sessions-list.js +6 -6
- package/dist/http/routes/sessions.d.ts +6 -1
- package/dist/http/routes/sessions.js +41 -10
- package/dist/http/routes/tasks.js +63 -21
- package/dist/http/run-read-face.d.ts +26 -0
- package/dist/http/run-read-face.js +67 -0
- package/dist/http/server.d.ts +4 -4
- package/dist/http/server.js +1 -2
- package/dist/http/submit-replay.d.ts +40 -0
- package/dist/http/submit-replay.js +22 -0
- package/dist/leader/endpoint.d.ts +10 -5
- package/dist/leader/endpoint.js +27 -5
- package/dist/observability/cost-quota.d.ts +51 -7
- package/dist/observability/cost-quota.js +35 -17
- package/dist/observability/fail-open.d.ts +1 -1
- package/dist/observability/fail-open.js +1 -1
- package/dist/observability/run-terminal.d.ts +5 -9
- package/dist/observability/run-terminal.js +2 -7
- package/dist/plugins/approval-ask-store-sql.js +1 -1
- package/dist/plugins/caching-session-store.d.ts +7 -2
- package/dist/plugins/caching-session-store.js +6 -0
- package/dist/plugins/device-store-sql.js +1 -1
- package/dist/plugins/ensure-index.js +22 -0
- package/dist/plugins/file-run-store.d.ts +24 -14
- package/dist/plugins/file-run-store.js +52 -12
- package/dist/plugins/leader-run-store-sql.d.ts +1 -11
- package/dist/plugins/leader-run-store-sql.js +1 -5
- package/dist/plugins/local-session-store.d.ts +35 -6
- package/dist/plugins/local-session-store.js +87 -18
- package/dist/plugins/memory-run-store.d.ts +13 -12
- package/dist/plugins/memory-run-store.js +25 -0
- package/dist/plugins/one-time-migrations.js +1 -1
- package/dist/plugins/pg-cost-quota.d.ts +1 -1
- package/dist/plugins/pg-cost-quota.js +2 -5
- package/dist/plugins/pg-pool.js +3 -0
- package/dist/plugins/pg-session-storage.d.ts +11 -1
- package/dist/plugins/pg-session-storage.js +44 -8
- package/dist/plugins/remote-env-k8s.d.ts +127 -12
- package/dist/plugins/remote-env-k8s.js +270 -27
- package/dist/plugins/run-store-sql.d.ts +26 -12
- package/dist/plugins/run-store-sql.js +50 -25
- package/dist/plugins/schema-consistency.d.ts +1 -13
- package/dist/plugins/schema-consistency.js +1 -41
- package/dist/plugins/schema-indexes.js +1 -0
- package/dist/plugins/schema-live-columns.d.ts +20 -0
- package/dist/plugins/schema-live-columns.js +42 -0
- package/dist/plugins/store-contracts.d.ts +40 -2
- package/dist/plugins/tidb-cost-quota.d.ts +1 -1
- package/dist/plugins/tidb-cost-quota.js +2 -5
- package/dist/plugins/tidb-pool.js +14 -3
- package/dist/plugins/tidb-session-store.d.ts +22 -2
- package/dist/plugins/tidb-session-store.js +33 -6
- package/dist/plugins/write-behind-counter.d.ts +12 -5
- package/dist/plugins/write-behind-counter.js +14 -7
- package/dist/prompts-domain-validate.js +3 -2
- package/dist/rules-consent.d.ts +2 -2
- package/dist/rules-consent.js +2 -2
- package/dist/run-local.js +31 -23
- package/dist/runs.d.ts +9 -0
- package/dist/runs.js +8 -0
- package/dist/runtime-governance.d.ts +3 -3
- package/dist/runtime-governance.js +2 -4
- package/dist/security.d.ts +19 -2
- package/dist/session-row.d.ts +53 -0
- package/dist/session-row.js +47 -0
- package/dist/session-sync.d.ts +6 -0
- package/dist/session-sync.js +5 -1
- package/dist/session-titler.d.ts +9 -5
- package/dist/session-titler.js +3 -1
- package/dist/spec-fields.js +2 -2
- package/dist/task-cwd.d.ts +0 -20
- package/dist/task-cwd.js +2 -10
- package/dist/tool-approval.d.ts +5 -7
- package/dist/trace/redact.d.ts +23 -50
- package/dist/trace/redact.js +42 -9
- package/dist/trace/self-minted-wire.d.ts +25 -0
- package/dist/trace/self-minted-wire.js +9 -0
- package/dist/web-admin-client/apply-effective.d.ts +10 -12
- package/dist/web-admin-client/apply-effective.js +2 -3
- package/dist/web-admin-client/candidate-gate.d.ts +69 -0
- package/dist/web-admin-client/candidate-gate.js +27 -0
- package/dist/web-admin-client/http-client.d.ts +2 -1
- package/dist/web-admin-client/http-client.js +2 -1
- package/dist/web-admin-client/pull-state.d.ts +6 -4
- package/dist/web-admin-client/read-warnings.d.ts +29 -5
- package/dist/web-admin-client/read-warnings.js +24 -4
- package/dist/web-admin-client/stage-limits.js +4 -1
- package/dist/workspace-space.d.ts +41 -0
- package/dist/workspace-space.js +5 -0
- package/package.json +3 -3
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { redactErrorMessage, redactHead, redactTerminalResult } from "../../observability/run-terminal.js";
|
|
2
2
|
import { runWithVerification, runCascade, taskNotificationDedupKey, uuidv7 } from "@sema-agent/core";
|
|
3
|
-
import { graftParkToolCallId, markChildrenStoppedByUserOnAbort, syncRefusalHttpStatus, HEARTBEAT_MS } from "../../runs.js";
|
|
3
|
+
import { graftParkToolCallId, markChildrenStoppedByUserOnAbort, syncRefusalHttpStatus, releaseKeyIfZeroWorkRefusal, HEARTBEAT_MS } from "../../runs.js";
|
|
4
4
|
import { sealResultForExit, outwardPlane } from "../../terminal.js";
|
|
5
5
|
import { withPrincipal } from "../../observability/principal-context.js";
|
|
6
6
|
import { fleetRunPublisher, fleetRunLabels, fleetRunResiduals, isFleetAgentTerminalNotification } from "../../fleet/fleet-bus.js";
|
|
@@ -15,12 +15,13 @@ import { recordResourceWindow } from "../../resource-window.js";
|
|
|
15
15
|
import { redactSecrets } from "../../trace/redact.js";
|
|
16
16
|
import { contextUsageEventData, turnEndEventData, toolStartEventData, toolEndEventData, taskProgressEventData, taskNotificationEventData, compactedEventData, diagnosticsEventData, brainStatusEventData, steeringInjectedEventData, compactionOutcomeEventData, workspaceChangedEventData, wiringManifestFaceEventData, toolRosterDeltaEventData, toolProgressEventData, toolDisclosureEventData, humanInputEventData, appendModelUsageDelta, attachModelUsage, textEndEventData, liveIdentityFields } from "../../trace/project.js";
|
|
17
17
|
import { cascadeConfig, runMeta } from "../run-meta.js";
|
|
18
|
-
import {
|
|
18
|
+
import { idemCallerOf, idemKeyOf, inflightKeyOf } from "../idempotency.js";
|
|
19
|
+
import { replayOfExistingRun } from "../submit-replay.js";
|
|
19
20
|
import { sendJson, sendError, sseHeaders } from "../send.js";
|
|
20
21
|
import { collectSsePreamble } from "../sse-log.js";
|
|
21
22
|
import { buildApprovalPreamble, buildApprovalPreambleSseFrames } from "../../approval-card.js";
|
|
22
23
|
import { createApprovalCardEmitter, resolveApprovalLeg, resolveStreamApprovalGate } from "../../tool-approval.js";
|
|
23
|
-
import { buildActiveRunConflict, toDoneFrameResult } from "../active-run-conflict.js";
|
|
24
|
+
import { buildActiveRunConflict, isActiveRunConflictBody, toDoneFrameResult } from "../active-run-conflict.js";
|
|
24
25
|
import { commitPlacement, devicePlacementErrorBody } from "../device-placement.js";
|
|
25
26
|
import { commitSessionInheritEnv } from "../session-inherit-env.js";
|
|
26
27
|
import { headerStr, gatedPrincipal, explicitOperatorOk } from "../principal-gate.js";
|
|
@@ -57,6 +58,18 @@ function sendSubmitResult(res, resp) {
|
|
|
57
58
|
: raw;
|
|
58
59
|
sendJson(res, 200, body);
|
|
59
60
|
}
|
|
61
|
+
function writeMetaFrame(res, taskId, sessionId) {
|
|
62
|
+
res.write(`event: meta\ndata: ${JSON.stringify({ type: "meta", taskId, ...(sessionId ? { sessionId } : {}) })}\n\n`);
|
|
63
|
+
}
|
|
64
|
+
function refusalDoneResult(body) {
|
|
65
|
+
if (isActiveRunConflictBody(body))
|
|
66
|
+
return toDoneFrameResult(body);
|
|
67
|
+
const b = body;
|
|
68
|
+
return { status: "failed", errorCode: String(b.errorCode), errorMessage: String(b.error) };
|
|
69
|
+
}
|
|
70
|
+
function doneResultOfOutcome(resp) {
|
|
71
|
+
return resp.status === 200 ? resp.body : refusalDoneResult(resp.body);
|
|
72
|
+
}
|
|
60
73
|
const MULTI_ATTEMPT_NO_EVENT_CHANNEL_NOTICE = "verify / cascade ran with their normal multi-attempt semantics, but the synchronous /v1/tasks endpoint has no event channel — no status/progress frames exist to replay on this shape. For status / progress / delegated-child frames and the terminal done, submit via POST /v1/runs and read GET /v1/runs/:id/events; note that verify / cascade legs carry no per-turn stream there either (no wiring_manifest / text / turn_end frames — the engine runs those legs as result-returning attempts).";
|
|
61
74
|
export async function handleTasks(req, res, match, ctx) {
|
|
62
75
|
const { deps } = ctx;
|
|
@@ -83,8 +96,24 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
83
96
|
const liveStream = match.decl.id === "tasks-submit-stream";
|
|
84
97
|
{
|
|
85
98
|
const rawIdem = headerStr(req.headers["idempotency-key"]);
|
|
86
|
-
const
|
|
87
|
-
const
|
|
99
|
+
const submitter = gatedPrincipal(req, deps.config);
|
|
100
|
+
const idemKey = rawIdem ? idemKeyOf(rawIdem, idemCallerOf(req, ctx), "run") : undefined;
|
|
101
|
+
const inflightKey = idemKey ? inflightKeyOf(idemKey, "sync") : undefined;
|
|
102
|
+
const cached = inflightKey ? idemCache.peek(inflightKey) : undefined;
|
|
103
|
+
const keyedRow = idemKey && !cached && deps.runStore ? await deps.runStore.getRunByIdem(idemKey) : undefined;
|
|
104
|
+
if (keyedRow && deps.runStore) {
|
|
105
|
+
const replay = await replayOfExistingRun(keyedRow, { principal: submitter }, "sync", deps, deps.runStore);
|
|
106
|
+
if (liveStream && replay.status === 200) {
|
|
107
|
+
sseHeaders(res, { "x-task-id": keyedRow.taskId });
|
|
108
|
+
writeMetaFrame(res, keyedRow.taskId, keyedRow.sessionId);
|
|
109
|
+
sseData(res, { type: "done", result: replay.body, replay: true });
|
|
110
|
+
res.end();
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
sendSubmitResult(res, replay);
|
|
114
|
+
}
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
88
117
|
if (cached) {
|
|
89
118
|
if (liveStream) {
|
|
90
119
|
sseHeaders(res);
|
|
@@ -97,7 +126,7 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
97
126
|
try {
|
|
98
127
|
const resp = await cached;
|
|
99
128
|
if (!res.writableEnded)
|
|
100
|
-
sseData(res, { type: "done", result: resp
|
|
129
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(resp), replay: true });
|
|
101
130
|
}
|
|
102
131
|
finally {
|
|
103
132
|
clearInterval(hb);
|
|
@@ -136,9 +165,8 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
136
165
|
let usageKey;
|
|
137
166
|
reqState.streamDetached = detachOnDisconnect;
|
|
138
167
|
sseHeaders(res, earlyDurableTid ? { "x-task-id": earlyDurableTid } : undefined);
|
|
139
|
-
if (earlyDurableTid)
|
|
140
|
-
res
|
|
141
|
-
}
|
|
168
|
+
if (earlyDurableTid)
|
|
169
|
+
writeMetaFrame(res, earlyDurableTid, prepared.spec.sessionId);
|
|
142
170
|
const ac = createRunAbort();
|
|
143
171
|
let closed = false;
|
|
144
172
|
let detachLogged = false;
|
|
@@ -229,23 +257,30 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
229
257
|
};
|
|
230
258
|
if (deps.runStore && prepared.spec.sessionId) {
|
|
231
259
|
const tid = earlyDurableTid ?? uuidv7();
|
|
232
|
-
const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
|
|
260
|
+
const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", { ...runMeta(prepared, source), idemKey: idemKey ?? null });
|
|
233
261
|
if (created.ok)
|
|
234
262
|
clearTurnActivity(tid);
|
|
235
263
|
if (created.ok)
|
|
236
264
|
recordResourceWindow(tid, prepared.spec.resourceSuspend);
|
|
237
265
|
if (created.ok)
|
|
238
266
|
deps.sessionTitler?.maybeTitle(prepared.spec.sessionId, prepared.spec.objective, typeof prepared.spec.model === "string" ? prepared.spec.model : prepared.spec.model?.id);
|
|
267
|
+
if (created.existing) {
|
|
268
|
+
const replay = await replayOfExistingRun(created.existing, { principal: submitter }, "sync", deps, deps.runStore);
|
|
269
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(replay), replay: true });
|
|
270
|
+
return replay;
|
|
271
|
+
}
|
|
239
272
|
if (!created.ok) {
|
|
240
273
|
const conflict = await buildActiveRunConflict({ runStore: deps.runStore, checkpointStore: deps.checkpointStore, governance: deps.config, runStaleSec: deps.config.runStaleSec, turnActivity: readTurnActivityMs }, prepared.spec.sessionId, created.activeTaskId);
|
|
241
|
-
|
|
242
|
-
|
|
274
|
+
const refused = { status: 409, body: conflict };
|
|
275
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(refused) });
|
|
276
|
+
return refused;
|
|
243
277
|
}
|
|
244
278
|
const placementReject = await commitPlacement(deps, deps.runStore, tid, prepared.bindingIntent);
|
|
245
279
|
if (placementReject) {
|
|
246
280
|
const body = devicePlacementErrorBody(placementReject);
|
|
247
|
-
|
|
248
|
-
|
|
281
|
+
const refused = { status: placementReject.status, body };
|
|
282
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(refused) });
|
|
283
|
+
return refused;
|
|
249
284
|
}
|
|
250
285
|
durableTaskId = tid;
|
|
251
286
|
}
|
|
@@ -280,7 +315,7 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
280
315
|
let fleetPub;
|
|
281
316
|
try {
|
|
282
317
|
fleetPub = durableTaskId
|
|
283
|
-
? fleetRunPublisher(deps.fleetBus, { runId: durableTaskId, scope: gatedPrincipal(req, deps.config) ?? "default", rootTaskId: durableTaskId, ...fleetRunLabels(prepared.spec.objective) })
|
|
318
|
+
? fleetRunPublisher(deps.fleetBus, { runId: durableTaskId, scope: gatedPrincipal(req, deps.config) ?? "default", rootTaskId: durableTaskId, sessionId: prepared.spec.sessionId, ...fleetRunLabels(prepared.spec.objective) })
|
|
284
319
|
: undefined;
|
|
285
320
|
}
|
|
286
321
|
catch (e) {
|
|
@@ -658,8 +693,10 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
658
693
|
await deps.runStore.setSuspended(durableTaskId, finalResult);
|
|
659
694
|
else if (finalResult && finalPlane?.status === "needs_review" && deps.checkpointStore)
|
|
660
695
|
await deps.runStore.setNeedsReview(durableTaskId, finalResult);
|
|
661
|
-
else if (finalResult && finalPlane)
|
|
696
|
+
else if (finalResult && finalPlane) {
|
|
662
697
|
await deps.runStore.setTerminal(durableTaskId, finalPlane.status, finalResult, finalPlane.errorMessage ?? null);
|
|
698
|
+
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, finalResult);
|
|
699
|
+
}
|
|
663
700
|
else {
|
|
664
701
|
putRewindAnchor();
|
|
665
702
|
const pending = await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId);
|
|
@@ -734,6 +771,7 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
734
771
|
}
|
|
735
772
|
const retryPlane = outwardPlane(finalResult.terminal);
|
|
736
773
|
await deps.runStore.setTerminal(durableTaskId, retryPlane.status, finalResult, retryPlane.errorMessage ?? null);
|
|
774
|
+
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, finalResult);
|
|
737
775
|
settleFleet(retryPlane.status, fleetRunResiduals(finalResult));
|
|
738
776
|
}
|
|
739
777
|
else if (backstopCause !== undefined) {
|
|
@@ -766,7 +804,7 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
766
804
|
};
|
|
767
805
|
counters.uncountedBillableInflight++;
|
|
768
806
|
try {
|
|
769
|
-
resp = await idemCache.run(
|
|
807
|
+
resp = await idemCache.run(inflightKey, async () => withRunReceipt(await runStreamSubmitLeg()));
|
|
770
808
|
}
|
|
771
809
|
catch (e) {
|
|
772
810
|
deps.logger?.error("stream_error", { closed, err: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
@@ -780,7 +818,7 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
780
818
|
deps.modelUsage?.clear(usageKey);
|
|
781
819
|
}
|
|
782
820
|
if (!ranLive && resp && !res.writableEnded) {
|
|
783
|
-
sseData(res, { type: "done", result: resp
|
|
821
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(resp), replay: true });
|
|
784
822
|
}
|
|
785
823
|
res.end();
|
|
786
824
|
}
|
|
@@ -792,11 +830,13 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
792
830
|
let durableHeartbeat;
|
|
793
831
|
if (deps.runStore && prepared.spec.sessionId) {
|
|
794
832
|
const tid = uuidv7();
|
|
795
|
-
const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
|
|
833
|
+
const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", { ...runMeta(prepared, source), idemKey: idemKey ?? null });
|
|
796
834
|
if (created.ok)
|
|
797
835
|
clearTurnActivity(tid);
|
|
798
836
|
if (created.ok)
|
|
799
837
|
recordResourceWindow(tid, prepared.spec.resourceSuspend);
|
|
838
|
+
if (created.existing)
|
|
839
|
+
return await replayOfExistingRun(created.existing, { principal: submitter }, "sync", deps, deps.runStore);
|
|
800
840
|
if (!created.ok)
|
|
801
841
|
return { status: 409, body: await buildActiveRunConflict({ runStore: deps.runStore, checkpointStore: deps.checkpointStore, governance: deps.config, runStaleSec: deps.config.runStaleSec, turnActivity: readTurnActivityMs }, prepared.spec.sessionId, created.activeTaskId) };
|
|
802
842
|
const placementReject = await commitPlacement(deps, deps.runStore, tid, prepared.bindingIntent);
|
|
@@ -872,8 +912,10 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
872
912
|
const resumeAtStatus = syncRefusalHttpStatus(result);
|
|
873
913
|
if (resumeAtStatus) {
|
|
874
914
|
const resumeAtPlane = outwardPlane(result.terminal);
|
|
875
|
-
if (durableTaskId && deps.runStore)
|
|
915
|
+
if (durableTaskId && deps.runStore) {
|
|
876
916
|
await deps.runStore.setTerminal(durableTaskId, "failed", sealResultForExit(result), resumeAtPlane.errorMessage ?? null);
|
|
917
|
+
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, result);
|
|
918
|
+
}
|
|
877
919
|
const retryAfterSec = resumeAtPlane.errorCode === "memory.admission_required" && result.retryAfterMs !== undefined
|
|
878
920
|
? Math.max(1, Math.ceil(result.retryAfterMs / 1000))
|
|
879
921
|
: undefined;
|
|
@@ -895,7 +937,7 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
895
937
|
noteRunOutcome(result);
|
|
896
938
|
return { status: 200, body: withSyncSubmitKeys(buildSyncSubmitBody(exitResult)) };
|
|
897
939
|
};
|
|
898
|
-
const resp = await idemCache.run(
|
|
940
|
+
const resp = await idemCache.run(inflightKey, async () => withRunReceipt(await runSyncSubmitLeg()));
|
|
899
941
|
sendSubmitResult(res, resp);
|
|
900
942
|
}
|
|
901
943
|
return;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type RunRecord } from "../plugins/store-contracts.js";
|
|
2
|
+
import type { RunStore } from "../plugins/store-backend.js";
|
|
3
|
+
import type { ServiceConfig } from "../config-types.js";
|
|
4
|
+
/** 读面要的部署配置两格(stale 折叠窗 + 基建计价)。 */
|
|
5
|
+
export interface RunFaceDeps {
|
|
6
|
+
config: Pick<ServiceConfig, "runStaleSec" | "infraCostRates">;
|
|
7
|
+
}
|
|
8
|
+
export declare function readRunFace(deps: RunFaceDeps, runStore: RunStore, run: RunRecord): Promise<{
|
|
9
|
+
result: import("../plugins/store-contracts.js").PersistedTaskResult | undefined;
|
|
10
|
+
supervisorCost: import("../observability/cost-taxonomy.js").SupervisorCostBreakdown | undefined;
|
|
11
|
+
suggestions: string[] | undefined;
|
|
12
|
+
errorCode: string | undefined;
|
|
13
|
+
error: string | undefined;
|
|
14
|
+
jobId: string | undefined;
|
|
15
|
+
source: string | undefined;
|
|
16
|
+
heldBy: string | null;
|
|
17
|
+
cancelRequested: boolean;
|
|
18
|
+
crossSliceUsage?: import("../resource-window.js").CrossSliceUsage | undefined;
|
|
19
|
+
msSinceLastActivity?: number | undefined;
|
|
20
|
+
taskId: string;
|
|
21
|
+
sessionId: string;
|
|
22
|
+
status: "running" | import("@sema-agent/core").TaskStatus;
|
|
23
|
+
}>;
|
|
24
|
+
/** run 行读面的值形 —— 从唯一投影口的返回型派生(不手抄键表)。 */
|
|
25
|
+
export type RunReadFace = Awaited<ReturnType<typeof readRunFace>>;
|
|
26
|
+
//# sourceMappingURL=run-read-face.d.ts.map
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { persistedPlaneOf } from "../terminal.js";
|
|
2
|
+
import { memoRedactedRead, memoRedactTerminalResult, redactErrorMessage, redactTerminalResult } from "../observability/run-terminal.js";
|
|
3
|
+
import { RUN_HOLDER_DEATH_CODES, isTerminalRunStatus } from "../plugins/store-contracts.js";
|
|
4
|
+
import { composeSupervisorCost, infraCost, infraUsageFromEvents, hasInfraPricing } from "../observability/cost-taxonomy.js";
|
|
5
|
+
import { recordFailOpen } from "../observability/fail-open.js";
|
|
6
|
+
import { readTurnActivityMs } from "../turn-activity.js";
|
|
7
|
+
import { buildCrossSliceUsage, readResourceWindow } from "../resource-window.js";
|
|
8
|
+
export async function readRunFace(deps, runStore, run) {
|
|
9
|
+
const taskId = run.taskId;
|
|
10
|
+
const stale = run.status === "running" && Date.now() - new Date(run.updatedAt).getTime() > deps.config.runStaleSec * 1000;
|
|
11
|
+
const infraRates = deps.config.infraCostRates;
|
|
12
|
+
const needCost = Boolean(run.result?.stats && !stale && infraRates && hasInfraPricing(infraRates));
|
|
13
|
+
const needSuggestions = !stale && run.status === "completed";
|
|
14
|
+
const resourceWindow = run.status === "running" && !stale ? readResourceWindow(run.taskId) : undefined;
|
|
15
|
+
let events;
|
|
16
|
+
let eventsUnavailable = false;
|
|
17
|
+
if (needCost || needSuggestions || resourceWindow !== undefined) {
|
|
18
|
+
if (runStore.getEvents) {
|
|
19
|
+
events = await runStore.getEvents(taskId, 0).catch(() => {
|
|
20
|
+
eventsUnavailable = true;
|
|
21
|
+
return [];
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
eventsUnavailable = true;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
let supervisorCost;
|
|
29
|
+
if (needCost && events && run.result?.stats) {
|
|
30
|
+
const durMs = new Date(run.updatedAt).getTime() - new Date(run.createdAt).getTime();
|
|
31
|
+
const usage = infraUsageFromEvents(events, Number.isFinite(durMs) ? durMs : 0);
|
|
32
|
+
const st = run.result.stats;
|
|
33
|
+
supervisorCost = composeSupervisorCost(st.costBreakdown ?? null, st.costMicroUsd ?? 0, infraCost(usage, infraRates));
|
|
34
|
+
}
|
|
35
|
+
let suggestions;
|
|
36
|
+
if (needSuggestions && events) {
|
|
37
|
+
const sug = [...events].reverse().find((e) => e.type === "suggestions");
|
|
38
|
+
const arr = sug?.data?.suggestions;
|
|
39
|
+
if (Array.isArray(arr))
|
|
40
|
+
suggestions = arr.map((s) => String(s));
|
|
41
|
+
}
|
|
42
|
+
const lastActivityAt = run.status === "running" && !stale ? readTurnActivityMs(run.taskId) : undefined;
|
|
43
|
+
let crossSliceUsage;
|
|
44
|
+
if (resourceWindow !== undefined) {
|
|
45
|
+
if (eventsUnavailable || events === undefined)
|
|
46
|
+
recordFailOpen("server.runs.cross-slice-usage-unavailable");
|
|
47
|
+
else
|
|
48
|
+
crossSliceUsage = buildCrossSliceUsage(resourceWindow, events);
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
taskId: run.taskId,
|
|
52
|
+
sessionId: run.sessionId,
|
|
53
|
+
status: stale ? "failed" : run.status,
|
|
54
|
+
...(lastActivityAt !== undefined ? { msSinceLastActivity: Math.max(0, Date.now() - lastActivityAt) } : {}),
|
|
55
|
+
...(crossSliceUsage !== undefined ? { crossSliceUsage } : {}),
|
|
56
|
+
result: isTerminalRunStatus(run.status) ? memoRedactTerminalResult("run.result", run.result ?? undefined) : redactTerminalResult(run.result ?? undefined),
|
|
57
|
+
supervisorCost,
|
|
58
|
+
suggestions,
|
|
59
|
+
errorCode: stale ? RUN_HOLDER_DEATH_CODES.stalled : (persistedPlaneOf(run.result).errorCode ?? run.errorCode ?? undefined),
|
|
60
|
+
error: stale ? "run stalled (instance lost?)" : isTerminalRunStatus(run.status) ? memoRedactedRead("run.error", run.error ?? undefined, redactErrorMessage) : redactErrorMessage(run.error ?? undefined),
|
|
61
|
+
jobId: run.jobId ?? undefined,
|
|
62
|
+
source: run.source ?? undefined,
|
|
63
|
+
heldBy: run.instanceId ?? null,
|
|
64
|
+
cancelRequested: run.cancelRequested,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=run-read-face.js.map
|
package/dist/http/server.d.ts
CHANGED
|
@@ -26,8 +26,6 @@ import type { Metrics } from "../observability/metrics.js";
|
|
|
26
26
|
import { type RateGate } from "../observability/rate-limit.js";
|
|
27
27
|
import type { QuotaTracker } from "../observability/cost-quota.js";
|
|
28
28
|
import type { ModelUsageTracker, PromptManifestTracker } from "../budget.js";
|
|
29
|
-
import { scopedIdempotencyKey } from "./idempotency.js";
|
|
30
|
-
export { scopedIdempotencyKey };
|
|
31
29
|
import type { ConsolidationSeatLabel, ConsolidationValve } from "../boot/memory-consolidation.js";
|
|
32
30
|
import { streamApprovals, checkQuestionAnswer } from "./routes/approvals-assistant.js";
|
|
33
31
|
export { streamApprovals, checkQuestionAnswer };
|
|
@@ -354,8 +352,10 @@ export interface ServiceSeamDeps {
|
|
|
354
352
|
* 缺席 = 本部署不是 host 车道(没有工厂读这一格)⇒ 提交点 no-op。装配 = `boot/stage-10-http-server.ts`(host 车道才给)。
|
|
355
353
|
*/
|
|
356
354
|
bindSessionInheritEnv?: import("./session-inherit-env.js").SessionInheritEnvCommitCtx["bindSessionInheritEnv"];
|
|
357
|
-
/** Audit回溯: current context (+ summary) for a session, plus its owner. Enables GET /v1/sessions/:id.
|
|
358
|
-
|
|
355
|
+
/** Audit回溯: current context (+ summary) for a session, plus its owner. Enables GET /v1/sessions/:id. `opts.reader` (the
|
|
356
|
+
* verified identity reading the view) ⇒ user message rows carry the rewind handle `taskId` (`audit.ts` `buildRewindHandles`);
|
|
357
|
+
* an embedder's one-parameter implementation stays assignable (it simply never projects handles). */
|
|
358
|
+
sessionAudit?: (sessionId: string, opts?: import("../audit.js").SessionAuditReadOpts) => Promise<({
|
|
359
359
|
owner: string | null;
|
|
360
360
|
} & Record<string, unknown>) | undefined>;
|
|
361
361
|
/** E21 session delete coordinator: purge a session's history (sessionStorage) PLUS the service-owned
|
package/dist/http/server.js
CHANGED
|
@@ -11,8 +11,7 @@ import {} from "../orchestration/workflow-completion-inbox.js";
|
|
|
11
11
|
import {} from "../fleet/fleet-bus.js";
|
|
12
12
|
import {} from "../tool-approval.js";
|
|
13
13
|
import {} from "../observability/rate-limit.js";
|
|
14
|
-
import { IdempotencyCache
|
|
15
|
-
export { scopedIdempotencyKey };
|
|
14
|
+
import { IdempotencyCache } from "./idempotency.js";
|
|
16
15
|
import { streamApprovals, checkQuestionAnswer } from "./routes/approvals-assistant.js";
|
|
17
16
|
export { streamApprovals, checkQuestionAnswer };
|
|
18
17
|
import { cascadeConfig } from "./run-meta.js";
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 提交幂等的**唯一回执函数** —— 一把提交钥匙命中了一条既有 run 行之后,这次提交拿到什么。
|
|
3
|
+
*
|
|
4
|
+
* ## 律(一条)
|
|
5
|
+
* 幂等 = 资源行上的唯一键。`task_run` 上有两把:`idem_key`(`Idempotency-Key`,唯一铸法 `idemKeyOf(raw, caller, "run")`)
|
|
6
|
+
* 与 `task_id`(调用方自铸的 `body.taskId`;A2A 由 messageId 派生)。哪一把命中、在哪条提交腿上命中、是请求进门时
|
|
7
|
+
* 读行命中还是建行撞唯一键时观察到,一律经**本函数**作答 —— 提交面不再有第二个判官(进程内缓存只做在飞合流,
|
|
8
|
+
* 见 `http/idempotency.ts`)。
|
|
9
|
+
*
|
|
10
|
+
* ## 判定(两条,别的一律不放)
|
|
11
|
+
* · 这次提交自带 taskId 而命中的行**不是**那一条(`Idempotency-Key` 指向别的行)⇒ 冲突;
|
|
12
|
+
* · 调用方读不到这一行(属主门 = `runOwnerReadable`,「这个调用方读得到这一行 run 吗」的唯一谓词)⇒ 冲突。
|
|
13
|
+
* 冲突 ⇒ `409 conflict.run_exists`,体里**不回显**那条行的任何标识(不给跨租户预言机)。
|
|
14
|
+
*
|
|
15
|
+
* ## 回执两形(按提交腿)
|
|
16
|
+
* · `durable`(`POST /v1/runs`、A2A `message/send`)—— `202 { taskId, sessionId, status }`(这条腿首答的既有形);
|
|
17
|
+
* · `sync`(`POST /v1/tasks`;`/v1/tasks/stream` 把同一份体放进 `done.result`)—— `200 { …run 行读面, replay: true }`
|
|
18
|
+
* (读面 = `GET /v1/runs/:id` 的 200 体,同一只投影口 `readRunFace`)。同步腿的首答是引擎结果,而重放时手里只有行;
|
|
19
|
+
* `replay: true` 是这一形的判别位。
|
|
20
|
+
*/
|
|
21
|
+
import type { RunRecord } from "../plugins/store-contracts.js";
|
|
22
|
+
import type { RunStore } from "../plugins/store-backend.js";
|
|
23
|
+
import type { ServiceConfig } from "../config-types.js";
|
|
24
|
+
import type { SubmitOutcome } from "./idempotency.js";
|
|
25
|
+
/** 提交腿的回执形(闭集;穷举 `switch`,加一形不写这里 = 编译红)。 */
|
|
26
|
+
export type SubmitReplayFace = "durable" | "sync";
|
|
27
|
+
/** 发起这次提交的一方。 */
|
|
28
|
+
export interface ReplayCaller {
|
|
29
|
+
/** 已验身份(`gatedPrincipal`)。 */
|
|
30
|
+
principal: string | undefined;
|
|
31
|
+
/** 这次提交自带的 taskId(`body.taskId`;A2A 派生值)。缺席 = 只凭 `Idempotency-Key` 命中。 */
|
|
32
|
+
clientTaskId?: string;
|
|
33
|
+
}
|
|
34
|
+
/** 一份提交回执是不是 {@link replayOfExistingRun} 铸的**冲突**体(判别位 = 机器码;铸点就在下面,码与它同文件)—— 把回执渲成别的协议形的
|
|
35
|
+
* 调用方(A2A)按它分句,不另抄码。 */
|
|
36
|
+
export declare function isRunExistsConflictBody(body: unknown): boolean;
|
|
37
|
+
export declare function replayOfExistingRun(row: RunRecord, caller: ReplayCaller, face: SubmitReplayFace, deps: {
|
|
38
|
+
config: ServiceConfig;
|
|
39
|
+
}, runStore: RunStore): Promise<SubmitOutcome>;
|
|
40
|
+
//# sourceMappingURL=submit-replay.d.ts.map
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { runOwnerReadable } from "./principal-gate.js";
|
|
2
|
+
import { readRunFace } from "./run-read-face.js";
|
|
3
|
+
export function isRunExistsConflictBody(body) {
|
|
4
|
+
return typeof body === "object" && body !== null && body.errorCode === "conflict.run_exists";
|
|
5
|
+
}
|
|
6
|
+
export async function replayOfExistingRun(row, caller, face, deps, runStore) {
|
|
7
|
+
const sameRun = caller.clientTaskId === undefined || caller.clientTaskId === row.taskId;
|
|
8
|
+
if (!sameRun || !runOwnerReadable(caller.principal, row.owner, deps.config)) {
|
|
9
|
+
return { status: 409, body: { error: "a run already exists for this submission's taskId or Idempotency-Key", errorCode: "conflict.run_exists" } };
|
|
10
|
+
}
|
|
11
|
+
switch (face) {
|
|
12
|
+
case "durable":
|
|
13
|
+
return { status: 202, body: { taskId: row.taskId, sessionId: row.sessionId, status: row.status } };
|
|
14
|
+
case "sync":
|
|
15
|
+
return { status: 200, body: { ...(await readRunFace(deps, runStore, row)), replay: true } };
|
|
16
|
+
default: {
|
|
17
|
+
const unknown = face;
|
|
18
|
+
throw new Error(`replayOfExistingRun: unknown submit face ${String(unknown)}`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=submit-replay.js.map
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { LeaderResult } from "./leader.js";
|
|
2
2
|
import { type LeaderRunRecord, type LeaderRunStatus, type NewLeaderRun, type LeaderRunTerminal } from "../plugins/leader-run-store-sql.js";
|
|
3
|
+
import { type IdemCaller } from "../http/idempotency.js";
|
|
3
4
|
/**
|
|
4
5
|
* endpoint 消费的 store 面(结构型,不是 nominal union):真实现是 `SqlLeaderRunStore` 的两个方言孪生,
|
|
5
6
|
* 测试用进程内假件。窄到只有 endpoint 真调的三个方法 —— 端点不该看见店的其余口。
|
|
@@ -10,6 +11,8 @@ export interface LeaderRunStore {
|
|
|
10
11
|
run: LeaderRunRecord;
|
|
11
12
|
}>;
|
|
12
13
|
getRun(id: string): Promise<LeaderRunRecord | undefined>;
|
|
14
|
+
/** 按提交幂等键(已 scope 的定宽 hash)读行 —— `GET /v1/leader?idempotencyKey=` 的读口。 */
|
|
15
|
+
getByIdem(idemKey: string): Promise<LeaderRunRecord | undefined>;
|
|
13
16
|
finishRun(id: string, patch: LeaderRunTerminal): Promise<boolean>;
|
|
14
17
|
}
|
|
15
18
|
export interface LeaderRequestBody {
|
|
@@ -36,11 +39,13 @@ export interface LeaderRun {
|
|
|
36
39
|
}
|
|
37
40
|
export interface LeaderEndpoint {
|
|
38
41
|
/** Returns a {status,body} to send, or null if the (method,url) is not a leader route. Kicks the background
|
|
39
|
-
* run for POST. `body` is the already-parsed request body (server reads it; undefined for GET). `
|
|
40
|
-
* (BL-4) is
|
|
41
|
-
* `idempotencyKey` is the caller's RAW `Idempotency-Key`
|
|
42
|
-
*
|
|
43
|
-
|
|
42
|
+
* run for POST. `body` is the already-parsed request body (server reads it; undefined for GET). `caller` is the
|
|
43
|
+
* request's identity (`idemCallerOf`): its verified principal (BL-4) is recorded as the run owner on POST and owner-checked
|
|
44
|
+
* on GET; the whole identity scopes the idempotency key. `idempotencyKey` is the caller's RAW `Idempotency-Key` (POST: the
|
|
45
|
+
* header; `GET /v1/leader`: the `idempotencyKey` query — the read port "which leader run did this key create"; scoped by the
|
|
46
|
+
* caller's identity bit `principal ?? source` before it reaches the store — the one key minter `idemKeyOf(raw, caller, "leader")`).
|
|
47
|
+
* ASYNC because the durable arm reads/writes SQL. */
|
|
48
|
+
handle(method: string, url: string, body: unknown, caller?: IdemCaller, idempotencyKey?: string | null): Promise<{
|
|
44
49
|
status: number;
|
|
45
50
|
body: object;
|
|
46
51
|
} | null>;
|
package/dist/leader/endpoint.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { uuidv7 } from "@sema-agent/core";
|
|
2
2
|
import { redactErrorMessage, redactHead, redactLeaderResult, redactLeaderResultWithFlag } from "../observability/run-terminal.js";
|
|
3
3
|
import { withPrincipal } from "../observability/principal-context.js";
|
|
4
|
-
import {
|
|
4
|
+
import {} from "../plugins/leader-run-store-sql.js";
|
|
5
|
+
import { idemKeyOf } from "../http/idempotency.js";
|
|
5
6
|
function statusForLeaderResult(result) {
|
|
6
7
|
if (result.conflict)
|
|
7
8
|
return "needs_human";
|
|
@@ -126,7 +127,9 @@ export function createLeaderEndpoint(runLeader, opts = {}) {
|
|
|
126
127
|
return { status: 200, body: { id: run.id, status: run.status, ...redactedView(run) } };
|
|
127
128
|
}
|
|
128
129
|
const NOT_FOUND = { status: 404, body: { error: "leader run not found", errorCode: "not_found.leader_run" } };
|
|
129
|
-
async function handle(method, url, body,
|
|
130
|
+
async function handle(method, url, body, caller, idempotencyKey) {
|
|
131
|
+
const requester = caller?.principal ?? null;
|
|
132
|
+
const idemCaller = caller ?? { principal: null, source: null };
|
|
130
133
|
if (method === "POST" && url === "/v1/leader") {
|
|
131
134
|
const b = body;
|
|
132
135
|
if (!b || typeof b.objective !== "string" || b.objective.trim() === "") {
|
|
@@ -138,13 +141,13 @@ export function createLeaderEndpoint(runLeader, opts = {}) {
|
|
|
138
141
|
return { status: 400, body: { error: `missing required field(s): ${missing.join(", ")}`, errorCode: "request.body_shape" } };
|
|
139
142
|
}
|
|
140
143
|
}
|
|
141
|
-
const owner = requester
|
|
144
|
+
const owner = requester;
|
|
142
145
|
let scopedIdem = null;
|
|
143
146
|
if (idempotencyKey != null) {
|
|
144
147
|
const bad = idemShapeError(idempotencyKey);
|
|
145
148
|
if (bad)
|
|
146
149
|
return { status: 400, body: { error: bad, errorCode: "request.body_shape" } };
|
|
147
|
-
scopedIdem =
|
|
150
|
+
scopedIdem = idemKeyOf(idempotencyKey, idemCaller, "leader");
|
|
148
151
|
}
|
|
149
152
|
reap();
|
|
150
153
|
if (opts.store) {
|
|
@@ -172,10 +175,29 @@ export function createLeaderEndpoint(runLeader, opts = {}) {
|
|
|
172
175
|
drive(id, b, owner);
|
|
173
176
|
return { status: 202, body: { leaderRunId: id, status: "running" } };
|
|
174
177
|
}
|
|
178
|
+
const ownerOk = (owner) => owner == null || owner === requester;
|
|
179
|
+
if (method === "GET" && url === "/v1/leader") {
|
|
180
|
+
if (idempotencyKey == null)
|
|
181
|
+
return { status: 400, body: { error: "idempotencyKey is required: pass the raw Idempotency-Key the submission was sent with", errorCode: "request.query_invalid" } };
|
|
182
|
+
const bad = idemShapeError(idempotencyKey);
|
|
183
|
+
if (bad)
|
|
184
|
+
return { status: 400, body: { error: bad, errorCode: "request.query_invalid" } };
|
|
185
|
+
const key = idemKeyOf(idempotencyKey, idemCaller, "leader");
|
|
186
|
+
if (opts.store) {
|
|
187
|
+
const run = await opts.store.getByIdem(key);
|
|
188
|
+
if (!run || !ownerOk(run.owner))
|
|
189
|
+
return NOT_FOUND;
|
|
190
|
+
return getBody(run);
|
|
191
|
+
}
|
|
192
|
+
const prior = memIdem.get(key);
|
|
193
|
+
const run = prior !== undefined ? runs.get(prior) : undefined;
|
|
194
|
+
if (!run || !ownerOk(run.owner))
|
|
195
|
+
return NOT_FOUND;
|
|
196
|
+
return getBody(run);
|
|
197
|
+
}
|
|
175
198
|
const m = method === "GET" ? LEADER_ID_RE.exec(url) : null;
|
|
176
199
|
if (m) {
|
|
177
200
|
const id = m[1];
|
|
178
|
-
const ownerOk = (owner) => owner == null || owner === (requester ?? null);
|
|
179
201
|
if (opts.store) {
|
|
180
202
|
const run = await opts.store.getRun(id);
|
|
181
203
|
if (!run || !ownerOk(run.owner))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Per-principal CUMULATIVE cost quota ([ref]/27 follow-up). The per-task `maxCostUsd` gate caps a
|
|
3
|
-
* SINGLE task; this caps a principal's spend ACROSS tasks within a
|
|
3
|
+
* SINGLE task; this caps a principal's spend ACROSS tasks within a clock-aligned fixed window. Spend is the
|
|
4
4
|
* authoritative integer `costMicroUsd` accumulated by the tracer (so it includes council/team
|
|
5
5
|
* sub-tasks, via the principal ALS context). A new task is REFUSED once the window's accumulated cost
|
|
6
6
|
* has crossed the ceiling — the in-flight task that crossed it still finishes (the per-task gate is the
|
|
@@ -9,14 +9,54 @@
|
|
|
9
9
|
* In-memory, per-replica — same trade as the request rate limiter. A multi-replica deployment under-
|
|
10
10
|
* counts across replicas (each tracks its own slice); a shared (TiDB) quota is the hardening step,
|
|
11
11
|
* mirroring TiDBBreakerState. Documented, not silently wrong.
|
|
12
|
+
*
|
|
13
|
+
* 🔴 **一只锚**:配额窗 = 时钟对齐桶,公式只在 {@link alignedWindow} 一处。内存腿({@link CostQuota})与
|
|
14
|
+
* SQL 腿(`plugins/write-behind-counter.ts`,cost quota 与 rate limiter 共用)同 import 它;决策体由
|
|
15
|
+
* {@link decideQuota} 一处铸(关断哨兵、退避秒数都在里面)。修前内存腿从该键**首笔花费**开锚、SQL 腿按
|
|
16
|
+
* 时钟对齐 —— 同一个 `COST_QUOTA_WINDOW_SEC` 两种窗形,同一个 principal 的窗末取决于部署拓扑。
|
|
17
|
+
*/
|
|
18
|
+
/** 一只对齐窗的读数:同一次时钟读出的桶号与桶末(决策与退避都从它派生 ⇒ 同一 `now`,零二次读钟)。 */
|
|
19
|
+
export interface AlignedWindow {
|
|
20
|
+
/** 读钟时刻(ms)。 */
|
|
21
|
+
readonly now: number;
|
|
22
|
+
/** 桶号 —— 记账键的一部分(内存腿的条目键、SQL 腿的 `window_bucket` 列)。 */
|
|
23
|
+
readonly bucket: number;
|
|
24
|
+
/** 当代桶末(ms,不含)= 下一桶起点;wire 上投影成 `windowResetsAt`。 */
|
|
25
|
+
readonly windowEndsAtMs: number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* 窗长(秒)的合法域 = 锚公式的定义域。env 腿(拒启)与 center 腿(整批拒)同调 {@link costQuotaWindowSecInRange},一个判据:
|
|
29
|
+
* · 下界 1 s —— 退避秒数与 `Retry-After` 以整秒计;极小正数会让 `now / windowMs` 溢成 Infinity(桶号与窗末无意义)。
|
|
30
|
+
* · 上界 1e10 s(≈ 317 年)—— 当代桶末必须是 `Date` 可表示的时刻(`windowResetsAt` 的 ISO 投影;Date 上界 8.64e15 ms)。
|
|
31
|
+
* 桶末 ≤ now + 窗长 ⇒ 1e10 s 下 ≤ now + 1e13 ms,离上界三个数量级;也盖得住拿超长窗当「终身额度」的用法。
|
|
12
32
|
*/
|
|
33
|
+
export declare const COST_QUOTA_WINDOW_SEC_RANGE: {
|
|
34
|
+
readonly min: 1;
|
|
35
|
+
readonly max: 10000000000;
|
|
36
|
+
};
|
|
37
|
+
export declare function costQuotaWindowSecInRange(sec: number): boolean;
|
|
38
|
+
/** 🔴 全仓**唯一**的桶公式(源码钉:`test/cost-quota.test.ts` 的「一只锚」格)。 */
|
|
39
|
+
export declare function alignedWindow(now: number, windowMs: number): AlignedWindow;
|
|
40
|
+
/** 到桶末的退避秒数(向上取整、不为负)—— 内存腿、SQL 腿、rate limiter 的唯一退避公式。 */
|
|
41
|
+
export declare function retryAfterSecUntil(w: AlignedWindow): number;
|
|
13
42
|
export interface QuotaDecision {
|
|
14
43
|
allowed: boolean;
|
|
15
44
|
usedMicroUsd: number;
|
|
16
45
|
limitMicroUsd: number;
|
|
17
|
-
/** Seconds until the current window rolls over (
|
|
46
|
+
/** Seconds until the current window rolls over (0 when allowed). Derived from {@link windowEndsAtMs} at the same `now`. */
|
|
18
47
|
retryAfterSec: number;
|
|
48
|
+
/** End of the current clock-aligned window (ms epoch, exclusive). ALWAYS a number — the OFF sentinel included
|
|
49
|
+
* (the window length is still a configured fact; it reports the current bucket's end). */
|
|
50
|
+
windowEndsAtMs: number;
|
|
19
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* 决策体的**唯一**铸点(三实现 —— 内存 / TiDB / PG —— 同调,各自零公式、零关断字面量)。
|
|
54
|
+
* `limitMicroUsd <= 0` = 关断哨兵:恒放行、如实报 `limitMicroUsd: 0`、**不调** `readUsed`(内存腿零窗表访问、
|
|
55
|
+
* SQL 腿零计数读 —— off 态零 SQL 的承诺不受影响)。`readUsed` 收的是本次读钟的桶号,读到的花费与桶末同源。
|
|
56
|
+
*/
|
|
57
|
+
export declare function decideQuota(limitMicroUsd: number, w: AlignedWindow, readUsed: (bucket: number) => number): QuotaDecision;
|
|
58
|
+
/** `windowEndsAtMs` 的 wire 投影(ISO 8601)—— `GET /v1/usage` 体与 429 `limit.cost_quota_exceeded` 体的同一只读法。 */
|
|
59
|
+
export declare function quotaWindowResetsAt(d: Pick<QuotaDecision, "windowEndsAtMs">): string;
|
|
20
60
|
/** The hot-path surface shared by the in-memory {@link CostQuota} (single replica) and the cross-replica
|
|
21
61
|
* {@link import("../plugins/tidb-cost-quota.js").TiDBCostQuota}. `key` is opaque (principal today;
|
|
22
62
|
* principal/team/feature later — [ref] N3), so both stores are key-based. */
|
|
@@ -34,28 +74,32 @@ export interface QuotaTracker {
|
|
|
34
74
|
}
|
|
35
75
|
export declare class CostQuota implements QuotaTracker {
|
|
36
76
|
private readonly now;
|
|
77
|
+
/** 记账条目 = 键 × 桶号(键里带桶号,值里也带 —— `reap` 按桶号比,不解析键)。读写都只碰当代桶;一个键跨桶时
|
|
78
|
+
* 旧桶条目留到 `reap`(与 SQL 腿「旧桶行留到 DELETE」同形)。 */
|
|
37
79
|
private windows;
|
|
38
80
|
/** 🔴 不再 `readonly`:限额/窗长是比较参数,换代只动它们,窗内已累计的花费一个不动([ref] §5)。 */
|
|
39
81
|
private limitMicroUsd;
|
|
40
82
|
private windowMs;
|
|
41
|
-
/** @param limitMicroUsd ceiling per window (integer micro-USD; `<=0` = OFF sentinel). @param windowMs
|
|
83
|
+
/** @param limitMicroUsd ceiling per window (integer micro-USD; `<=0` = OFF sentinel). @param windowMs clock-aligned window length. */
|
|
42
84
|
constructor(limitMicroUsd: number, windowMs: number, now?: () => number);
|
|
43
85
|
/**
|
|
44
86
|
* 值换代。两件事**语义不同**,别混:
|
|
45
87
|
* · **限额**换代 = 纯比较参数 ⇒ 窗内已累计的花费一个不动([ref] §5 定案:调小当场更严)。
|
|
46
88
|
* · **窗长**换代 = 换**记账周期** ⇒ 当代累计重开。
|
|
47
89
|
*
|
|
48
|
-
* 🔴 后者是 codex R1-F4(真 finding,红先):SQL 腿的窗长是**桶键的一部分**(
|
|
90
|
+
* 🔴 后者是 codex R1-F4(真 finding,红先):SQL 腿的窗长是**桶键的一部分**(桶号由 {@link alignedWindow} 算),
|
|
49
91
|
* 换窗长后当代桶天然重算 ⇒ 已累计不跟着搬家;内存腿修前却保留原窗口累计。于是**同一条发布**在
|
|
50
92
|
* 内存部署上继续拒绝超额 principal、在 PG/TiDB 部署上重新放行一整个额度 —— 后端相关的成本 fail-open。
|
|
51
|
-
* 统一取 SQL 腿那一侧(唯一能在两条腿上都实现的语义:SQL 侧无法把旧桶累计搬进新桶)
|
|
93
|
+
* 统一取 SQL 腿那一侧(唯一能在两条腿上都实现的语义:SQL 侧无法把旧桶累计搬进新桶)。内存腿按桶号记账后
|
|
94
|
+
* 仍要显式清:两个窗长在同一时刻可能算出同一个桶号(SQL 腿的这一形是既有差异,见 `CONFIG-HOT-RELOAD.md`)。
|
|
52
95
|
*/
|
|
53
96
|
setLimits(next: QuotaLimits): void;
|
|
54
97
|
/** Is this principal under quota right now? (Reads only — does not consume.) */
|
|
55
98
|
check(key: string): QuotaDecision;
|
|
56
|
-
/** Add spend for a principal
|
|
99
|
+
/** Add spend for a principal into the CURRENT aligned bucket. */
|
|
57
100
|
add(key: string, micro: number): void;
|
|
58
|
-
/** Drop
|
|
101
|
+
/** Drop entries of buckets older than the current one (called from the background reaper) to bound memory.
|
|
102
|
+
* Never the current bucket; a bucket AHEAD of the clock (a backwards clock step) survives until the clock passes it. */
|
|
59
103
|
reap(): number;
|
|
60
104
|
}
|
|
61
105
|
//# sourceMappingURL=cost-quota.d.ts.map
|