@flowdesk/opencode-plugin 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-task-runner.d.ts +27 -0
- package/dist/agent-task-runner.d.ts.map +1 -0
- package/dist/agent-task-runner.js +390 -0
- package/dist/agent-task-runner.js.map +1 -0
- package/dist/bootstrap-installer.d.ts +3 -0
- package/dist/bootstrap-installer.d.ts.map +1 -1
- package/dist/bootstrap-installer.js +153 -7
- package/dist/bootstrap-installer.js.map +1 -1
- package/dist/command-handlers.d.ts +3 -0
- package/dist/command-handlers.d.ts.map +1 -1
- package/dist/command-handlers.js +38 -4
- package/dist/command-handlers.js.map +1 -1
- package/dist/controlled-write-tool.d.ts +49 -0
- package/dist/controlled-write-tool.d.ts.map +1 -0
- package/dist/controlled-write-tool.js +296 -0
- package/dist/controlled-write-tool.js.map +1 -0
- package/dist/local-adapter.d.ts.map +1 -1
- package/dist/local-adapter.js +19 -0
- package/dist/local-adapter.js.map +1 -1
- package/dist/managed-dispatch-adapter.d.ts +3 -0
- package/dist/managed-dispatch-adapter.d.ts.map +1 -1
- package/dist/managed-dispatch-adapter.js +179 -27
- package/dist/managed-dispatch-adapter.js.map +1 -1
- package/dist/provider-usage-live-tool.d.ts +17 -0
- package/dist/provider-usage-live-tool.d.ts.map +1 -1
- package/dist/provider-usage-live-tool.js +317 -5
- package/dist/provider-usage-live-tool.js.map +1 -1
- package/dist/quick-reviewer-run.d.ts +16 -2
- package/dist/quick-reviewer-run.d.ts.map +1 -1
- package/dist/quick-reviewer-run.js +228 -72
- package/dist/quick-reviewer-run.js.map +1 -1
- package/dist/runtime-reviewer-execution-bridge.d.ts +21 -0
- package/dist/runtime-reviewer-execution-bridge.d.ts.map +1 -1
- package/dist/runtime-reviewer-execution-bridge.js +284 -1
- package/dist/runtime-reviewer-execution-bridge.js.map +1 -1
- package/dist/server.d.ts +72 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +816 -77
- package/dist/server.js.map +1 -1
- package/dist/shared/with-timeout.d.ts +12 -0
- package/dist/shared/with-timeout.d.ts.map +1 -0
- package/dist/shared/with-timeout.js +31 -0
- package/dist/shared/with-timeout.js.map +1 -0
- package/dist/stall-recovery.d.ts +214 -0
- package/dist/stall-recovery.d.ts.map +1 -0
- package/dist/stall-recovery.js +1257 -0
- package/dist/stall-recovery.js.map +1 -0
- package/dist/status-live-tool.d.ts +28 -0
- package/dist/status-live-tool.d.ts.map +1 -1
- package/dist/status-live-tool.js +306 -1
- package/dist/status-live-tool.js.map +1 -1
- package/dist/tui-usage-snapshot.d.ts +30 -0
- package/dist/tui-usage-snapshot.d.ts.map +1 -0
- package/dist/tui-usage-snapshot.js +216 -0
- package/dist/tui-usage-snapshot.js.map +1 -0
- package/dist/tui.d.ts +7 -0
- package/dist/tui.d.ts.map +1 -0
- package/dist/tui.js +103 -0
- package/dist/tui.js.map +1 -0
- package/dist/workflow-dispatch-plan-tool.d.ts +47 -0
- package/dist/workflow-dispatch-plan-tool.d.ts.map +1 -0
- package/dist/workflow-dispatch-plan-tool.js +251 -0
- package/dist/workflow-dispatch-plan-tool.js.map +1 -0
- package/dist/workflow-dispatch-tool.d.ts +56 -0
- package/dist/workflow-dispatch-tool.d.ts.map +1 -0
- package/dist/workflow-dispatch-tool.js +276 -0
- package/dist/workflow-dispatch-tool.js.map +1 -0
- package/dist/workflow-scheduler.d.ts +19 -0
- package/dist/workflow-scheduler.d.ts.map +1 -0
- package/dist/workflow-scheduler.js +43 -0
- package/dist/workflow-scheduler.js.map +1 -0
- package/package.json +10 -2
|
@@ -0,0 +1,1257 @@
|
|
|
1
|
+
import { applyFlowDeskSessionEvidenceWriteIntentsV1, prepareFlowDeskSessionEvidenceWriteIntentV1, reloadFlowDeskSessionEvidenceV1, projectFlowDeskLaneStallV1, } from "@flowdesk/core";
|
|
2
|
+
import { createHmac, createHash, timingSafeEqual } from "node:crypto";
|
|
3
|
+
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { launchFlowDeskInjectedSdkRuntimeLaneFromPlanV1, } from "./managed-dispatch-adapter.js";
|
|
6
|
+
import { FLOWDESK_TIMEOUT_DEFAULTS, FlowDeskTimeoutError, withTimeout, } from "./shared/with-timeout.js";
|
|
7
|
+
import { executeFlowDeskAgentTaskV1 } from "./agent-task-runner.js";
|
|
8
|
+
export async function checkSdkSessionApiHealthV1(client, sessionId, timeouts = {}) {
|
|
9
|
+
if (typeof client.session.messages !== "function") {
|
|
10
|
+
return { status: "unknown", reason: "sdk_messages_not_available" };
|
|
11
|
+
}
|
|
12
|
+
try {
|
|
13
|
+
await withTimeout(client.session.messages({ path: { id: sessionId } }), timeouts.sessionReadMs ?? FLOWDESK_TIMEOUT_DEFAULTS.sessionReadMs, "session.messages");
|
|
14
|
+
return { status: "api_responsive" };
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
if (error instanceof FlowDeskTimeoutError) {
|
|
18
|
+
return {
|
|
19
|
+
status: "api_timeout",
|
|
20
|
+
reason: "messages_api_did_not_respond_within_threshold",
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
return { status: "unknown", reason: "messages_api_error" };
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const FLOWDESK_ABORT_WORKFLOW_PREFIXES = [
|
|
27
|
+
"workflow-quick-reviewer-",
|
|
28
|
+
"workflow-quick-fallback-",
|
|
29
|
+
"workflow-stall-recovery-",
|
|
30
|
+
"workflow-provider-usage-",
|
|
31
|
+
];
|
|
32
|
+
const TERMINAL_LANE_STATES = new Set([
|
|
33
|
+
"complete",
|
|
34
|
+
"incomplete",
|
|
35
|
+
"no_output",
|
|
36
|
+
"missing_verdict",
|
|
37
|
+
"tool_calls_only_no_verdict",
|
|
38
|
+
"aborted",
|
|
39
|
+
"invocation_failed",
|
|
40
|
+
"timeout",
|
|
41
|
+
"late_output",
|
|
42
|
+
"orphaned",
|
|
43
|
+
]);
|
|
44
|
+
const ABORT_ELIGIBLE_LANE_STATES = new Set(["created", "running"]);
|
|
45
|
+
function isRecord(value) {
|
|
46
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
47
|
+
}
|
|
48
|
+
function isLaneLifecycleRecord(value) {
|
|
49
|
+
return isRecord(value) && value.schema_version === "flowdesk.lane_lifecycle_record.v1";
|
|
50
|
+
}
|
|
51
|
+
function workflowPrefixAllowed(workflowId) {
|
|
52
|
+
return FLOWDESK_ABORT_WORKFLOW_PREFIXES.some((prefix) => workflowId.startsWith(prefix));
|
|
53
|
+
}
|
|
54
|
+
function latestLifecycle(records) {
|
|
55
|
+
return [...records].sort((a, b) => new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime())[0];
|
|
56
|
+
}
|
|
57
|
+
function canonicalJson(value) {
|
|
58
|
+
if (value === null)
|
|
59
|
+
return "null";
|
|
60
|
+
if (typeof value === "string" || typeof value === "boolean")
|
|
61
|
+
return JSON.stringify(value);
|
|
62
|
+
if (typeof value === "number") {
|
|
63
|
+
if (!Number.isFinite(value))
|
|
64
|
+
throw new Error("non-finite canonical JSON number");
|
|
65
|
+
return JSON.stringify(value);
|
|
66
|
+
}
|
|
67
|
+
if (Array.isArray(value))
|
|
68
|
+
return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
|
|
69
|
+
if (isRecord(value)) {
|
|
70
|
+
return `{${Object.keys(value)
|
|
71
|
+
.sort()
|
|
72
|
+
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
|
|
73
|
+
.join(",")}}`;
|
|
74
|
+
}
|
|
75
|
+
throw new Error("unsupported canonical JSON value");
|
|
76
|
+
}
|
|
77
|
+
function sha256Hex(text) {
|
|
78
|
+
return createHash("sha256").update(text.replaceAll("\r\n", "\n"), "utf8").digest("hex");
|
|
79
|
+
}
|
|
80
|
+
function safeToken(value) {
|
|
81
|
+
const token = value.replaceAll(/[^A-Za-z0-9_.:-]/g, "-").slice(0, 80);
|
|
82
|
+
return token.length > 0 ? token : "unknown";
|
|
83
|
+
}
|
|
84
|
+
function timestampToken(date) {
|
|
85
|
+
return date.toISOString().replace(/[^0-9A-Za-z]/g, "");
|
|
86
|
+
}
|
|
87
|
+
function isGuardSignOff(value) {
|
|
88
|
+
return (isRecord(value) &&
|
|
89
|
+
value.schema_version === "flowdesk.guard_sign_off.v1" &&
|
|
90
|
+
typeof value.sign_off_id === "string" &&
|
|
91
|
+
typeof value.created_at === "string" &&
|
|
92
|
+
typeof value.target_markdown_sha256 === "string" &&
|
|
93
|
+
typeof value.p6_safe === "boolean" &&
|
|
94
|
+
typeof value.nonce === "string" &&
|
|
95
|
+
typeof value.hmac_sha256 === "string" &&
|
|
96
|
+
(value.expires_at === undefined || typeof value.expires_at === "string") &&
|
|
97
|
+
value.dispatch_authority_enabled === false);
|
|
98
|
+
}
|
|
99
|
+
export function verifyGuardSignOffHmacV1(input) {
|
|
100
|
+
if (!isGuardSignOff(input.signOff))
|
|
101
|
+
return { ok: false, reason: "guard_sign_off_schema_invalid" };
|
|
102
|
+
if (typeof input.hmacKey !== "string" || input.hmacKey.length < 16)
|
|
103
|
+
return { ok: false, reason: "guard_hmac_key_missing" };
|
|
104
|
+
if (input.signOff.p6_safe !== true)
|
|
105
|
+
return { ok: false, reason: "guard_sign_off_not_p6_safe" };
|
|
106
|
+
if (input.signOff.expires_at !== undefined) {
|
|
107
|
+
const expiresAtMs = Date.parse(input.signOff.expires_at);
|
|
108
|
+
if (!Number.isFinite(expiresAtMs))
|
|
109
|
+
return { ok: false, reason: "guard_sign_off_expiry_invalid" };
|
|
110
|
+
if ((input.now ?? new Date()).getTime() >= expiresAtMs)
|
|
111
|
+
return { ok: false, reason: "guard_sign_off_expired" };
|
|
112
|
+
}
|
|
113
|
+
if (input.signOff.target_markdown_sha256 !== sha256Hex(input.markdownText))
|
|
114
|
+
return { ok: false, reason: "guard_sign_off_markdown_digest_mismatch" };
|
|
115
|
+
const { hmac_sha256: _hmac, ...unsigned } = input.signOff;
|
|
116
|
+
const expected = createHmac("sha256", input.hmacKey)
|
|
117
|
+
.update(canonicalJson(unsigned), "utf8")
|
|
118
|
+
.digest("hex");
|
|
119
|
+
const actualBuffer = Buffer.from(input.signOff.hmac_sha256, "hex");
|
|
120
|
+
const expectedBuffer = Buffer.from(expected, "hex");
|
|
121
|
+
if (actualBuffer.length !== expectedBuffer.length || !timingSafeEqual(actualBuffer, expectedBuffer))
|
|
122
|
+
return { ok: false, reason: "guard_sign_off_hmac_mismatch" };
|
|
123
|
+
return { ok: true, signOff: input.signOff };
|
|
124
|
+
}
|
|
125
|
+
export function computeGuardSignOffHmacV1(input) {
|
|
126
|
+
return createHmac("sha256", input.hmacKey)
|
|
127
|
+
.update(canonicalJson(input.unsignedSignOff), "utf8")
|
|
128
|
+
.digest("hex");
|
|
129
|
+
}
|
|
130
|
+
function loadGuardSignOffFromRoot(rootDir, signOffPath) {
|
|
131
|
+
const sidecarPath = signOffPath ?? join(rootDir, "docs/adr/0002-sdk-surface-verification.guard_sign_off.json");
|
|
132
|
+
if (!existsSync(sidecarPath))
|
|
133
|
+
return undefined;
|
|
134
|
+
const signOff = JSON.parse(readFileSync(sidecarPath, "utf8"));
|
|
135
|
+
const markdownPath = sidecarPath.replace(/\.guard_sign_off\.json$/, ".md");
|
|
136
|
+
if (!existsSync(markdownPath))
|
|
137
|
+
return undefined;
|
|
138
|
+
return { signOff, markdownText: readFileSync(markdownPath, "utf8") };
|
|
139
|
+
}
|
|
140
|
+
export function isAutoAbortEnabledV1(input) {
|
|
141
|
+
if (input.config.autoAbortOnStall !== true)
|
|
142
|
+
return { enabled: false, reason: "auto_abort_not_configured" };
|
|
143
|
+
const keyFromConfig = input.config.guardHmacKey;
|
|
144
|
+
const keyFromEnv = input.env?.FLOWDESK_GUARD_HMAC_KEY ?? process.env.FLOWDESK_GUARD_HMAC_KEY;
|
|
145
|
+
if (input.config.productionMode === true && keyFromConfig === undefined && keyFromEnv !== undefined)
|
|
146
|
+
return { enabled: false, reason: "env_guard_hmac_key_rejected_in_production" };
|
|
147
|
+
const loaded = input.loadedSignOff ?? loadGuardSignOffFromRoot(input.rootDir, input.config.guardSignOffPath);
|
|
148
|
+
if (loaded === undefined)
|
|
149
|
+
return { enabled: false, reason: "guard_sign_off_missing" };
|
|
150
|
+
const verified = verifyGuardSignOffHmacV1({
|
|
151
|
+
signOff: loaded.signOff,
|
|
152
|
+
markdownText: loaded.markdownText,
|
|
153
|
+
hmacKey: keyFromConfig ?? keyFromEnv,
|
|
154
|
+
now: input.now,
|
|
155
|
+
});
|
|
156
|
+
if (!verified.ok)
|
|
157
|
+
return { enabled: false, reason: verified.reason };
|
|
158
|
+
return { enabled: true, sign_off_id: verified.signOff.sign_off_id };
|
|
159
|
+
}
|
|
160
|
+
function isPendingAbortWarning(value) {
|
|
161
|
+
return isRecord(value) && value.schema_version === "flowdesk.pending_abort_warning.v1";
|
|
162
|
+
}
|
|
163
|
+
function isPendingAbortCancel(value) {
|
|
164
|
+
return isRecord(value) && value.schema_version === "flowdesk.pending_abort_cancel.v1";
|
|
165
|
+
}
|
|
166
|
+
function latestPendingAbortWarning(records) {
|
|
167
|
+
return [...records].sort((a, b) => Date.parse(b.warning_issued_at) - Date.parse(a.warning_issued_at))[0];
|
|
168
|
+
}
|
|
169
|
+
function writeEvidence(input) {
|
|
170
|
+
const prepared = prepareFlowDeskSessionEvidenceWriteIntentV1({
|
|
171
|
+
workflowId: input.workflowId,
|
|
172
|
+
evidenceId: input.evidenceId,
|
|
173
|
+
record: input.record,
|
|
174
|
+
});
|
|
175
|
+
if (!prepared.ok || prepared.writeIntent === undefined)
|
|
176
|
+
return false;
|
|
177
|
+
return applyFlowDeskSessionEvidenceWriteIntentsV1(input.rootDir, [prepared.writeIntent]).ok;
|
|
178
|
+
}
|
|
179
|
+
function warningRecord(input) {
|
|
180
|
+
return {
|
|
181
|
+
schema_version: "flowdesk.pending_abort_warning.v1",
|
|
182
|
+
warning_id: input.warningId,
|
|
183
|
+
workflow_id: input.workflowId,
|
|
184
|
+
lane_id: input.laneId,
|
|
185
|
+
warning_issued_at: input.issuedAt.toISOString(),
|
|
186
|
+
expires_at: input.expiresAt.toISOString(),
|
|
187
|
+
cancel_command: `/flowdesk-abort ${input.laneId} cancel`,
|
|
188
|
+
status: input.status,
|
|
189
|
+
dispatch_authority_enabled: false,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
export function validateAndAbortFlowDeskLaneEvidenceV1(input) {
|
|
193
|
+
if (!workflowPrefixAllowed(input.workflow_id)) {
|
|
194
|
+
return { status: "blocked", reason: "workflow_prefix_not_allowed" };
|
|
195
|
+
}
|
|
196
|
+
const reloaded = reloadFlowDeskSessionEvidenceV1({
|
|
197
|
+
rootDir: input.rootDir,
|
|
198
|
+
workflowId: input.workflow_id,
|
|
199
|
+
});
|
|
200
|
+
if (!reloaded.ok)
|
|
201
|
+
return { status: "blocked", reason: "workflow_not_found" };
|
|
202
|
+
const laneRecords = [];
|
|
203
|
+
for (const entry of reloaded.entries) {
|
|
204
|
+
if (isLaneLifecycleRecord(entry.record) && entry.record.lane_id === input.lane_id) {
|
|
205
|
+
laneRecords.push(entry.record);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
if (laneRecords.length === 0)
|
|
209
|
+
return { status: "blocked", reason: "lane_not_found" };
|
|
210
|
+
const hasExplicitOwnership = laneRecords.some((record) => record.spawned_by === "flowdesk");
|
|
211
|
+
if (input.requireExplicitOwnership === true && !hasExplicitOwnership) {
|
|
212
|
+
return { status: "blocked", reason: "not_explicitly_flowdesk_owned" };
|
|
213
|
+
}
|
|
214
|
+
const hasLegacyOwnership = laneRecords.some((record) => record.spawned_by === undefined);
|
|
215
|
+
if (!hasExplicitOwnership && !hasLegacyOwnership) {
|
|
216
|
+
return { status: "blocked", reason: "not_flowdesk_owned" };
|
|
217
|
+
}
|
|
218
|
+
const latest = latestLifecycle(laneRecords);
|
|
219
|
+
if (TERMINAL_LANE_STATES.has(latest.state)) {
|
|
220
|
+
return { status: "blocked", reason: "lane_already_terminal", current_state: latest.state };
|
|
221
|
+
}
|
|
222
|
+
if (!ABORT_ELIGIBLE_LANE_STATES.has(latest.state)) {
|
|
223
|
+
return { status: "blocked", reason: "lane_not_eligible", current_state: latest.state };
|
|
224
|
+
}
|
|
225
|
+
const observedAt = (input.now?.() ?? new Date()).toISOString();
|
|
226
|
+
const evidenceId = `lifecycle-abort-${input.lane_id}-${observedAt.replace(/[^0-9A-Za-z]/g, "")}`;
|
|
227
|
+
const abortRecord = {
|
|
228
|
+
...latest,
|
|
229
|
+
state: "aborted",
|
|
230
|
+
updated_at: observedAt,
|
|
231
|
+
verdict_ref: undefined,
|
|
232
|
+
spawned_by: latest.spawned_by ?? "flowdesk",
|
|
233
|
+
};
|
|
234
|
+
const prepared = prepareFlowDeskSessionEvidenceWriteIntentV1({
|
|
235
|
+
workflowId: input.workflow_id,
|
|
236
|
+
evidenceId,
|
|
237
|
+
record: abortRecord,
|
|
238
|
+
});
|
|
239
|
+
if (!prepared.ok || prepared.writeIntent === undefined) {
|
|
240
|
+
return { status: "write_failed", reason: "abort_evidence_prepare_failed" };
|
|
241
|
+
}
|
|
242
|
+
const applyResult = applyFlowDeskSessionEvidenceWriteIntentsV1(input.rootDir, [prepared.writeIntent]);
|
|
243
|
+
if (!applyResult.ok)
|
|
244
|
+
return { status: "write_failed", reason: "abort_evidence_write_failed" };
|
|
245
|
+
const verify = reloadFlowDeskSessionEvidenceV1({
|
|
246
|
+
rootDir: input.rootDir,
|
|
247
|
+
workflowId: input.workflow_id,
|
|
248
|
+
});
|
|
249
|
+
const persisted = verify.ok && verify.entries.some((entry) => entry.evidenceClass === "lane_lifecycle" &&
|
|
250
|
+
entry.evidenceId === evidenceId &&
|
|
251
|
+
isLaneLifecycleRecord(entry.record) &&
|
|
252
|
+
entry.record.state === "aborted");
|
|
253
|
+
if (!persisted)
|
|
254
|
+
return { status: "write_failed", reason: "abort_evidence_not_persisted" };
|
|
255
|
+
return {
|
|
256
|
+
status: "aborted",
|
|
257
|
+
lane_id: input.lane_id,
|
|
258
|
+
lifecycle_evidence_id: evidenceId,
|
|
259
|
+
reason: `user-requested-abort at ${observedAt} via /flowdesk-abort`,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
export function cancelPendingAbortWarningEvidenceV1(input) {
|
|
263
|
+
const now = input.now?.() ?? new Date();
|
|
264
|
+
const cancelId = `cancel-${safeToken(input.lane_id)}-${timestampToken(now)}`;
|
|
265
|
+
const cancelRecord = {
|
|
266
|
+
schema_version: "flowdesk.pending_abort_cancel.v1",
|
|
267
|
+
cancel_id: cancelId,
|
|
268
|
+
warning_id_ref: input.warning_id_ref,
|
|
269
|
+
workflow_id: input.workflow_id,
|
|
270
|
+
lane_id: input.lane_id,
|
|
271
|
+
cancelled_at: now.toISOString(),
|
|
272
|
+
cancel_reason: "user_requested_via_command",
|
|
273
|
+
cancel_actor: "user",
|
|
274
|
+
dispatch_authority_enabled: false,
|
|
275
|
+
};
|
|
276
|
+
const cancelWritten = writeEvidence({
|
|
277
|
+
rootDir: input.rootDir,
|
|
278
|
+
workflowId: input.workflow_id,
|
|
279
|
+
evidenceId: cancelId,
|
|
280
|
+
record: cancelRecord,
|
|
281
|
+
});
|
|
282
|
+
if (!cancelWritten)
|
|
283
|
+
return { status: "blocked", reason: "pending_abort_cancel_write_failed" };
|
|
284
|
+
const tombstoneId = `warning-cancelled-${safeToken(input.lane_id)}-${timestampToken(now)}`;
|
|
285
|
+
const tombstone = warningRecord({
|
|
286
|
+
warningId: tombstoneId,
|
|
287
|
+
workflowId: input.workflow_id,
|
|
288
|
+
laneId: input.lane_id,
|
|
289
|
+
issuedAt: now,
|
|
290
|
+
expiresAt: now,
|
|
291
|
+
status: "cancelled",
|
|
292
|
+
});
|
|
293
|
+
const tombstoneWritten = writeEvidence({
|
|
294
|
+
rootDir: input.rootDir,
|
|
295
|
+
workflowId: input.workflow_id,
|
|
296
|
+
evidenceId: tombstoneId,
|
|
297
|
+
record: tombstone,
|
|
298
|
+
});
|
|
299
|
+
if (!tombstoneWritten)
|
|
300
|
+
return { status: "blocked", reason: "pending_abort_cancel_tombstone_write_failed" };
|
|
301
|
+
return { status: "warning_cancelled", warning_id: input.warning_id_ref, cancel_id: cancelId };
|
|
302
|
+
}
|
|
303
|
+
export function evaluateGuardedAutoAbortHookV1(input) {
|
|
304
|
+
const now = input.now?.() ?? new Date();
|
|
305
|
+
const enablement = isAutoAbortEnabledV1({
|
|
306
|
+
config: input.config,
|
|
307
|
+
rootDir: input.rootDir,
|
|
308
|
+
now,
|
|
309
|
+
loadedSignOff: input.loadedSignOff,
|
|
310
|
+
env: input.env,
|
|
311
|
+
});
|
|
312
|
+
if (!enablement.enabled)
|
|
313
|
+
return { status: "manual_recommended", reason: enablement.reason };
|
|
314
|
+
if (!input.stallConfirmed)
|
|
315
|
+
return { status: "noop", reason: "stall_not_confirmed" };
|
|
316
|
+
if (input.sdkSessionHealth.status !== "api_timeout") {
|
|
317
|
+
return { status: "manual_recommended", reason: `sdk_session_health_${input.sdkSessionHealth.status}` };
|
|
318
|
+
}
|
|
319
|
+
const reloaded = reloadFlowDeskSessionEvidenceV1({
|
|
320
|
+
rootDir: input.rootDir,
|
|
321
|
+
workflowId: input.workflow_id,
|
|
322
|
+
});
|
|
323
|
+
if (!reloaded.ok)
|
|
324
|
+
return { status: "blocked", reason: "session_evidence_reload_failed" };
|
|
325
|
+
const warnings = [];
|
|
326
|
+
for (const entry of reloaded.entries) {
|
|
327
|
+
if (isPendingAbortWarning(entry.record) &&
|
|
328
|
+
entry.record.workflow_id === input.workflow_id &&
|
|
329
|
+
entry.record.lane_id === input.lane_id) {
|
|
330
|
+
warnings.push(entry.record);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const latestWarning = latestPendingAbortWarning(warnings);
|
|
334
|
+
if (latestWarning !== undefined && latestWarning.status === "cancelled")
|
|
335
|
+
return { status: "manual_recommended", reason: "pending_abort_cancelled" };
|
|
336
|
+
if (latestWarning !== undefined && latestWarning.status === "executed")
|
|
337
|
+
return { status: "noop", reason: "pending_abort_already_executed" };
|
|
338
|
+
if (latestWarning !== undefined && latestWarning.status === "tombstoned")
|
|
339
|
+
return { status: "noop", reason: "pending_abort_tombstoned" };
|
|
340
|
+
if (latestWarning !== undefined) {
|
|
341
|
+
const cancels = [];
|
|
342
|
+
for (const entry of reloaded.entries) {
|
|
343
|
+
if (isPendingAbortCancel(entry.record) && entry.record.warning_id_ref === latestWarning.warning_id) {
|
|
344
|
+
cancels.push(entry.record);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
if (cancels.length > 0) {
|
|
348
|
+
return cancelPendingAbortWarningEvidenceV1({
|
|
349
|
+
rootDir: input.rootDir,
|
|
350
|
+
workflow_id: input.workflow_id,
|
|
351
|
+
lane_id: input.lane_id,
|
|
352
|
+
warning_id_ref: latestWarning.warning_id,
|
|
353
|
+
now: () => now,
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
if (now.getTime() < Date.parse(latestWarning.expires_at)) {
|
|
357
|
+
return {
|
|
358
|
+
status: "warning_pending",
|
|
359
|
+
warning_id: latestWarning.warning_id,
|
|
360
|
+
expires_at: latestWarning.expires_at,
|
|
361
|
+
cancel_command: latestWarning.cancel_command,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
const abort = validateAndAbortFlowDeskLaneEvidenceV1({
|
|
365
|
+
rootDir: input.rootDir,
|
|
366
|
+
workflow_id: input.workflow_id,
|
|
367
|
+
lane_id: input.lane_id,
|
|
368
|
+
now: () => now,
|
|
369
|
+
requireExplicitOwnership: true,
|
|
370
|
+
});
|
|
371
|
+
if (abort.status !== "aborted")
|
|
372
|
+
return { status: "blocked", reason: abort.reason };
|
|
373
|
+
const executedId = `warning-executed-${safeToken(input.lane_id)}-${timestampToken(now)}`;
|
|
374
|
+
const executed = warningRecord({
|
|
375
|
+
warningId: executedId,
|
|
376
|
+
workflowId: input.workflow_id,
|
|
377
|
+
laneId: input.lane_id,
|
|
378
|
+
issuedAt: now,
|
|
379
|
+
expiresAt: now,
|
|
380
|
+
status: "executed",
|
|
381
|
+
});
|
|
382
|
+
const executedWritten = writeEvidence({
|
|
383
|
+
rootDir: input.rootDir,
|
|
384
|
+
workflowId: input.workflow_id,
|
|
385
|
+
evidenceId: executedId,
|
|
386
|
+
record: executed,
|
|
387
|
+
});
|
|
388
|
+
if (!executedWritten)
|
|
389
|
+
return { status: "blocked", reason: "pending_abort_executed_tombstone_write_failed" };
|
|
390
|
+
return {
|
|
391
|
+
status: "auto_abort_executed",
|
|
392
|
+
warning_id: latestWarning.warning_id,
|
|
393
|
+
lifecycle_evidence_id: abort.lifecycle_evidence_id,
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
const warningId = `warning-${safeToken(input.lane_id)}-${timestampToken(now)}`;
|
|
397
|
+
const expiresAt = new Date(now.getTime() + Math.max(10_000, input.config.preAbortWarningMs ?? 60_000));
|
|
398
|
+
const pending = warningRecord({
|
|
399
|
+
warningId,
|
|
400
|
+
workflowId: input.workflow_id,
|
|
401
|
+
laneId: input.lane_id,
|
|
402
|
+
issuedAt: now,
|
|
403
|
+
expiresAt,
|
|
404
|
+
status: "pending",
|
|
405
|
+
});
|
|
406
|
+
const warningWritten = writeEvidence({
|
|
407
|
+
rootDir: input.rootDir,
|
|
408
|
+
workflowId: input.workflow_id,
|
|
409
|
+
evidenceId: warningId,
|
|
410
|
+
record: pending,
|
|
411
|
+
});
|
|
412
|
+
if (!warningWritten)
|
|
413
|
+
return { status: "blocked", reason: "pending_abort_warning_write_failed" };
|
|
414
|
+
return {
|
|
415
|
+
status: "warning_issued",
|
|
416
|
+
warning_id: warningId,
|
|
417
|
+
expires_at: expiresAt.toISOString(),
|
|
418
|
+
cancel_command: pending.cancel_command,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
// ---------------------------------------------------------------------------
|
|
422
|
+
// P7 Guarded Auto-Retry helpers
|
|
423
|
+
// ---------------------------------------------------------------------------
|
|
424
|
+
function isReviewerLaneContext(value) {
|
|
425
|
+
return isRecord(value) && value.schema_version === "flowdesk.reviewer_lane_context.v1";
|
|
426
|
+
}
|
|
427
|
+
function isAgentTaskContext(value) {
|
|
428
|
+
return isRecord(value) && value.schema_version === "flowdesk.agent_task_context.v1";
|
|
429
|
+
}
|
|
430
|
+
function isTaskFailed(value) {
|
|
431
|
+
return isRecord(value) && value.schema_version === "flowdesk.task_failed.v1";
|
|
432
|
+
}
|
|
433
|
+
function isTaskResult(value) {
|
|
434
|
+
return isRecord(value) && value.schema_version === "flowdesk.task_result.v1";
|
|
435
|
+
}
|
|
436
|
+
function isPendingRetryPlan(value) {
|
|
437
|
+
return isRecord(value) && value.schema_version === "flowdesk.pending_retry_plan.v1";
|
|
438
|
+
}
|
|
439
|
+
function isRetryExecuted(value) {
|
|
440
|
+
return isRecord(value) && value.schema_version === "flowdesk.retry_executed.v1";
|
|
441
|
+
}
|
|
442
|
+
function isRetryFailed(value) {
|
|
443
|
+
return isRecord(value) && value.schema_version === "flowdesk.retry_failed.v1";
|
|
444
|
+
}
|
|
445
|
+
function latestLifecycleByLane(records) {
|
|
446
|
+
const byLane = new Map();
|
|
447
|
+
for (const record of records) {
|
|
448
|
+
const existing = byLane.get(record.lane_id);
|
|
449
|
+
if (existing === undefined ||
|
|
450
|
+
Date.parse(record.updated_at) > Date.parse(existing.updated_at) ||
|
|
451
|
+
(Date.parse(record.updated_at) === Date.parse(existing.updated_at) && record.state > existing.state)) {
|
|
452
|
+
byLane.set(record.lane_id, record);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return byLane;
|
|
456
|
+
}
|
|
457
|
+
function latestTaskFailedByLane(records) {
|
|
458
|
+
const byLane = new Map();
|
|
459
|
+
for (const record of records) {
|
|
460
|
+
const existing = byLane.get(record.lane_id);
|
|
461
|
+
if (existing === undefined || Date.parse(record.created_at) >= Date.parse(existing.created_at)) {
|
|
462
|
+
byLane.set(record.lane_id, record);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
return byLane;
|
|
466
|
+
}
|
|
467
|
+
function latestTaskResultByLane(records) {
|
|
468
|
+
const byLane = new Map();
|
|
469
|
+
for (const record of records) {
|
|
470
|
+
const existing = byLane.get(record.lane_id);
|
|
471
|
+
if (existing === undefined || Date.parse(record.created_at) >= Date.parse(existing.created_at)) {
|
|
472
|
+
byLane.set(record.lane_id, record);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
return byLane;
|
|
476
|
+
}
|
|
477
|
+
function agentTaskContextByLane(records) {
|
|
478
|
+
const byLane = new Map();
|
|
479
|
+
for (const record of records)
|
|
480
|
+
byLane.set(record.lane_id, record);
|
|
481
|
+
return byLane;
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* Safe local cleanup/backfill for legacy `flowdesk_agent_task_run` lanes.
|
|
485
|
+
*
|
|
486
|
+
* Older agent-task results/failures could persist `task_result.v1` or
|
|
487
|
+
* `task_failed.v1` while the latest `lane_lifecycle` remained
|
|
488
|
+
* `created`/`running`, which made status cards keep reporting stale active
|
|
489
|
+
* lanes. This helper only writes terminal lifecycle evidence when existing
|
|
490
|
+
* durable evidence already proves the agent-task ended.
|
|
491
|
+
* It never launches, aborts, retries, enables dispatch authority, or rewrites
|
|
492
|
+
* existing evidence.
|
|
493
|
+
*/
|
|
494
|
+
export function backfillTerminalAgentTaskFailedLanesV1(input) {
|
|
495
|
+
const reloaded = reloadFlowDeskSessionEvidenceV1({ rootDir: input.rootDir, workflowId: input.workflowId });
|
|
496
|
+
if (!reloaded.ok)
|
|
497
|
+
return { status: "backfill_skipped", workflowId: input.workflowId, reason: "session_evidence_reload_failed" };
|
|
498
|
+
const lifecycles = [];
|
|
499
|
+
const taskResults = [];
|
|
500
|
+
const taskFailures = [];
|
|
501
|
+
const contexts = [];
|
|
502
|
+
for (const entry of reloaded.entries) {
|
|
503
|
+
if (isLaneLifecycleRecord(entry.record))
|
|
504
|
+
lifecycles.push(entry.record);
|
|
505
|
+
else if (isTaskResult(entry.record))
|
|
506
|
+
taskResults.push(entry.record);
|
|
507
|
+
else if (isTaskFailed(entry.record))
|
|
508
|
+
taskFailures.push(entry.record);
|
|
509
|
+
else if (isAgentTaskContext(entry.record))
|
|
510
|
+
contexts.push(entry.record);
|
|
511
|
+
}
|
|
512
|
+
const latestLifecycle = latestLifecycleByLane(lifecycles);
|
|
513
|
+
const latestResult = latestTaskResultByLane(taskResults);
|
|
514
|
+
const latestFailure = latestTaskFailedByLane(taskFailures);
|
|
515
|
+
const contextByLane = agentTaskContextByLane(contexts);
|
|
516
|
+
const observedAt = (input.now ?? new Date()).toISOString();
|
|
517
|
+
const token = timestampToken(input.now ?? new Date());
|
|
518
|
+
const terminalEvidenceIds = [];
|
|
519
|
+
let lanesScanned = 0;
|
|
520
|
+
for (const [laneId, result] of latestResult) {
|
|
521
|
+
const latest = latestLifecycle.get(laneId);
|
|
522
|
+
if (latest === undefined)
|
|
523
|
+
continue;
|
|
524
|
+
lanesScanned++;
|
|
525
|
+
if (!ABORT_ELIGIBLE_LANE_STATES.has(latest.state))
|
|
526
|
+
continue;
|
|
527
|
+
const context = contextByLane.get(laneId);
|
|
528
|
+
const evidenceId = `lifecycle-agent-task-terminal-backfill-${safeToken(laneId)}-${token}`;
|
|
529
|
+
const terminalRecord = {
|
|
530
|
+
...latest,
|
|
531
|
+
workflow_id: result.workflow_id,
|
|
532
|
+
lane_id: laneId,
|
|
533
|
+
agent_ref: context?.agent_ref ?? result.agent_ref ?? latest.agent_ref,
|
|
534
|
+
provider_qualified_model_id: context?.provider_qualified_model_id ?? result.provider_qualified_model_id ?? latest.provider_qualified_model_id,
|
|
535
|
+
parent_session_ref: context?.parent_session_ref ?? latest.parent_session_ref,
|
|
536
|
+
state: "incomplete",
|
|
537
|
+
verdict_ref: undefined,
|
|
538
|
+
output_ref: `output-${safeToken(result.task_id)}`,
|
|
539
|
+
updated_at: observedAt,
|
|
540
|
+
spawned_by: latest.spawned_by ?? "flowdesk",
|
|
541
|
+
dispatch_authority_enabled: false,
|
|
542
|
+
providerCall: false,
|
|
543
|
+
actualLaneLaunch: false,
|
|
544
|
+
runtimeExecution: false,
|
|
545
|
+
};
|
|
546
|
+
if (writeEvidence({ rootDir: input.rootDir, workflowId: input.workflowId, evidenceId, record: terminalRecord })) {
|
|
547
|
+
terminalEvidenceIds.push(evidenceId);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
for (const [laneId, failed] of latestFailure) {
|
|
551
|
+
if (latestResult.has(laneId))
|
|
552
|
+
continue;
|
|
553
|
+
const latest = latestLifecycle.get(laneId);
|
|
554
|
+
if (latest === undefined)
|
|
555
|
+
continue;
|
|
556
|
+
lanesScanned++;
|
|
557
|
+
if (!ABORT_ELIGIBLE_LANE_STATES.has(latest.state))
|
|
558
|
+
continue;
|
|
559
|
+
const context = contextByLane.get(laneId);
|
|
560
|
+
const state = failed.failure_category === "no_response" ? "no_output" : "invocation_failed";
|
|
561
|
+
const evidenceId = `lifecycle-agent-task-terminal-backfill-${safeToken(laneId)}-${token}`;
|
|
562
|
+
const terminalRecord = {
|
|
563
|
+
...latest,
|
|
564
|
+
workflow_id: failed.workflow_id,
|
|
565
|
+
lane_id: laneId,
|
|
566
|
+
agent_ref: context?.agent_ref ?? failed.agent_ref ?? latest.agent_ref,
|
|
567
|
+
provider_qualified_model_id: context?.provider_qualified_model_id ?? failed.provider_qualified_model_id ?? latest.provider_qualified_model_id,
|
|
568
|
+
parent_session_ref: context?.parent_session_ref ?? latest.parent_session_ref,
|
|
569
|
+
state,
|
|
570
|
+
verdict_ref: undefined,
|
|
571
|
+
output_ref: undefined,
|
|
572
|
+
updated_at: observedAt,
|
|
573
|
+
spawned_by: latest.spawned_by ?? "flowdesk",
|
|
574
|
+
dispatch_authority_enabled: false,
|
|
575
|
+
providerCall: false,
|
|
576
|
+
actualLaneLaunch: false,
|
|
577
|
+
runtimeExecution: false,
|
|
578
|
+
};
|
|
579
|
+
if (writeEvidence({ rootDir: input.rootDir, workflowId: input.workflowId, evidenceId, record: terminalRecord })) {
|
|
580
|
+
terminalEvidenceIds.push(evidenceId);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
return {
|
|
584
|
+
status: "backfill_completed",
|
|
585
|
+
workflowId: input.workflowId,
|
|
586
|
+
lanesScanned,
|
|
587
|
+
lanesTerminalized: terminalEvidenceIds.length,
|
|
588
|
+
terminalLifecycleEvidenceIds: terminalEvidenceIds,
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* On startup/reload: any `pending_retry_plan` in `launched` state with no matching
|
|
593
|
+
* `retry_executed` or `retry_failed` within 10 minutes of `created_at` is reconciled
|
|
594
|
+
* as `retry_failed(indeterminate_launch)`.
|
|
595
|
+
*/
|
|
596
|
+
export function reconcileStalePendingRetryPlansV1(input) {
|
|
597
|
+
const now = input.now ?? new Date();
|
|
598
|
+
const staleThresholdMs = input.staleThresholdMs ?? 600_000; // 10 minutes default
|
|
599
|
+
const reloaded = reloadFlowDeskSessionEvidenceV1({
|
|
600
|
+
rootDir: input.rootDir,
|
|
601
|
+
workflowId: input.workflowId,
|
|
602
|
+
});
|
|
603
|
+
if (!reloaded.ok)
|
|
604
|
+
return;
|
|
605
|
+
for (const entry of reloaded.entries) {
|
|
606
|
+
if (!isPendingRetryPlan(entry.record))
|
|
607
|
+
continue;
|
|
608
|
+
const plan = entry.record;
|
|
609
|
+
if (plan.status !== "launched")
|
|
610
|
+
continue;
|
|
611
|
+
// Check if stale: created_at + staleThresholdMs < now
|
|
612
|
+
const createdAtMs = Date.parse(plan.created_at);
|
|
613
|
+
if (!Number.isFinite(createdAtMs))
|
|
614
|
+
continue;
|
|
615
|
+
if (now.getTime() < createdAtMs + staleThresholdMs)
|
|
616
|
+
continue;
|
|
617
|
+
// Check if any terminal evidence exists for new_lane_id
|
|
618
|
+
const hasTerminal = reloaded.entries.some((e) => {
|
|
619
|
+
if (isRetryExecuted(e.record) && e.record.new_lane_id === plan.new_lane_id)
|
|
620
|
+
return true;
|
|
621
|
+
if (isRetryFailed(e.record) && e.record.new_lane_id === plan.new_lane_id)
|
|
622
|
+
return true;
|
|
623
|
+
return false;
|
|
624
|
+
});
|
|
625
|
+
if (hasTerminal)
|
|
626
|
+
continue;
|
|
627
|
+
// Write retry_failed(indeterminate_launch)
|
|
628
|
+
const failedId = `retry-failed-indeterminate-${safeToken(plan.new_lane_id)}-${timestampToken(now)}`;
|
|
629
|
+
const failedRecord = {
|
|
630
|
+
schema_version: "flowdesk.retry_failed.v1",
|
|
631
|
+
workflow_id: plan.workflow_id,
|
|
632
|
+
original_lane_id: plan.original_lane_id,
|
|
633
|
+
new_lane_id: plan.new_lane_id,
|
|
634
|
+
retry_attempt: plan.retry_attempt,
|
|
635
|
+
failure_category: "indeterminate_launch",
|
|
636
|
+
redacted_reason: "pending_retry_plan launched state stale without terminal evidence",
|
|
637
|
+
created_at: now.toISOString(),
|
|
638
|
+
dispatch_authority_enabled: false,
|
|
639
|
+
};
|
|
640
|
+
writeEvidence({
|
|
641
|
+
rootDir: input.rootDir,
|
|
642
|
+
workflowId: input.workflowId,
|
|
643
|
+
evidenceId: failedId,
|
|
644
|
+
record: failedRecord,
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* Build a minimal FlowDeskRuntimeLaneLaunchPlanV1-compatible structure
|
|
650
|
+
* from reviewer lane context evidence, for use in retry launch.
|
|
651
|
+
*/
|
|
652
|
+
function buildRetryLaunchPlanFromContextV1(context, newLaneId, parentSessionId) {
|
|
653
|
+
const token = timestampToken(new Date());
|
|
654
|
+
return {
|
|
655
|
+
schema_version: "flowdesk.runtime_lane_launch_plan.v1",
|
|
656
|
+
ok: true,
|
|
657
|
+
errors: [],
|
|
658
|
+
launch_request_id: `launch-request-retry-${context.perspective}-${token}`,
|
|
659
|
+
workflow_id: context.workflow_id,
|
|
660
|
+
attempt_id: context.original_attempt_id,
|
|
661
|
+
lane_id: newLaneId,
|
|
662
|
+
state: "launch_ready",
|
|
663
|
+
blocked_labels: [],
|
|
664
|
+
parent_session_ref: `ses-${parentSessionId}`,
|
|
665
|
+
agent_ref: context.agent_ref,
|
|
666
|
+
provider_qualified_model_id: context.provider_qualified_model_id,
|
|
667
|
+
launch_reason: "reviewer_fanout",
|
|
668
|
+
pre_launch_audit_ref: `audit-retry-pre-launch-${context.perspective}-${token}`,
|
|
669
|
+
lane_launch_approval_ref: `approval-retry-lane-launch-${context.perspective}-${token}`,
|
|
670
|
+
durable_evidence_root_ref: `evidence-root-retry-${token}`,
|
|
671
|
+
lifecycle_evidence_class: "lane_lifecycle",
|
|
672
|
+
exact_binding_confirmed: true,
|
|
673
|
+
sdk_client_required: true,
|
|
674
|
+
launch_attempted: false,
|
|
675
|
+
dispatch_authority_enabled: false,
|
|
676
|
+
providerCall: false,
|
|
677
|
+
actualLaneLaunch: false,
|
|
678
|
+
runtimeExecution: false,
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
function parentSessionIdFromRef(parentSessionRef) {
|
|
682
|
+
return parentSessionRef.startsWith("ses-") && parentSessionRef.length > "ses-".length
|
|
683
|
+
? parentSessionRef.slice("ses-".length)
|
|
684
|
+
: undefined;
|
|
685
|
+
}
|
|
686
|
+
function writeRetryFailedV1(input) {
|
|
687
|
+
writeEvidence({
|
|
688
|
+
rootDir: input.rootDir,
|
|
689
|
+
workflowId: input.workflowId,
|
|
690
|
+
evidenceId: input.evidenceId,
|
|
691
|
+
record: input.record,
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
/**
|
|
695
|
+
* Evaluate guarded auto-retry hook following the execution order from the design doc exactly.
|
|
696
|
+
* Called after `evaluateGuardedAutoAbortHookV1` returns `auto_abort_executed`.
|
|
697
|
+
*/
|
|
698
|
+
export async function evaluateGuardedAutoRetryHookV1(input) {
|
|
699
|
+
const now = input.now ?? new Date();
|
|
700
|
+
const maxAutoRetries = Math.min(2, Math.max(1, input.config.maxAutoRetries ?? 1));
|
|
701
|
+
const timeoutMs = input.timeoutMs ?? 30_000;
|
|
702
|
+
// Step 1: Check opt-in
|
|
703
|
+
if (input.config.autoRetryAfterAbort !== true) {
|
|
704
|
+
return { status: "auto_retry_not_configured", reason: "opt_in_false" };
|
|
705
|
+
}
|
|
706
|
+
// Step 2: Re-verify Guard HMAC
|
|
707
|
+
const guardLoaded = loadGuardSignOffFromRoot(input.rootDir, input.config.guardSignOffPath);
|
|
708
|
+
if (guardLoaded === undefined) {
|
|
709
|
+
return { status: "auto_retry_disabled", reason: "guard_unverified" };
|
|
710
|
+
}
|
|
711
|
+
const guardVerified = verifyGuardSignOffHmacV1({
|
|
712
|
+
signOff: guardLoaded.signOff,
|
|
713
|
+
markdownText: guardLoaded.markdownText,
|
|
714
|
+
hmacKey: input.config.guardHmacKey,
|
|
715
|
+
now,
|
|
716
|
+
});
|
|
717
|
+
if (!guardVerified.ok) {
|
|
718
|
+
return { status: "auto_retry_disabled", reason: "guard_unverified" };
|
|
719
|
+
}
|
|
720
|
+
// Step 3: Check SDK client availability
|
|
721
|
+
if (input.client === undefined || typeof input.client.session?.create !== "function") {
|
|
722
|
+
const failedId = `retry-failed-sdk-unavailable-${safeToken(input.laneId)}-${timestampToken(now)}`;
|
|
723
|
+
writeEvidence({
|
|
724
|
+
rootDir: input.rootDir,
|
|
725
|
+
workflowId: input.workflowId,
|
|
726
|
+
evidenceId: failedId,
|
|
727
|
+
record: {
|
|
728
|
+
schema_version: "flowdesk.retry_failed.v1",
|
|
729
|
+
workflow_id: input.workflowId,
|
|
730
|
+
original_lane_id: input.laneId,
|
|
731
|
+
retry_attempt: 1,
|
|
732
|
+
failure_category: "sdk_unavailable",
|
|
733
|
+
redacted_reason: "sdk_client_missing_or_session_create_unavailable",
|
|
734
|
+
created_at: now.toISOString(),
|
|
735
|
+
dispatch_authority_enabled: false,
|
|
736
|
+
},
|
|
737
|
+
});
|
|
738
|
+
return {
|
|
739
|
+
status: "retry_failed",
|
|
740
|
+
failureCategory: "sdk_unavailable",
|
|
741
|
+
redactedReason: "sdk_client_missing_or_session_create_unavailable",
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
// Step 4: Load retry context for laneId. Reviewer context remains the
|
|
745
|
+
// preferred P7 path; generic agent tasks fall back to agent_task_context.v1.
|
|
746
|
+
const reloaded = reloadFlowDeskSessionEvidenceV1({
|
|
747
|
+
rootDir: input.rootDir,
|
|
748
|
+
workflowId: input.workflowId,
|
|
749
|
+
});
|
|
750
|
+
if (!reloaded.ok) {
|
|
751
|
+
return { status: "auto_retry_disabled", reason: "context_missing" };
|
|
752
|
+
}
|
|
753
|
+
const reviewerContextEntry = reloaded.entries.find((e) => isReviewerLaneContext(e.record) && e.record.lane_id === input.laneId);
|
|
754
|
+
const agentTaskContextEntry = reviewerContextEntry === undefined
|
|
755
|
+
? reloaded.entries.find((e) => isAgentTaskContext(e.record) && e.record.lane_id === input.laneId)
|
|
756
|
+
: undefined;
|
|
757
|
+
const contextEntry = reviewerContextEntry ?? agentTaskContextEntry;
|
|
758
|
+
if (contextEntry === undefined) {
|
|
759
|
+
return { status: "auto_retry_disabled", reason: "context_missing" };
|
|
760
|
+
}
|
|
761
|
+
const isAgentTaskRetry = reviewerContextEntry === undefined;
|
|
762
|
+
const reviewerContext = isAgentTaskRetry ? undefined : contextEntry.record;
|
|
763
|
+
const agentTaskContext = isAgentTaskRetry ? contextEntry.record : undefined;
|
|
764
|
+
const context = reviewerContext ?? agentTaskContext;
|
|
765
|
+
// Step 5: Verify context.redaction_version present
|
|
766
|
+
if (!context.redaction_version || typeof context.redaction_version !== "string" || context.redaction_version.trim().length === 0) {
|
|
767
|
+
return { status: "auto_retry_disabled", reason: "context_redaction_invalid" };
|
|
768
|
+
}
|
|
769
|
+
// Step 6: Verify context.workflow_id === workflowId and context-specific invariants.
|
|
770
|
+
const VALID_PERSPECTIVES = new Set(["policy_security", "architecture", "verification_implementation"]);
|
|
771
|
+
if (context.workflow_id !== input.workflowId) {
|
|
772
|
+
return { status: "auto_retry_disabled", reason: "invariant_violated" };
|
|
773
|
+
}
|
|
774
|
+
if (reviewerContext !== undefined && !VALID_PERSPECTIVES.has(reviewerContext.perspective)) {
|
|
775
|
+
return { status: "auto_retry_disabled", reason: "invariant_violated" };
|
|
776
|
+
}
|
|
777
|
+
if (agentTaskContext !== undefined && agentTaskContext.prompt_text_truncated === true) {
|
|
778
|
+
return { status: "auto_retry_disabled", reason: "invariant_violated" };
|
|
779
|
+
}
|
|
780
|
+
// Step 7: Count cap — retry_executed + retry_failed + pending_retry_plan(pending|launched) for laneId
|
|
781
|
+
const retryExecutedCount = reloaded.entries.filter((e) => isRetryExecuted(e.record) && e.record.original_lane_id === input.laneId).length;
|
|
782
|
+
const retryFailedCount = reloaded.entries.filter((e) => isRetryFailed(e.record) && e.record.original_lane_id === input.laneId).length;
|
|
783
|
+
const pendingActiveCount = reloaded.entries.filter((e) => {
|
|
784
|
+
if (!isPendingRetryPlan(e.record))
|
|
785
|
+
return false;
|
|
786
|
+
const plan = e.record;
|
|
787
|
+
return plan.original_lane_id === input.laneId && (plan.status === "pending" || plan.status === "launched");
|
|
788
|
+
}).length;
|
|
789
|
+
const retriesUsed = retryExecutedCount + retryFailedCount + pendingActiveCount;
|
|
790
|
+
if (retriesUsed >= maxAutoRetries) {
|
|
791
|
+
const capFailedId = `retry-failed-cap-${safeToken(input.laneId)}-${timestampToken(now)}`;
|
|
792
|
+
writeEvidence({
|
|
793
|
+
rootDir: input.rootDir,
|
|
794
|
+
workflowId: input.workflowId,
|
|
795
|
+
evidenceId: capFailedId,
|
|
796
|
+
record: {
|
|
797
|
+
schema_version: "flowdesk.retry_failed.v1",
|
|
798
|
+
workflow_id: input.workflowId,
|
|
799
|
+
original_lane_id: input.laneId,
|
|
800
|
+
retry_attempt: retriesUsed + 1,
|
|
801
|
+
failure_category: "cap_reached",
|
|
802
|
+
redacted_reason: `retry_cap_reached(max=${maxAutoRetries},used=${retriesUsed})`,
|
|
803
|
+
created_at: now.toISOString(),
|
|
804
|
+
dispatch_authority_enabled: false,
|
|
805
|
+
},
|
|
806
|
+
});
|
|
807
|
+
return { status: "auto_retry_disabled", reason: "cap_reached", retriesUsed };
|
|
808
|
+
}
|
|
809
|
+
// Step 8: Check no pending_retry_plan(pending|launched) already exists for laneId
|
|
810
|
+
if (pendingActiveCount > 0) {
|
|
811
|
+
return { status: "auto_retry_disabled", reason: "concurrent_retry_in_progress" };
|
|
812
|
+
}
|
|
813
|
+
// Step 9: Verify lane_lifecycle terminal state = aborted for laneId (monotonic check)
|
|
814
|
+
const lifecycleEntries = reloaded.entries.filter((e) => e.evidenceClass === "lane_lifecycle" && isRecord(e.record) && e.record.lane_id === input.laneId);
|
|
815
|
+
if (lifecycleEntries.length === 0) {
|
|
816
|
+
return { status: "auto_retry_disabled", reason: "lane_not_terminal_aborted" };
|
|
817
|
+
}
|
|
818
|
+
const latestLifecycle = lifecycleEntries
|
|
819
|
+
.map((e) => e.record)
|
|
820
|
+
.sort((a, b) => new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime())[0];
|
|
821
|
+
if (latestLifecycle.state !== "aborted") {
|
|
822
|
+
return { status: "auto_retry_disabled", reason: "lane_not_terminal_aborted" };
|
|
823
|
+
}
|
|
824
|
+
// Step 10: Generate newLaneId and pendingRetryEvidenceId
|
|
825
|
+
const retryAttempt = retriesUsed + 1;
|
|
826
|
+
const retryToken = timestampToken(now);
|
|
827
|
+
const newLaneId = `lane-retry-${safeToken(input.laneId)}-${retryToken}`;
|
|
828
|
+
const pendingRetryEvidenceId = `pending-retry-${safeToken(input.laneId)}-${retryToken}`;
|
|
829
|
+
// Step 11: Write pending_retry_plan.v1(status=pending) — IDEMPOTENCY FENCE — before any SDK call
|
|
830
|
+
const guardSignOffExpiry = guardLoaded.signOff && isRecord(guardLoaded.signOff) && typeof guardLoaded.signOff.expires_at === "string"
|
|
831
|
+
? Date.parse(guardLoaded.signOff.expires_at)
|
|
832
|
+
: now.getTime() + 30 * 24 * 60 * 60 * 1000; // 30 days default
|
|
833
|
+
const pendingExpiresAt = new Date(Math.min(guardSignOffExpiry, now.getTime() + 60 * 60 * 1000)); // 1h max
|
|
834
|
+
const pendingRecord = {
|
|
835
|
+
schema_version: "flowdesk.pending_retry_plan.v1",
|
|
836
|
+
workflow_id: input.workflowId,
|
|
837
|
+
original_lane_id: input.laneId,
|
|
838
|
+
new_lane_id: newLaneId,
|
|
839
|
+
retry_attempt: retryAttempt,
|
|
840
|
+
context_evidence_id: contextEntry.evidenceId,
|
|
841
|
+
abort_evidence_id: input.abortEvidenceId,
|
|
842
|
+
status: "pending",
|
|
843
|
+
created_at: now.toISOString(),
|
|
844
|
+
expires_at: pendingExpiresAt.toISOString(),
|
|
845
|
+
dispatch_authority_enabled: false,
|
|
846
|
+
};
|
|
847
|
+
const pendingWritten = writeEvidence({
|
|
848
|
+
rootDir: input.rootDir,
|
|
849
|
+
workflowId: input.workflowId,
|
|
850
|
+
evidenceId: pendingRetryEvidenceId,
|
|
851
|
+
record: pendingRecord,
|
|
852
|
+
});
|
|
853
|
+
if (!pendingWritten) {
|
|
854
|
+
return {
|
|
855
|
+
status: "retry_failed",
|
|
856
|
+
failureCategory: "invariant_violated",
|
|
857
|
+
redactedReason: "pending_retry_plan_write_failed",
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
if (agentTaskContext !== undefined) {
|
|
861
|
+
const retryParentSessionId = parentSessionIdFromRef(agentTaskContext.parent_session_ref);
|
|
862
|
+
if (retryParentSessionId === undefined) {
|
|
863
|
+
const failedId = `retry-failed-agent-task-parent-${safeToken(newLaneId)}-${retryToken}`;
|
|
864
|
+
writeRetryFailedV1({
|
|
865
|
+
rootDir: input.rootDir,
|
|
866
|
+
workflowId: input.workflowId,
|
|
867
|
+
evidenceId: failedId,
|
|
868
|
+
record: {
|
|
869
|
+
schema_version: "flowdesk.retry_failed.v1",
|
|
870
|
+
workflow_id: input.workflowId,
|
|
871
|
+
original_lane_id: input.laneId,
|
|
872
|
+
new_lane_id: newLaneId,
|
|
873
|
+
retry_attempt: retryAttempt,
|
|
874
|
+
failure_category: "invariant_violated",
|
|
875
|
+
redacted_reason: "agent_task_context_parent_session_ref_invalid",
|
|
876
|
+
created_at: now.toISOString(),
|
|
877
|
+
dispatch_authority_enabled: false,
|
|
878
|
+
},
|
|
879
|
+
});
|
|
880
|
+
writeEvidence({
|
|
881
|
+
rootDir: input.rootDir,
|
|
882
|
+
workflowId: input.workflowId,
|
|
883
|
+
evidenceId: `${pendingRetryEvidenceId}-failed`,
|
|
884
|
+
record: { ...pendingRecord, status: "failed" },
|
|
885
|
+
});
|
|
886
|
+
return {
|
|
887
|
+
status: "retry_failed",
|
|
888
|
+
failureCategory: "invariant_violated",
|
|
889
|
+
redactedReason: "agent_task_context_parent_session_ref_invalid",
|
|
890
|
+
};
|
|
891
|
+
}
|
|
892
|
+
const taskResult = await executeFlowDeskAgentTaskV1({
|
|
893
|
+
workflowId: input.workflowId,
|
|
894
|
+
taskId: agentTaskContext.task_id,
|
|
895
|
+
laneId: newLaneId,
|
|
896
|
+
agentRef: agentTaskContext.agent_ref,
|
|
897
|
+
providerQualifiedModelId: agentTaskContext.provider_qualified_model_id,
|
|
898
|
+
promptText: agentTaskContext.prompt_text,
|
|
899
|
+
parentSessionId: retryParentSessionId,
|
|
900
|
+
rootDir: input.rootDir,
|
|
901
|
+
client: input.client,
|
|
902
|
+
timeoutMs: timeoutMs,
|
|
903
|
+
});
|
|
904
|
+
if (taskResult.status === "task_failed") {
|
|
905
|
+
const failedId = `retry-failed-agent-task-${safeToken(newLaneId)}-${retryToken}`;
|
|
906
|
+
writeRetryFailedV1({
|
|
907
|
+
rootDir: input.rootDir,
|
|
908
|
+
workflowId: input.workflowId,
|
|
909
|
+
evidenceId: failedId,
|
|
910
|
+
record: {
|
|
911
|
+
schema_version: "flowdesk.retry_failed.v1",
|
|
912
|
+
workflow_id: input.workflowId,
|
|
913
|
+
original_lane_id: input.laneId,
|
|
914
|
+
new_lane_id: newLaneId,
|
|
915
|
+
retry_attempt: retryAttempt,
|
|
916
|
+
failure_category: taskResult.failureCategory === "no_response" ? "indeterminate_launch" : "sdk_create_failed",
|
|
917
|
+
redacted_reason: `agent_task_retry_${taskResult.failureCategory}`,
|
|
918
|
+
created_at: now.toISOString(),
|
|
919
|
+
dispatch_authority_enabled: false,
|
|
920
|
+
},
|
|
921
|
+
});
|
|
922
|
+
writeEvidence({
|
|
923
|
+
rootDir: input.rootDir,
|
|
924
|
+
workflowId: input.workflowId,
|
|
925
|
+
evidenceId: `${pendingRetryEvidenceId}-failed`,
|
|
926
|
+
record: { ...pendingRecord, status: "failed" },
|
|
927
|
+
});
|
|
928
|
+
return {
|
|
929
|
+
status: "retry_failed",
|
|
930
|
+
failureCategory: taskResult.failureCategory,
|
|
931
|
+
redactedReason: taskResult.redactedReason,
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
const executedId = `retry-executed-${safeToken(newLaneId)}-${retryToken}`;
|
|
935
|
+
const executedRecord = {
|
|
936
|
+
schema_version: "flowdesk.retry_executed.v1",
|
|
937
|
+
workflow_id: input.workflowId,
|
|
938
|
+
original_lane_id: input.laneId,
|
|
939
|
+
new_lane_id: newLaneId,
|
|
940
|
+
retry_attempt: retryAttempt,
|
|
941
|
+
retry_kind: "agent_task",
|
|
942
|
+
task_id: agentTaskContext.task_id,
|
|
943
|
+
provider_qualified_model_id: agentTaskContext.provider_qualified_model_id,
|
|
944
|
+
new_parent_session_ref: agentTaskContext.parent_session_ref,
|
|
945
|
+
created_at: now.toISOString(),
|
|
946
|
+
dispatch_authority_enabled: false,
|
|
947
|
+
};
|
|
948
|
+
writeEvidence({
|
|
949
|
+
rootDir: input.rootDir,
|
|
950
|
+
workflowId: input.workflowId,
|
|
951
|
+
evidenceId: executedId,
|
|
952
|
+
record: executedRecord,
|
|
953
|
+
});
|
|
954
|
+
writeEvidence({
|
|
955
|
+
rootDir: input.rootDir,
|
|
956
|
+
workflowId: input.workflowId,
|
|
957
|
+
evidenceId: `${pendingRetryEvidenceId}-launched`,
|
|
958
|
+
record: { ...pendingRecord, status: "launched" },
|
|
959
|
+
});
|
|
960
|
+
return {
|
|
961
|
+
status: "retry_launched",
|
|
962
|
+
newLaneId,
|
|
963
|
+
pendingRetryEvidenceId,
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
// Step 12: Call launchFlowDeskInjectedSdkRuntimeLaneFromPlanV1
|
|
967
|
+
const launchPlan = buildRetryLaunchPlanFromContextV1(reviewerContext, newLaneId, input.parentSessionId);
|
|
968
|
+
let launchResult;
|
|
969
|
+
try {
|
|
970
|
+
const launchPromise = launchFlowDeskInjectedSdkRuntimeLaneFromPlanV1({
|
|
971
|
+
client: input.client,
|
|
972
|
+
launchPlan: launchPlan,
|
|
973
|
+
request: {
|
|
974
|
+
allowActualLaneLaunch: true,
|
|
975
|
+
parentSessionId: input.parentSessionId,
|
|
976
|
+
promptText: reviewerContext.prompt_text,
|
|
977
|
+
dispatchMethod: "prompt",
|
|
978
|
+
},
|
|
979
|
+
});
|
|
980
|
+
launchResult = await withTimeout(launchPromise, timeoutMs, "retry_lane_launch");
|
|
981
|
+
}
|
|
982
|
+
catch (launchErr) {
|
|
983
|
+
// SDK call itself threw — treat as sdk_create_failed
|
|
984
|
+
const failedId = `retry-failed-launch-err-${safeToken(newLaneId)}-${retryToken}`;
|
|
985
|
+
const failedRecord = {
|
|
986
|
+
schema_version: "flowdesk.retry_failed.v1",
|
|
987
|
+
workflow_id: input.workflowId,
|
|
988
|
+
original_lane_id: input.laneId,
|
|
989
|
+
new_lane_id: newLaneId,
|
|
990
|
+
retry_attempt: retryAttempt,
|
|
991
|
+
failure_category: "sdk_create_failed",
|
|
992
|
+
redacted_reason: "sdk_launch_threw_exception",
|
|
993
|
+
created_at: now.toISOString(),
|
|
994
|
+
dispatch_authority_enabled: false,
|
|
995
|
+
};
|
|
996
|
+
writeEvidence({
|
|
997
|
+
rootDir: input.rootDir,
|
|
998
|
+
workflowId: input.workflowId,
|
|
999
|
+
evidenceId: failedId,
|
|
1000
|
+
record: failedRecord,
|
|
1001
|
+
});
|
|
1002
|
+
// Update pending plan to failed
|
|
1003
|
+
const failedPendingRecord = { ...pendingRecord, status: "failed" };
|
|
1004
|
+
writeEvidence({
|
|
1005
|
+
rootDir: input.rootDir,
|
|
1006
|
+
workflowId: input.workflowId,
|
|
1007
|
+
evidenceId: `${pendingRetryEvidenceId}-failed`,
|
|
1008
|
+
record: failedPendingRecord,
|
|
1009
|
+
});
|
|
1010
|
+
return {
|
|
1011
|
+
status: "retry_failed",
|
|
1012
|
+
failureCategory: "sdk_create_failed",
|
|
1013
|
+
redactedReason: "sdk_launch_threw_exception",
|
|
1014
|
+
};
|
|
1015
|
+
}
|
|
1016
|
+
// Step 13/14: Handle session.create success but promptAsync rejection, or create failure
|
|
1017
|
+
if (launchResult.status !== "lane_launch_started") {
|
|
1018
|
+
const isCreateAttempted = launchResult.createAttempted;
|
|
1019
|
+
const failureCategory = isCreateAttempted
|
|
1020
|
+
? "sdk_prompt_rejected"
|
|
1021
|
+
: "sdk_create_failed";
|
|
1022
|
+
const failedId = `retry-failed-${failureCategory}-${safeToken(newLaneId)}-${retryToken}`;
|
|
1023
|
+
const failedRecord = {
|
|
1024
|
+
schema_version: "flowdesk.retry_failed.v1",
|
|
1025
|
+
workflow_id: input.workflowId,
|
|
1026
|
+
original_lane_id: input.laneId,
|
|
1027
|
+
new_lane_id: newLaneId,
|
|
1028
|
+
retry_attempt: retryAttempt,
|
|
1029
|
+
failure_category: failureCategory,
|
|
1030
|
+
redacted_reason: `sdk_launch_${launchResult.status}`,
|
|
1031
|
+
created_at: now.toISOString(),
|
|
1032
|
+
dispatch_authority_enabled: false,
|
|
1033
|
+
};
|
|
1034
|
+
writeEvidence({
|
|
1035
|
+
rootDir: input.rootDir,
|
|
1036
|
+
workflowId: input.workflowId,
|
|
1037
|
+
evidenceId: failedId,
|
|
1038
|
+
record: failedRecord,
|
|
1039
|
+
});
|
|
1040
|
+
// Update pending plan to failed
|
|
1041
|
+
const failedPendingRecord = { ...pendingRecord, status: "failed" };
|
|
1042
|
+
writeEvidence({
|
|
1043
|
+
rootDir: input.rootDir,
|
|
1044
|
+
workflowId: input.workflowId,
|
|
1045
|
+
evidenceId: `${pendingRetryEvidenceId}-failed`,
|
|
1046
|
+
record: failedPendingRecord,
|
|
1047
|
+
});
|
|
1048
|
+
return {
|
|
1049
|
+
status: "retry_failed",
|
|
1050
|
+
failureCategory,
|
|
1051
|
+
redactedReason: `sdk_launch_${launchResult.status}`,
|
|
1052
|
+
};
|
|
1053
|
+
}
|
|
1054
|
+
// Step 15: Full success — write retry_executed.v1 + update pending to launched
|
|
1055
|
+
const newParentSessionRef = launchResult.childSessionRef ?? `ses-${input.parentSessionId}`;
|
|
1056
|
+
const executedId = `retry-executed-${safeToken(newLaneId)}-${retryToken}`;
|
|
1057
|
+
const executedRecord = {
|
|
1058
|
+
schema_version: "flowdesk.retry_executed.v1",
|
|
1059
|
+
workflow_id: input.workflowId,
|
|
1060
|
+
original_lane_id: input.laneId,
|
|
1061
|
+
new_lane_id: newLaneId,
|
|
1062
|
+
retry_attempt: retryAttempt,
|
|
1063
|
+
retry_kind: "reviewer_lane",
|
|
1064
|
+
perspective: reviewerContext.perspective,
|
|
1065
|
+
provider_qualified_model_id: reviewerContext.provider_qualified_model_id,
|
|
1066
|
+
new_parent_session_ref: newParentSessionRef,
|
|
1067
|
+
original_attempt_id: reviewerContext.original_attempt_id,
|
|
1068
|
+
created_at: now.toISOString(),
|
|
1069
|
+
dispatch_authority_enabled: false,
|
|
1070
|
+
};
|
|
1071
|
+
writeEvidence({
|
|
1072
|
+
rootDir: input.rootDir,
|
|
1073
|
+
workflowId: input.workflowId,
|
|
1074
|
+
evidenceId: executedId,
|
|
1075
|
+
record: executedRecord,
|
|
1076
|
+
});
|
|
1077
|
+
// Update pending plan to launched
|
|
1078
|
+
const launchedPendingRecord = { ...pendingRecord, status: "launched" };
|
|
1079
|
+
writeEvidence({
|
|
1080
|
+
rootDir: input.rootDir,
|
|
1081
|
+
workflowId: input.workflowId,
|
|
1082
|
+
evidenceId: `${pendingRetryEvidenceId}-launched`,
|
|
1083
|
+
record: launchedPendingRecord,
|
|
1084
|
+
});
|
|
1085
|
+
return {
|
|
1086
|
+
status: "retry_launched",
|
|
1087
|
+
newLaneId,
|
|
1088
|
+
pendingRetryEvidenceId,
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
// Module-level flag to prevent concurrent cycles
|
|
1092
|
+
let _isWatchdogCycleRunning = false;
|
|
1093
|
+
const FLOWDESK_SESSION_EVIDENCE_ROOT = ".flowdesk/sessions";
|
|
1094
|
+
function listWatchdogWorkflowIds(rootDir) {
|
|
1095
|
+
const sessionsDir = join(rootDir, FLOWDESK_SESSION_EVIDENCE_ROOT);
|
|
1096
|
+
if (!existsSync(sessionsDir))
|
|
1097
|
+
return [];
|
|
1098
|
+
let entries;
|
|
1099
|
+
try {
|
|
1100
|
+
entries = readdirSync(sessionsDir);
|
|
1101
|
+
}
|
|
1102
|
+
catch {
|
|
1103
|
+
return [];
|
|
1104
|
+
}
|
|
1105
|
+
const result = [];
|
|
1106
|
+
for (const name of entries) {
|
|
1107
|
+
const candidatePath = join(sessionsDir, name);
|
|
1108
|
+
try {
|
|
1109
|
+
const stat = statSync(candidatePath);
|
|
1110
|
+
if (stat.isDirectory())
|
|
1111
|
+
result.push(name);
|
|
1112
|
+
}
|
|
1113
|
+
catch {
|
|
1114
|
+
// skip unreadable entries
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
return result;
|
|
1118
|
+
}
|
|
1119
|
+
export async function runFlowDeskWatchdogCycleV1(input) {
|
|
1120
|
+
const cycleAt = (input.now ?? new Date()).toISOString();
|
|
1121
|
+
// Check Guard HMAC first (before concurrency check per the plan spec)
|
|
1122
|
+
const guardLoaded = loadGuardSignOffFromRoot(input.rootDir, input.config.guardSignOffPath);
|
|
1123
|
+
if (guardLoaded === undefined) {
|
|
1124
|
+
return {
|
|
1125
|
+
cycleAt,
|
|
1126
|
+
guardValid: false,
|
|
1127
|
+
lanesChecked: 0,
|
|
1128
|
+
lanesAborted: 0,
|
|
1129
|
+
lanesRetried: 0,
|
|
1130
|
+
lanesFailed: 0,
|
|
1131
|
+
skippedReason: "guard_invalid",
|
|
1132
|
+
};
|
|
1133
|
+
}
|
|
1134
|
+
const guardVerified = verifyGuardSignOffHmacV1({
|
|
1135
|
+
signOff: guardLoaded.signOff,
|
|
1136
|
+
markdownText: guardLoaded.markdownText,
|
|
1137
|
+
hmacKey: input.config.guardHmacKey,
|
|
1138
|
+
now: input.now,
|
|
1139
|
+
});
|
|
1140
|
+
if (!guardVerified.ok) {
|
|
1141
|
+
return {
|
|
1142
|
+
cycleAt,
|
|
1143
|
+
guardValid: false,
|
|
1144
|
+
lanesChecked: 0,
|
|
1145
|
+
lanesAborted: 0,
|
|
1146
|
+
lanesRetried: 0,
|
|
1147
|
+
lanesFailed: 0,
|
|
1148
|
+
skippedReason: "guard_invalid",
|
|
1149
|
+
};
|
|
1150
|
+
}
|
|
1151
|
+
// Check concurrent execution flag
|
|
1152
|
+
if (_isWatchdogCycleRunning) {
|
|
1153
|
+
return {
|
|
1154
|
+
cycleAt,
|
|
1155
|
+
guardValid: true,
|
|
1156
|
+
lanesChecked: 0,
|
|
1157
|
+
lanesAborted: 0,
|
|
1158
|
+
lanesRetried: 0,
|
|
1159
|
+
lanesFailed: 0,
|
|
1160
|
+
skippedReason: "cycle_already_running",
|
|
1161
|
+
};
|
|
1162
|
+
}
|
|
1163
|
+
_isWatchdogCycleRunning = true;
|
|
1164
|
+
let lanesChecked = 0;
|
|
1165
|
+
let lanesAborted = 0;
|
|
1166
|
+
let lanesRetried = 0;
|
|
1167
|
+
let lanesFailed = 0;
|
|
1168
|
+
try {
|
|
1169
|
+
const workflowIds = listWatchdogWorkflowIds(input.rootDir);
|
|
1170
|
+
const now = input.now ?? new Date();
|
|
1171
|
+
for (const workflowId of workflowIds) {
|
|
1172
|
+
backfillTerminalAgentTaskFailedLanesV1({
|
|
1173
|
+
rootDir: input.rootDir,
|
|
1174
|
+
workflowId,
|
|
1175
|
+
now,
|
|
1176
|
+
});
|
|
1177
|
+
// Reload evidence for this workflow
|
|
1178
|
+
const reloaded = reloadFlowDeskSessionEvidenceV1({
|
|
1179
|
+
rootDir: input.rootDir,
|
|
1180
|
+
workflowId,
|
|
1181
|
+
});
|
|
1182
|
+
if (!reloaded.ok)
|
|
1183
|
+
continue;
|
|
1184
|
+
// Run stall projection
|
|
1185
|
+
const stallProjection = projectFlowDeskLaneStallV1({
|
|
1186
|
+
workflowId,
|
|
1187
|
+
reload: reloaded,
|
|
1188
|
+
observedAt: now.toISOString(),
|
|
1189
|
+
});
|
|
1190
|
+
// Find stalled lanes
|
|
1191
|
+
const stalledEntries = stallProjection.entries.filter((entry) => entry.classification === "stalled");
|
|
1192
|
+
for (const stalledEntry of stalledEntries) {
|
|
1193
|
+
lanesChecked++;
|
|
1194
|
+
try {
|
|
1195
|
+
// Run guarded auto-abort
|
|
1196
|
+
const autoAbort = evaluateGuardedAutoAbortHookV1({
|
|
1197
|
+
rootDir: input.rootDir,
|
|
1198
|
+
workflow_id: workflowId,
|
|
1199
|
+
lane_id: stalledEntry.laneId,
|
|
1200
|
+
config: input.config,
|
|
1201
|
+
stallConfirmed: true,
|
|
1202
|
+
sdkSessionHealth: { status: "api_timeout", reason: "watchdog_cycle_stall_detected" },
|
|
1203
|
+
now: () => now,
|
|
1204
|
+
loadedSignOff: guardLoaded,
|
|
1205
|
+
});
|
|
1206
|
+
if (autoAbort.status === "auto_abort_executed") {
|
|
1207
|
+
lanesAborted++;
|
|
1208
|
+
// Run guarded auto-retry if configured
|
|
1209
|
+
if (input.config.autoRetryAfterAbort === true &&
|
|
1210
|
+
input.client !== undefined) {
|
|
1211
|
+
try {
|
|
1212
|
+
const retryResult = await evaluateGuardedAutoRetryHookV1({
|
|
1213
|
+
config: input.config,
|
|
1214
|
+
rootDir: input.rootDir,
|
|
1215
|
+
workflowId,
|
|
1216
|
+
laneId: stalledEntry.laneId,
|
|
1217
|
+
abortEvidenceId: autoAbort.lifecycle_evidence_id,
|
|
1218
|
+
client: input.client,
|
|
1219
|
+
parentSessionId: input.parentSessionId,
|
|
1220
|
+
now,
|
|
1221
|
+
});
|
|
1222
|
+
if (retryResult.status === "retry_launched") {
|
|
1223
|
+
lanesRetried++;
|
|
1224
|
+
}
|
|
1225
|
+
else if (retryResult.status === "retry_failed" ||
|
|
1226
|
+
retryResult.status === "auto_retry_disabled") {
|
|
1227
|
+
// Not a failure of the watchdog cycle itself
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
catch {
|
|
1231
|
+
// Retry evaluation is best-effort; do not count as lanesFailed
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
else if (autoAbort.status === "blocked") {
|
|
1236
|
+
lanesFailed++;
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
catch {
|
|
1240
|
+
lanesFailed++;
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
finally {
|
|
1246
|
+
_isWatchdogCycleRunning = false;
|
|
1247
|
+
}
|
|
1248
|
+
return {
|
|
1249
|
+
cycleAt,
|
|
1250
|
+
guardValid: true,
|
|
1251
|
+
lanesChecked,
|
|
1252
|
+
lanesAborted,
|
|
1253
|
+
lanesRetried,
|
|
1254
|
+
lanesFailed,
|
|
1255
|
+
};
|
|
1256
|
+
}
|
|
1257
|
+
//# sourceMappingURL=stall-recovery.js.map
|