@awak-app/simy-cli 0.7.21 → 0.7.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -17
- package/package.json +3 -3
- package/src/agent.js +28 -8
- package/src/cli-contract.js +1 -1
- package/src/codex-plugins/simy/hooks/delivery-loop.mjs +3 -1
- package/src/codex-plugins/simy/hooks/outcome-progress.mjs +1 -1
- package/src/codex-plugins/simy/hooks/update-dispatch-gate.mjs +89 -6
- package/src/coding-run-automatic-recovery.js +40 -0
- package/src/console/app.js +5 -0
- package/src/delivery-hooks/command.js +12 -0
- package/src/delivery-hooks/supervisor.js +14 -4
- package/src/desktop-access.js +1 -1
- package/src/desktop-clipboard.js +23 -0
- package/src/desktop-host.js +76 -9
- package/src/orchestrator/audit.js +2 -2
- package/src/orchestrator/independent-audit.js +3 -3
- package/src/orchestrator/instruction.js +1 -0
- package/src/orchestrator/loop.js +19 -12
- package/src/orchestrator/recovery.js +12 -7
- package/src/orchestrator/retry.js +7 -1
- package/src/provider-links.js +14 -4
- package/src/runner.js +24 -0
- package/src/sqm/agent-skill.js +4 -2
- package/src/sqm/failure-circuit.js +39 -7
- package/src/sqm/provider-command.js +1 -1
- package/src/sqm/provider-plugin-repair.js +105 -0
- package/src/sqm/provider-plugin-status.js +20 -8
- package/src/sqm/review-worker.js +30 -8
- package/src/update-detachment/history-recovery.js +51 -0
- package/src/update-detachment/runtime.js +3 -3
package/src/desktop-host.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { remoteOperationFailure } from './session-supervision/remote-errors.js';
|
|
2
|
+
import { copyDesktopText } from './desktop-clipboard.js';
|
|
2
3
|
import { createRemoteHostManager } from './session-supervision/remote-hosts.js';
|
|
3
4
|
import { readDesktopAutoUpdate, writeDesktopAutoUpdate } from './desktop-update-settings.js';
|
|
4
5
|
import { createUpdateRuntime } from './update-detachment/runtime.js';
|
|
@@ -15,6 +16,7 @@ import { randomUUID } from "node:crypto";
|
|
|
15
16
|
import { readCodexStatistics, unavailableStatistics } from './session-supervision/statistics.js';
|
|
16
17
|
import { readClaudeStatistics } from './session-supervision/claude-sessions.js';
|
|
17
18
|
import { inspectProviderPlugins, unavailablePluginStatus } from './sqm/provider-plugin-status.js';
|
|
19
|
+
import { repairProviderPlugin } from './sqm/provider-plugin-repair.js';
|
|
18
20
|
import { openCodexAppServer } from './session-supervision/codex-app-server.js';
|
|
19
21
|
import { conversationSqmChecks } from './session-supervision/sqm-presentation.js';
|
|
20
22
|
import { stopOwnedDesktopHost } from "./desktop-force-quit.js";
|
|
@@ -22,7 +24,7 @@ import { stopDesktopWorkForUpdate } from './desktop-update-stop.js';
|
|
|
22
24
|
import { desktopAccessReady, desktopActionAllowed, requireDesktopAccess, validDesktopAccessPath } from './desktop-access.js';
|
|
23
25
|
import { createActionDeadlineService } from './session-supervision/action-deadline.js';
|
|
24
26
|
import { deliveryHookStatus, openHookSetup } from "./delivery-hooks/desktop.js";
|
|
25
|
-
import { controlDelivery, verifyDelivery, createRecordReader, updateRecord, recordProviderArchive, TERMINAL } from "./codex-plugins/simy/hooks/delivery-loop.mjs";
|
|
27
|
+
import { controlDelivery, verifyDelivery, createRecordReader, readRecord, updateRecord, recordProviderArchive, TERMINAL } from "./codex-plugins/simy/hooks/delivery-loop.mjs";
|
|
26
28
|
import { createManagedSupervisor } from "./delivery-hooks/supervisor.js";
|
|
27
29
|
import { createActionAssociationHost } from "./session-supervision/action-association-host.js";
|
|
28
30
|
import { createStopNotifier } from "./session-supervision/stop-notifications.js";
|
|
@@ -104,6 +106,7 @@ function desktopActionError(action, error) {
|
|
|
104
106
|
};
|
|
105
107
|
if (messages[error?.code]) return messages[error.code];
|
|
106
108
|
}
|
|
109
|
+
if (action === 'authorization_copy') return 'Could not copy the connection link. Reopen the connection page and try again.';
|
|
107
110
|
if (action === 'authorization_open' || error?.code === 'desktop_browser_unavailable') return "Could not open your browser. Open the verification link again, or check your default browser.";
|
|
108
111
|
if (error?.code === "desktop_access_required") return "Sign in to an organization with SQM and Developer access before using SIMY.";
|
|
109
112
|
if (action === "codex_hooks" || action === "claude_hooks") return "Could not open the setup guide. Check your default browser, or use /hooks inside the provider.";
|
|
@@ -408,6 +411,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
408
411
|
const link = await requireProviderLink({ root: deliveryRoot, provider: record.provider });
|
|
409
412
|
return JSON.stringify([scope, link.epoch]);
|
|
410
413
|
}, readRecords: () => recordReader.list(),
|
|
414
|
+
readCurrentRecord: (provider, session) => readRecord(deliveryRoot, provider, session),
|
|
411
415
|
beforeClaim: record => actionAssociations.prepareCached?.(record) || record });
|
|
412
416
|
let shuttingDown = null;
|
|
413
417
|
let commands;
|
|
@@ -503,6 +507,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
503
507
|
const changeProviderLink = async (provider, connected) => {
|
|
504
508
|
if (!PROVIDER_LINKS.includes(provider)) throw new Error("Unsupported SIMY provider connection.");
|
|
505
509
|
if (providerLinkBusy.has(provider)) throw new Error("SIMY connection is already changing.");
|
|
510
|
+
providerRepairControllers.get(provider)?.abort();
|
|
506
511
|
providerLinkBusy.add(provider);
|
|
507
512
|
providerLinkFeedback[provider] = { state: connected ? "connecting" : "disconnecting" };
|
|
508
513
|
let phase = "read_settings";
|
|
@@ -527,7 +532,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
527
532
|
phase = "authorize_connect";
|
|
528
533
|
await authorizeWork();
|
|
529
534
|
phase = "save_connect";
|
|
530
|
-
providerLinks = await writeProviderLink(root, provider, true);
|
|
535
|
+
providerLinks = await writeProviderLink(root, provider, true, { repairInvalidSettings: true });
|
|
531
536
|
}
|
|
532
537
|
delete providerLinkFeedback[provider];
|
|
533
538
|
try { await agent.refreshProviderStatus?.(); }
|
|
@@ -575,6 +580,11 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
575
580
|
const providerAuthBusy = new Set();
|
|
576
581
|
let pluginInstallationActive = false, pluginSetupEpoch = 0;
|
|
577
582
|
let providerPlugins = { codex: { state: 'checking' }, claude: { state: 'checking' } };
|
|
583
|
+
const providerRepairs = {}, providerRepairControllers = new Map(), providerRepairJobs = new Set();
|
|
584
|
+
const providerPluginEpochs = { codex: 0, claude: 0 };
|
|
585
|
+
const pluginDiagnostic = ({ provider, reason, elapsedMs }) => {
|
|
586
|
+
diagnostics.write(`provider plugin inspection provider=${provider} reason=${reason} elapsedMs=${elapsedMs}`);
|
|
587
|
+
};
|
|
578
588
|
let pluginInspection = null, lastPluginInspectionAt = -Infinity;
|
|
579
589
|
let pluginInspectionAbort = new AbortController();
|
|
580
590
|
const refreshProviderPlugins = () => {
|
|
@@ -582,9 +592,17 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
582
592
|
if (pluginInspection) return pluginInspection;
|
|
583
593
|
if (pluginInspectionAbort.signal.aborted) pluginInspectionAbort = new AbortController();
|
|
584
594
|
lastPluginInspectionAt = Date.now();
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
595
|
+
const epochs = { ...providerPluginEpochs };
|
|
596
|
+
const apply = value => {
|
|
597
|
+
for (const provider of PROVIDER_LINKS) {
|
|
598
|
+
if (!providerRepairControllers.has(provider) && epochs[provider] === providerPluginEpochs[provider]) {
|
|
599
|
+
providerPlugins[provider] = value[provider] || unavailablePluginStatus();
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
};
|
|
603
|
+
pluginInspection = Promise.resolve().then(() => (dependencies.inspectProviderPlugins || inspectProviderPlugins)({ signal: pluginInspectionAbort.signal, details: true, onDiagnostic: pluginDiagnostic }))
|
|
604
|
+
.then(apply)
|
|
605
|
+
.catch(() => apply({ codex: unavailablePluginStatus(), claude: unavailablePluginStatus() }))
|
|
588
606
|
.finally(() => { pluginInspection = null; if (!closed && !relay) void publish().catch(() => {}); });
|
|
589
607
|
return pluginInspection;
|
|
590
608
|
};
|
|
@@ -608,6 +626,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
608
626
|
}]));
|
|
609
627
|
status.pluginSetup = { ...pluginSetup };
|
|
610
628
|
status.providerPlugins = structuredClone(providerPlugins);
|
|
629
|
+
status.providerRepairs = structuredClone(providerRepairs);
|
|
611
630
|
for (const provider of ["codex", "claude"]) {
|
|
612
631
|
if (providerAuth[provider]?.state === "awaiting_sign_in" && !providerAuthBusy.has(provider) && status.providers?.[provider]?.state === "ready") delete providerAuth[provider];
|
|
613
632
|
}
|
|
@@ -804,6 +823,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
804
823
|
clearInterval(timer);
|
|
805
824
|
teamThreads?.close();
|
|
806
825
|
pluginInspectionAbort.abort();
|
|
826
|
+
for (const controller of providerRepairControllers.values()) controller.abort();
|
|
807
827
|
activeStatisticsRead?.controller.abort();
|
|
808
828
|
discoveryStopped = true;
|
|
809
829
|
encodedStatus = null; lastStatus = null;
|
|
@@ -812,6 +832,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
812
832
|
// reader must never postpone cancellation of an owned Provider process.
|
|
813
833
|
const results = await Promise.allSettled([stopDiscovery({ drain: true }), publishing, connectionPublishing,
|
|
814
834
|
pluginInspection,
|
|
835
|
+
...providerRepairJobs,
|
|
815
836
|
remoteHosts?.close(), actionAssociations.close(), automaticContinuation?.close(), analysisManager?.close(),
|
|
816
837
|
managedSupervisor.close(), recordReader.close(), stopWorkflows(),
|
|
817
838
|
agent ? createAgentShutdown(agent)() : undefined]);
|
|
@@ -1163,7 +1184,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
1163
1184
|
}
|
|
1164
1185
|
|
|
1165
1186
|
let commandProviderLink;
|
|
1166
|
-
if (command.provider && !["refresh", "provider_connect", "provider_disconnect", "delivery_stop", "delivery_pause", "delivery_reject", "session_untrack", "session_unmonitor"].includes(action) &&
|
|
1187
|
+
if (command.provider && !["provider_repair", "refresh", "provider_connect", "provider_disconnect", "delivery_stop", "delivery_pause", "delivery_reject", "session_untrack", "session_unmonitor"].includes(action) &&
|
|
1167
1188
|
!(action === "session_set_auto_continue" && command.enabled === false)) {
|
|
1168
1189
|
commandProviderLink = await requireProviderLink({ root: deliveryRoot, provider: command.provider });
|
|
1169
1190
|
}
|
|
@@ -1188,7 +1209,48 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
1188
1209
|
await remoteHosts.save({ alias: command.alias, label: command.label }, action === 'remote_host_remove');
|
|
1189
1210
|
emit({ type: 'result', id: command.id, ok: true }); await publish(); return;
|
|
1190
1211
|
}
|
|
1191
|
-
if (action ===
|
|
1212
|
+
if (action === 'provider_repair') {
|
|
1213
|
+
const provider = command.provider;
|
|
1214
|
+
if (!PROVIDER_LINKS.includes(provider)) throw new Error('Unsupported provider.');
|
|
1215
|
+
if (providerRepairControllers.has(provider) || providerLinkBusy.has(provider)) throw new Error('Provider repair is already running.');
|
|
1216
|
+
const controller = new AbortController();
|
|
1217
|
+
providerRepairControllers.set(provider, controller);
|
|
1218
|
+
providerPluginEpochs[provider]++;
|
|
1219
|
+
providerRepairs[provider] = { state: 'repairing' };
|
|
1220
|
+
const beganAt = Date.now();
|
|
1221
|
+
let repairJob;
|
|
1222
|
+
try {
|
|
1223
|
+
await publish();
|
|
1224
|
+
repairJob = Promise.resolve().then(() => (dependencies.repairProviderPlugin || repairProviderPlugin)({
|
|
1225
|
+
provider, signal: controller.signal, onDiagnostic: pluginDiagnostic,
|
|
1226
|
+
authorize: async () => {
|
|
1227
|
+
if (closed || relay || controller.signal.aborted) throw Object.assign(new Error('Repair cancelled.'), { reason: 'cancelled' });
|
|
1228
|
+
await authorizeWork(commandScope);
|
|
1229
|
+
},
|
|
1230
|
+
}));
|
|
1231
|
+
providerRepairJobs.add(repairJob);
|
|
1232
|
+
const result = await repairJob;
|
|
1233
|
+
if (closed || controller.signal.aborted) throw Object.assign(new Error('Repair cancelled.'), { reason: 'cancelled' });
|
|
1234
|
+
await authorizeWork(commandScope);
|
|
1235
|
+
if (result?.plugin?.state !== 'enabled') throw Object.assign(new Error('Repair was not verified.'), { reason: 'verification_failed' });
|
|
1236
|
+
providerPlugins[provider] = result.plugin;
|
|
1237
|
+
providerRepairs[provider] = { state: 'repaired' };
|
|
1238
|
+
diagnostics.write(`provider plugin repair provider=${provider} result=repaired elapsedMs=${Date.now() - beganAt}`);
|
|
1239
|
+
} catch (error) {
|
|
1240
|
+
const reasons = ['timeout', 'cli_missing', 'invalid_inventory', 'command_failed', 'verification_failed',
|
|
1241
|
+
'scope_requires_attention', 'source_conflict', 'busy', 'permission_denied', 'cancelled'];
|
|
1242
|
+
const reason = reasons.includes(error?.reason) ? error.reason : 'command_failed';
|
|
1243
|
+
providerRepairs[provider] = { state: 'failed', reason };
|
|
1244
|
+
diagnostics.write(`provider plugin repair provider=${provider} result=failed reason=${reason} elapsedMs=${Date.now() - beganAt}`);
|
|
1245
|
+
throw error;
|
|
1246
|
+
} finally {
|
|
1247
|
+
providerRepairJobs.delete(repairJob);
|
|
1248
|
+
providerRepairControllers.delete(provider);
|
|
1249
|
+
await publish();
|
|
1250
|
+
}
|
|
1251
|
+
emit({ type: 'result', id: command.id, ok: true });
|
|
1252
|
+
return;
|
|
1253
|
+
} else if (action === "provider_connect" || action === "provider_disconnect") {
|
|
1192
1254
|
await changeProviderLink(command.provider, action === "provider_connect");
|
|
1193
1255
|
emit({ type: "result", id: command.id, ok: true });
|
|
1194
1256
|
return;
|
|
@@ -1270,6 +1332,11 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
1270
1332
|
else if (action === "sqm_set_engine_fallback") await agent.setSqmEngineFallback({ enabled: command.enabled, allowedProviders: command.allowedProviders });
|
|
1271
1333
|
else if (action === "sqm_retry_environment") await agent.retrySqmEnvironment();
|
|
1272
1334
|
else if (action === "organization_cancel") await agent.cancelOrganizationSelection();
|
|
1335
|
+
else if (action === 'authorization_copy') {
|
|
1336
|
+
const url = agent.getPendingAuthorizationUrl?.() || await agent.requestDesktopLogin();
|
|
1337
|
+
if (!url || new URL(url).origin !== webOrigin || new URL(url).pathname !== '/local-cli/connect') throw new Error('No valid connection link.');
|
|
1338
|
+
await (dependencies.copyText || copyDesktopText)(url);
|
|
1339
|
+
}
|
|
1273
1340
|
else if (action === "login" || action === "organization_select" || action === "authorization_open") {
|
|
1274
1341
|
if (action === "organization_select" && (typeof command.organizationId !== "string" || !command.organizationId)) throw new Error("Select an organization first.");
|
|
1275
1342
|
const url = action === "authorization_open" ? (agent.getPendingAuthorizationUrl?.() ?? agent.retryAuthorization?.())
|
|
@@ -1596,9 +1663,9 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
1596
1663
|
};
|
|
1597
1664
|
if (command.action === "quit" || command.action === "prepare_update_restart" || command.action === "update" && (command.force === true || stoppingForUpdate)) {
|
|
1598
1665
|
void execute();
|
|
1599
|
-
} else if (["login", "organization_select", "organization_cancel", "authorization_open"].includes(command.action)) {
|
|
1666
|
+
} else if (["login", "organization_select", "organization_cancel", "authorization_open", "authorization_copy"].includes(command.action)) {
|
|
1600
1667
|
authorizationQueue = authorizationQueue.then(execute).catch(() => {});
|
|
1601
|
-
} else if (/^(codex|claude)_login$/.test(command.action || "") || ["provider_connect", "provider_disconnect"].includes(command.action)) {
|
|
1668
|
+
} else if (/^(codex|claude)_login$/.test(command.action || "") || ["provider_repair", "provider_connect", "provider_disconnect"].includes(command.action)) {
|
|
1602
1669
|
// Opening an auth terminal must not wait for an unrelated provider turn.
|
|
1603
1670
|
// Per-provider busy guards prevent overlapping connection changes.
|
|
1604
1671
|
void execute();
|
|
@@ -101,11 +101,11 @@ export async function auditAttempt(charter, attempt, { previousAttempt = null, s
|
|
|
101
101
|
target: check.id,
|
|
102
102
|
explanation: check.detail,
|
|
103
103
|
repairability:
|
|
104
|
-
check.id === "executor_succeeded" && attempt.raw_output.
|
|
104
|
+
check.id === "executor_succeeded" && attempt.raw_output?.blocked && attempt.raw_output?.blocker_kind === "human"
|
|
105
105
|
? "manual"
|
|
106
106
|
: check.repairability,
|
|
107
107
|
auto_fix_hint:
|
|
108
|
-
check.id === "executor_succeeded" && attempt.raw_output.
|
|
108
|
+
check.id === "executor_succeeded" && attempt.raw_output?.blocked && attempt.raw_output?.blocker_kind === "human"
|
|
109
109
|
? null
|
|
110
110
|
: `Resolve ${check.label.toLowerCase()} and check the result.`,
|
|
111
111
|
}));
|
|
@@ -12,7 +12,7 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
12
12
|
return buildWorkflowReportingInstruction({ auditor: true }) + "\n" + buildAcceptedHumanGuidanceInstruction(charter) + "\n" + section("requested_review", `Review the original request and actual behavior read-only. Report concrete defects, regressions and missing user outcomes. No preparation matrix, citation package, assurance score or mandatory screenshot record is required. Do not certify unobserved results or grant permission.
|
|
13
13
|
Request: ${JSON.stringify({ requirement: charter.requirement, criteria: acceptanceCriterionRows(charter), mustNot: charter.must_not, experience: charter.completion_contract?.experience })}
|
|
14
14
|
Current result: ${JSON.stringify({ summary: attempt.summary ?? null, residual_risks: attempt.residual_risks, tests: attempt.tests_run, observations: attempt.criterion_evidence, current: attempt.observed_evidence })}
|
|
15
|
-
Return SIMY_AUDIT_JSON: with {passed,summary,findings,criterion_results,coverage_review:{user_outcome:{passed,verification_method,observed_result}}}. Check the original requested result even if the listed criteria mention only code or tests. Each criterion result describes criterion_id,passed,verification_method,observed_result. Each finding candidate must contain passed. Only passed=false entries are findings; never report passed=true evidence as a finding. Findings describe the concrete remaining defect and necessary repair.`);
|
|
15
|
+
Return SIMY_AUDIT_JSON: with {passed,summary,findings,criterion_results,coverage_review:{user_outcome:{passed,verification_method,observed_result}}}. Check the original requested result even if the listed criteria mention only code or tests. Each criterion result describes criterion_id,passed,verification_method,observed_result. Each finding candidate must contain passed. Only passed=false entries are findings; never report passed=true evidence as a finding. Findings describe the concrete remaining defect and necessary repair. Set blocker_kind to machine for technical failures, external for unavailable dependencies, and human only for actual missing authorization, authentication interaction or an explicitly user-reserved decision. A legacy repairability=manual label or missing report does not establish a human dependency.`);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
export function buildIndependentAudit(execution, charter = null, attempt = null) {
|
|
@@ -60,11 +60,11 @@ export function buildIndependentAudit(execution, charter = null, attempt = null)
|
|
|
60
60
|
severity: cleanString(item.severity).toLowerCase() || "major",
|
|
61
61
|
target: cleanString(item.target) || "pull_request",
|
|
62
62
|
explanation: cleanString(item.explanation) || "Independent audit finding.",
|
|
63
|
-
repairability: item.
|
|
63
|
+
repairability: item.blocker_kind === "human" ? "manual" : "auto",
|
|
64
64
|
disposition: ["defer", "dismiss"].includes(item.disposition) && cleanString(item.explanation)
|
|
65
65
|
? item.disposition : "fix",
|
|
66
66
|
blocker_kind: ["machine", "human", "external"].includes(item.blocker_kind)
|
|
67
|
-
? item.blocker_kind :
|
|
67
|
+
? item.blocker_kind : "machine",
|
|
68
68
|
auto_fix_hint: cleanString(item.auto_fix_hint) || null,
|
|
69
69
|
}));
|
|
70
70
|
const findings = evaluations.filter((item) => item.passed === false);
|
|
@@ -13,6 +13,7 @@ export function buildCodingInstruction(
|
|
|
13
13
|
},
|
|
14
14
|
) {
|
|
15
15
|
return [
|
|
16
|
+
section("automatic_recovery", "Continue authorized work without procedural sign-off. Repair technical failures automatically. For an actual blocker, report blocker_kind=machine for technical repair, external for an unavailable dependency, or human for missing authorization, authentication interaction or an explicitly reserved decision. Preserve native permissions and explicit stops; blocked=true alone does not require human input."),
|
|
16
17
|
section("role", "Complete the user's requested outcome in this repository."),
|
|
17
18
|
buildWorkflowReportingInstruction(),
|
|
18
19
|
buildAcceptedHumanGuidanceInstruction(charter),
|
package/src/orchestrator/loop.js
CHANGED
|
@@ -113,7 +113,7 @@ async function runCodingLoopOwned({
|
|
|
113
113
|
previousAttempt,
|
|
114
114
|
previousFindings,
|
|
115
115
|
promptInterventions,
|
|
116
|
-
});
|
|
116
|
+
}) + (snapshot.metadata?.automatic_recovery ? '\nAutomatic recovery of the same unfinished task: inspect the current failure and previous attempts, test a different cause or reversible repair, then return to the original requested work. Do not repeat an unchanged failing operation or ask the user to certify technical correctness. Preserve existing permissions and limits.' : '');
|
|
117
117
|
|
|
118
118
|
let attempt = recoveredAttempt;
|
|
119
119
|
if (!attempt) {
|
|
@@ -574,7 +574,18 @@ async function handleFailedAudit({
|
|
|
574
574
|
}) {
|
|
575
575
|
snapshot.charter.thread_state.failure_count += 1;
|
|
576
576
|
const diagnosticCircuit = evaluateRetryCircuit({ attempt, previousAttempt: snapshot.attempts.at(-2) ?? null, findings });
|
|
577
|
-
|
|
577
|
+
// A new commit does not resolve an unchanged review failure. Criteria can all
|
|
578
|
+
// pass while a required review finding still prevents acceptance. Compare
|
|
579
|
+
// concrete details: a shared generic code does not identify the same defect.
|
|
580
|
+
const priorAttempt = snapshot.attempts.at(-2);
|
|
581
|
+
const repeatedReview = !attempt.outcome_progress?.has_progress &&
|
|
582
|
+
diagnosticCircuit.previous_fingerprint?.review_findings_sha256 === diagnosticCircuit.fingerprint.review_findings_sha256 &&
|
|
583
|
+
priorAttempt?.outcome_progress?.objective_digest === attempt.outcome_progress?.objective_digest;
|
|
584
|
+
attempt.audit_retry_streak = repeatedReview ? (priorAttempt.audit_retry_streak || 1) + 1 : 1;
|
|
585
|
+
const reviewStalled = !requiresHuman && attempt.audit?.passed === true && attempt.independent_audit?.passed === false &&
|
|
586
|
+
attempt.audit_retry_streak >= 3 && attempt.outcome_progress?.status === "assessed" &&
|
|
587
|
+
!attempt.outcome_progress?.stalled && !attempt.outcome_progress?.assessment_unavailable;
|
|
588
|
+
const technicalStall = reviewStalled || attempt.outcome_progress?.stalled || attempt.outcome_progress?.assessment_unavailable || diagnosticCircuit.open && !attempt.outcome_progress?.has_progress;
|
|
578
589
|
if (technicalStall && !requiresHuman) {
|
|
579
590
|
const state = snapshot.charter.thread_state;
|
|
580
591
|
const key = JSON.stringify([state.objective_contract?.current_digest, attempt.outcome_progress?.unproven_ids || [], findings.filter(row => row.code !== 'AI_STRATEGY_CHANGE_REQUIRED').map(row => row.code).sort()]);
|
|
@@ -590,18 +601,18 @@ async function handleFailedAudit({
|
|
|
590
601
|
return false;
|
|
591
602
|
}
|
|
592
603
|
}
|
|
593
|
-
if (attempt.outcome_progress?.stalled) {
|
|
594
|
-
const unresolved = attempt.outcome_progress
|
|
604
|
+
if (reviewStalled || attempt.outcome_progress?.stalled) {
|
|
605
|
+
const unresolved = attempt.outcome_progress?.unproven_ids || [];
|
|
595
606
|
snapshot.final_audit = {
|
|
596
607
|
...(snapshot.final_audit || {}),
|
|
597
608
|
passed: false,
|
|
598
|
-
summary: "Independent review found no progress on the remaining user outcomes.",
|
|
609
|
+
summary: reviewStalled ? "Automatic diagnosis did not resolve the repeated review failure." : "Independent review found no progress on the remaining user outcomes.",
|
|
599
610
|
findings: [...(snapshot.final_audit?.findings || []), {
|
|
600
611
|
passed: false,
|
|
601
|
-
code: "OUTCOME_PROGRESS_STALLED",
|
|
612
|
+
code: reviewStalled ? "REVIEW_PROGRESS_STALLED" : "OUTCOME_PROGRESS_STALLED",
|
|
602
613
|
severity: "blocker",
|
|
603
614
|
target: "acceptance_criteria",
|
|
604
|
-
explanation: `The same user outcomes remain unproven after another attempt: ${unresolved.join(", ")}. Reassess the approach against the approved objective before more work.`,
|
|
615
|
+
explanation: reviewStalled ? "The same review findings remain after automatic diagnosis and another repair attempt. Preserve the failure for machine recovery; human certification cannot resolve it." : `The same user outcomes remain unproven after another attempt: ${unresolved.join(", ")}. Reassess the approach against the approved objective before more work.`,
|
|
605
616
|
repairability: "auto",
|
|
606
617
|
blocker_kind: "machine",
|
|
607
618
|
}],
|
|
@@ -618,11 +629,7 @@ async function handleFailedAudit({
|
|
|
618
629
|
await publish(snapshot, "blocked", onUpdate);
|
|
619
630
|
return true;
|
|
620
631
|
}
|
|
621
|
-
const retryCircuit =
|
|
622
|
-
attempt,
|
|
623
|
-
previousAttempt: snapshot.attempts.at(-2) ?? null,
|
|
624
|
-
findings,
|
|
625
|
-
});
|
|
632
|
+
const retryCircuit = diagnosticCircuit;
|
|
626
633
|
if (retryCircuit.open && !attempt.outcome_progress?.has_progress &&
|
|
627
634
|
!(attempt.audit?.passed && attempt.outcome_progress?.status === "insufficient_evidence" &&
|
|
628
635
|
!attempt.outcome_progress?.failed_ids?.length)) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { suggestedProviderTokenBudget, MAX_PROVIDER_TOKEN_BUDGET } from "./budget.js";
|
|
1
|
+
import { suggestedProviderTokenBudget, MAX_PROVIDER_TOKEN_BUDGET, budgetState } from "./budget.js";
|
|
2
2
|
|
|
3
3
|
const RECOVERABLE_STATES = new Set(["waiting_human", "blocked", "failed"]);
|
|
4
4
|
|
|
@@ -8,13 +8,17 @@ export function recoveryContractForEvent({ snapshot, state, message, detail = {}
|
|
|
8
8
|
|
|
9
9
|
const reasonCode = recoveryReasonCode(state, message, detail);
|
|
10
10
|
const attemptNumber = positiveInteger(detail.attempt_number);
|
|
11
|
-
const
|
|
11
|
+
const machineStopped = state === "blocked" && snapshot?.final_audit?.requires_human === false &&
|
|
12
|
+
["outcome_progress_stalled", "retry_circuit_open"].includes(detail.code);
|
|
13
|
+
const actions = machineStopped ? [] : recoveryActions(reasonCode, detail.budget);
|
|
14
|
+
const automaticRecovery = machineStopped && !budgetState(snapshot).exhausted;
|
|
12
15
|
return {
|
|
13
16
|
schema_version: 1,
|
|
14
17
|
state,
|
|
15
|
-
reason_code: reasonCode,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
reason_code: automaticRecovery ? 'machine_recovery_pending' : machineStopped ? "machine_recovery_exhausted" : reasonCode,
|
|
19
|
+
...(machineStopped ? { requires_user_action: false } : {}),
|
|
20
|
+
user_message: automaticRecovery ? 'The task remains unfinished. SIMY will retry this same task after a recovery cooldown while its connection is enabled. No manual confirmation is required.' : machineStopped ? "Automatic diagnosis and repair attempts could not resolve the technical failure. The task remains unfinished; failure details are retained. No human approval or success declaration is required." : isSafetyCeiling(reasonCode) ? safetyCeilingRecoveryMessage(reasonCode, detail.budget) : recoveryMessage(reasonCode),
|
|
21
|
+
primary_action_id: actions[0]?.id ?? null,
|
|
18
22
|
actions,
|
|
19
23
|
technical_details: {
|
|
20
24
|
event_code: clean(detail.code) || null,
|
|
@@ -104,9 +108,10 @@ export function isRecoveryContract(value) {
|
|
|
104
108
|
value.schema_version === 1 &&
|
|
105
109
|
typeof value.reason_code === "string" &&
|
|
106
110
|
typeof value.user_message === "string" &&
|
|
107
|
-
typeof value.primary_action_id === "string" &&
|
|
108
111
|
Array.isArray(value.actions) &&
|
|
109
|
-
value.actions.length > 0
|
|
112
|
+
(typeof value.primary_action_id === "string" && value.actions.length > 0 ||
|
|
113
|
+
value.state === "blocked" && ['machine_recovery_pending', 'machine_recovery_exhausted'].includes(value.reason_code) &&
|
|
114
|
+
value.requires_user_action === false && value.primary_action_id === null && value.actions.length === 0),
|
|
110
115
|
);
|
|
111
116
|
}
|
|
112
117
|
|
|
@@ -39,6 +39,7 @@ export function buildRetryFingerprint(attempt, findings) {
|
|
|
39
39
|
commit_sha: commitSha,
|
|
40
40
|
finding_codes: normalizedFindings.map((finding) => finding.code),
|
|
41
41
|
findings_sha256: sha256(findingPayload),
|
|
42
|
+
review_findings_sha256: sha256(stableJson(normalizeFindings(findings, true))),
|
|
42
43
|
evidence_sha256: sha256(evidencePayload),
|
|
43
44
|
};
|
|
44
45
|
}
|
|
@@ -52,7 +53,7 @@ function findingsForAttempt(attempt) {
|
|
|
52
53
|
];
|
|
53
54
|
}
|
|
54
55
|
|
|
55
|
-
function normalizeFindings(findings) {
|
|
56
|
+
function normalizeFindings(findings, includeDetails = false) {
|
|
56
57
|
return (Array.isArray(findings) ? findings : [])
|
|
57
58
|
.filter((finding) => finding && typeof finding === "object")
|
|
58
59
|
.map((finding) => ({
|
|
@@ -62,6 +63,11 @@ function normalizeFindings(findings) {
|
|
|
62
63
|
severity: cleanString(finding.severity).toLowerCase() || null,
|
|
63
64
|
target: cleanString(finding.target) || null,
|
|
64
65
|
repairability: cleanString(finding.repairability).toLowerCase() || null,
|
|
66
|
+
...(includeDetails ? {
|
|
67
|
+
explanation: cleanString(finding.explanation) || null,
|
|
68
|
+
auto_fix_hint: cleanString(finding.auto_fix_hint) || null,
|
|
69
|
+
blocker_kind: cleanString(finding.blocker_kind) || null,
|
|
70
|
+
} : {}),
|
|
65
71
|
}))
|
|
66
72
|
.sort(compareStableValues);
|
|
67
73
|
}
|
package/src/provider-links.js
CHANGED
|
@@ -14,12 +14,13 @@ async function readSettings(root) {
|
|
|
14
14
|
let payload;
|
|
15
15
|
try { payload = JSON.parse(await readFile(join(root, 'provider-links.json'), 'utf8')); }
|
|
16
16
|
catch (error) { if (error.code === 'ENOENT') return defaults(); throw error; }
|
|
17
|
-
if (payload?.schemaVersion !== 1
|
|
17
|
+
if (payload?.schemaVersion !== 1) throw new Error('Unsupported SIMY provider connection settings version.');
|
|
18
|
+
if (!payload.providers || PROVIDER_LINKS.some(provider => {
|
|
18
19
|
const value = payload.providers[provider];
|
|
19
20
|
return !value || typeof value.connected !== 'boolean' || typeof value.requiresConsent !== 'boolean' ||
|
|
20
21
|
value.epoch !== null && (typeof value.epoch !== 'string' || !value.epoch || value.epoch.length > 128) ||
|
|
21
22
|
value.consentEpoch !== undefined && (value.consentEpoch !== value.epoch || !value.consentEpoch || value.connected !== false || typeof value.enabled !== 'boolean');
|
|
22
|
-
})) throw new Error('Invalid SIMY provider connection settings.');
|
|
23
|
+
})) throw Object.assign(new Error('Invalid SIMY provider connection settings.'), { code: 'invalid_provider_link_settings' });
|
|
23
24
|
return Object.fromEntries(PROVIDER_LINKS.map(provider => [provider, {
|
|
24
25
|
connected: payload.providers[provider].consentEpoch ? payload.providers[provider].enabled : payload.providers[provider].connected,
|
|
25
26
|
...(payload.providers[provider].consentEpoch ? { consentEpoch: payload.providers[provider].consentEpoch } : {}),
|
|
@@ -52,13 +53,22 @@ export async function requireProviderLink({ root = join(defaultRoot(), 'delivery
|
|
|
52
53
|
return link;
|
|
53
54
|
}
|
|
54
55
|
|
|
55
|
-
export function writeProviderLink(root = defaultRoot(), provider, connected) {
|
|
56
|
+
export function writeProviderLink(root = defaultRoot(), provider, connected, { repairInvalidSettings = false } = {}) {
|
|
56
57
|
validateProvider(provider);
|
|
57
58
|
if (typeof connected !== 'boolean') throw new Error('Choose whether this provider is connected to SIMY.');
|
|
58
59
|
const key = resolve(root);
|
|
59
60
|
const previous = writeQueues.get(key) || Promise.resolve();
|
|
60
61
|
const pending = previous.catch(() => {}).then(async () => {
|
|
61
|
-
|
|
62
|
+
let providers;
|
|
63
|
+
try { providers = await readSettings(root); }
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (!repairInvalidSettings || !connected || !(error instanceof SyntaxError || error.code === 'invalid_provider_link_settings')) throw error;
|
|
66
|
+
// Only an explicit Connect action restores this provider. Retain the
|
|
67
|
+
// damaged file and leave all other providers disconnected.
|
|
68
|
+
const original = await readFile(join(root, 'provider-links.json'));
|
|
69
|
+
await writeFile(join(root, `provider-links.invalid-${randomUUID()}.json`), original, { mode: 0o600, flag: 'wx' });
|
|
70
|
+
providers = disconnected();
|
|
71
|
+
}
|
|
62
72
|
// Reconnection restores discovery only. Session execution consent is cleared
|
|
63
73
|
// separately and must be selected again, including after a host restart.
|
|
64
74
|
const epoch = randomUUID();
|
package/src/runner.js
CHANGED
|
@@ -59,6 +59,7 @@ import {
|
|
|
59
59
|
assertRecoveryActor, completedWorkerResult, recoveryActorBinding, refreshRecoveryAudit, refreshRecoveryBudgetAudit,
|
|
60
60
|
} from "./coding-run-recovery.js";
|
|
61
61
|
import { workflowRunEnv } from "./dev-work-events.js";
|
|
62
|
+
import { canRecoverCodingRun, codingRecoveryDue, scheduleCodingRecovery } from './coding-run-automatic-recovery.js';
|
|
62
63
|
|
|
63
64
|
const execFileAsync = promisify(execFile);
|
|
64
65
|
const MAX_LOCAL_LOG_LINES = 1_000;
|
|
@@ -321,6 +322,7 @@ export function restoreRun({ snapshot, session, apiOrigin, localPath = null, loc
|
|
|
321
322
|
run.startedAt = restoredSnapshot.created_at || null;
|
|
322
323
|
run.completedAt = restoredSnapshot.updated_at || null;
|
|
323
324
|
run.controlState = restoredState === "stopped" ? "stopped" : "waiting_human";
|
|
325
|
+
if (canRecoverCodingRun(run)) run.controlState = 'complete';
|
|
324
326
|
run.restoredFromState = restoredFromState;
|
|
325
327
|
run.restoredObjectiveUnverified = objectiveUnverified;
|
|
326
328
|
run.restoredCompletedWorkerResult = Boolean(recoverableResult);
|
|
@@ -430,6 +432,23 @@ export async function continueLocalCodingRun(run, guidance, dependencies = {}) {
|
|
|
430
432
|
return executeLocalCodingRun(run, dependencies, { humanGuidance: message, resume: true });
|
|
431
433
|
}
|
|
432
434
|
|
|
435
|
+
// Automated recovery is not human guidance and does not reset stop intent,
|
|
436
|
+
// usage, the objective, or the existing task identity.
|
|
437
|
+
export async function recoverLocalCodingRun(run, dependencies = {}, now = Date.now()) {
|
|
438
|
+
if (!canRecoverCodingRun(run)) return run.snapshot;
|
|
439
|
+
if (!run.snapshot.metadata?.automatic_recovery) {
|
|
440
|
+
scheduleCodingRecovery(run.snapshot, now);
|
|
441
|
+
if (run.localStateRoot) await saveCodingRunSnapshot(run.localStateRoot, run.snapshot);
|
|
442
|
+
return run.snapshot;
|
|
443
|
+
}
|
|
444
|
+
if (!codingRecoveryDue(run, now)) return run.snapshot;
|
|
445
|
+
assertRunCanContinue(run);
|
|
446
|
+
run.completedAt = null;
|
|
447
|
+
run.controlState = 'running';
|
|
448
|
+
run.resumeCount += 1;
|
|
449
|
+
return executeLocalCodingRun(run, dependencies, { resume: true });
|
|
450
|
+
}
|
|
451
|
+
|
|
433
452
|
function assertRunCanContinue(run) {
|
|
434
453
|
if (run.operation || run.child) throw new Error("The selected Agentic Loop run is still active.");
|
|
435
454
|
if (!RESUMABLE_STATES.has(run.status)) {
|
|
@@ -1324,6 +1343,11 @@ async function updateRun(run, state) {
|
|
|
1324
1343
|
run.snapshot.state = state;
|
|
1325
1344
|
run.snapshot.metadata = { ...run.snapshot.metadata, pr_lifecycle_state: state };
|
|
1326
1345
|
run.snapshot.updated_at = new Date().toISOString();
|
|
1346
|
+
if (state === 'blocked' && run.snapshot.final_audit?.requires_human === false &&
|
|
1347
|
+
['outcome_progress_stalled', 'retry_circuit_open'].includes(run.snapshot.events.at(-1)?.detail?.code) &&
|
|
1348
|
+
!budgetState(run.snapshot).exhausted) {
|
|
1349
|
+
scheduleCodingRecovery(run.snapshot);
|
|
1350
|
+
}
|
|
1327
1351
|
run.completedAt = TERMINAL_STATES.has(state) ? run.snapshot.updated_at : null;
|
|
1328
1352
|
if (state === "waiting_human" || state === "pr_ready_for_review") {
|
|
1329
1353
|
run.controlState = "waiting_human";
|
package/src/sqm/agent-skill.js
CHANGED
|
@@ -100,14 +100,16 @@ export async function inspectSqmAgentSkill({ home = homedir() } = {}) {
|
|
|
100
100
|
};
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
-
export async function installSqmAgentSkill({ home = homedir(), platforms = Object.keys(TARGETS) } = {}) {
|
|
103
|
+
export async function installSqmAgentSkill({ home = homedir(), platforms = Object.keys(TARGETS), migrateLegacyGate = true } = {}) {
|
|
104
104
|
const requested = [...new Set(platforms)];
|
|
105
105
|
if (!requested.length || requested.some((platform) => !Object.hasOwn(TARGETS, platform))) {
|
|
106
106
|
throw new Error("platforms must contain codex and/or claude");
|
|
107
107
|
}
|
|
108
108
|
if (requested.includes("codex")) await installCodexPlugin({ home });
|
|
109
109
|
if (requested.includes("claude")) await installClaudePlugin({ home });
|
|
110
|
-
|
|
110
|
+
// Explicit single-provider repair must not migrate another provider's hooks.
|
|
111
|
+
// Ordinary setup and workflow updates retain the existing migration behavior.
|
|
112
|
+
if (migrateLegacyGate) await retireLegacySqmGate({ home });
|
|
111
113
|
for (const platform of requested) {
|
|
112
114
|
await atomicText(path.join(home, ...TARGETS[platform]), SKILL_CONTENT, 0o600);
|
|
113
115
|
}
|
|
@@ -6,6 +6,10 @@ import { CLI_VERSION } from "../cli-contract.js";
|
|
|
6
6
|
import { defaultSessionRoot } from "../session-store.js";
|
|
7
7
|
|
|
8
8
|
export const SQM_FAILURE_PAUSE_THRESHOLD = 6;
|
|
9
|
+
export const SQM_RECOVERY_DELAY_MS = 5 * 60_000;
|
|
10
|
+
const MAX_RECOVERY_DELAY_MS = 30 * 60_000;
|
|
11
|
+
const exhausted = state => state.failures >= SQM_FAILURE_PAUSE_THRESHOLD || (state.providerFailures || 0) >= 2;
|
|
12
|
+
const recoveryDelay = state => Math.min(MAX_RECOVERY_DELAY_MS, SQM_RECOVERY_DELAY_MS * 2 ** Math.min(state.recoveryAttempts || 0, 3));
|
|
9
13
|
const scopeKey = scope => JSON.stringify([scope?.webOrigin, scope?.accountId, scope?.organizationId]);
|
|
10
14
|
const validVersion = value => typeof value === "string" && /^\d+\.\d+\.\d+$/.test(value);
|
|
11
15
|
|
|
@@ -21,6 +25,9 @@ export function createSqmFailureCircuitStore({ root = defaultSessionRoot() } = {
|
|
|
21
25
|
const state = JSON.parse(await readFile(path(scope), "utf8"));
|
|
22
26
|
if (!validVersion(state.version) || !Number.isInteger(state.failures) || state.failures < 0 || state.failures > SQM_FAILURE_PAUSE_THRESHOLD || (state.inFlight !== undefined && typeof state.inFlight !== "boolean") || (state.providerFailures !== undefined && (!Number.isInteger(state.providerFailures) || state.providerFailures < 0 || state.providerFailures > 2)) || (state.providerRetryAt !== undefined && (!Number.isFinite(state.providerRetryAt) || state.providerRetryAt < 0)))
|
|
23
27
|
throw new Error("Invalid saved SQM failure state.");
|
|
28
|
+
if (state.recoveryAt !== undefined && (!Number.isFinite(state.recoveryAt) || state.recoveryAt < 0) ||
|
|
29
|
+
state.recoveryAttempts !== undefined && (!Number.isSafeInteger(state.recoveryAttempts) || state.recoveryAttempts < 0))
|
|
30
|
+
throw new Error("Invalid saved SQM recovery state.");
|
|
24
31
|
return state;
|
|
25
32
|
} catch (error) { if (error.code === "ENOENT") return null; throw error; }
|
|
26
33
|
},
|
|
@@ -37,12 +44,17 @@ export function createSqmFailureCircuitStore({ root = defaultSessionRoot() } = {
|
|
|
37
44
|
|
|
38
45
|
// One counter per organization on this machine, shared by both local engines.
|
|
39
46
|
// Neither reconnecting nor an empty claim demonstrates successful analysis.
|
|
40
|
-
export function createSqmFailureCircuit({ version = CLI_VERSION, store = null } = {}) {
|
|
47
|
+
export function createSqmFailureCircuit({ version = CLI_VERSION, store = null, now = Date.now } = {}) {
|
|
41
48
|
if (!validVersion(version)) throw new Error("SQM failure circuit requires a SIMY version.");
|
|
42
49
|
const states = new Map();
|
|
43
50
|
const unreadableScopes = new Set();
|
|
51
|
+
const scheduleRecovery = state => {
|
|
52
|
+
if (!exhausted(state) || state.recoveryAt > now()) return false;
|
|
53
|
+
state.recoveryAt = Math.max(state.providerRetryAt || 0, now() + recoveryDelay(state));
|
|
54
|
+
return true;
|
|
55
|
+
};
|
|
44
56
|
const save = async (scope, state) => {
|
|
45
|
-
try { await store?.write(scope, { version: state.version, failures: state.failures, inFlight: state.inFlight === true, ...(state.providerFailures ? { providerFailures: state.providerFailures, providerRetryAt: state.providerRetryAt } : {}) }); state.storageError = false; }
|
|
57
|
+
try { await store?.write(scope, { version: state.version, failures: state.failures, inFlight: state.inFlight === true, ...(state.recoveryAt !== undefined ? { recoveryAt: state.recoveryAt, recoveryAttempts: state.recoveryAttempts || 0 } : {}), ...(state.providerFailures ? { providerFailures: state.providerFailures, providerRetryAt: state.providerRetryAt } : {}) }); state.storageError = false; }
|
|
46
58
|
catch (error) { state.storageError = true; throw error; }
|
|
47
59
|
};
|
|
48
60
|
const load = async scope => {
|
|
@@ -52,7 +64,7 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
52
64
|
let saved;
|
|
53
65
|
try { saved = await store?.read(scope); unreadableScopes.delete(key); }
|
|
54
66
|
catch (error) { unreadableScopes.add(key); throw error; }
|
|
55
|
-
state = saved ? { version: saved.version, failures: saved.failures, inFlight: saved.inFlight === true, providerFailures: saved.providerFailures || 0, providerRetryAt: saved.providerRetryAt || 0 } : { version, failures: 0, inFlight: false };
|
|
67
|
+
state = saved ? { version: saved.version, failures: saved.failures, inFlight: saved.inFlight === true, providerFailures: saved.providerFailures || 0, providerRetryAt: saved.providerRetryAt || 0, ...(saved.recoveryAt !== undefined ? { recoveryAt: saved.recoveryAt, recoveryAttempts: saved.recoveryAttempts || 0 } : {}) } : { version, failures: 0, inFlight: false };
|
|
56
68
|
states.set(key, state);
|
|
57
69
|
// An unfinished review from this same version means the last host exited
|
|
58
70
|
// without a result or graceful cancellation. Count it once on restart.
|
|
@@ -60,6 +72,7 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
60
72
|
state.failures = Math.min(SQM_FAILURE_PAUSE_THRESHOLD, state.failures + 1);
|
|
61
73
|
if (state.providerFailures === 1) state.providerFailures = 2;
|
|
62
74
|
state.inFlight = false;
|
|
75
|
+
scheduleRecovery(state);
|
|
63
76
|
await save(scope, state);
|
|
64
77
|
}
|
|
65
78
|
}
|
|
@@ -67,20 +80,35 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
67
80
|
state.version = version;
|
|
68
81
|
state.providerFailures = 0; state.providerRetryAt = 0;
|
|
69
82
|
state.failures = 0;
|
|
83
|
+
delete state.recoveryAt; delete state.recoveryAttempts;
|
|
70
84
|
state.inFlight = false;
|
|
71
85
|
await save(scope, state);
|
|
72
86
|
} else if (state.storageError) await save(scope, state);
|
|
87
|
+
// Migrate a legacy update-only pause once, retaining counters and scope.
|
|
88
|
+
if (exhausted(state) && state.recoveryAt === undefined) {
|
|
89
|
+
scheduleRecovery(state); await save(scope, state);
|
|
90
|
+
}
|
|
73
91
|
return state;
|
|
74
92
|
};
|
|
75
93
|
return {
|
|
76
|
-
async canClaim(scope,
|
|
94
|
+
async canClaim(scope, time = now()) {
|
|
77
95
|
const state = await load(scope);
|
|
78
|
-
|
|
96
|
+
if (time < (state.providerRetryAt || 0) || state.inFlight) return false;
|
|
97
|
+
if (!exhausted(state)) return true;
|
|
98
|
+
if (time < state.recoveryAt) return false;
|
|
99
|
+
// Reserve the next cooldown before claiming. Empty claims, reconnects and
|
|
100
|
+
// cancelled work never reset failures or allow a burst of recovery turns.
|
|
101
|
+
state.recoveryAttempts = Math.min((state.recoveryAttempts || 0) + 1, 3);
|
|
102
|
+
state.recoveryAt = time + recoveryDelay(state);
|
|
103
|
+
await save(scope, state);
|
|
104
|
+
return true;
|
|
79
105
|
},
|
|
80
106
|
async providerFailure(scope, retryAt, retryAttempt = 0) {
|
|
81
107
|
const state = await load(scope);
|
|
82
108
|
state.providerFailures = retryAttempt === 1 ? 2 : Math.min(2, (state.providerFailures || 0) + 1);
|
|
83
109
|
state.providerRetryAt = retryAt;
|
|
110
|
+
scheduleRecovery(state);
|
|
111
|
+
if (state.recoveryAt !== undefined) state.recoveryAt = Math.max(state.recoveryAt, retryAt);
|
|
84
112
|
await save(scope, state);
|
|
85
113
|
},
|
|
86
114
|
async providerRetryUsed(scope) {
|
|
@@ -104,12 +132,14 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
104
132
|
state.failures = Math.min(SQM_FAILURE_PAUSE_THRESHOLD, state.failures + 1);
|
|
105
133
|
if (state.inFlight && state.providerFailures === 1) state.providerFailures = 2;
|
|
106
134
|
state.inFlight = false;
|
|
135
|
+
scheduleRecovery(state);
|
|
107
136
|
await save(scope, state);
|
|
108
137
|
},
|
|
109
138
|
async success(scope) {
|
|
110
139
|
const state = await load(scope);
|
|
111
140
|
state.failures = 0;
|
|
112
141
|
state.providerFailures = 0; state.providerRetryAt = 0;
|
|
142
|
+
delete state.recoveryAt; delete state.recoveryAttempts;
|
|
113
143
|
state.inFlight = false;
|
|
114
144
|
await save(scope, state);
|
|
115
145
|
},
|
|
@@ -117,10 +147,12 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
117
147
|
const state = states.get(scopeKey(scope));
|
|
118
148
|
const paused = (state?.failures || 0) >= SQM_FAILURE_PAUSE_THRESHOLD || (state?.providerFailures || 0) >= 2;
|
|
119
149
|
return { consecutive_failures: state?.failures || 0,
|
|
120
|
-
paused_until_update:
|
|
150
|
+
paused_until_update: false,
|
|
151
|
+
recovery_pending: paused && state?.inFlight !== true,
|
|
152
|
+
next_retry_at: paused ? state.recoveryAt ?? null : null,
|
|
121
153
|
provider_retry_exhausted: (state?.providerFailures || 0) >= 2,
|
|
122
154
|
provider_recovery_pending: state?.providerFailures === 1,
|
|
123
|
-
paused_at_version:
|
|
155
|
+
paused_at_version: null,
|
|
124
156
|
storage_unavailable: state?.storageError === true || unreadableScopes.has(scopeKey(scope)) };
|
|
125
157
|
},
|
|
126
158
|
};
|
|
@@ -16,7 +16,7 @@ export function runProviderCommand(command, args, { env, cwd, timeout = 10_000,
|
|
|
16
16
|
};
|
|
17
17
|
const timer = setTimeout(() => {
|
|
18
18
|
child.kill("SIGTERM");
|
|
19
|
-
finish(new Error("Provider plugin command timed out."));
|
|
19
|
+
finish(Object.assign(new Error("Provider plugin command timed out."), { code: 'provider_timeout' }));
|
|
20
20
|
}, timeout);
|
|
21
21
|
timer.unref?.();
|
|
22
22
|
const capture = (stream, append) => {
|