@awak-app/simy-cli 0.7.22 → 0.7.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -17
- package/package.json +2 -2
- package/src/agent.js +28 -8
- package/src/cli-contract.js +1 -1
- package/src/codex-plugins/simy/hooks/delivery-loop.mjs +3 -1
- package/src/codex-plugins/simy/hooks/outcome-progress.mjs +1 -1
- package/src/codex-plugins/simy/hooks/update-dispatch-gate.mjs +89 -6
- package/src/coding-run-automatic-recovery.js +40 -0
- package/src/console/app.js +5 -0
- package/src/delivery-hooks/command.js +12 -0
- package/src/delivery-hooks/supervisor.js +14 -4
- package/src/desktop-access.js +1 -1
- package/src/desktop-clipboard.js +23 -0
- package/src/desktop-host.js +11 -3
- package/src/orchestrator/audit.js +2 -2
- package/src/orchestrator/independent-audit.js +3 -3
- package/src/orchestrator/instruction.js +1 -0
- package/src/orchestrator/loop.js +19 -12
- package/src/orchestrator/recovery.js +12 -7
- package/src/orchestrator/retry.js +7 -1
- package/src/provider-links.js +14 -4
- package/src/runner.js +24 -0
- package/src/sqm/failure-circuit.js +39 -7
- package/src/sqm/review-worker.js +30 -8
- package/src/update-detachment/history-recovery.js +51 -0
- package/src/update-detachment/runtime.js +3 -3
package/README.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# SIMY CLI
|
|
2
2
|
|
|
3
|
+
**Copy connection URL / 接続用URLをコピー** copies the current SIMY authentication
|
|
4
|
+
link, including its pending organization selection. It is available while waiting
|
|
5
|
+
for the browser and in settings. Paste it into Chrome if automatic opening fails.
|
|
6
|
+
**Copy workspace URL** continues to copy only the Web workspace address.
|
|
7
|
+
This signs the desktop into SIMY; browser automation uses its own provider connection.
|
|
8
|
+
|
|
9
|
+
Technical coding failures retain their task and retry after a persisted 5–30 minute
|
|
10
|
+
cooldown. Automatic recovery preserves the original goal, usage limits, provider
|
|
11
|
+
connection and explicit stops. Enabling automatic continuation can also adopt an
|
|
12
|
+
ordinary loop waiting for an external dependency, starting with read-only inspection.
|
|
13
|
+
A damaged provider-link file can be repaired by **Connect SIMY** for that provider;
|
|
14
|
+
the original file is retained and the other provider stays disconnected.
|
|
15
|
+
|
|
3
16
|
### Experimental delegated approval and input bridge
|
|
4
17
|
|
|
5
18
|
`simy delivery interactions proxy --policy-file /absolute/private/policy.json`
|
|
@@ -93,17 +106,15 @@ Both providers share one analysis deadline. A deterministic provider failure pau
|
|
|
93
106
|
new claims and retains failed history until environment/settings change or **Retry
|
|
94
107
|
analysis** is selected.
|
|
95
108
|
|
|
96
|
-
After **6 consecutive SQM analysis failures**, this machine
|
|
97
|
-
SQM
|
|
98
|
-
counter; a completed analysis resets it.
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
The native organization menu shows when an update is required. This pause does
|
|
106
|
-
not change server analysis consent or stop ordinary implementation tasks.
|
|
109
|
+
After **6 consecutive SQM analysis failures**, this machine waits before its next
|
|
110
|
+
SQM claim for the current Web/account/organization. Codex and Claude share the
|
|
111
|
+
counter; a completed analysis resets it. The cooldown increases from 5 to 30
|
|
112
|
+
minutes and survives restart. Empty polls and reconnects do not reset failures.
|
|
113
|
+
Retries resume automatically while analysis and the provider connection remain
|
|
114
|
+
enabled. Authentication requiring the account holder remains a sign-in wait.
|
|
115
|
+
The server's durable recovery migration requeues the same failed job and retains
|
|
116
|
+
its input, history and attempt count; it is required to recover jobs previously
|
|
117
|
+
held by the server's lifetime retry cap.
|
|
107
118
|
|
|
108
119
|
Agentic Loop preflight accepts the task's existing requirement, acceptance
|
|
109
120
|
criteria, required checks and charter context alongside the repository/provider.
|
|
@@ -1020,12 +1031,15 @@ records cannot be recovered.
|
|
|
1020
1031
|
History remains visible with local analysis off and after restarting. A saved
|
|
1021
1032
|
in-flight analysis is shown as Interrupted until this client claims it again;
|
|
1022
1033
|
it does not contribute to the running count. The web workspace owns retry status.
|
|
1023
|
-
Model/provider execution failures permit
|
|
1024
|
-
cooldown, even when the app/account identity is unchanged.
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1034
|
+
Model/provider execution failures permit a recovery claim after a five-minute
|
|
1035
|
+
cooldown, even when the app/account identity is unchanged. Repeated provider
|
|
1036
|
+
failures or six consecutive analysis failures schedule further automatic recovery
|
|
1037
|
+
with a persisted 5–30 minute backoff. Reconnects, restarts, empty claims and
|
|
1038
|
+
cancelled work do not reset the failure counters or cooldown. Only a successful
|
|
1039
|
+
review or a newer SIMY version clears the circuit. Authentication interaction,
|
|
1040
|
+
local installation, SIMY access and persistence requirements remain enforced;
|
|
1041
|
+
waiting does not grant permission. Turning analysis off or disconnecting its
|
|
1042
|
+
provider still prevents new work.
|
|
1029
1043
|
|
|
1030
1044
|
Only summary metadata is retained, without analysis prompts, source inputs,
|
|
1031
1045
|
model responses or credentials. Historical summaries are not a replacement for
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@awak-app/simy-cli",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.23",
|
|
4
4
|
"description": "Local SIMY executor for non-coding Local Tasks and Agentic Loop runs.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
],
|
|
13
13
|
"scripts": {
|
|
14
14
|
"start": "node ./src/index.js",
|
|
15
|
-
"test": "node --test --test-concurrency=2",
|
|
15
|
+
"test": "node --import ./test/isolate-simy-home.mjs --test --test-concurrency=2",
|
|
16
16
|
"test:e2e:console": "node ./scripts/console-e2e.js",
|
|
17
17
|
"test:e2e:work-activity": "node ./scripts/work-activity-e2e.js",
|
|
18
18
|
"test:e2e:sqm-complete-loop": "node ./scripts/sqm-complete-loop-e2e.js",
|
package/src/agent.js
CHANGED
|
@@ -18,6 +18,7 @@ import { createDesktopAccessPolicy, desktopAccessReady, requireDesktopAccess } f
|
|
|
18
18
|
import { readSqmEnabled, writeSqmEnabled, readSqmSettings, writeSqmEngineFallback } from "./desktop-sqm-settings.js";
|
|
19
19
|
import { createServer } from "node:http";
|
|
20
20
|
import { createHash, randomBytes } from "node:crypto";
|
|
21
|
+
import { canRecoverCodingRun, codingRecoveryDue, codingRecoveryLinksMatch } from './coding-run-automatic-recovery.js';
|
|
21
22
|
import { createReadStream } from "node:fs";
|
|
22
23
|
import { dirname, join, resolve } from "node:path";
|
|
23
24
|
|
|
@@ -35,6 +36,7 @@ import {
|
|
|
35
36
|
prepareLocalCodingRunRecovery,
|
|
36
37
|
reconcileLocalCodingRun,
|
|
37
38
|
recoverCompletedLocalCodingRun,
|
|
39
|
+
recoverLocalCodingRun,
|
|
38
40
|
queueLocalCodingGuidance,
|
|
39
41
|
recheckLocalCodingRun,
|
|
40
42
|
publishRestoredLocalCodingRun,
|
|
@@ -545,7 +547,7 @@ export async function startAgent({
|
|
|
545
547
|
return inspection;
|
|
546
548
|
};
|
|
547
549
|
|
|
548
|
-
const requireRunProvider = async (run, { backend = run.request.backend } = {}) => {
|
|
550
|
+
const requireRunProvider = async (run, { backend = run.request.backend, automatic = false } = {}) => {
|
|
549
551
|
const scope = await authorizeDesktopWork();
|
|
550
552
|
const inspection = await inspectReadyBackend(backend, { force: true });
|
|
551
553
|
await authorizeDesktopWork(scope);
|
|
@@ -555,18 +557,19 @@ export async function startAgent({
|
|
|
555
557
|
code: `desktop_executor_${inspection.status}`,
|
|
556
558
|
});
|
|
557
559
|
}
|
|
558
|
-
await bindRunAuthorization(run, scope, inspection, backend);
|
|
560
|
+
await bindRunAuthorization(run, scope, inspection, backend, !automatic);
|
|
559
561
|
};
|
|
560
562
|
|
|
561
563
|
const authorizeProviderDispatch = async (backend, scope, link) => {
|
|
562
564
|
await authorizeDesktopWork(scope);
|
|
563
565
|
await requireLinkedProvider(backend, link);
|
|
564
566
|
};
|
|
565
|
-
const bindRunAuthorization = async (run, scope, inspection, backend = run.request.backend) => {
|
|
567
|
+
const bindRunAuthorization = async (run, scope, inspection, backend = run.request.backend, rememberConnection = true) => {
|
|
566
568
|
const links = await readProviderLinks(sessionRoot);
|
|
567
569
|
links[backend] = inspectedLinks.get(inspection);
|
|
568
570
|
await authorizeProviderDispatch(backend, scope, links[backend]);
|
|
569
571
|
runAuthorizations.set(run, { scope, links });
|
|
572
|
+
if (rememberConnection) run.snapshot.metadata.provider_link_epochs = Object.fromEntries(Object.entries(links).map(([provider, link]) => [provider, link.epoch]));
|
|
570
573
|
};
|
|
571
574
|
const providerRuntimeOptions = (task, options = {}, scope, link) => {
|
|
572
575
|
const guard = () => authorizeProviderDispatch(task.backend, scope, link);
|
|
@@ -868,13 +871,29 @@ export async function startAgent({
|
|
|
868
871
|
}
|
|
869
872
|
};
|
|
870
873
|
|
|
874
|
+
const automaticCodingRecoveries = new WeakSet();
|
|
871
875
|
const monitorCodingRuns = () => {
|
|
872
876
|
if (!desktopWorkAllowed()) return Promise.resolve();
|
|
873
|
-
if (!
|
|
877
|
+
if (!isSessionValid(session, Date.now(), apiOrigin)) return Promise.resolve();
|
|
874
878
|
if (codingRunMonitorInFlight) return codingRunMonitorInFlight;
|
|
875
879
|
codingRunMonitorInFlight = (async () => {
|
|
876
880
|
for (const run of registry.list()) {
|
|
877
|
-
if (!
|
|
881
|
+
if (canRecoverCodingRun(run) && !automaticCodingRecoveries.has(run) && (!run.snapshot.metadata?.automatic_recovery || codingRecoveryDue(run))) {
|
|
882
|
+
automaticCodingRecoveries.add(run);
|
|
883
|
+
void withUpdateWork(updateManager, async () => {
|
|
884
|
+
if (!sameRunScope(run.session, session) || run.apiOrigin !== apiOrigin) return;
|
|
885
|
+
const saved = { charter: run.snapshot.charter, metadata: { ...run.snapshot.metadata } };
|
|
886
|
+
if (!codingRecoveryLinksMatch(saved, await readProviderLinks(sessionRoot))) return;
|
|
887
|
+
await requireRunProvider(run, { automatic: true });
|
|
888
|
+
if (!codingRecoveryLinksMatch(saved, await readProviderLinks(sessionRoot))) return;
|
|
889
|
+
// Each run owns its operation promise; a slow repair must not
|
|
890
|
+
// prevent the monitor from advancing independent tasks.
|
|
891
|
+
await recoverLocalCodingRun(run, codingRuntimeOptions(run));
|
|
892
|
+
}).catch(error => reportStartupConnectionError(error, quiet))
|
|
893
|
+
.finally(() => automaticCodingRecoveries.delete(run));
|
|
894
|
+
continue;
|
|
895
|
+
}
|
|
896
|
+
if (!daemon || !["pr_ready_for_review", "merge_ready"].includes(run.status) || run.operation || run.child) continue;
|
|
878
897
|
try {
|
|
879
898
|
await (dependencies.recheckCodingRun || recheckAuthorizedCodingRun)(run);
|
|
880
899
|
} catch (error) {
|
|
@@ -887,7 +906,7 @@ export async function startAgent({
|
|
|
887
906
|
|
|
888
907
|
const startCodingRunMonitor = () => {
|
|
889
908
|
if (!desktopWorkAllowed()) return;
|
|
890
|
-
if (
|
|
909
|
+
if (codingRunMonitorTimer) return;
|
|
891
910
|
codingRunMonitorTimer = setInterval(() => {
|
|
892
911
|
void monitorCodingRuns().catch((error) => reportStartupConnectionError(error, quiet));
|
|
893
912
|
}, dependencies.codingRunMonitorIntervalMs || CODING_RUN_MONITOR_INTERVAL_MS);
|
|
@@ -1140,6 +1159,7 @@ export async function startAgent({
|
|
|
1140
1159
|
getHistoryScope: () => sqmHistoryScope(apiOrigin, session),
|
|
1141
1160
|
getRecoveryKey: (provider) => providerRecoveryRevisions[provider] ?? 0,
|
|
1142
1161
|
recordHistory: (scope, item) => sqmHistory.record(scope, item),
|
|
1162
|
+
readHistory: scope => sqmHistory.items(scope),
|
|
1143
1163
|
onHistoryError: () => { if (!quiet) console.error("Could not save the local SQM analysis history."); },
|
|
1144
1164
|
getProvider: () => !desktopWorkAllowed() || availableCapabilities?.features?.sqm_local_review !== true ? null : ["codex", "claude"].find(
|
|
1145
1165
|
(provider) => providerLinks[provider]?.connected === true && availableCapabilities?.backends?.[provider] === true &&
|
|
@@ -2353,7 +2373,7 @@ export async function startAgent({
|
|
|
2353
2373
|
: [],
|
|
2354
2374
|
required_checks: Array.isArray(body.required_checks) ? body.required_checks : [],
|
|
2355
2375
|
independent_review_required: body.independent_review_required === true,
|
|
2356
|
-
require_human_approval: body.require_human_approval
|
|
2376
|
+
require_human_approval: body.require_human_approval === true,
|
|
2357
2377
|
design_summary:
|
|
2358
2378
|
typeof body.design_summary === "string" ? body.design_summary : "",
|
|
2359
2379
|
design_review_approved_by:
|
|
@@ -3049,7 +3069,7 @@ export async function startAgent({
|
|
|
3049
3069
|
expected_tests: [],
|
|
3050
3070
|
expected_evidence: [],
|
|
3051
3071
|
required_checks: [],
|
|
3052
|
-
require_human_approval: true,
|
|
3072
|
+
require_human_approval: input.require_human_approval === true,
|
|
3053
3073
|
must_not: [],
|
|
3054
3074
|
attachments: await referenceLocalAttachmentPaths(input.attachmentPaths || []),
|
|
3055
3075
|
};
|
package/src/cli-contract.js
CHANGED
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
} from "./durable-local-task-steps-contract.js";
|
|
15
15
|
import { buildExecutionCapabilityContract } from "./execution-capability-contract.js";
|
|
16
16
|
|
|
17
|
-
export const CLI_VERSION = "0.7.
|
|
17
|
+
export const CLI_VERSION = "0.7.23";
|
|
18
18
|
export const CLI_API_CONTRACT_VERSION = 10;
|
|
19
19
|
|
|
20
20
|
export function narrativeImageCapabilityMetadata() {
|
|
@@ -118,7 +118,9 @@ const text = (value, max = 4000) => typeof value === 'string' ? value.trim().sli
|
|
|
118
118
|
const exactText = value => typeof value === 'string' && Buffer.byteLength(value, 'utf8') <= 1000 ? value : null;
|
|
119
119
|
export function canUpgradeDelivery(run) {
|
|
120
120
|
return Boolean(run && !run.managed && text(run.id) && text(run.objective) && Number.isFinite(run.startedAt) &&
|
|
121
|
-
Number.isSafeInteger(run.maxFailures) && run.maxFailures > 0 &&
|
|
121
|
+
Number.isSafeInteger(run.maxFailures) && run.maxFailures > 0 &&
|
|
122
|
+
(['active', 'paused', 'recovery_required'].includes(run.state) ||
|
|
123
|
+
run.state === 'blocked' && run.blocker === 'external_dependency'));
|
|
122
124
|
}
|
|
123
125
|
const contextText = (value, max = 500) => text(value, max).replace(/\b(?:sk|key)-[A-Za-z0-9_-]{12,}\b|\bBearer\s+\S+|\b(?:password|token|api[_-]?key)\s*[:=]\s*\S+/gi, '[redacted]');
|
|
124
126
|
export function recordPath(root, provider, session) {
|
|
@@ -17,7 +17,7 @@ function classifyBlockers(findings) {
|
|
|
17
17
|
const kind = ["machine", "human", "external"].includes(finding.blocker_kind)
|
|
18
18
|
? finding.blocker_kind
|
|
19
19
|
: finding.repairability === "external" ? "external"
|
|
20
|
-
:
|
|
20
|
+
: "machine";
|
|
21
21
|
blockers[kind].push({
|
|
22
22
|
code: text(finding.code) || "OUTCOME_UNPROVEN",
|
|
23
23
|
target: text(finding.target) || null,
|
|
@@ -11,9 +11,10 @@ export const detachmentRoot = () => resolve(process.env.SIMY_HOME?.trim() || joi
|
|
|
11
11
|
export const rootFromDelivery = root => basename(resolve(root)) === 'delivery-hooks' ? dirname(resolve(root)) : resolve(root);
|
|
12
12
|
const path = root => join(resolve(root), 'update-detachment', 'gate.json');
|
|
13
13
|
function read(root) {
|
|
14
|
-
const state = readState(path(root), { versions: [1, 2] }) || { schemaVersion: 1, revision: 0, generation: 0, connection: 'connected', attempts: {}, receipts: {}, events: {} };
|
|
14
|
+
const state = readState(path(root), { versions: [1, 2, 3] }) || { schemaVersion: 1, revision: 0, generation: 0, connection: 'connected', attempts: {}, receipts: {}, events: {} };
|
|
15
15
|
if (!Number.isSafeInteger(state.generation) || !Number.isSafeInteger(state.revision) || !CONNECTION_STATES.includes(state.connection) || !['attempts', 'receipts', 'events'].every(key => isMap(state[key])) || (state.hookOwners !== undefined && !isMap(state.hookOwners)) || !Object.values(state.attempts).every(attempt => isMap(attempt) && typeof attempt.id === 'string' && Number.isSafeInteger(attempt.generation) && ['intent','sending','sent','provider_identified','result_saved','confirmed'].includes(attempt.phase))) fail('invalid_state');
|
|
16
|
-
if ((state.schemaVersion
|
|
16
|
+
if ((state.schemaVersion >= 2 && ![2, 3].includes(state.minimumDispatchProtocol)) || (state.minimumDispatchProtocol !== undefined && ![2, 3].includes(state.minimumDispatchProtocol))) fail('writer_protocol_incompatible');
|
|
17
|
+
validateRecoveryFences(state);
|
|
17
18
|
validateTaskFences(state, root);
|
|
18
19
|
return state;
|
|
19
20
|
}
|
|
@@ -31,7 +32,7 @@ export function transitionConnection({ root = detachmentRoot(), expectedRevision
|
|
|
31
32
|
if (state.revision !== expectedRevision) fail('revision_conflict');
|
|
32
33
|
const allowed = { connected: ['detaching'], ready: ['connected', 'detaching'], detaching: ['detached', 'recovery_required'], detached: ['reconciling'], reconciling: ['ready', 'recovery_required'], recovery_required: ['reconciling', 'detaching'] };
|
|
33
34
|
if (!allowed[state.connection]?.includes(connection)) fail('invalid_transition');
|
|
34
|
-
if (connection === 'detaching') { state.generation++; state.schemaVersion = 2; state.minimumDispatchProtocol = DISPATCH_PROTOCOL; }
|
|
35
|
+
if (connection === 'detaching') { state.generation++; state.schemaVersion = Math.max(2, state.schemaVersion); state.minimumDispatchProtocol = DISPATCH_PROTOCOL; }
|
|
35
36
|
state.connection = connection; state.revision++;
|
|
36
37
|
if (eventId) state.events[digest(eventId)] = { revision: state.revision };
|
|
37
38
|
atomicSave(path(root), state); return state;
|
|
@@ -221,7 +222,7 @@ export function prepareUpgradeFence({ root = detachmentRoot(), expectedRevision,
|
|
|
221
222
|
const state = read(root); validateWriters(state, writers);
|
|
222
223
|
if (state.revision !== expectedRevision) fail('revision_conflict');
|
|
223
224
|
if (!['connected', 'ready', 'recovery_required'].includes(state.connection)) fail('invalid_transition');
|
|
224
|
-
state.generation++; state.schemaVersion = 2; state.minimumDispatchProtocol = DISPATCH_PROTOCOL;
|
|
225
|
+
state.generation++; state.schemaVersion = Math.max(2, state.schemaVersion); state.minimumDispatchProtocol = DISPATCH_PROTOCOL;
|
|
225
226
|
state.connection = 'detaching'; state.revision++; atomicSave(path(root), state); return state;
|
|
226
227
|
});
|
|
227
228
|
}
|
|
@@ -252,6 +253,12 @@ function validateTaskFences(state, root) {
|
|
|
252
253
|
}
|
|
253
254
|
}
|
|
254
255
|
function assertTaskAdmissionState(state, { taskId, identity = null }) {
|
|
256
|
+
for (const item of Object.values(state.recoveryFences?.items || {})) {
|
|
257
|
+
if (item.taskHash === digest(taskId) || identity && (item.identityHash === digest(identity) ||
|
|
258
|
+
item.sessions.some(target => target.provider === identity.provider && target.session === identity.session))) {
|
|
259
|
+
fail('dispatch_history_quarantined');
|
|
260
|
+
}
|
|
261
|
+
}
|
|
255
262
|
const records = state.taskFences?.records || {};
|
|
256
263
|
if (!identity) {
|
|
257
264
|
if (Object.values(records).some(item => item.taskId === taskId)) fail('task_admission_identity_required');
|
|
@@ -276,7 +283,9 @@ export function readTaskAdmission({ root = detachmentRoot(), identity, taskId })
|
|
|
276
283
|
identity = storedFenceIdentity(identity, root);
|
|
277
284
|
if (identity.root !== realpathSync(root)) fail('identity_root_mismatch');
|
|
278
285
|
const state = read(root), record = state.taskFences?.records[fenceKey(identity, taskId)] || null;
|
|
279
|
-
|
|
286
|
+
let historyAdmitted = true;
|
|
287
|
+
try { assertTaskAdmissionState(state, {taskId, identity}); } catch (error) { if (error.code !== 'dispatch_history_quarantined' && error.code !== 'task_admission_blocked') throw error; historyAdmitted = false; }
|
|
288
|
+
return { revision: state.revision, generation: state.generation, record, canResume: historyAdmitted && ['connected', 'ready'].includes(state.connection) && (!record || record.status === 'admitted' && record.generation === state.generation && admissionWriterMatches(state, record)) };
|
|
280
289
|
}
|
|
281
290
|
function checkAdmissionOwner(state, owner, identity, taskId, expectedRevision) {
|
|
282
291
|
if (!owner || owner.root !== identity.root || owner.protocol !== DISPATCH_PROTOCOL || owner.generation !== state.generation) fail('owner_fenced');
|
|
@@ -295,7 +304,7 @@ export function setTaskAdmissionFence({ owner, identity, taskId, expectedRevisio
|
|
|
295
304
|
state.taskFences ||= { version: 1, records: {} };
|
|
296
305
|
state.taskFences.records[key] = { version: 1, identity, identityHash: digest(identity), taskId, taskHash: digest(taskId), generation: state.generation, revision: (previous?.revision || 0) + 1, status: 'blocked', reason: { ...reason }, createdAt: Date.now(), admission: null };
|
|
297
306
|
// v2 writers validate minimumDispatchProtocol and cannot ignore taskFences.
|
|
298
|
-
state.schemaVersion = 2; state.minimumDispatchProtocol = 3; state.revision++;
|
|
307
|
+
state.schemaVersion = Math.max(2, state.schemaVersion); state.minimumDispatchProtocol = 3; state.revision++;
|
|
299
308
|
atomicSave(path(identity.root), state); return state;
|
|
300
309
|
});
|
|
301
310
|
}
|
|
@@ -324,3 +333,77 @@ export async function admitTaskDispatch({ owner, identity, taskId, expectedRevis
|
|
|
324
333
|
atomicSave(path(identity.root), state); return state;
|
|
325
334
|
} finally { release(); }
|
|
326
335
|
}
|
|
336
|
+
|
|
337
|
+
// Historical evidence without a provable result is retained, never replayed or
|
|
338
|
+
// called completed. A schema fence makes every pre-recovery binary fail closed.
|
|
339
|
+
function validateRecoveryFences(state) {
|
|
340
|
+
if (state.schemaVersion !== 3 && state.recoveryFences === undefined) return;
|
|
341
|
+
const recovery = state.recoveryFences;
|
|
342
|
+
if (state.schemaVersion !== 3 || !recovery || recovery.version !== 1 || !isMap(recovery.items) ||
|
|
343
|
+
typeof recovery.auditRef !== 'string' || !/^[a-f0-9]{64}$/.test(recovery.auditDigest || '')) fail('recovery_fence_invalid');
|
|
344
|
+
for (const [key, item] of Object.entries(recovery.items)) {
|
|
345
|
+
const attempt = state.attempts[key];
|
|
346
|
+
if (!attempt || item.taskHash !== attempt.taskHash || item.identityHash !== attempt.identityHash ||
|
|
347
|
+
!/^[a-f0-9]{64}$/.test(item.attemptDigest || '') || !Array.isArray(item.sessions) ||
|
|
348
|
+
item.sessions.some(target => !['codex','claude'].includes(target.provider) || typeof target.session !== 'string' || !target.session)) fail('recovery_fence_invalid');
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
export function unresolvedDispatchAttempts(state) {
|
|
352
|
+
return Object.entries(state.attempts).filter(([key, attempt]) => attempt.phase !== 'confirmed' &&
|
|
353
|
+
state.recoveryFences?.items?.[key]?.attemptDigest !== digest(attempt)).map(([, attempt]) => attempt);
|
|
354
|
+
}
|
|
355
|
+
export function reconcileDispatchHistory({root = detachmentRoot(), expectedRevision, expectedGeneration, auditRef, auditDigest}) {
|
|
356
|
+
// Exclude a live update/reconnect owner as well as simultaneous dispatches.
|
|
357
|
+
const release = acquireLock(join(root, 'update-detachment', 'coordinator'));
|
|
358
|
+
try { return withLock(path(root), () => {
|
|
359
|
+
const state = read(root);
|
|
360
|
+
if (state.revision !== expectedRevision || state.generation !== expectedGeneration) fail('revision_conflict');
|
|
361
|
+
if (state.connection !== 'recovery_required') fail('reconciliation_required');
|
|
362
|
+
const auditPath = resolve(auditRef || '');
|
|
363
|
+
if (!auditPath.startsWith(resolve(root, 'update-detachment', 'recovery') + '/') || realpathSync(auditPath) !== auditPath) fail('recovery_audit_invalid');
|
|
364
|
+
const bytes = readFileSync(auditPath);
|
|
365
|
+
if (createHash('sha256').update(bytes).digest('hex') !== auditDigest) fail('recovery_audit_changed');
|
|
366
|
+
const audit = JSON.parse(bytes);
|
|
367
|
+
if (audit.version !== 1 || audit.gateDigest !== digest(state) || !Array.isArray(audit.items) ||
|
|
368
|
+
audit.items.length !== Object.keys(state.attempts).length) fail('recovery_audit_invalid');
|
|
369
|
+
const items = {}, seen = new Set(), settled = [];
|
|
370
|
+
for (const item of audit.items) {
|
|
371
|
+
const key = digest(item.attemptId), attempt = state.attempts[key];
|
|
372
|
+
if (!attempt || seen.has(key) || item.attemptDigest !== digest(attempt)) fail('recovery_attempt_changed');
|
|
373
|
+
seen.add(key);
|
|
374
|
+
if (item.disposition === 'already_confirmed' && attempt.phase === 'confirmed') continue;
|
|
375
|
+
if (item.disposition === 'saved_terminal_result') {
|
|
376
|
+
const evidence = item.evidence;
|
|
377
|
+
if (!evidence || !attempt.provenance || evidence.ref !== attempt.provenance.recordRef ||
|
|
378
|
+
!Object.values(state.receipts).some(receipt => receipt.attemptId === attempt.id && receipt.generation === attempt.generation && receipt.resultRef === evidence.ref)) fail('recovery_result_unbound');
|
|
379
|
+
if (!resolve(evidence.ref).startsWith(resolve(root, 'direct-tasks') + '/') || realpathSync(evidence.ref) !== evidence.ref) fail('recovery_result_scope');
|
|
380
|
+
const source = readFileSync(evidence.ref);
|
|
381
|
+
if (createHash('sha256').update(source).digest('hex') !== evidence.sha256) fail('recovery_result_changed');
|
|
382
|
+
const result = JSON.parse(source), match = /^(.*):provider:([1-9][0-9]*)$/.exec(attempt.id);
|
|
383
|
+
if (!match || result.id !== match[1] || digest(result.id) !== attempt.taskHash ||
|
|
384
|
+
result.id !== attempt.provenance.taskId || result.invocation_count !== Number(match[2]) ||
|
|
385
|
+
!['succeeded','failed','stopped'].includes(result.status)) fail('recovery_result_unbound');
|
|
386
|
+
settled.push(attempt);
|
|
387
|
+
} else if (item.disposition === 'retain_blocked_unknown') {
|
|
388
|
+
const sessions = [];
|
|
389
|
+
if (attempt.provenance?.providerSession) sessions.push({provider:attempt.provenance.context.provider,session:attempt.provenance.providerSession});
|
|
390
|
+
// Recover the parent identity only from its original canonical record.
|
|
391
|
+
const ref = attempt.provenance?.recordRef;
|
|
392
|
+
if (ref?.startsWith(resolve(root, 'delivery-hooks') + '/')) {
|
|
393
|
+
try { const record = JSON.parse(readFileSync(ref));
|
|
394
|
+
if (record.run?.managed?.taskId === attempt.provenance.taskId && record.provider === attempt.provenance.context.provider &&
|
|
395
|
+
basename(ref) === createHash('sha256').update(record.provider + '\0' + record.session).digest('hex') + '.json') sessions.push({provider:record.provider,session:record.session});
|
|
396
|
+
} catch { /* Missing parent never authorizes an inferred identity. */ }
|
|
397
|
+
}
|
|
398
|
+
items[key] = {taskHash:attempt.taskHash,identityHash:attempt.identityHash,attemptDigest:digest(attempt),sessions};
|
|
399
|
+
} else fail('recovery_disposition_invalid');
|
|
400
|
+
}
|
|
401
|
+
for (const attempt of settled) attempt.phase = 'confirmed';
|
|
402
|
+
state.schemaVersion = 3; state.minimumDispatchProtocol = 3;
|
|
403
|
+
state.generation++; state.revision++;
|
|
404
|
+
state.recoveryFences = {version:1,auditRef:auditPath,auditDigest,items};
|
|
405
|
+
validateRecoveryFences(state);
|
|
406
|
+
atomicSave(path(root), state);
|
|
407
|
+
return {settled:settled.length,blocked:Object.keys(items).length,revision:state.revision,generation:state.generation,replayAttempts:false};
|
|
408
|
+
}); } finally { release(); }
|
|
409
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { budgetState } from './orchestrator/budget.js';
|
|
2
|
+
|
|
3
|
+
const BASE_DELAY = 5 * 60_000;
|
|
4
|
+
const MAX_DELAY = 30 * 60_000;
|
|
5
|
+
const MACHINE_STOPS = new Set(['outcome_progress_stalled', 'retry_circuit_open']);
|
|
6
|
+
|
|
7
|
+
export function canRecoverCodingRun(run) {
|
|
8
|
+
const snapshot = run.snapshot;
|
|
9
|
+
return run.status === 'blocked' && !run.stopRequested &&
|
|
10
|
+
!['paused', 'stopped', 'stopping'].includes(run.controlState) &&
|
|
11
|
+
!snapshot.metadata?.local_stop_requested && !run.operation && !run.child &&
|
|
12
|
+
snapshot.final_audit?.requires_human === false &&
|
|
13
|
+
MACHINE_STOPS.has(snapshot.events?.at(-1)?.detail?.code) &&
|
|
14
|
+
!budgetState(snapshot).exhausted;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function scheduleCodingRecovery(snapshot, now = Date.now()) {
|
|
18
|
+
const count = (snapshot.metadata?.automatic_recovery?.count || 0) + 1;
|
|
19
|
+
snapshot.metadata ??= {};
|
|
20
|
+
snapshot.metadata.automatic_recovery = {
|
|
21
|
+
count,
|
|
22
|
+
next_retry_at: now + Math.min(MAX_DELAY, BASE_DELAY * 2 ** Math.min(count - 1, 3)),
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function codingRecoveryDue(run, now = Date.now()) {
|
|
27
|
+
return canRecoverCodingRun(run) &&
|
|
28
|
+
Number.isFinite(run.snapshot.metadata?.automatic_recovery?.next_retry_at) &&
|
|
29
|
+
now >= run.snapshot.metadata.automatic_recovery.next_retry_at;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function codingRecoveryLinksMatch(snapshot, links) {
|
|
33
|
+
return [snapshot.charter.backend, snapshot.charter.audit_backend].filter(Boolean).every(provider => {
|
|
34
|
+
const link = links[provider];
|
|
35
|
+
const saved = snapshot.metadata?.provider_link_epochs;
|
|
36
|
+
return link?.connected === true && (saved && Object.hasOwn(saved, provider)
|
|
37
|
+
? saved[provider] === link.epoch
|
|
38
|
+
: link.epoch === null && link.requiresConsent === false);
|
|
39
|
+
});
|
|
40
|
+
}
|
package/src/console/app.js
CHANGED
|
@@ -1147,6 +1147,7 @@ function statusLabel(run) {
|
|
|
1147
1147
|
if (run.controlState === "stopping") return "Stopping";
|
|
1148
1148
|
if (run.controlState === "stopped" || run.status === "stopped") return "Stopped";
|
|
1149
1149
|
if (run.status === "pr_ready_for_review") return "PR ready · Review needed";
|
|
1150
|
+
if (run.status === 'blocked' && run.snapshot?.final_audit?.requires_human === false) return 'Blocked';
|
|
1150
1151
|
if (run.controlState === "waiting_human" || run.status === "waiting_human") {
|
|
1151
1152
|
return "Waiting for you";
|
|
1152
1153
|
}
|
|
@@ -1181,6 +1182,10 @@ function nextActionForRun(run, repositoryMissing = false) {
|
|
|
1181
1182
|
text: "Review the final evidence and merge the pull request when ready.",
|
|
1182
1183
|
};
|
|
1183
1184
|
}
|
|
1185
|
+
const latestRecovery = [...(run.snapshot?.events || [])].reverse().find(event => event.detail?.recovery)?.detail.recovery;
|
|
1186
|
+
if (run.status === "blocked" && ['machine_recovery_pending', 'machine_recovery_exhausted'].includes(latestRecovery?.reason_code) && latestRecovery.requires_user_action === false) {
|
|
1187
|
+
return { tone: latestRecovery.reason_code === 'machine_recovery_pending' ? 'yellow' : 'red', text: latestRecovery.user_message };
|
|
1188
|
+
}
|
|
1184
1189
|
if (run.controlState === "waiting_human" || run.status === "waiting_human") {
|
|
1185
1190
|
return { tone: "yellow", text: "Press i, type your guidance or approval, then press Enter." };
|
|
1186
1191
|
}
|
|
@@ -45,6 +45,18 @@ function parseDeliveryOptions(args, allowed) {
|
|
|
45
45
|
|
|
46
46
|
export async function runDeliveryCommand(argv, { root = rootDirectory(), cwd = process.cwd(), webOrigin, input = process.stdin, output = console.log } = {}) {
|
|
47
47
|
const [action, ...args] = argv;
|
|
48
|
+
if (action === 'history-recovery') {
|
|
49
|
+
const [operation,...flags] = args;
|
|
50
|
+
const options = parseDeliveryOptions(flags,new Set(['audit','digest']));
|
|
51
|
+
const recovery = await import('../update-detachment/history-recovery.js');
|
|
52
|
+
const parentRoot = dirname(root);
|
|
53
|
+
let result;
|
|
54
|
+
if (operation === 'plan') result = await recovery.planDispatchHistoryRecovery({root:parentRoot});
|
|
55
|
+
else if (operation === 'apply') result = await recovery.applyDispatchHistoryRecovery({root:parentRoot,auditRef:options.audit,auditDigest:options.digest});
|
|
56
|
+
else if (operation === 'reconnect') result = await recovery.reconnectRecoveredHistory({root:parentRoot,origin:webOrigin});
|
|
57
|
+
else throw new Error('Use simy delivery history-recovery plan|apply --audit PATH --digest SHA256|reconnect.');
|
|
58
|
+
output(JSON.stringify(result,null,2)); return 0;
|
|
59
|
+
}
|
|
48
60
|
if (action === 'auto') {
|
|
49
61
|
const [operation, ...flags] = args;
|
|
50
62
|
if (!['on', 'off', 'status'].includes(operation)) throw new Error('Use simy delivery auto on|off|status --provider codex|claude in the project directory.');
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { captureDispatchProvenance } from '../update-detachment/provenance.js';
|
|
2
|
-
import { captureDispatchOwner, assertTaskAdmission, dispatchWithFence, queueDispatchReceipt, rootFromDelivery, observeDispatchEvent } from '../update-detachment/dispatch-gate.js';
|
|
2
|
+
import { captureDispatchOwner, assertTaskAdmission, dispatchWithFence, queueDispatchReceipt, confirmDispatch, rootFromDelivery, observeDispatchEvent } from '../update-detachment/dispatch-gate.js';
|
|
3
3
|
import { bindCompletionArtifacts } from '../session-supervision/completion-artifacts.js';
|
|
4
4
|
import { recoveryWorkProposal, admitRecoveryWork } from '../session-supervision/recovery-frontier.js';
|
|
5
5
|
import { safeCheckpointDiagnostic } from './checkpoint-diagnostic.js';
|
|
@@ -480,7 +480,7 @@ export async function tickManagedTasks({ root, webOrigin, now = Date.now(), exec
|
|
|
480
480
|
else await checkpointDelivery({ root, provider: record.provider, session: record.session, cwd: record.cwd,
|
|
481
481
|
checkpoint: result.checkpoint || parseAgentCheckpoint(result.output, record.session), expectedLease: leaseId, toolEvidence: result.toolEvidence || [], sourceContext: result.sourceContext, controllerEvidence: result.controllerEvidence, now: Date.now() });
|
|
482
482
|
if (result.dispatchAttemptId) queueDispatchReceipt({ root: rootFromDelivery(root), attemptId: result.dispatchAttemptId, receiptId: `checkpoint:${leaseId}`, resultRef: `delivery-record:${record.provider}:${record.session}:${leaseId}` });
|
|
483
|
-
await updateRecord(root, record.provider, record.session, current => {
|
|
483
|
+
const checkpointed = await updateRecord(root, record.provider, record.session, current => {
|
|
484
484
|
if (current?.run?.managed?.executionAttempt?.id !== leaseId) return current;
|
|
485
485
|
current.run.managed.executionAttempt.phase = 'checkpointed';
|
|
486
486
|
current.run.managed.executionAttempt.checkpointRevision = current.run.revision;
|
|
@@ -501,6 +501,14 @@ export async function tickManagedTasks({ root, webOrigin, now = Date.now(), exec
|
|
|
501
501
|
}
|
|
502
502
|
return current;
|
|
503
503
|
});
|
|
504
|
+
// Settle this successful native turn while its exact durable checkpoint
|
|
505
|
+
// is still current. A later worker replaces executionAttempt, so waiting
|
|
506
|
+
// until an update boundary would lose the original result association.
|
|
507
|
+
const savedAttempt = checkpointed?.run?.managed?.executionAttempt;
|
|
508
|
+
if (result.dispatchAttemptId && checkpointed?.run?.id === claimed.run.id &&
|
|
509
|
+
savedAttempt?.id === leaseId && result.dispatchAttemptId === leaseId &&
|
|
510
|
+
savedAttempt.phase === 'checkpointed' && savedAttempt.checkpointRevision === checkpointed.run.revision)
|
|
511
|
+
confirmDispatch({ root: rootFromDelivery(root), attemptId: result.dispatchAttemptId });
|
|
504
512
|
} catch (error) { result = { ...result, code: -1, errorCode: 'checkpoint_rejected', diagnostic: safeCheckpointDiagnostic(error) }; }
|
|
505
513
|
}
|
|
506
514
|
await updateRecordAfterControl(root, record.provider, record.session, async current => {
|
|
@@ -657,8 +665,10 @@ export function createManagedSupervisor(options) {
|
|
|
657
665
|
// A queue entry is a wakeup, not authority or a saved action. Re-read
|
|
658
666
|
// consent, revisions and task identity at admission and each claim.
|
|
659
667
|
const readRecords = async () => {
|
|
660
|
-
const rows = options.
|
|
661
|
-
|
|
668
|
+
const rows = options.readCurrentRecord
|
|
669
|
+
? [await options.readCurrentRecord(record.provider, record.session)]
|
|
670
|
+
: options.readRecords ? await options.readRecords()
|
|
671
|
+
: [await readRecord(options.root, record.provider, record.session)];
|
|
662
672
|
return rows.filter(row => row?.provider === record.provider && row.session === record.session &&
|
|
663
673
|
row.run?.id === record.run.id && row.run?.managed?.taskId === record.run.managed.taskId);
|
|
664
674
|
};
|
package/src/desktop-access.js
CHANGED
|
@@ -7,7 +7,7 @@ export function validDesktopAccessPath(path, slug) {
|
|
|
7
7
|
return path === expected || path === `${expected}/`;
|
|
8
8
|
}
|
|
9
9
|
export const desktopAccessReady = value => value?.state === 'ready';
|
|
10
|
-
const recoveryActions = new Set(['refresh', 'reconnect', 'login', 'authorization_open', 'organization_select', 'organization_cancel', 'organizations_refresh',
|
|
10
|
+
const recoveryActions = new Set(['refresh', 'reconnect', 'login', 'authorization_open', 'authorization_copy', 'organization_select', 'organization_cancel', 'organizations_refresh',
|
|
11
11
|
'check_update', 'auto_update_set_enabled', 'update', 'prepare_update_restart', 'quit', 'task_stop', 'delivery_stop', 'delivery_pause', 'delivery_reject',
|
|
12
12
|
'provider_disconnect', 'desktop_access_refresh', 'desktop_access_settings', 'desktop_access_members']);
|
|
13
13
|
export function desktopActionAllowed(access, command) {
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
|
|
3
|
+
// Pass private links over stdin, never in process arguments, status, or logs.
|
|
4
|
+
export function copyDesktopText(text, { platform = process.platform, spawnImpl = spawn } = {}) {
|
|
5
|
+
const command = platform === 'darwin' ? ['pbcopy', []] : platform === 'win32'
|
|
6
|
+
? ['powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', '$inputText = [Console]::In.ReadToEnd(); Set-Clipboard -Value $inputText -ErrorAction Stop']]
|
|
7
|
+
: ['xclip', ['-selection', 'clipboard']];
|
|
8
|
+
return new Promise((resolve, reject) => {
|
|
9
|
+
const child = spawnImpl(command[0], command[1], { stdio: ['pipe', 'ignore', 'ignore'], windowsHide: true });
|
|
10
|
+
const timer = setTimeout(() => { child.kill(); finish(false); }, 5000);
|
|
11
|
+
let settled = false;
|
|
12
|
+
function finish(ok) {
|
|
13
|
+
if (settled) return;
|
|
14
|
+
settled = true; clearTimeout(timer);
|
|
15
|
+
if (ok) resolve();
|
|
16
|
+
else { child.kill(); reject(Object.assign(new Error('Clipboard unavailable.'), { code: 'desktop_clipboard_unavailable' })); }
|
|
17
|
+
}
|
|
18
|
+
child.on('error', () => finish(false));
|
|
19
|
+
child.on('close', code => finish(code === 0));
|
|
20
|
+
child.stdin.on('error', () => finish(false));
|
|
21
|
+
child.stdin.end(text);
|
|
22
|
+
});
|
|
23
|
+
}
|
package/src/desktop-host.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { remoteOperationFailure } from './session-supervision/remote-errors.js';
|
|
2
|
+
import { copyDesktopText } from './desktop-clipboard.js';
|
|
2
3
|
import { createRemoteHostManager } from './session-supervision/remote-hosts.js';
|
|
3
4
|
import { readDesktopAutoUpdate, writeDesktopAutoUpdate } from './desktop-update-settings.js';
|
|
4
5
|
import { createUpdateRuntime } from './update-detachment/runtime.js';
|
|
@@ -23,7 +24,7 @@ import { stopDesktopWorkForUpdate } from './desktop-update-stop.js';
|
|
|
23
24
|
import { desktopAccessReady, desktopActionAllowed, requireDesktopAccess, validDesktopAccessPath } from './desktop-access.js';
|
|
24
25
|
import { createActionDeadlineService } from './session-supervision/action-deadline.js';
|
|
25
26
|
import { deliveryHookStatus, openHookSetup } from "./delivery-hooks/desktop.js";
|
|
26
|
-
import { controlDelivery, verifyDelivery, createRecordReader, updateRecord, recordProviderArchive, TERMINAL } from "./codex-plugins/simy/hooks/delivery-loop.mjs";
|
|
27
|
+
import { controlDelivery, verifyDelivery, createRecordReader, readRecord, updateRecord, recordProviderArchive, TERMINAL } from "./codex-plugins/simy/hooks/delivery-loop.mjs";
|
|
27
28
|
import { createManagedSupervisor } from "./delivery-hooks/supervisor.js";
|
|
28
29
|
import { createActionAssociationHost } from "./session-supervision/action-association-host.js";
|
|
29
30
|
import { createStopNotifier } from "./session-supervision/stop-notifications.js";
|
|
@@ -105,6 +106,7 @@ function desktopActionError(action, error) {
|
|
|
105
106
|
};
|
|
106
107
|
if (messages[error?.code]) return messages[error.code];
|
|
107
108
|
}
|
|
109
|
+
if (action === 'authorization_copy') return 'Could not copy the connection link. Reopen the connection page and try again.';
|
|
108
110
|
if (action === 'authorization_open' || error?.code === 'desktop_browser_unavailable') return "Could not open your browser. Open the verification link again, or check your default browser.";
|
|
109
111
|
if (error?.code === "desktop_access_required") return "Sign in to an organization with SQM and Developer access before using SIMY.";
|
|
110
112
|
if (action === "codex_hooks" || action === "claude_hooks") return "Could not open the setup guide. Check your default browser, or use /hooks inside the provider.";
|
|
@@ -409,6 +411,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
409
411
|
const link = await requireProviderLink({ root: deliveryRoot, provider: record.provider });
|
|
410
412
|
return JSON.stringify([scope, link.epoch]);
|
|
411
413
|
}, readRecords: () => recordReader.list(),
|
|
414
|
+
readCurrentRecord: (provider, session) => readRecord(deliveryRoot, provider, session),
|
|
412
415
|
beforeClaim: record => actionAssociations.prepareCached?.(record) || record });
|
|
413
416
|
let shuttingDown = null;
|
|
414
417
|
let commands;
|
|
@@ -529,7 +532,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
529
532
|
phase = "authorize_connect";
|
|
530
533
|
await authorizeWork();
|
|
531
534
|
phase = "save_connect";
|
|
532
|
-
providerLinks = await writeProviderLink(root, provider, true);
|
|
535
|
+
providerLinks = await writeProviderLink(root, provider, true, { repairInvalidSettings: true });
|
|
533
536
|
}
|
|
534
537
|
delete providerLinkFeedback[provider];
|
|
535
538
|
try { await agent.refreshProviderStatus?.(); }
|
|
@@ -1329,6 +1332,11 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
1329
1332
|
else if (action === "sqm_set_engine_fallback") await agent.setSqmEngineFallback({ enabled: command.enabled, allowedProviders: command.allowedProviders });
|
|
1330
1333
|
else if (action === "sqm_retry_environment") await agent.retrySqmEnvironment();
|
|
1331
1334
|
else if (action === "organization_cancel") await agent.cancelOrganizationSelection();
|
|
1335
|
+
else if (action === 'authorization_copy') {
|
|
1336
|
+
const url = agent.getPendingAuthorizationUrl?.() || await agent.requestDesktopLogin();
|
|
1337
|
+
if (!url || new URL(url).origin !== webOrigin || new URL(url).pathname !== '/local-cli/connect') throw new Error('No valid connection link.');
|
|
1338
|
+
await (dependencies.copyText || copyDesktopText)(url);
|
|
1339
|
+
}
|
|
1332
1340
|
else if (action === "login" || action === "organization_select" || action === "authorization_open") {
|
|
1333
1341
|
if (action === "organization_select" && (typeof command.organizationId !== "string" || !command.organizationId)) throw new Error("Select an organization first.");
|
|
1334
1342
|
const url = action === "authorization_open" ? (agent.getPendingAuthorizationUrl?.() ?? agent.retryAuthorization?.())
|
|
@@ -1655,7 +1663,7 @@ export async function startDesktopHost({ input = process.stdin, output = process
|
|
|
1655
1663
|
};
|
|
1656
1664
|
if (command.action === "quit" || command.action === "prepare_update_restart" || command.action === "update" && (command.force === true || stoppingForUpdate)) {
|
|
1657
1665
|
void execute();
|
|
1658
|
-
} else if (["login", "organization_select", "organization_cancel", "authorization_open"].includes(command.action)) {
|
|
1666
|
+
} else if (["login", "organization_select", "organization_cancel", "authorization_open", "authorization_copy"].includes(command.action)) {
|
|
1659
1667
|
authorizationQueue = authorizationQueue.then(execute).catch(() => {});
|
|
1660
1668
|
} else if (/^(codex|claude)_login$/.test(command.action || "") || ["provider_repair", "provider_connect", "provider_disconnect"].includes(command.action)) {
|
|
1661
1669
|
// Opening an auth terminal must not wait for an unrelated provider turn.
|
|
@@ -101,11 +101,11 @@ export async function auditAttempt(charter, attempt, { previousAttempt = null, s
|
|
|
101
101
|
target: check.id,
|
|
102
102
|
explanation: check.detail,
|
|
103
103
|
repairability:
|
|
104
|
-
check.id === "executor_succeeded" && attempt.raw_output.
|
|
104
|
+
check.id === "executor_succeeded" && attempt.raw_output?.blocked && attempt.raw_output?.blocker_kind === "human"
|
|
105
105
|
? "manual"
|
|
106
106
|
: check.repairability,
|
|
107
107
|
auto_fix_hint:
|
|
108
|
-
check.id === "executor_succeeded" && attempt.raw_output.
|
|
108
|
+
check.id === "executor_succeeded" && attempt.raw_output?.blocked && attempt.raw_output?.blocker_kind === "human"
|
|
109
109
|
? null
|
|
110
110
|
: `Resolve ${check.label.toLowerCase()} and check the result.`,
|
|
111
111
|
}));
|
|
@@ -12,7 +12,7 @@ export function buildIndependentAuditInstruction(charter, attempt) {
|
|
|
12
12
|
return buildWorkflowReportingInstruction({ auditor: true }) + "\n" + buildAcceptedHumanGuidanceInstruction(charter) + "\n" + section("requested_review", `Review the original request and actual behavior read-only. Report concrete defects, regressions and missing user outcomes. No preparation matrix, citation package, assurance score or mandatory screenshot record is required. Do not certify unobserved results or grant permission.
|
|
13
13
|
Request: ${JSON.stringify({ requirement: charter.requirement, criteria: acceptanceCriterionRows(charter), mustNot: charter.must_not, experience: charter.completion_contract?.experience })}
|
|
14
14
|
Current result: ${JSON.stringify({ summary: attempt.summary ?? null, residual_risks: attempt.residual_risks, tests: attempt.tests_run, observations: attempt.criterion_evidence, current: attempt.observed_evidence })}
|
|
15
|
-
Return SIMY_AUDIT_JSON: with {passed,summary,findings,criterion_results,coverage_review:{user_outcome:{passed,verification_method,observed_result}}}. Check the original requested result even if the listed criteria mention only code or tests. Each criterion result describes criterion_id,passed,verification_method,observed_result. Each finding candidate must contain passed. Only passed=false entries are findings; never report passed=true evidence as a finding. Findings describe the concrete remaining defect and necessary repair.`);
|
|
15
|
+
Return SIMY_AUDIT_JSON: with {passed,summary,findings,criterion_results,coverage_review:{user_outcome:{passed,verification_method,observed_result}}}. Check the original requested result even if the listed criteria mention only code or tests. Each criterion result describes criterion_id,passed,verification_method,observed_result. Each finding candidate must contain passed. Only passed=false entries are findings; never report passed=true evidence as a finding. Findings describe the concrete remaining defect and necessary repair. Set blocker_kind to machine for technical failures, external for unavailable dependencies, and human only for actual missing authorization, authentication interaction or an explicitly user-reserved decision. A legacy repairability=manual label or missing report does not establish a human dependency.`);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
export function buildIndependentAudit(execution, charter = null, attempt = null) {
|
|
@@ -60,11 +60,11 @@ export function buildIndependentAudit(execution, charter = null, attempt = null)
|
|
|
60
60
|
severity: cleanString(item.severity).toLowerCase() || "major",
|
|
61
61
|
target: cleanString(item.target) || "pull_request",
|
|
62
62
|
explanation: cleanString(item.explanation) || "Independent audit finding.",
|
|
63
|
-
repairability: item.
|
|
63
|
+
repairability: item.blocker_kind === "human" ? "manual" : "auto",
|
|
64
64
|
disposition: ["defer", "dismiss"].includes(item.disposition) && cleanString(item.explanation)
|
|
65
65
|
? item.disposition : "fix",
|
|
66
66
|
blocker_kind: ["machine", "human", "external"].includes(item.blocker_kind)
|
|
67
|
-
? item.blocker_kind :
|
|
67
|
+
? item.blocker_kind : "machine",
|
|
68
68
|
auto_fix_hint: cleanString(item.auto_fix_hint) || null,
|
|
69
69
|
}));
|
|
70
70
|
const findings = evaluations.filter((item) => item.passed === false);
|
|
@@ -13,6 +13,7 @@ export function buildCodingInstruction(
|
|
|
13
13
|
},
|
|
14
14
|
) {
|
|
15
15
|
return [
|
|
16
|
+
section("automatic_recovery", "Continue authorized work without procedural sign-off. Repair technical failures automatically. For an actual blocker, report blocker_kind=machine for technical repair, external for an unavailable dependency, or human for missing authorization, authentication interaction or an explicitly reserved decision. Preserve native permissions and explicit stops; blocked=true alone does not require human input."),
|
|
16
17
|
section("role", "Complete the user's requested outcome in this repository."),
|
|
17
18
|
buildWorkflowReportingInstruction(),
|
|
18
19
|
buildAcceptedHumanGuidanceInstruction(charter),
|
package/src/orchestrator/loop.js
CHANGED
|
@@ -113,7 +113,7 @@ async function runCodingLoopOwned({
|
|
|
113
113
|
previousAttempt,
|
|
114
114
|
previousFindings,
|
|
115
115
|
promptInterventions,
|
|
116
|
-
});
|
|
116
|
+
}) + (snapshot.metadata?.automatic_recovery ? '\nAutomatic recovery of the same unfinished task: inspect the current failure and previous attempts, test a different cause or reversible repair, then return to the original requested work. Do not repeat an unchanged failing operation or ask the user to certify technical correctness. Preserve existing permissions and limits.' : '');
|
|
117
117
|
|
|
118
118
|
let attempt = recoveredAttempt;
|
|
119
119
|
if (!attempt) {
|
|
@@ -574,7 +574,18 @@ async function handleFailedAudit({
|
|
|
574
574
|
}) {
|
|
575
575
|
snapshot.charter.thread_state.failure_count += 1;
|
|
576
576
|
const diagnosticCircuit = evaluateRetryCircuit({ attempt, previousAttempt: snapshot.attempts.at(-2) ?? null, findings });
|
|
577
|
-
|
|
577
|
+
// A new commit does not resolve an unchanged review failure. Criteria can all
|
|
578
|
+
// pass while a required review finding still prevents acceptance. Compare
|
|
579
|
+
// concrete details: a shared generic code does not identify the same defect.
|
|
580
|
+
const priorAttempt = snapshot.attempts.at(-2);
|
|
581
|
+
const repeatedReview = !attempt.outcome_progress?.has_progress &&
|
|
582
|
+
diagnosticCircuit.previous_fingerprint?.review_findings_sha256 === diagnosticCircuit.fingerprint.review_findings_sha256 &&
|
|
583
|
+
priorAttempt?.outcome_progress?.objective_digest === attempt.outcome_progress?.objective_digest;
|
|
584
|
+
attempt.audit_retry_streak = repeatedReview ? (priorAttempt.audit_retry_streak || 1) + 1 : 1;
|
|
585
|
+
const reviewStalled = !requiresHuman && attempt.audit?.passed === true && attempt.independent_audit?.passed === false &&
|
|
586
|
+
attempt.audit_retry_streak >= 3 && attempt.outcome_progress?.status === "assessed" &&
|
|
587
|
+
!attempt.outcome_progress?.stalled && !attempt.outcome_progress?.assessment_unavailable;
|
|
588
|
+
const technicalStall = reviewStalled || attempt.outcome_progress?.stalled || attempt.outcome_progress?.assessment_unavailable || diagnosticCircuit.open && !attempt.outcome_progress?.has_progress;
|
|
578
589
|
if (technicalStall && !requiresHuman) {
|
|
579
590
|
const state = snapshot.charter.thread_state;
|
|
580
591
|
const key = JSON.stringify([state.objective_contract?.current_digest, attempt.outcome_progress?.unproven_ids || [], findings.filter(row => row.code !== 'AI_STRATEGY_CHANGE_REQUIRED').map(row => row.code).sort()]);
|
|
@@ -590,18 +601,18 @@ async function handleFailedAudit({
|
|
|
590
601
|
return false;
|
|
591
602
|
}
|
|
592
603
|
}
|
|
593
|
-
if (attempt.outcome_progress?.stalled) {
|
|
594
|
-
const unresolved = attempt.outcome_progress
|
|
604
|
+
if (reviewStalled || attempt.outcome_progress?.stalled) {
|
|
605
|
+
const unresolved = attempt.outcome_progress?.unproven_ids || [];
|
|
595
606
|
snapshot.final_audit = {
|
|
596
607
|
...(snapshot.final_audit || {}),
|
|
597
608
|
passed: false,
|
|
598
|
-
summary: "Independent review found no progress on the remaining user outcomes.",
|
|
609
|
+
summary: reviewStalled ? "Automatic diagnosis did not resolve the repeated review failure." : "Independent review found no progress on the remaining user outcomes.",
|
|
599
610
|
findings: [...(snapshot.final_audit?.findings || []), {
|
|
600
611
|
passed: false,
|
|
601
|
-
code: "OUTCOME_PROGRESS_STALLED",
|
|
612
|
+
code: reviewStalled ? "REVIEW_PROGRESS_STALLED" : "OUTCOME_PROGRESS_STALLED",
|
|
602
613
|
severity: "blocker",
|
|
603
614
|
target: "acceptance_criteria",
|
|
604
|
-
explanation: `The same user outcomes remain unproven after another attempt: ${unresolved.join(", ")}. Reassess the approach against the approved objective before more work.`,
|
|
615
|
+
explanation: reviewStalled ? "The same review findings remain after automatic diagnosis and another repair attempt. Preserve the failure for machine recovery; human certification cannot resolve it." : `The same user outcomes remain unproven after another attempt: ${unresolved.join(", ")}. Reassess the approach against the approved objective before more work.`,
|
|
605
616
|
repairability: "auto",
|
|
606
617
|
blocker_kind: "machine",
|
|
607
618
|
}],
|
|
@@ -618,11 +629,7 @@ async function handleFailedAudit({
|
|
|
618
629
|
await publish(snapshot, "blocked", onUpdate);
|
|
619
630
|
return true;
|
|
620
631
|
}
|
|
621
|
-
const retryCircuit =
|
|
622
|
-
attempt,
|
|
623
|
-
previousAttempt: snapshot.attempts.at(-2) ?? null,
|
|
624
|
-
findings,
|
|
625
|
-
});
|
|
632
|
+
const retryCircuit = diagnosticCircuit;
|
|
626
633
|
if (retryCircuit.open && !attempt.outcome_progress?.has_progress &&
|
|
627
634
|
!(attempt.audit?.passed && attempt.outcome_progress?.status === "insufficient_evidence" &&
|
|
628
635
|
!attempt.outcome_progress?.failed_ids?.length)) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { suggestedProviderTokenBudget, MAX_PROVIDER_TOKEN_BUDGET } from "./budget.js";
|
|
1
|
+
import { suggestedProviderTokenBudget, MAX_PROVIDER_TOKEN_BUDGET, budgetState } from "./budget.js";
|
|
2
2
|
|
|
3
3
|
const RECOVERABLE_STATES = new Set(["waiting_human", "blocked", "failed"]);
|
|
4
4
|
|
|
@@ -8,13 +8,17 @@ export function recoveryContractForEvent({ snapshot, state, message, detail = {}
|
|
|
8
8
|
|
|
9
9
|
const reasonCode = recoveryReasonCode(state, message, detail);
|
|
10
10
|
const attemptNumber = positiveInteger(detail.attempt_number);
|
|
11
|
-
const
|
|
11
|
+
const machineStopped = state === "blocked" && snapshot?.final_audit?.requires_human === false &&
|
|
12
|
+
["outcome_progress_stalled", "retry_circuit_open"].includes(detail.code);
|
|
13
|
+
const actions = machineStopped ? [] : recoveryActions(reasonCode, detail.budget);
|
|
14
|
+
const automaticRecovery = machineStopped && !budgetState(snapshot).exhausted;
|
|
12
15
|
return {
|
|
13
16
|
schema_version: 1,
|
|
14
17
|
state,
|
|
15
|
-
reason_code: reasonCode,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
reason_code: automaticRecovery ? 'machine_recovery_pending' : machineStopped ? "machine_recovery_exhausted" : reasonCode,
|
|
19
|
+
...(machineStopped ? { requires_user_action: false } : {}),
|
|
20
|
+
user_message: automaticRecovery ? 'The task remains unfinished. SIMY will retry this same task after a recovery cooldown while its connection is enabled. No manual confirmation is required.' : machineStopped ? "Automatic diagnosis and repair attempts could not resolve the technical failure. The task remains unfinished; failure details are retained. No human approval or success declaration is required." : isSafetyCeiling(reasonCode) ? safetyCeilingRecoveryMessage(reasonCode, detail.budget) : recoveryMessage(reasonCode),
|
|
21
|
+
primary_action_id: actions[0]?.id ?? null,
|
|
18
22
|
actions,
|
|
19
23
|
technical_details: {
|
|
20
24
|
event_code: clean(detail.code) || null,
|
|
@@ -104,9 +108,10 @@ export function isRecoveryContract(value) {
|
|
|
104
108
|
value.schema_version === 1 &&
|
|
105
109
|
typeof value.reason_code === "string" &&
|
|
106
110
|
typeof value.user_message === "string" &&
|
|
107
|
-
typeof value.primary_action_id === "string" &&
|
|
108
111
|
Array.isArray(value.actions) &&
|
|
109
|
-
value.actions.length > 0
|
|
112
|
+
(typeof value.primary_action_id === "string" && value.actions.length > 0 ||
|
|
113
|
+
value.state === "blocked" && ['machine_recovery_pending', 'machine_recovery_exhausted'].includes(value.reason_code) &&
|
|
114
|
+
value.requires_user_action === false && value.primary_action_id === null && value.actions.length === 0),
|
|
110
115
|
);
|
|
111
116
|
}
|
|
112
117
|
|
|
@@ -39,6 +39,7 @@ export function buildRetryFingerprint(attempt, findings) {
|
|
|
39
39
|
commit_sha: commitSha,
|
|
40
40
|
finding_codes: normalizedFindings.map((finding) => finding.code),
|
|
41
41
|
findings_sha256: sha256(findingPayload),
|
|
42
|
+
review_findings_sha256: sha256(stableJson(normalizeFindings(findings, true))),
|
|
42
43
|
evidence_sha256: sha256(evidencePayload),
|
|
43
44
|
};
|
|
44
45
|
}
|
|
@@ -52,7 +53,7 @@ function findingsForAttempt(attempt) {
|
|
|
52
53
|
];
|
|
53
54
|
}
|
|
54
55
|
|
|
55
|
-
function normalizeFindings(findings) {
|
|
56
|
+
function normalizeFindings(findings, includeDetails = false) {
|
|
56
57
|
return (Array.isArray(findings) ? findings : [])
|
|
57
58
|
.filter((finding) => finding && typeof finding === "object")
|
|
58
59
|
.map((finding) => ({
|
|
@@ -62,6 +63,11 @@ function normalizeFindings(findings) {
|
|
|
62
63
|
severity: cleanString(finding.severity).toLowerCase() || null,
|
|
63
64
|
target: cleanString(finding.target) || null,
|
|
64
65
|
repairability: cleanString(finding.repairability).toLowerCase() || null,
|
|
66
|
+
...(includeDetails ? {
|
|
67
|
+
explanation: cleanString(finding.explanation) || null,
|
|
68
|
+
auto_fix_hint: cleanString(finding.auto_fix_hint) || null,
|
|
69
|
+
blocker_kind: cleanString(finding.blocker_kind) || null,
|
|
70
|
+
} : {}),
|
|
65
71
|
}))
|
|
66
72
|
.sort(compareStableValues);
|
|
67
73
|
}
|
package/src/provider-links.js
CHANGED
|
@@ -14,12 +14,13 @@ async function readSettings(root) {
|
|
|
14
14
|
let payload;
|
|
15
15
|
try { payload = JSON.parse(await readFile(join(root, 'provider-links.json'), 'utf8')); }
|
|
16
16
|
catch (error) { if (error.code === 'ENOENT') return defaults(); throw error; }
|
|
17
|
-
if (payload?.schemaVersion !== 1
|
|
17
|
+
if (payload?.schemaVersion !== 1) throw new Error('Unsupported SIMY provider connection settings version.');
|
|
18
|
+
if (!payload.providers || PROVIDER_LINKS.some(provider => {
|
|
18
19
|
const value = payload.providers[provider];
|
|
19
20
|
return !value || typeof value.connected !== 'boolean' || typeof value.requiresConsent !== 'boolean' ||
|
|
20
21
|
value.epoch !== null && (typeof value.epoch !== 'string' || !value.epoch || value.epoch.length > 128) ||
|
|
21
22
|
value.consentEpoch !== undefined && (value.consentEpoch !== value.epoch || !value.consentEpoch || value.connected !== false || typeof value.enabled !== 'boolean');
|
|
22
|
-
})) throw new Error('Invalid SIMY provider connection settings.');
|
|
23
|
+
})) throw Object.assign(new Error('Invalid SIMY provider connection settings.'), { code: 'invalid_provider_link_settings' });
|
|
23
24
|
return Object.fromEntries(PROVIDER_LINKS.map(provider => [provider, {
|
|
24
25
|
connected: payload.providers[provider].consentEpoch ? payload.providers[provider].enabled : payload.providers[provider].connected,
|
|
25
26
|
...(payload.providers[provider].consentEpoch ? { consentEpoch: payload.providers[provider].consentEpoch } : {}),
|
|
@@ -52,13 +53,22 @@ export async function requireProviderLink({ root = join(defaultRoot(), 'delivery
|
|
|
52
53
|
return link;
|
|
53
54
|
}
|
|
54
55
|
|
|
55
|
-
export function writeProviderLink(root = defaultRoot(), provider, connected) {
|
|
56
|
+
export function writeProviderLink(root = defaultRoot(), provider, connected, { repairInvalidSettings = false } = {}) {
|
|
56
57
|
validateProvider(provider);
|
|
57
58
|
if (typeof connected !== 'boolean') throw new Error('Choose whether this provider is connected to SIMY.');
|
|
58
59
|
const key = resolve(root);
|
|
59
60
|
const previous = writeQueues.get(key) || Promise.resolve();
|
|
60
61
|
const pending = previous.catch(() => {}).then(async () => {
|
|
61
|
-
|
|
62
|
+
let providers;
|
|
63
|
+
try { providers = await readSettings(root); }
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (!repairInvalidSettings || !connected || !(error instanceof SyntaxError || error.code === 'invalid_provider_link_settings')) throw error;
|
|
66
|
+
// Only an explicit Connect action restores this provider. Retain the
|
|
67
|
+
// damaged file and leave all other providers disconnected.
|
|
68
|
+
const original = await readFile(join(root, 'provider-links.json'));
|
|
69
|
+
await writeFile(join(root, `provider-links.invalid-${randomUUID()}.json`), original, { mode: 0o600, flag: 'wx' });
|
|
70
|
+
providers = disconnected();
|
|
71
|
+
}
|
|
62
72
|
// Reconnection restores discovery only. Session execution consent is cleared
|
|
63
73
|
// separately and must be selected again, including after a host restart.
|
|
64
74
|
const epoch = randomUUID();
|
package/src/runner.js
CHANGED
|
@@ -59,6 +59,7 @@ import {
|
|
|
59
59
|
assertRecoveryActor, completedWorkerResult, recoveryActorBinding, refreshRecoveryAudit, refreshRecoveryBudgetAudit,
|
|
60
60
|
} from "./coding-run-recovery.js";
|
|
61
61
|
import { workflowRunEnv } from "./dev-work-events.js";
|
|
62
|
+
import { canRecoverCodingRun, codingRecoveryDue, scheduleCodingRecovery } from './coding-run-automatic-recovery.js';
|
|
62
63
|
|
|
63
64
|
const execFileAsync = promisify(execFile);
|
|
64
65
|
const MAX_LOCAL_LOG_LINES = 1_000;
|
|
@@ -321,6 +322,7 @@ export function restoreRun({ snapshot, session, apiOrigin, localPath = null, loc
|
|
|
321
322
|
run.startedAt = restoredSnapshot.created_at || null;
|
|
322
323
|
run.completedAt = restoredSnapshot.updated_at || null;
|
|
323
324
|
run.controlState = restoredState === "stopped" ? "stopped" : "waiting_human";
|
|
325
|
+
if (canRecoverCodingRun(run)) run.controlState = 'complete';
|
|
324
326
|
run.restoredFromState = restoredFromState;
|
|
325
327
|
run.restoredObjectiveUnverified = objectiveUnverified;
|
|
326
328
|
run.restoredCompletedWorkerResult = Boolean(recoverableResult);
|
|
@@ -430,6 +432,23 @@ export async function continueLocalCodingRun(run, guidance, dependencies = {}) {
|
|
|
430
432
|
return executeLocalCodingRun(run, dependencies, { humanGuidance: message, resume: true });
|
|
431
433
|
}
|
|
432
434
|
|
|
435
|
+
// Automated recovery is not human guidance and does not reset stop intent,
|
|
436
|
+
// usage, the objective, or the existing task identity.
|
|
437
|
+
export async function recoverLocalCodingRun(run, dependencies = {}, now = Date.now()) {
|
|
438
|
+
if (!canRecoverCodingRun(run)) return run.snapshot;
|
|
439
|
+
if (!run.snapshot.metadata?.automatic_recovery) {
|
|
440
|
+
scheduleCodingRecovery(run.snapshot, now);
|
|
441
|
+
if (run.localStateRoot) await saveCodingRunSnapshot(run.localStateRoot, run.snapshot);
|
|
442
|
+
return run.snapshot;
|
|
443
|
+
}
|
|
444
|
+
if (!codingRecoveryDue(run, now)) return run.snapshot;
|
|
445
|
+
assertRunCanContinue(run);
|
|
446
|
+
run.completedAt = null;
|
|
447
|
+
run.controlState = 'running';
|
|
448
|
+
run.resumeCount += 1;
|
|
449
|
+
return executeLocalCodingRun(run, dependencies, { resume: true });
|
|
450
|
+
}
|
|
451
|
+
|
|
433
452
|
function assertRunCanContinue(run) {
|
|
434
453
|
if (run.operation || run.child) throw new Error("The selected Agentic Loop run is still active.");
|
|
435
454
|
if (!RESUMABLE_STATES.has(run.status)) {
|
|
@@ -1324,6 +1343,11 @@ async function updateRun(run, state) {
|
|
|
1324
1343
|
run.snapshot.state = state;
|
|
1325
1344
|
run.snapshot.metadata = { ...run.snapshot.metadata, pr_lifecycle_state: state };
|
|
1326
1345
|
run.snapshot.updated_at = new Date().toISOString();
|
|
1346
|
+
if (state === 'blocked' && run.snapshot.final_audit?.requires_human === false &&
|
|
1347
|
+
['outcome_progress_stalled', 'retry_circuit_open'].includes(run.snapshot.events.at(-1)?.detail?.code) &&
|
|
1348
|
+
!budgetState(run.snapshot).exhausted) {
|
|
1349
|
+
scheduleCodingRecovery(run.snapshot);
|
|
1350
|
+
}
|
|
1327
1351
|
run.completedAt = TERMINAL_STATES.has(state) ? run.snapshot.updated_at : null;
|
|
1328
1352
|
if (state === "waiting_human" || state === "pr_ready_for_review") {
|
|
1329
1353
|
run.controlState = "waiting_human";
|
|
@@ -6,6 +6,10 @@ import { CLI_VERSION } from "../cli-contract.js";
|
|
|
6
6
|
import { defaultSessionRoot } from "../session-store.js";
|
|
7
7
|
|
|
8
8
|
export const SQM_FAILURE_PAUSE_THRESHOLD = 6;
|
|
9
|
+
export const SQM_RECOVERY_DELAY_MS = 5 * 60_000;
|
|
10
|
+
const MAX_RECOVERY_DELAY_MS = 30 * 60_000;
|
|
11
|
+
const exhausted = state => state.failures >= SQM_FAILURE_PAUSE_THRESHOLD || (state.providerFailures || 0) >= 2;
|
|
12
|
+
const recoveryDelay = state => Math.min(MAX_RECOVERY_DELAY_MS, SQM_RECOVERY_DELAY_MS * 2 ** Math.min(state.recoveryAttempts || 0, 3));
|
|
9
13
|
const scopeKey = scope => JSON.stringify([scope?.webOrigin, scope?.accountId, scope?.organizationId]);
|
|
10
14
|
const validVersion = value => typeof value === "string" && /^\d+\.\d+\.\d+$/.test(value);
|
|
11
15
|
|
|
@@ -21,6 +25,9 @@ export function createSqmFailureCircuitStore({ root = defaultSessionRoot() } = {
|
|
|
21
25
|
const state = JSON.parse(await readFile(path(scope), "utf8"));
|
|
22
26
|
if (!validVersion(state.version) || !Number.isInteger(state.failures) || state.failures < 0 || state.failures > SQM_FAILURE_PAUSE_THRESHOLD || (state.inFlight !== undefined && typeof state.inFlight !== "boolean") || (state.providerFailures !== undefined && (!Number.isInteger(state.providerFailures) || state.providerFailures < 0 || state.providerFailures > 2)) || (state.providerRetryAt !== undefined && (!Number.isFinite(state.providerRetryAt) || state.providerRetryAt < 0)))
|
|
23
27
|
throw new Error("Invalid saved SQM failure state.");
|
|
28
|
+
if (state.recoveryAt !== undefined && (!Number.isFinite(state.recoveryAt) || state.recoveryAt < 0) ||
|
|
29
|
+
state.recoveryAttempts !== undefined && (!Number.isSafeInteger(state.recoveryAttempts) || state.recoveryAttempts < 0))
|
|
30
|
+
throw new Error("Invalid saved SQM recovery state.");
|
|
24
31
|
return state;
|
|
25
32
|
} catch (error) { if (error.code === "ENOENT") return null; throw error; }
|
|
26
33
|
},
|
|
@@ -37,12 +44,17 @@ export function createSqmFailureCircuitStore({ root = defaultSessionRoot() } = {
|
|
|
37
44
|
|
|
38
45
|
// One counter per organization on this machine, shared by both local engines.
|
|
39
46
|
// Neither reconnecting nor an empty claim demonstrates successful analysis.
|
|
40
|
-
export function createSqmFailureCircuit({ version = CLI_VERSION, store = null } = {}) {
|
|
47
|
+
export function createSqmFailureCircuit({ version = CLI_VERSION, store = null, now = Date.now } = {}) {
|
|
41
48
|
if (!validVersion(version)) throw new Error("SQM failure circuit requires a SIMY version.");
|
|
42
49
|
const states = new Map();
|
|
43
50
|
const unreadableScopes = new Set();
|
|
51
|
+
const scheduleRecovery = state => {
|
|
52
|
+
if (!exhausted(state) || state.recoveryAt > now()) return false;
|
|
53
|
+
state.recoveryAt = Math.max(state.providerRetryAt || 0, now() + recoveryDelay(state));
|
|
54
|
+
return true;
|
|
55
|
+
};
|
|
44
56
|
const save = async (scope, state) => {
|
|
45
|
-
try { await store?.write(scope, { version: state.version, failures: state.failures, inFlight: state.inFlight === true, ...(state.providerFailures ? { providerFailures: state.providerFailures, providerRetryAt: state.providerRetryAt } : {}) }); state.storageError = false; }
|
|
57
|
+
try { await store?.write(scope, { version: state.version, failures: state.failures, inFlight: state.inFlight === true, ...(state.recoveryAt !== undefined ? { recoveryAt: state.recoveryAt, recoveryAttempts: state.recoveryAttempts || 0 } : {}), ...(state.providerFailures ? { providerFailures: state.providerFailures, providerRetryAt: state.providerRetryAt } : {}) }); state.storageError = false; }
|
|
46
58
|
catch (error) { state.storageError = true; throw error; }
|
|
47
59
|
};
|
|
48
60
|
const load = async scope => {
|
|
@@ -52,7 +64,7 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
52
64
|
let saved;
|
|
53
65
|
try { saved = await store?.read(scope); unreadableScopes.delete(key); }
|
|
54
66
|
catch (error) { unreadableScopes.add(key); throw error; }
|
|
55
|
-
state = saved ? { version: saved.version, failures: saved.failures, inFlight: saved.inFlight === true, providerFailures: saved.providerFailures || 0, providerRetryAt: saved.providerRetryAt || 0 } : { version, failures: 0, inFlight: false };
|
|
67
|
+
state = saved ? { version: saved.version, failures: saved.failures, inFlight: saved.inFlight === true, providerFailures: saved.providerFailures || 0, providerRetryAt: saved.providerRetryAt || 0, ...(saved.recoveryAt !== undefined ? { recoveryAt: saved.recoveryAt, recoveryAttempts: saved.recoveryAttempts || 0 } : {}) } : { version, failures: 0, inFlight: false };
|
|
56
68
|
states.set(key, state);
|
|
57
69
|
// An unfinished review from this same version means the last host exited
|
|
58
70
|
// without a result or graceful cancellation. Count it once on restart.
|
|
@@ -60,6 +72,7 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
60
72
|
state.failures = Math.min(SQM_FAILURE_PAUSE_THRESHOLD, state.failures + 1);
|
|
61
73
|
if (state.providerFailures === 1) state.providerFailures = 2;
|
|
62
74
|
state.inFlight = false;
|
|
75
|
+
scheduleRecovery(state);
|
|
63
76
|
await save(scope, state);
|
|
64
77
|
}
|
|
65
78
|
}
|
|
@@ -67,20 +80,35 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
67
80
|
state.version = version;
|
|
68
81
|
state.providerFailures = 0; state.providerRetryAt = 0;
|
|
69
82
|
state.failures = 0;
|
|
83
|
+
delete state.recoveryAt; delete state.recoveryAttempts;
|
|
70
84
|
state.inFlight = false;
|
|
71
85
|
await save(scope, state);
|
|
72
86
|
} else if (state.storageError) await save(scope, state);
|
|
87
|
+
// Migrate a legacy update-only pause once, retaining counters and scope.
|
|
88
|
+
if (exhausted(state) && state.recoveryAt === undefined) {
|
|
89
|
+
scheduleRecovery(state); await save(scope, state);
|
|
90
|
+
}
|
|
73
91
|
return state;
|
|
74
92
|
};
|
|
75
93
|
return {
|
|
76
|
-
async canClaim(scope,
|
|
94
|
+
async canClaim(scope, time = now()) {
|
|
77
95
|
const state = await load(scope);
|
|
78
|
-
|
|
96
|
+
if (time < (state.providerRetryAt || 0) || state.inFlight) return false;
|
|
97
|
+
if (!exhausted(state)) return true;
|
|
98
|
+
if (time < state.recoveryAt) return false;
|
|
99
|
+
// Reserve the next cooldown before claiming. Empty claims, reconnects and
|
|
100
|
+
// cancelled work never reset failures or allow a burst of recovery turns.
|
|
101
|
+
state.recoveryAttempts = Math.min((state.recoveryAttempts || 0) + 1, 3);
|
|
102
|
+
state.recoveryAt = time + recoveryDelay(state);
|
|
103
|
+
await save(scope, state);
|
|
104
|
+
return true;
|
|
79
105
|
},
|
|
80
106
|
async providerFailure(scope, retryAt, retryAttempt = 0) {
|
|
81
107
|
const state = await load(scope);
|
|
82
108
|
state.providerFailures = retryAttempt === 1 ? 2 : Math.min(2, (state.providerFailures || 0) + 1);
|
|
83
109
|
state.providerRetryAt = retryAt;
|
|
110
|
+
scheduleRecovery(state);
|
|
111
|
+
if (state.recoveryAt !== undefined) state.recoveryAt = Math.max(state.recoveryAt, retryAt);
|
|
84
112
|
await save(scope, state);
|
|
85
113
|
},
|
|
86
114
|
async providerRetryUsed(scope) {
|
|
@@ -104,12 +132,14 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
104
132
|
state.failures = Math.min(SQM_FAILURE_PAUSE_THRESHOLD, state.failures + 1);
|
|
105
133
|
if (state.inFlight && state.providerFailures === 1) state.providerFailures = 2;
|
|
106
134
|
state.inFlight = false;
|
|
135
|
+
scheduleRecovery(state);
|
|
107
136
|
await save(scope, state);
|
|
108
137
|
},
|
|
109
138
|
async success(scope) {
|
|
110
139
|
const state = await load(scope);
|
|
111
140
|
state.failures = 0;
|
|
112
141
|
state.providerFailures = 0; state.providerRetryAt = 0;
|
|
142
|
+
delete state.recoveryAt; delete state.recoveryAttempts;
|
|
113
143
|
state.inFlight = false;
|
|
114
144
|
await save(scope, state);
|
|
115
145
|
},
|
|
@@ -117,10 +147,12 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
|
|
|
117
147
|
const state = states.get(scopeKey(scope));
|
|
118
148
|
const paused = (state?.failures || 0) >= SQM_FAILURE_PAUSE_THRESHOLD || (state?.providerFailures || 0) >= 2;
|
|
119
149
|
return { consecutive_failures: state?.failures || 0,
|
|
120
|
-
paused_until_update:
|
|
150
|
+
paused_until_update: false,
|
|
151
|
+
recovery_pending: paused && state?.inFlight !== true,
|
|
152
|
+
next_retry_at: paused ? state.recoveryAt ?? null : null,
|
|
121
153
|
provider_retry_exhausted: (state?.providerFailures || 0) >= 2,
|
|
122
154
|
provider_recovery_pending: state?.providerFailures === 1,
|
|
123
|
-
paused_at_version:
|
|
155
|
+
paused_at_version: null,
|
|
124
156
|
storage_unavailable: state?.storageError === true || unreadableScopes.has(scopeKey(scope)) };
|
|
125
157
|
},
|
|
126
158
|
};
|
package/src/sqm/review-worker.js
CHANGED
|
@@ -115,11 +115,12 @@ export function createSqmLocalReviewWorker({
|
|
|
115
115
|
getHistoryScope = () => null,
|
|
116
116
|
getWorkspaceUrl = () => null,
|
|
117
117
|
recordHistory = async () => {},
|
|
118
|
+
readHistory = async () => [],
|
|
118
119
|
onHistoryError = () => {},
|
|
119
120
|
now = () => Date.now(),
|
|
120
121
|
providerFailureCooldownMs = 5 * 60_000,
|
|
121
122
|
getRecoveryKey = () => null,
|
|
122
|
-
failureCircuit = createSqmFailureCircuit(),
|
|
123
|
+
failureCircuit = createSqmFailureCircuit({ now }),
|
|
123
124
|
} = {}) {
|
|
124
125
|
if (!api || typeof api.request !== "function") throw new TypeError("api.request is required");
|
|
125
126
|
let activeJobId = null;
|
|
@@ -158,6 +159,7 @@ export function createSqmLocalReviewWorker({
|
|
|
158
159
|
item.state = state; item.finishedAt = new Date().toISOString();
|
|
159
160
|
item.result = state === "completed" ? "SQM review submitted." : "";
|
|
160
161
|
item.error = state === "failed" ? formatFailure(failure, failureReported) : "";
|
|
162
|
+
if (state === 'completed') for (const field of ['errorCode', 'errorStage', 'httpStatus', 'providerDiagnostic']) delete item[field];
|
|
161
163
|
if (failure) { item.phase = failure.stage; item.errorStage = failure.stage; item.errorCode = failure.code; item.httpStatus = failure.status; item.providerDiagnostic = failure.providerDiagnostic; }
|
|
162
164
|
await persist(scope, item);
|
|
163
165
|
}
|
|
@@ -273,20 +275,22 @@ export function createSqmLocalReviewWorker({
|
|
|
273
275
|
snapshot() {
|
|
274
276
|
const pause = currentPause();
|
|
275
277
|
const circuit = failureCircuit.snapshot(getHistoryScope());
|
|
278
|
+
const requiresInteraction = pause && !pause.recoverable && pause.recoveryKey === getRecoveryKey(getProvider());
|
|
279
|
+
const waitingForRecovery = circuit.recovery_pending && !requiresInteraction && !activeJobId;
|
|
276
280
|
return { active_job_id: activeJobId, polling: Boolean(pollPromise), closed, suspended,
|
|
277
|
-
paused: pausedEnvironment !== null || circuit.
|
|
278
|
-
pause_reason:
|
|
281
|
+
paused: !activeJobId && (pausedEnvironment !== null || circuit.recovery_pending || circuit.storage_unavailable),
|
|
282
|
+
pause_reason: activeJobId ? null : requiresInteraction ? pause.code : waitingForRecovery ? "automatic_recovery_pending" : circuit.storage_unavailable ? "failure_state_unavailable" : pauseReason,
|
|
279
283
|
...circuit, issue: issueContext === contextKey(getProvider(), getHistoryScope()) ? issue : null,
|
|
280
|
-
...(pause ? { provider_pause: { code: pause.code, status: pause.status,
|
|
281
|
-
retry_at:
|
|
282
|
-
message:
|
|
284
|
+
...(pause && !activeJobId ? { provider_pause: { code: pause.code, status: pause.status,
|
|
285
|
+
retry_at: new Date(Math.max(pause.retryAt, circuit.next_retry_at || 0)).toISOString(),
|
|
286
|
+
message: waitingForRecovery ? "SIMY will automatically retry after the recovery cooldown. No update or manual confirmation is required." : pause.message } } : {}) };
|
|
283
287
|
},
|
|
284
288
|
desktopTasks() { return { activeCount: activeJobId ? 1 : 0, items: visibleHistory().map(item => ({ ...item, canStop: item.id === activeJobId })) }; },
|
|
285
289
|
desktopTaskDetail(key) {
|
|
286
290
|
const item = visibleHistory().find(item => item.key === key);
|
|
287
291
|
return item ? { key, instruction: `SQM local review · ${item.provider}`, result: item.state === "completed" ? "SQM review submitted." : "", error: item.error || "", links: item.webUrl ? [{ label: "Analysis details", url: item.webUrl }] : [] } : null;
|
|
288
292
|
},
|
|
289
|
-
retry() { if (closed || suspended || pollPromise || failureCircuit.snapshot(getHistoryScope()).
|
|
293
|
+
retry() { if (closed || suspended || pollPromise || failureCircuit.snapshot(getHistoryScope()).recovery_pending) return false; pausedEnvironment = null; pauseReason = null; pauses.delete(contextKey(getProvider(), getHistoryScope())); issue = null; onChange(); return true; },
|
|
290
294
|
resetHistory() { if (!pollPromise) { history.splice(0); issue = null; pausedEnvironment = null; pauseReason = null; } },
|
|
291
295
|
poll() {
|
|
292
296
|
if (closed || suspended || pollPromise) return pollPromise || Promise.resolve(null);
|
|
@@ -295,7 +299,8 @@ export function createSqmLocalReviewWorker({
|
|
|
295
299
|
const pause = currentPause();
|
|
296
300
|
// Known provider failures already have a bounded cooldown. Do not let
|
|
297
301
|
// the environment gate suppress their recovery probe indefinitely.
|
|
298
|
-
if (
|
|
302
|
+
if (pause?.recoveryKey === getRecoveryKey(getProvider()) &&
|
|
303
|
+
(!pause.recoverable || now() < pause.retryAt)) return Promise.resolve(null);
|
|
299
304
|
pausedEnvironment = null; pauseReason = null;
|
|
300
305
|
} else if (pausedEnvironment !== null) {
|
|
301
306
|
pausedEnvironment = null; pauseReason = null; pauses.delete(contextKey(getProvider(), getHistoryScope())); issue = null;
|
|
@@ -323,6 +328,23 @@ export function createSqmLocalReviewWorker({
|
|
|
323
328
|
lease_seconds: leaseSeconds,
|
|
324
329
|
}, { signal: reviewController.signal });
|
|
325
330
|
if (issue) { issue = null; onChange(); }
|
|
331
|
+
if (claim?.recovered === true && claim.completed === true && typeof claim.job_id === 'string') {
|
|
332
|
+
// The server completed this exact saved result without re-running a
|
|
333
|
+
// provider. Reflect the original job's completion locally as well.
|
|
334
|
+
if (!history.some(item => item.id === claim.job_id)) {
|
|
335
|
+
let saved;
|
|
336
|
+
try { saved = (await readHistory(historyScope)).find(item => item.key === `sqm:${claim.job_id}`); }
|
|
337
|
+
catch { onHistoryError(); }
|
|
338
|
+
const item = { ...saved, id: claim.job_id, key: `sqm:${claim.job_id}`, kind: 'analysis',
|
|
339
|
+
label: saved?.label || `SQM · ${claim.job_id.slice(0, 8)}`, provider: saved?.provider || provider,
|
|
340
|
+
startedAt: saved?.startedAt || new Date().toISOString(), webUrl: saved?.webUrl || analysisTaskUrl(workspaceUrl, claim.job_id) };
|
|
341
|
+
history.unshift(item); historyScopes.set(item, JSON.stringify(historyScope)); history.splice(50);
|
|
342
|
+
}
|
|
343
|
+
await updateTask(claim.job_id, 'completed', historyScope);
|
|
344
|
+
await failureCircuit.success(historyScope);
|
|
345
|
+
pauses.delete(key);
|
|
346
|
+
return claim;
|
|
347
|
+
}
|
|
326
348
|
if (!claim?.claimed) {
|
|
327
349
|
// A successful empty claim proves SIMY access recovered, but it says
|
|
328
350
|
// nothing about a local provider that has not executed again yet.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import {readFile,writeFile,mkdir,realpath} from 'node:fs/promises';
|
|
2
|
+
import {join,resolve} from 'node:path';
|
|
3
|
+
import {createHash,randomUUID} from 'node:crypto';
|
|
4
|
+
import {digest} from './state-store.js';
|
|
5
|
+
import {readDispatchGate,reconcileDispatchHistory} from './dispatch-gate.js';
|
|
6
|
+
import {createUpdateRuntime} from './runtime.js';
|
|
7
|
+
import {createUpdateCoordinator} from './coordinator.js';
|
|
8
|
+
import {readSession,isSessionValid} from '../session-store.js';
|
|
9
|
+
import {webApiUrl,webApiHeaders} from '../web-api.js';
|
|
10
|
+
import {acquireDesktopProcessLock} from '../desktop-process-lock.js';
|
|
11
|
+
import {listRecords} from '../codex-plugins/simy/hooks/delivery-loop.mjs';
|
|
12
|
+
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
|
13
|
+
export async function planDispatchHistoryRecovery({root}) {
|
|
14
|
+
root=await realpath(root);const state=readDispatchGate(root),items=[];
|
|
15
|
+
for(const attempt of Object.values(state.attempts)) {
|
|
16
|
+
const item={attemptId:attempt.id,attemptDigest:digest(attempt),disposition:attempt.phase==='confirmed'?'already_confirmed':'retain_blocked_unknown'};
|
|
17
|
+
const ref=attempt.provenance?.recordRef;
|
|
18
|
+
if(attempt.phase==='result_saved' && ref?.startsWith(resolve(root,'direct-tasks')+'/')) {
|
|
19
|
+
try {
|
|
20
|
+
const bytes=await readFile(ref),result=JSON.parse(bytes),match=/^(.*):provider:([1-9][0-9]*)$/.exec(attempt.id);
|
|
21
|
+
if(await realpath(ref)===ref && match && result.id===match[1] && result.id===attempt.provenance.taskId && digest(result.id)===attempt.taskHash &&
|
|
22
|
+
result.invocation_count===Number(match[2]) && ['succeeded','failed','stopped'].includes(result.status) && Object.values(state.receipts).some(r=>r.attemptId===attempt.id && r.generation===attempt.generation && r.resultRef===ref)) {
|
|
23
|
+
item.disposition='saved_terminal_result';item.evidence={ref,sha256:sha(bytes),status:result.status};
|
|
24
|
+
}
|
|
25
|
+
} catch { /* Unknown effects remain blocked, including removed test output. */ }
|
|
26
|
+
}
|
|
27
|
+
items.push(item);
|
|
28
|
+
}
|
|
29
|
+
const audit={version:1,gateDigest:digest(state),expectedRevision:state.revision,expectedGeneration:state.generation,items};
|
|
30
|
+
const directory=join(root,'update-detachment','recovery');await mkdir(directory,{recursive:true,mode:0o700});
|
|
31
|
+
const auditRef=join(directory,randomUUID()+'.json'),bytes=JSON.stringify(audit,null,2)+'\n';await writeFile(auditRef,bytes,{flag:'wx',mode:0o600});
|
|
32
|
+
return {auditRef,auditDigest:sha(bytes),revision:state.revision,generation:state.generation,results:items.filter(i=>i.disposition==='saved_terminal_result').length,blocked:items.filter(i=>i.disposition==='retain_blocked_unknown').length,replayAttempts:false};
|
|
33
|
+
}
|
|
34
|
+
export async function applyDispatchHistoryRecovery({root,auditRef,auditDigest}) {
|
|
35
|
+
root=await realpath(root);const lock=await acquireDesktopProcessLock(root);
|
|
36
|
+
try {const audit=JSON.parse(await readFile(auditRef,'utf8'));return reconcileDispatchHistory({root,auditRef,auditDigest,expectedRevision:audit.expectedRevision,expectedGeneration:audit.expectedGeneration});}
|
|
37
|
+
finally {await lock.release();}
|
|
38
|
+
}
|
|
39
|
+
export async function reconnectRecoveredHistory({root,origin='https://app.simy.one'}) {
|
|
40
|
+
root=await realpath(root);const lock=await acquireDesktopProcessLock(root);
|
|
41
|
+
try {
|
|
42
|
+
const session=await readSession(origin,root);if(!isSessionValid(session,Date.now(),origin))throw Error('Existing device sign-in required.');
|
|
43
|
+
const response=await fetch(webApiUrl('desktop-access',{webOrigin:origin,apiBaseUrl:session.api_base_url}),{headers:webApiHeaders(session),signal:AbortSignal.timeout(10000)});
|
|
44
|
+
const access=await response.json();if(!response.ok || access.state!=='ready' || access.organizationId!==session.organization_id)throw Error('Current server entitlement required.');
|
|
45
|
+
const runtime=createUpdateRuntime({root,origin,readContext:()=>({actor:{account:session.account_id,device:session.device_id,origin},runs:[],directTasks:[],localTasks:[]}),listRecords:()=>listRecords(join(root,'delivery-hooks'))});
|
|
46
|
+
// Reconciliation reads exact results or retained fences. It never resumes a task.
|
|
47
|
+
const coordinator=createUpdateCoordinator({store:runtime.store,gate:runtime.gate,boundary:runtime.boundary,reconcile:runtime.reconcile,initialState:'recovery_required'});
|
|
48
|
+
if(digest(await readSession(origin,root))!==digest(session))throw Error('Device connection changed.');
|
|
49
|
+
return await coordinator.reconnect();
|
|
50
|
+
} finally {await lock.release();}
|
|
51
|
+
}
|
|
@@ -6,7 +6,7 @@ import { execFile } from 'node:child_process';
|
|
|
6
6
|
import { promisify } from 'node:util';
|
|
7
7
|
import { readFile, lstat, readlink, realpath } from 'node:fs/promises';
|
|
8
8
|
import { atomicSave, readState, digest, withLock } from './state-store.js';
|
|
9
|
-
import { readDispatchGate, adoptHookOwner, DISPATCH_PROTOCOL, confirmDispatch, flushDispatchIntake, readTaskAdmission } from './dispatch-gate.js';
|
|
9
|
+
import { readDispatchGate, unresolvedDispatchAttempts, adoptHookOwner, DISPATCH_PROTOCOL, confirmDispatch, flushDispatchIntake, readTaskAdmission } from './dispatch-gate.js';
|
|
10
10
|
import { createLedgerCoordinatorAdapters } from './ledger-adapter.js';
|
|
11
11
|
import { listDispatchRegistrations, normalizeDispatchIdentity } from './provenance.js';
|
|
12
12
|
import { reconcileSession } from './reconcile.js';
|
|
@@ -182,7 +182,7 @@ export function createUpdateRuntime({ root, origin, readContext, listRecords, ob
|
|
|
182
182
|
confirmDispatch({root,attemptId:attempt.id});
|
|
183
183
|
}
|
|
184
184
|
ledger=readDispatchGate(root);
|
|
185
|
-
const unresolved=
|
|
185
|
+
const unresolved=unresolvedDispatchAttempts(ledger).filter(attempt=> !workers.some(task=>task.handle.attemptId===attempt.id && attempt.taskHash===digest(task.taskId) && attempt.identityHash===digest(task.identity) && attempt.generation===task.generation));
|
|
186
186
|
if(unresolved.length)return {safe:false,liveExecutions:0,pendingWrites:0,unsettledPermissions:0};
|
|
187
187
|
const currentGate=readDispatchGate(root);
|
|
188
188
|
if(currentGate.connection!=='detaching' || currentGate.generation!==startingGate.generation)throw failure('boundary_owner_changed');
|
|
@@ -194,7 +194,7 @@ export function createUpdateRuntime({ root, origin, readContext, listRecords, ob
|
|
|
194
194
|
const reconcile=async()=> {
|
|
195
195
|
const actor=context().actor, assessments=[];
|
|
196
196
|
const workers=(transaction?.tasks || []).filter(task=>task.handle);
|
|
197
|
-
if(
|
|
197
|
+
if(unresolvedDispatchAttempts(readDispatchGate(root)).some(attempt=> !workers.some(task=>task.handle.attemptId===attempt.id)))return {verified:false,unresolvedAttempts:1};
|
|
198
198
|
for (const task of transaction?.tasks || []) {
|
|
199
199
|
if(task.handle) {
|
|
200
200
|
if(actor?.account!==task.identity.account || actor?.origin!==task.identity.origin || actor?.device!==task.identity.device || task.identity.root!==root)return {verified:false,unresolvedAttempts:1};
|