mtech-ai-local-runner 0.1.57 → 0.1.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/src/daemon/execution-coordinator.js +102 -50
- package/dist/src/daemon/heartbeat-scheduler.js +5 -4
- package/dist/src/daemon/runtime.js +4 -2
- package/dist/src/gateway-client.js +80 -18
- package/dist/src/publication/failure.js +5 -4
- package/dist/src/publication/recovery.js +58 -4
- package/dist/src/safety/execution-failure.js +6 -0
- package/dist/src/safety/transport-diagnostic.js +15 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -911,3 +911,11 @@ not evidence of logout. These diagnostics do not bypass per-job admission.
|
|
|
911
911
|
Use `local_replace({path, old, new})` for a small approved edit after reading the file: `old` must be nonempty and match exactly once, including whitespace. Zero or overlapping/multiple matches are rejected with distinct errors and no write. Replacement text must not be a patch; the result retains the same scoped write, size, link and revocation protections as `local_edit`. `local_edit` remains a complete-file replacement API.
|
|
912
912
|
|
|
913
913
|
Both CLI adapters require the five-tool inventory. Gateway tools descriptors and runtime pins must match this release; CLI release manifests and the standalone binary stay fixed. In `gateway_publish_v1`, missing registered tests is expected: report not_run/unverified, apply only the smallest source-evidenced correction, disclose uncertainty and seal for a human-reviewed Gateway PR. Historical contracts and attempts are not edited. Reverb 0.1.49 is retained; installation, deployment and real model execution remain separate approvals.
|
|
914
|
+
|
|
915
|
+
## 0.1.58: publication completion and diagnostic compatibility
|
|
916
|
+
|
|
917
|
+
Publication cursor synchronization now validates the exact server proof for the selected mode: `progress/server_publication` with `unverified` and `CI not_run` for `gateway_publish_v1`, or the existing verified test-result form for `gateway_ci_v1`. It checks the frozen Job, current host context and completed publication fingerprints before advancing the cursor. No model output can grant publication authority.
|
|
918
|
+
|
|
919
|
+
Cursor failures retain recovery state and report `LOCAL_FIX_PUBLICATION_CURSOR_REJECTED` / `cursor_sync`. Runner advertisement failures use `RUNNER_HEARTBEAT_SEND_FAILED`; execution Job heartbeat failures use `JOB_HEARTBEAT_FAILED`. Diagnostics expose only bounded elapsed milliseconds, HTTP status and allowlisted OS codes. The existing fence/cancellation policy is unchanged. This package does not repair expired leases or existing completed-PR projections by itself, and installing it does not authorize a model rerun.
|
|
920
|
+
|
|
921
|
+
For `gateway_publish_v1` only, a matched final `server_reconciliation` completed event acknowledges the Gateway's atomic Job/binding completion. The Runner reads once before cursor/terminal work, and once on a terminal-path rejection if necessary, then skips duplicate terminal and Job writes only when every identity/publication field matches the stored server response. Unknown evidence remains suspended for recovery. The Loop's paused/human-review projection remains entirely Gateway-owned; unverified tests remain unverified. This does not execute or archive the existing40bf recovery.
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { completionRejectionDiagnostic } from '../adapters/completion-report.js';
|
|
2
|
+
import { publicationError } from '../publication/failure.js';
|
|
2
3
|
import { PublicationOutcomeUnknownError } from '../publication/publisher.js';
|
|
3
4
|
import { assertP1Bindings, hasP1, p1ModelEvent } from '../verification/p1-execution-host.js';
|
|
4
|
-
import { executionError, executionFailure } from '../safety/execution-failure.js';
|
|
5
|
+
import { executionError, executionFailure, jobHeartbeatError } from '../safety/execution-failure.js';
|
|
5
6
|
import { isUnsupportedSupervisorCommand } from '../safety/unsupported-supervisor-command.js';
|
|
6
7
|
import { requireSecurityRequirements } from '../safety/security-requirements.js';
|
|
7
8
|
import { GatewayToolError } from "../gateway-client.js";
|
|
@@ -194,6 +195,8 @@ export class ExecutionCoordinator {
|
|
|
194
195
|
const controlStopped = new Promise((_, reject) => { rejectControlStop = reject; });
|
|
195
196
|
void controlStopped.catch(() => { });
|
|
196
197
|
let publicationPending = false;
|
|
198
|
+
let serverCompletionConfirmed = false, serverCompletionCheckFailed = false;
|
|
199
|
+
let publicationRecoveryState;
|
|
197
200
|
let commandCursor = 0;
|
|
198
201
|
let controlsStopped = false;
|
|
199
202
|
let controlsInFlight;
|
|
@@ -315,14 +318,19 @@ export class ExecutionCoordinator {
|
|
|
315
318
|
await readControls();
|
|
316
319
|
requireControlFence();
|
|
317
320
|
if (!requestedTerminal && this.deps.gateway.heartbeat) {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
321
|
+
try {
|
|
322
|
+
const renewed = await fenced(this.deps.gateway.heartbeat(binding.bindingId, job, controlAbort.signal));
|
|
323
|
+
requireControlFence();
|
|
324
|
+
const expiry = validateJobLease(renewed, job, true);
|
|
325
|
+
if (!['running', 'review_pending'].includes(renewed.job.status))
|
|
326
|
+
throw new GatewayToolError('Running Job lease status is invalid', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID');
|
|
327
|
+
this.deps.gateway.confirmJobLease?.(job, expiry);
|
|
328
|
+
leaseExpiresAt = expiry;
|
|
329
|
+
armLeaseBoundary();
|
|
330
|
+
}
|
|
331
|
+
catch (error) {
|
|
332
|
+
throw jobHeartbeatError(error);
|
|
333
|
+
}
|
|
326
334
|
}
|
|
327
335
|
})().catch((error) => {
|
|
328
336
|
failFence(error);
|
|
@@ -334,6 +342,17 @@ export class ExecutionCoordinator {
|
|
|
334
342
|
let sequence = 0;
|
|
335
343
|
let checkpointSeen = false;
|
|
336
344
|
let eventUncertain = false;
|
|
345
|
+
const recognizeServerCompletion = async () => {
|
|
346
|
+
if (!localFixPublication || serverCompletionCheckFailed || job.taskPackage.local_fix_scope?.verification_mode !== 'gateway_publish_v1' || !this.deps.gateway.confirmServerPublicationCompletion)
|
|
347
|
+
return false;
|
|
348
|
+
try {
|
|
349
|
+
return await this.deps.gateway.confirmServerPublicationCompletion(binding.bindingId, sequence, job, localFixPublication);
|
|
350
|
+
}
|
|
351
|
+
catch (error) {
|
|
352
|
+
serverCompletionCheckFailed = true;
|
|
353
|
+
throw publicationError('server_completion', 'LOCAL_FIX_SERVER_COMPLETION_REJECTED', true, error);
|
|
354
|
+
}
|
|
355
|
+
};
|
|
337
356
|
const stop = async (ignoreControlFailure = false) => {
|
|
338
357
|
controlsStopped = true;
|
|
339
358
|
const cancelled = stopExecutor();
|
|
@@ -514,18 +533,31 @@ export class ExecutionCoordinator {
|
|
|
514
533
|
executionStage = 'local_fix_publication';
|
|
515
534
|
if (!checkpointSeen)
|
|
516
535
|
await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
|
|
517
|
-
|
|
536
|
+
publicationRecoveryState = { jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: 'publication_pending', bindingId: binding.bindingId, eventCursor: sequence, workspaceManifest: workspace.manifest, publicationRecovery: { version: 1, job, checkoutPath: workspace.path, bindingId: binding.bindingId, completedEvent: event, gatewayContext: this.deps.gateway.publicationRecoveryContext?.(binding.bindingId), securityReceipt: adapter.securityReceipt?.(), freezeConfirmed: lifecycle.frozen } };
|
|
537
|
+
await publicationStep(() => this.deps.state.save(publicationRecoveryState));
|
|
518
538
|
publicationPending = true;
|
|
519
|
-
localFixPublication = await publicationStep(() => fixSession.finalize(event));
|
|
539
|
+
localFixPublication = await publicationStep(async () => { localFixPublication = await fixSession.finalize(event); return localFixPublication; });
|
|
540
|
+
publicationRecoveryState = { ...publicationRecoveryState, publicationRecovery: { ...publicationRecoveryState.publicationRecovery, completedPublication: structuredClone(localFixPublication) } };
|
|
541
|
+
try {
|
|
542
|
+
await this.deps.state.save(publicationRecoveryState);
|
|
543
|
+
}
|
|
544
|
+
catch (error) {
|
|
545
|
+
throw publicationError('journal', 'LOCAL_FIX_PUBLICATION_OUTCOME_UNKNOWN', true, error);
|
|
546
|
+
}
|
|
547
|
+
if (await recognizeServerCompletion()) {
|
|
548
|
+
serverCompletionConfirmed = true;
|
|
549
|
+
terminal = event;
|
|
550
|
+
break;
|
|
551
|
+
}
|
|
520
552
|
if (this.deps.gateway.syncPublicationCursor) {
|
|
521
553
|
try {
|
|
522
|
-
const cursor = await publicationStep(() => this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence));
|
|
554
|
+
const cursor = await publicationStep(() => this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence, job, localFixPublication));
|
|
523
555
|
if (!Number.isSafeInteger(cursor) || cursor < sequence)
|
|
524
556
|
throw new Error('Invalid event cursor');
|
|
525
557
|
sequence = cursor;
|
|
526
558
|
}
|
|
527
|
-
catch {
|
|
528
|
-
throw
|
|
559
|
+
catch (error) {
|
|
560
|
+
throw publicationError('cursor_sync', 'LOCAL_FIX_PUBLICATION_CURSOR_REJECTED', true, error);
|
|
529
561
|
}
|
|
530
562
|
}
|
|
531
563
|
if (controlsInFlight)
|
|
@@ -563,18 +595,20 @@ export class ExecutionCoordinator {
|
|
|
563
595
|
await iterator.return?.();
|
|
564
596
|
}
|
|
565
597
|
}
|
|
566
|
-
await stop();
|
|
567
|
-
if (
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
terminal
|
|
598
|
+
await stop(serverCompletionConfirmed);
|
|
599
|
+
if (!serverCompletionConfirmed) {
|
|
600
|
+
if (publicationPending && requestedTerminal)
|
|
601
|
+
throw new PublicationOutcomeUnknownError();
|
|
602
|
+
if (requestedTerminal)
|
|
603
|
+
terminal = requestedTerminal;
|
|
604
|
+
if (!terminal) {
|
|
605
|
+
terminal = { eventType: "failed", summary: "adapter ended without a terminal event" };
|
|
606
|
+
}
|
|
607
|
+
terminal = daemonOutcome(terminal).event;
|
|
608
|
+
if (terminal.eventType === 'completed' && !checkpointSeen)
|
|
609
|
+
await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
|
|
610
|
+
await append(terminal);
|
|
573
611
|
}
|
|
574
|
-
terminal = daemonOutcome(terminal).event;
|
|
575
|
-
if (terminal.eventType === 'completed' && !checkpointSeen)
|
|
576
|
-
await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
|
|
577
|
-
await append(terminal);
|
|
578
612
|
}
|
|
579
613
|
catch (error) {
|
|
580
614
|
try {
|
|
@@ -593,36 +627,47 @@ export class ExecutionCoordinator {
|
|
|
593
627
|
catch { /* logging must not change terminal reconciliation */ }
|
|
594
628
|
throw cleanupError;
|
|
595
629
|
}
|
|
596
|
-
if (
|
|
630
|
+
if (!serverCompletionCheckFailed) {
|
|
597
631
|
try {
|
|
598
|
-
|
|
632
|
+
if (await recognizeServerCompletion())
|
|
633
|
+
serverCompletionConfirmed = true;
|
|
634
|
+
}
|
|
635
|
+
catch (reconciliationError) {
|
|
636
|
+
error = reconciliationError;
|
|
599
637
|
}
|
|
600
|
-
catch { /* fixed diagnostics must not alter reconciliation */ }
|
|
601
|
-
// The provider may already have committed/pushed/opened a PR. Do not
|
|
602
|
-
// release this run as failed or let restart begin another model session.
|
|
603
|
-
throw new Error('Local publication outcome requires reconciliation');
|
|
604
|
-
}
|
|
605
|
-
if (requestedTerminal && !heartbeatFailure && !eventUncertain) {
|
|
606
|
-
terminal = daemonOutcome(requestedTerminal).event;
|
|
607
638
|
}
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
this.
|
|
639
|
+
if (!serverCompletionConfirmed) {
|
|
640
|
+
if (error instanceof PublicationOutcomeUnknownError || (publicationPending && (heartbeatFailure || requestedTerminal))) {
|
|
641
|
+
try {
|
|
642
|
+
this.deps.reportExecutionFailure?.(executionFailure(error, executionStage));
|
|
643
|
+
}
|
|
644
|
+
catch { /* fixed diagnostics must not alter reconciliation */ }
|
|
645
|
+
// The provider may already have committed/pushed/opened a PR. Do not
|
|
646
|
+
// release this run as failed or let restart begin another model session.
|
|
647
|
+
throw new Error('Local publication outcome requires reconciliation');
|
|
616
648
|
}
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
649
|
+
if (requestedTerminal && !heartbeatFailure && !eventUncertain) {
|
|
650
|
+
terminal = daemonOutcome(requestedTerminal).event;
|
|
651
|
+
}
|
|
652
|
+
else {
|
|
653
|
+
// Do not complete a job whose terminal evidence was not acknowledged.
|
|
654
|
+
// A safe bounded reason prevents raw adapter/transport errors leaking.
|
|
655
|
+
const diagnostic = executionFailure(error, executionStage);
|
|
656
|
+
failureDiagnostic = diagnostic;
|
|
657
|
+
// Log before reporting so an unavailable Gateway does not erase the cause.
|
|
658
|
+
try {
|
|
659
|
+
this.deps.reportExecutionFailure?.(structuredClone(diagnostic));
|
|
660
|
+
}
|
|
661
|
+
catch { /* logging must not change terminal reconciliation */ }
|
|
662
|
+
if (heartbeatFailure)
|
|
663
|
+
throw error;
|
|
664
|
+
if (eventUncertain || ['GATEWAY_TOOL_REJECTED', 'GATEWAY_TRANSPORT_ERROR', 'WORK_LOOP_CANCELLED'].includes(diagnostic.code)
|
|
665
|
+
|| diagnostic.stage === 'terminal_evidence')
|
|
666
|
+
throw error;
|
|
667
|
+
terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
|
|
668
|
+
}
|
|
669
|
+
await append(terminal);
|
|
624
670
|
}
|
|
625
|
-
await append(terminal);
|
|
626
671
|
}
|
|
627
672
|
finally {
|
|
628
673
|
controlsStopped = true;
|
|
@@ -634,6 +679,13 @@ export class ExecutionCoordinator {
|
|
|
634
679
|
clearTimeout(leaseBoundary);
|
|
635
680
|
await fixSession?.close();
|
|
636
681
|
}
|
|
682
|
+
if (serverCompletionConfirmed) {
|
|
683
|
+
await this.deps.gateway.releaseServerCompletedJob?.(job.id);
|
|
684
|
+
await this.deps.state.clear();
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
if (!terminal)
|
|
688
|
+
throw publicationError('server_completion', 'LOCAL_FIX_SERVER_COMPLETION_REJECTED', true);
|
|
637
689
|
const { status, reason } = daemonOutcome(terminal);
|
|
638
690
|
const diagnostic = status === "failed" || status === "blocked" ? completionReportDiagnostic(terminal.payload?.completion_report) : undefined;
|
|
639
691
|
const reportDiagnostic = diagnostic && (status === "blocked"
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { transportDiagnostic } from '../safety/transport-diagnostic.js';
|
|
1
2
|
export class HeartbeatScheduler {
|
|
2
3
|
deps;
|
|
3
4
|
lastKey;
|
|
@@ -33,8 +34,8 @@ export class HeartbeatScheduler {
|
|
|
33
34
|
this.lastKey = key;
|
|
34
35
|
this.lastSentAt = now;
|
|
35
36
|
}
|
|
36
|
-
catch {
|
|
37
|
-
this.report("
|
|
37
|
+
catch (error) {
|
|
38
|
+
this.report("RUNNER_HEARTBEAT_SEND_FAILED", transportDiagnostic(error));
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
41
|
/** Keeps ticking until stop(). The first tick comes one interval in: the caller sends the first heartbeat itself. */
|
|
@@ -54,9 +55,9 @@ export class HeartbeatScheduler {
|
|
|
54
55
|
clearTimeout(this.timer);
|
|
55
56
|
this.release();
|
|
56
57
|
}
|
|
57
|
-
report(code) {
|
|
58
|
+
report(code, diagnostic) {
|
|
58
59
|
try {
|
|
59
|
-
this.deps.reportError?.(code);
|
|
60
|
+
this.deps.reportError?.(code, diagnostic);
|
|
60
61
|
}
|
|
61
62
|
catch { /* Logging cannot stop heartbeats. */ }
|
|
62
63
|
}
|
|
@@ -103,7 +103,9 @@ export async function buildDaemon(input) {
|
|
|
103
103
|
launchGrant: job.launchGrant, workLoopId: job.workLoopId, checkoutPath: job.checkoutPath,
|
|
104
104
|
}, signal),
|
|
105
105
|
publicationRecoveryContext: bindingId => gateway.exportLocalPublicationRecoveryContext(bindingId),
|
|
106
|
-
syncPublicationCursor: (bindingId, cursor) => gateway.syncLocalPublicationCursor(bindingId, cursor),
|
|
106
|
+
syncPublicationCursor: (bindingId, cursor, job, receipt) => gateway.syncLocalPublicationCursor(bindingId, cursor, job, receipt),
|
|
107
|
+
confirmServerPublicationCompletion: (bindingId, cursor, job, receipt) => gateway.confirmServerPublicationCompletion(bindingId, cursor, job, receipt),
|
|
108
|
+
releaseServerCompletedJob: async (jobId) => { await production?.releaseJob(jobId); pendingCodexClaim = undefined; registeredCodexJob = undefined; },
|
|
107
109
|
readCommands: (bindingId, cursor, signal) => gateway.readCommands(bindingId, cursor, signal),
|
|
108
110
|
acknowledgeCommand: (bindingId, sequence, signal) => gateway.acknowledgeCommand(bindingId, sequence, signal),
|
|
109
111
|
appendEvent: (bindingId, event) => gateway.appendEvent(bindingId, event),
|
|
@@ -217,7 +219,7 @@ export async function buildDaemon(input) {
|
|
|
217
219
|
};
|
|
218
220
|
},
|
|
219
221
|
send: async (payload) => { await gateway.heartbeatLocalRunner(payload); },
|
|
220
|
-
reportError: code => { process.stderr.write(`${JSON.stringify({ event: "heartbeat_failed", code })}\n`); },
|
|
222
|
+
reportError: (code, diagnostic) => { process.stderr.write(`${JSON.stringify({ event: "heartbeat_failed", code, heartbeat_channel: "runner", timestamp: new Date().toISOString(), ...diagnostic })}\n`); },
|
|
221
223
|
});
|
|
222
224
|
const stopping = new AbortController();
|
|
223
225
|
const closureAcks = new PendingClosureAcks();
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { publicationError } from './publication/failure.js';
|
|
2
|
+
import { issueTransportDiagnostic } from './safety/transport-diagnostic.js';
|
|
1
3
|
import { createHash } from "node:crypto";
|
|
2
4
|
import { v5 as uuidv5 } from "uuid";
|
|
3
5
|
import { bindingApprovalIdFromEnv, redact } from "./config.js";
|
|
@@ -173,6 +175,8 @@ export class McpGatewayClient {
|
|
|
173
175
|
throw new Error("No token provider or static token configured");
|
|
174
176
|
}
|
|
175
177
|
async callTool(name, args, signal) {
|
|
178
|
+
const started = performance.now();
|
|
179
|
+
let httpStatus;
|
|
176
180
|
let failureCode = "GATEWAY_TRANSPORT_ERROR";
|
|
177
181
|
try {
|
|
178
182
|
const token = await this.resolveToken();
|
|
@@ -183,6 +187,7 @@ export class McpGatewayClient {
|
|
|
183
187
|
headers: { Authorization: `Bearer ${token}`, "content-type": "application/json" },
|
|
184
188
|
body: JSON.stringify({ jsonrpc: "2.0", id: ++this.requestId, method: "tools/call", params: { name, arguments: args } }),
|
|
185
189
|
});
|
|
190
|
+
httpStatus = response.status;
|
|
186
191
|
signal?.throwIfAborted();
|
|
187
192
|
if (!response.ok) {
|
|
188
193
|
failureCode = `GATEWAY_HTTP_${response.status}`;
|
|
@@ -222,7 +227,9 @@ export class McpGatewayClient {
|
|
|
222
227
|
}
|
|
223
228
|
const leaseToken = this.context?.leaseToken ?? this.initialLeaseToken ?? "";
|
|
224
229
|
sanitized = redact(sanitized, leaseToken);
|
|
225
|
-
|
|
230
|
+
const wrapped = new GatewayToolError(redact(sanitized, this.bindingApprovalId ?? ""), name, failureCode);
|
|
231
|
+
issueTransportDiagnostic(wrapped, error, performance.now() - started, httpStatus);
|
|
232
|
+
throw wrapped;
|
|
226
233
|
}
|
|
227
234
|
}
|
|
228
235
|
async resolveCompatibility(input) {
|
|
@@ -619,26 +626,81 @@ export class McpGatewayClient {
|
|
|
619
626
|
}
|
|
620
627
|
/** Account for the server-authored verification event before appending a
|
|
621
628
|
* terminal event. Never infer a cursor from model output or a local counter. */
|
|
622
|
-
async syncLocalPublicationCursor(bindingId, afterSequence) {
|
|
623
|
-
const
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
629
|
+
async syncLocalPublicationCursor(bindingId, afterSequence, job, receipt) {
|
|
630
|
+
const reject = () => new Error('LOCAL_PUBLICATION_EVENT_CURSOR_REJECTED');
|
|
631
|
+
const mode = nested(job?.taskPackage, 'local_fix_scope.verification_mode');
|
|
632
|
+
if (!Number.isSafeInteger(afterSequence) || afterSequence < 0 || (mode !== undefined && !['gateway_ci_v1', 'gateway_publish_v1'].includes(String(mode))) || receipt?.status !== 'completed' || receipt.verification_mode !== mode)
|
|
633
|
+
throw reject();
|
|
634
|
+
const context = (await this.localPublicationContext({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, launchGrant: job.launchGrant, bindingId })).context;
|
|
635
|
+
const expected = { binding_id: bindingId, runner_id: this.runnerId, job_id: job.id, generation: job.generation, work_loop_id: job.workLoopId, attempt_id: nested(job.taskPackage, 'local_fix_scope.attempt_id'), repository_key: nested(job.taskPackage, 'repository.key'), base_commit_sha: nested(job.taskPackage, 'repository.base_commit_sha'), working_branch: nested(job.taskPackage, 'repository.working_branch'), contract_fingerprint: job.gatewayContractFingerprint, task_package_fingerprint: job.gatewayTaskPackageFingerprint, launch_fingerprint: job.launchFingerprint };
|
|
636
|
+
if (!context || Object.entries(expected).some(([k, v]) => v === undefined || v === '' || context[k] !== v))
|
|
637
|
+
throw reject();
|
|
638
|
+
const identity = [...Object.keys(expected), 'actor_id', 'execution_contract_id', 'repository_lease_id'];
|
|
639
|
+
if (identity.some(k => k === 'generation' ? !Number.isSafeInteger(context[k]) : typeof context[k] !== 'string' || !context[k]))
|
|
640
|
+
throw reject();
|
|
641
|
+
const hashes = ['candidate_fingerprint', 'snapshot_fingerprint', 'verification_fingerprint', 'diff_hash', 'changed_file_manifest_hash'];
|
|
642
|
+
if (hashes.some(k => typeof receipt[k] !== 'string' || !/^[a-f0-9]{64}$/.test(receipt[k])))
|
|
643
|
+
throw reject();
|
|
644
|
+
const page = await this.callTool('agent_runtime_events_read', { binding_id: bindingId, after_sequence: afterSequence, limit: 100 });
|
|
645
|
+
const events = page.events;
|
|
646
|
+
if (page.terminal || !Array.isArray(events) || events.length === 0 || events.length >= 100)
|
|
647
|
+
throw reject();
|
|
648
|
+
let cursor = afterSequence, proofs = 0;
|
|
649
|
+
for (const e of events) {
|
|
650
|
+
if (!Number.isSafeInteger(e.sequence) || e.sequence !== cursor + 1 || isTerminalEventType(String(e.event_type)))
|
|
651
|
+
throw reject();
|
|
652
|
+
cursor = e.sequence;
|
|
653
|
+
const proof = e.payload?.local_publication;
|
|
654
|
+
if (!proof || typeof proof !== 'object' || Array.isArray(proof) || identity.some(k => proof[k] !== context[k]) || hashes.some(k => proof[k] !== receipt[k]) || proof.verification_mode !== mode || typeof proof.gateway_mac !== 'string' || !/^[a-f0-9]{64}$/.test(proof.gateway_mac))
|
|
655
|
+
throw reject();
|
|
656
|
+
const publish = mode === 'gateway_publish_v1';
|
|
657
|
+
if (e.event_type !== (publish ? 'progress' : 'test_result') || !(publish ? e.stage === 'server_publication' : mode === 'gateway_ci_v1' ? ['server_ci_verification', 'local_publication_verification'].includes(String(e.stage)) : e.stage === 'local_publication_verification') || proof.verification_status !== (publish ? 'unverified' : 'passed') || (publish && nested(proof, 'ci.status') !== 'not_run'))
|
|
658
|
+
throw reject();
|
|
659
|
+
proofs++;
|
|
637
660
|
}
|
|
638
|
-
if (
|
|
639
|
-
throw
|
|
661
|
+
if (proofs !== 1)
|
|
662
|
+
throw reject();
|
|
640
663
|
return cursor;
|
|
641
664
|
}
|
|
665
|
+
/** Read-only acknowledgement of Route B's atomic server completion. No lease,
|
|
666
|
+
* context restoration, replay or publication write is authorized here. */
|
|
667
|
+
async confirmServerPublicationCompletion(bindingId, afterSequence, job, receipt) {
|
|
668
|
+
if (nested(job.taskPackage, 'local_fix_scope.verification_mode') !== 'gateway_publish_v1')
|
|
669
|
+
return false;
|
|
670
|
+
try {
|
|
671
|
+
if (!Number.isSafeInteger(job.generation) || job.generation < 1 || typeof job.id !== 'string' || !job.id || bindingId !== uuidv5(`local-agent-job:${job.id}:${job.generation}`, uuidv5.URL)
|
|
672
|
+
|| !Number.isSafeInteger(afterSequence) || afterSequence < 0 || receipt.status !== 'completed' || receipt.verification_mode !== 'gateway_publish_v1' || receipt.verification_status !== 'unverified' || nested(receipt, 'ci.status') !== 'not_run'
|
|
673
|
+
|| typeof receipt.publication_id !== 'string' || !receipt.publication_id || typeof receipt.commit_sha !== 'string' || !/^[a-f0-9]{40}$/.test(receipt.commit_sha) || typeof receipt.pr_url !== 'string' || !receipt.pr_url)
|
|
674
|
+
throw Error();
|
|
675
|
+
const page = await this.callTool('agent_runtime_events_read', { binding_id: bindingId, after_sequence: afterSequence, limit: 100 });
|
|
676
|
+
const events = page.events;
|
|
677
|
+
if (typeof page.terminal !== 'boolean' || !Array.isArray(events) || events.length >= 100)
|
|
678
|
+
throw Error();
|
|
679
|
+
let cursor = afterSequence;
|
|
680
|
+
for (const e of events) {
|
|
681
|
+
if (!Number.isSafeInteger(e.sequence) || e.sequence !== cursor + 1)
|
|
682
|
+
throw Error();
|
|
683
|
+
cursor = e.sequence;
|
|
684
|
+
}
|
|
685
|
+
if (page.next_cursor !== undefined && page.next_cursor !== cursor)
|
|
686
|
+
throw Error();
|
|
687
|
+
const terminals = events.filter(e => ['completed', 'failed', 'blocked', 'cancelled'].includes(String(e.event_type)));
|
|
688
|
+
if (!page.terminal) {
|
|
689
|
+
if (terminals.length)
|
|
690
|
+
throw Error();
|
|
691
|
+
return false;
|
|
692
|
+
}
|
|
693
|
+
const last = events.at(-1), payload = last?.payload;
|
|
694
|
+
if (terminals.length !== 1 || terminals[0] !== last || last?.event_type !== 'completed' || !payload || payload.provenance !== 'server_reconciliation' || payload.reason !== 'completed_publication_without_terminal'
|
|
695
|
+
|| payload.job_id !== job.id || payload.generation !== job.generation || !Number.isSafeInteger(payload.generation)
|
|
696
|
+
|| ['publication_id', 'commit_sha', 'pr_url'].some(k => payload[k] !== receipt[k]))
|
|
697
|
+
throw Error();
|
|
698
|
+
return true;
|
|
699
|
+
}
|
|
700
|
+
catch (error) {
|
|
701
|
+
throw publicationError('server_completion', 'LOCAL_FIX_SERVER_COMPLETION_REJECTED', true, error);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
642
704
|
/** Host-only publication envelope. This client is never a model MCP relay. */
|
|
643
705
|
localPublicationFence(input) {
|
|
644
706
|
return { job_id: input.jobId, generation: input.generation, lease_token: input.leaseToken,
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
import { transportDiagnostic } from '../safety/transport-diagnostic.js';
|
|
1
2
|
import { PublicationOutcomeUnknownError } from './publisher.js';
|
|
2
|
-
export const PUBLICATION_STEPS = ['validation_report', 'current_fence', 'snapshot_observe', 'snapshot_materialize', 'candidate_limits', 'delta_format', 'delta_modes', 'delta_scope', 'candidate_files', 'journal', 'security_receipt', 'stage', 'advance', 'result', 'cleanup'];
|
|
3
|
-
export const PUBLICATION_CODES = ['LOCAL_FIX_CHANGE_LIMIT_REJECTED', 'LOCAL_FIX_PAYLOAD_LIMIT_REJECTED', 'LOCAL_FIX_CHANGE_MEASUREMENT_REJECTED', 'LOCAL_FIX_CHECK_REJECTED', 'LOCAL_FIX_REPORT_REJECTED', 'LOCAL_FIX_DIFF_LIMIT_REJECTED', 'LOCAL_FIX_FILE_COUNT_REJECTED', 'LOCAL_FIX_FILE_LIMIT_REJECTED', 'LOCAL_FIX_FILE_ENCODING_REJECTED', 'LOCAL_FIX_PUBLICATION_OUTCOME_UNKNOWN', 'LOCAL_FIX_CLEANUP_FAILED'];
|
|
3
|
+
export const PUBLICATION_STEPS = ['validation_report', 'current_fence', 'snapshot_observe', 'snapshot_materialize', 'candidate_limits', 'delta_format', 'delta_modes', 'delta_scope', 'candidate_files', 'journal', 'security_receipt', 'stage', 'advance', 'result', 'cleanup', 'cursor_sync', 'server_completion'];
|
|
4
|
+
export const PUBLICATION_CODES = ['LOCAL_FIX_CHANGE_LIMIT_REJECTED', 'LOCAL_FIX_PAYLOAD_LIMIT_REJECTED', 'LOCAL_FIX_CHANGE_MEASUREMENT_REJECTED', 'LOCAL_FIX_CHECK_REJECTED', 'LOCAL_FIX_REPORT_REJECTED', 'LOCAL_FIX_DIFF_LIMIT_REJECTED', 'LOCAL_FIX_FILE_COUNT_REJECTED', 'LOCAL_FIX_FILE_LIMIT_REJECTED', 'LOCAL_FIX_FILE_ENCODING_REJECTED', 'LOCAL_FIX_PUBLICATION_OUTCOME_UNKNOWN', 'LOCAL_FIX_CLEANUP_FAILED', 'LOCAL_FIX_PUBLICATION_CURSOR_REJECTED', 'LOCAL_FIX_SERVER_COMPLETION_REJECTED'];
|
|
4
5
|
const issued = new WeakMap();
|
|
5
6
|
const cleanupFailures = new WeakSet();
|
|
6
7
|
/** Host vocabulary only; serialized fields/messages never carry diagnostic authority. */
|
|
7
|
-
export function publicationError(step, code = 'LOCAL_FIX_CHECK_REJECTED', unknown = false) {
|
|
8
|
+
export function publicationError(step, code = 'LOCAL_FIX_CHECK_REJECTED', unknown = false, origin) {
|
|
8
9
|
const safeStep = PUBLICATION_STEPS.includes(step) ? step : 'result';
|
|
9
10
|
const safeCode = PUBLICATION_CODES.includes(code) ? code : 'LOCAL_FIX_CHECK_REJECTED';
|
|
10
11
|
const e = unknown ? new PublicationOutcomeUnknownError() : new Error('LOCAL_FIX_SERVER_CI_REJECTED');
|
|
11
|
-
issued.set(e, Object.freeze({ stage: 'local_fix_publication', code: safeCode, publication_step: safeStep }));
|
|
12
|
+
issued.set(e, Object.freeze({ stage: 'local_fix_publication', code: safeCode, publication_step: safeStep, ...transportDiagnostic(origin) }));
|
|
12
13
|
return e;
|
|
13
14
|
}
|
|
14
15
|
export function trustedPublicationFailure(error) { const d = issued.get(error); return d ? { ...d } : undefined; }
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { fingerprint } from '../daemon/execution-coordinator.js';
|
|
2
2
|
import { PROTOCOL_VERSION } from '../protocol.js';
|
|
3
|
+
import { publicationError } from './failure.js';
|
|
3
4
|
import { PublicationOutcomeUnknownError } from './publisher.js';
|
|
4
5
|
/** Explicit one-Job operation. No adapter factory/model/claim/automatic retry.
|
|
5
6
|
* Existing publication intents reconcile before effects; expired/revoked server
|
|
@@ -12,12 +13,41 @@ export async function recoverLocalFixPublication(input) {
|
|
|
12
13
|
if (!job || job.id !== s.jobId || job.generation !== s.generation || job.leaseToken !== s.leaseToken || bindingId !== s.bindingId || event?.eventType !== 'completed' || typeof r.checkoutPath !== 'string' || job.gatewayTaskPackageFingerprint !== fingerprint(job.taskPackage))
|
|
13
14
|
throw Error('LOCAL_PUBLICATION_RECOVERY_REJECTED');
|
|
14
15
|
const fence = { jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, launchGrant: job.launchGrant, bindingId };
|
|
16
|
+
const confirmed = async (receipt, cursor = s.eventCursor ?? 0) => {
|
|
17
|
+
if (!receipt || !input.gateway.confirmServerPublicationCompletion || job.taskPackage.local_fix_scope?.verification_mode !== 'gateway_publish_v1')
|
|
18
|
+
return false;
|
|
19
|
+
return input.gateway.confirmServerPublicationCompletion(bindingId, cursor, job, receipt);
|
|
20
|
+
};
|
|
21
|
+
if (s.phase === 'publication_terminal_pending') {
|
|
22
|
+
const pending = s.publicationTerminal;
|
|
23
|
+
if (!pending || pending.event.bindingId !== bindingId || pending.event.eventType !== 'completed' || pending.completion.jobId !== job.id || pending.completion.generation !== job.generation || pending.completion.leaseToken !== job.leaseToken)
|
|
24
|
+
throw Error('LOCAL_PUBLICATION_RECOVERY_REJECTED');
|
|
25
|
+
}
|
|
26
|
+
const savedReceipt = (s.phase === 'publication_terminal_pending' ? s.publicationTerminal?.completion.result.local_fix_publication : r.completedPublication);
|
|
27
|
+
if (await confirmed(savedReceipt)) {
|
|
28
|
+
await input.state.clear();
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
const appendOrConfirm = async (event, receipt) => {
|
|
32
|
+
try {
|
|
33
|
+
await input.gateway.appendEvent(bindingId, event);
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
if (await confirmed(receipt, event.sequence - 1))
|
|
38
|
+
return true;
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
15
42
|
if (s.phase === 'publication_terminal_pending') {
|
|
16
43
|
const pending = s.publicationTerminal;
|
|
17
44
|
if (!pending || pending.event.bindingId !== bindingId || pending.event.eventType !== 'completed' || pending.completion.jobId !== job.id || pending.completion.generation !== job.generation || pending.completion.leaseToken !== job.leaseToken)
|
|
18
45
|
throw Error('LOCAL_PUBLICATION_RECOVERY_REJECTED');
|
|
19
46
|
await input.gateway.restoreLocalPublicationRecoveryContext(fence, r.gatewayContext, true);
|
|
20
|
-
await
|
|
47
|
+
if (await appendOrConfirm(pending.event, savedReceipt)) {
|
|
48
|
+
await input.state.clear();
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
21
51
|
await input.state.save({ ...s, phase: 'terminal_pending', completionRequest: pending.completion });
|
|
22
52
|
await input.gateway.completeLocalJob(pending.completion);
|
|
23
53
|
await input.state.clear();
|
|
@@ -33,14 +63,38 @@ export async function recoverLocalFixPublication(input) {
|
|
|
33
63
|
try {
|
|
34
64
|
session = await input.host.prepare({ job, checkoutPath: r.checkoutPath, bindingId, lifecycle: { isFrozen: () => true, isCurrent: () => current, securityReceipt: () => r.securityReceipt } });
|
|
35
65
|
const receipt = await session.finalize(event);
|
|
66
|
+
const completedState = { ...s, publicationRecovery: { ...r, completedPublication: structuredClone(receipt) } };
|
|
67
|
+
try {
|
|
68
|
+
await input.state.save(completedState);
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
throw publicationError('journal', 'LOCAL_FIX_PUBLICATION_OUTCOME_UNKNOWN', true, error);
|
|
72
|
+
}
|
|
73
|
+
if (await confirmed(receipt)) {
|
|
74
|
+
await session.close();
|
|
75
|
+
session = undefined;
|
|
76
|
+
await input.state.clear();
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
36
79
|
if (heartbeatError)
|
|
37
80
|
throw new PublicationOutcomeUnknownError();
|
|
38
|
-
|
|
81
|
+
let cursor;
|
|
82
|
+
try {
|
|
83
|
+
cursor = await input.gateway.syncLocalPublicationCursor(bindingId, s.eventCursor ?? 0, job, receipt);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
throw publicationError('cursor_sync', 'LOCAL_FIX_PUBLICATION_CURSOR_REJECTED', true, error);
|
|
87
|
+
}
|
|
39
88
|
const payload = { ...event.payload, local_fix_publication: receipt, security_receipt: r.securityReceipt, diff_hash: receipt.diff_hash, changed_file_manifest_hash: receipt.changed_file_manifest_hash };
|
|
40
89
|
const normalized = { protocolVersion: PROTOCOL_VERSION, bindingId, sequence: cursor + 1, eventType: 'completed', summary: event.summary, payload, idempotencyKey: `local-job:${job.id}:${job.generation}:event:${cursor + 1}`, occurredAt: new Date().toISOString() };
|
|
41
90
|
const completionRequest = { jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status: 'completed', result: { binding_id: bindingId, event_type: 'completed', summary: event.summary, local_fix_publication: receipt, security_receipt: r.securityReceipt } };
|
|
42
|
-
await input.state.save({ ...
|
|
43
|
-
await
|
|
91
|
+
await input.state.save({ ...completedState, phase: 'publication_terminal_pending', publicationTerminal: { event: normalized, completion: completionRequest } });
|
|
92
|
+
if (await appendOrConfirm(normalized, receipt)) {
|
|
93
|
+
await session.close();
|
|
94
|
+
session = undefined;
|
|
95
|
+
await input.state.clear();
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
44
98
|
await input.state.save({ ...s, phase: 'terminal_pending', completionRequest });
|
|
45
99
|
await input.gateway.completeLocalJob(completionRequest);
|
|
46
100
|
await input.state.clear();
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { transportDiagnostic } from './transport-diagnostic.js';
|
|
1
2
|
import { trustedPublicationCleanupFailure, trustedPublicationFailure } from '../publication/failure.js';
|
|
2
3
|
import { trustedServerCiFailure } from '../publication/server-ci-error.js';
|
|
3
4
|
import { trustedLaunchObservation } from '../process/owned-launch-observer.js';
|
|
@@ -29,6 +30,11 @@ export const STARTUP_OBSERVATIONS = Object.freeze([
|
|
|
29
30
|
'cleanup_guardian_root_exited', 'cleanup_guardian_root_running',
|
|
30
31
|
]);
|
|
31
32
|
const issued = new WeakMap();
|
|
33
|
+
export function jobHeartbeatError(error) {
|
|
34
|
+
const wrapped = new Error('JOB_HEARTBEAT_FAILED');
|
|
35
|
+
issued.set(wrapped, Object.freeze({ stage: 'heartbeat', code: 'JOB_HEARTBEAT_FAILED', heartbeat_channel: 'job', ...transportDiagnostic(error) }));
|
|
36
|
+
return wrapped;
|
|
37
|
+
}
|
|
32
38
|
export function executionFailure(error, stage, fallback = 'EXECUTION_FAILED') {
|
|
33
39
|
if (error && typeof error === 'object') {
|
|
34
40
|
const ci = trustedServerCiFailure(error);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const issued = new WeakMap();
|
|
2
|
+
const allowed = new Set(['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EAI_AGAIN', 'EPIPE', 'ENETUNREACH', 'EHOSTUNREACH', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_SOCKET', 'ABORT_ERR']);
|
|
3
|
+
function own(value, key) { try {
|
|
4
|
+
return value && typeof value === 'object' ? Object.getOwnPropertyDescriptor(value, key)?.value : undefined;
|
|
5
|
+
}
|
|
6
|
+
catch {
|
|
7
|
+
return undefined;
|
|
8
|
+
} }
|
|
9
|
+
export function issueTransportDiagnostic(target, error, elapsed, status) {
|
|
10
|
+
const code = own(error, 'code') ?? own(own(error, 'cause'), 'code');
|
|
11
|
+
const d = { elapsed_ms: Number.isFinite(elapsed) ? Math.min(86400000, Math.max(0, Math.trunc(elapsed))) : 0, ...(typeof status === 'number' && Number.isInteger(status) && status >= 100 && status <= 599 ? { http_status: status } : {}), ...(typeof code === 'string' && allowed.has(code) ? { os_error: code } : {}) };
|
|
12
|
+
issued.set(target, Object.freeze(d));
|
|
13
|
+
}
|
|
14
|
+
export function transportDiagnostic(error) { if (!error || typeof error !== 'object')
|
|
15
|
+
return; const d = issued.get(error); return d ? { ...d } : undefined; }
|