@superjackfruitlabs/salaam 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +12 -11
- package/dist/src/acp-outcome.js +177 -17
- package/dist/src/acp.js +41 -7
- package/dist/src/agent-profiles.js +45 -0
- package/dist/src/agentpod-authority.js +169 -11
- package/dist/src/agentpod-default-config.js +12 -0
- package/dist/src/agentpod-work-decision-protocol.js +40 -6
- package/dist/src/agentpod-work-decisions.js +108 -4
- package/dist/src/agentpod-work.js +1214 -127
- package/dist/src/board-reads.js +106 -0
- package/dist/src/children.js +83 -0
- package/dist/src/cli-help.js +27 -2
- package/dist/src/cli.js +87 -3
- package/dist/src/command-network.js +200 -0
- package/dist/src/command-sandbox.js +217 -0
- package/dist/src/execution-evidence.js +33 -1
- package/dist/src/host-bind.js +111 -0
- package/dist/src/host-breaker.js +88 -0
- package/dist/src/host-start.js +41 -1
- package/dist/src/host.js +192 -6
- package/dist/src/house-extension.js +162 -57
- package/dist/src/local-coding.js +214 -18
- package/dist/src/local-edit-helper.js +25 -0
- package/dist/src/local-undo.js +25 -16
- package/dist/src/managed.js +1 -1
- package/dist/src/operator-mcp-cli.js +107 -0
- package/dist/src/operator-mcp.js +342 -0
- package/dist/src/persona.js +31 -0
- package/dist/src/pi.js +22 -1
- package/dist/src/protocol.js +5 -2
- package/dist/src/rate-limit-retries.js +62 -0
- package/dist/src/recurring-work.js +1 -0
- package/dist/src/repository-snapshots.js +5 -2
- package/dist/src/run-grants.js +62 -0
- package/dist/src/service.js +141 -6
- package/dist/src/station-checkpoints.js +139 -0
- package/dist/src/station-config.js +8 -1
- package/dist/src/station-files.js +20 -1
- package/dist/src/station-task-verification.js +6 -3
- package/dist/src/station-work.js +315 -22
- package/dist/src/store.js +19 -0
- package/dist/src/supervisor.js +177 -5
- package/dist/src/task-budget.js +39 -3
- package/dist/src/task-critic.js +115 -0
- package/dist/src/task-driver.js +46 -2
- package/dist/src/task-policy.js +44 -12
- package/dist/src/task-verification.js +42 -9
- package/dist/src/task-work.js +47 -0
- package/dist/src/tool-identity.js +69 -0
- package/dist/src/usage.js +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/src/acp-outcome.ts +169 -19
- package/src/acp.ts +53 -7
- package/src/agent-profiles.ts +48 -0
- package/src/agentpod-authority.ts +234 -15
- package/src/agentpod-default-config.ts +16 -0
- package/src/agentpod-work-decision-protocol.ts +69 -8
- package/src/agentpod-work-decisions.ts +123 -5
- package/src/agentpod-work.ts +1455 -152
- package/src/board-reads.ts +148 -0
- package/src/children.ts +112 -0
- package/src/cli-help.ts +27 -2
- package/src/cli.ts +109 -3
- package/src/command-network.ts +215 -0
- package/src/command-sandbox.ts +273 -0
- package/src/execution-evidence.ts +31 -1
- package/src/host-bind.ts +156 -0
- package/src/host-breaker.ts +89 -0
- package/src/host-start.ts +51 -1
- package/src/host.ts +244 -6
- package/src/house-extension.ts +174 -56
- package/src/local-coding.ts +270 -18
- package/src/local-edit-helper.ts +26 -0
- package/src/local-undo.ts +29 -14
- package/src/managed.ts +2 -2
- package/src/operator-mcp-cli.ts +108 -0
- package/src/operator-mcp.ts +420 -0
- package/src/persona.ts +37 -0
- package/src/pi.ts +23 -1
- package/src/protocol.ts +6 -1
- package/src/rate-limit-retries.ts +69 -0
- package/src/recurring-work.ts +1 -0
- package/src/repository-snapshots.ts +5 -2
- package/src/run-grants.ts +107 -0
- package/src/service.ts +222 -6
- package/src/station-checkpoints.ts +183 -0
- package/src/station-config.ts +10 -1
- package/src/station-files.ts +20 -1
- package/src/station-task-verification.ts +6 -3
- package/src/station-work.ts +369 -20
- package/src/store.ts +18 -0
- package/src/supervisor.ts +203 -4
- package/src/task-budget.ts +48 -3
- package/src/task-critic.ts +171 -0
- package/src/task-driver.ts +49 -2
- package/src/task-policy.ts +65 -13
- package/src/task-verification.ts +53 -9
- package/src/task-work.ts +75 -0
- package/src/tool-identity.ts +110 -0
- package/src/usage.ts +1 -1
package/src/agentpod-work.ts
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import { BoardReads } from './board-reads.js';
|
|
2
|
+
import type { StationWork } from './station-work.js';
|
|
3
|
+
import { StationCheckpoints } from './station-checkpoints.js';
|
|
4
|
+
import { requiredChecks } from './task-policy.js';
|
|
5
|
+
import { RATE_LIMIT_STOP, RateLimitRetries } from './rate-limit-retries.js';
|
|
6
|
+
import { planUpdate, toolMetadata, toolTitle } from './acp-outcome.js';
|
|
1
7
|
import type { Store } from './store.js';
|
|
2
8
|
import type { AgentPodAuthority } from './agentpod-authority.js';
|
|
3
9
|
import type { TaskWork, WorkTask } from './task-work.js';
|
|
@@ -29,6 +35,8 @@ export interface AgentPodWorkReceipt {
|
|
|
29
35
|
* adapter must supply its own session and independently admitted work identity.
|
|
30
36
|
* This receipt records task admission, never successful work or permission to run.
|
|
31
37
|
*/
|
|
38
|
+
/** Extra progression steps a strict grant may carry, only ever spent on re-running a step. */
|
|
39
|
+
export const RERUN_ALLOWANCE = 3;
|
|
32
40
|
export class AgentPodWork {
|
|
33
41
|
constructor(
|
|
34
42
|
readonly store: Store,
|
|
@@ -53,6 +61,7 @@ export class AgentPodWork {
|
|
|
53
61
|
objective: task.objective,
|
|
54
62
|
constraints: task.constraints,
|
|
55
63
|
acceptance: task.acceptance,
|
|
64
|
+
...(task.completion_gate ? { completion_gate: task.completion_gate } : {}),
|
|
56
65
|
deliverables: task.deliverables,
|
|
57
66
|
limits: task.limits,
|
|
58
67
|
steps: task.steps.map(({ key, title, instruction, depends_on, checks }) => ({
|
|
@@ -349,12 +358,16 @@ export class AgentPodWork {
|
|
|
349
358
|
offered,
|
|
350
359
|
})
|
|
351
360
|
: null;
|
|
352
|
-
|
|
361
|
+
const multi =
|
|
362
|
+
!simple && !history
|
|
363
|
+
? this.multiOfferReviewExpiry(root.session_id, task, runtime, offered)
|
|
364
|
+
: null;
|
|
365
|
+
if (!simple && !history && !multi)
|
|
353
366
|
throw new ServiceError(
|
|
354
367
|
'work_settlement_unsafe',
|
|
355
368
|
'Only exact expired unanswered unapplied work with closed historical evidence can be settled',
|
|
356
369
|
);
|
|
357
|
-
if (history)
|
|
370
|
+
if (history || multi)
|
|
358
371
|
this.store.event(
|
|
359
372
|
root.session_id,
|
|
360
373
|
'agentpod.work_expiry_evidence',
|
|
@@ -367,7 +380,7 @@ export class AgentPodWork {
|
|
|
367
380
|
request_id: request,
|
|
368
381
|
work_request_id: workRequest,
|
|
369
382
|
producer_terminal_digest: terminalDigest,
|
|
370
|
-
history,
|
|
383
|
+
...(history ? { history } : { several_review_offers: multi }),
|
|
371
384
|
}),
|
|
372
385
|
},
|
|
373
386
|
request,
|
|
@@ -393,6 +406,618 @@ export class AgentPodWork {
|
|
|
393
406
|
},
|
|
394
407
|
);
|
|
395
408
|
}
|
|
409
|
+
/** Several unanswered review offers in one attempt: each one expired, or was invalidated by the
|
|
410
|
+
* owner's revocation (a hub restart), before anyone answered it. Every operation was cancelled
|
|
411
|
+
* before dispatch, the only station calls are closed reads, and nothing ran after the revoke. */
|
|
412
|
+
private multiOfferReviewExpiry(
|
|
413
|
+
session: string,
|
|
414
|
+
task: WorkTask,
|
|
415
|
+
runtime: Record<string, any> | undefined,
|
|
416
|
+
offered: Record<string, any>,
|
|
417
|
+
) {
|
|
418
|
+
if (
|
|
419
|
+
task.state !== 'blocked' ||
|
|
420
|
+
task.steps.some((s) => !['pending', 'cancelled'].includes(s.state)) ||
|
|
421
|
+
!runtime ||
|
|
422
|
+
runtime.auto_enabled ||
|
|
423
|
+
runtime.reserved_requests ||
|
|
424
|
+
runtime.reserved_tokens ||
|
|
425
|
+
runtime.verification_runs
|
|
426
|
+
)
|
|
427
|
+
return null;
|
|
428
|
+
const attempts = this.store.db
|
|
429
|
+
.prepare('SELECT * FROM attempts WHERE session_id=?')
|
|
430
|
+
.all(session);
|
|
431
|
+
const attempt = attempts[0];
|
|
432
|
+
if (
|
|
433
|
+
attempts.length !== 1 ||
|
|
434
|
+
!attempt ||
|
|
435
|
+
attempt.state !== 'cancelled' ||
|
|
436
|
+
attempt.resolved_by !== null ||
|
|
437
|
+
attempt.result !== null
|
|
438
|
+
)
|
|
439
|
+
return null;
|
|
440
|
+
// Cancelled by the node's revocation of this session, which names exactly this attempt.
|
|
441
|
+
const cancellations = this.store.db
|
|
442
|
+
.prepare('SELECT * FROM cancellations WHERE attempt_id=?')
|
|
443
|
+
.all(String(attempt.id));
|
|
444
|
+
const revocations = this.store.db
|
|
445
|
+
.prepare('SELECT * FROM session_revocations WHERE session_id=?')
|
|
446
|
+
.all(session);
|
|
447
|
+
if (
|
|
448
|
+
cancellations.length !== 1 ||
|
|
449
|
+
cancellations[0]!.requested_by !== 'agentpod-node' ||
|
|
450
|
+
revocations.length !== 1
|
|
451
|
+
)
|
|
452
|
+
return null;
|
|
453
|
+
const revocation = JSON.parse(String(revocations[0]!.receipt));
|
|
454
|
+
if (
|
|
455
|
+
revocation.revoked !== true ||
|
|
456
|
+
canonical(revocation.attempt_ids) !== canonical([attempt.id]) ||
|
|
457
|
+
canonical(revocation.unresolved_operations) !== canonical([])
|
|
458
|
+
)
|
|
459
|
+
return null;
|
|
460
|
+
const events = this.store.db
|
|
461
|
+
.prepare('SELECT seq,kind,payload,causation_id FROM events WHERE session_id=? ORDER BY seq')
|
|
462
|
+
.all(session);
|
|
463
|
+
const one = (kind: string) => {
|
|
464
|
+
const found = events.filter((e) => e.kind === kind);
|
|
465
|
+
return found.length === 1 ? found[0]! : null;
|
|
466
|
+
};
|
|
467
|
+
const requested = one('attempt.cancel_requested'),
|
|
468
|
+
revoked = one('execution.revoked'),
|
|
469
|
+
cancelled = one('attempt.cancelled');
|
|
470
|
+
if (
|
|
471
|
+
!requested ||
|
|
472
|
+
!revoked ||
|
|
473
|
+
!cancelled ||
|
|
474
|
+
requested.causation_id !== 'agentpod-revocation' ||
|
|
475
|
+
!(
|
|
476
|
+
Number(requested.seq) < Number(revoked.seq) && Number(revoked.seq) < Number(cancelled.seq)
|
|
477
|
+
) ||
|
|
478
|
+
events.some((e) => e.kind === 'tool.started' && Number(e.seq) > Number(requested.seq))
|
|
479
|
+
)
|
|
480
|
+
return null;
|
|
481
|
+
// Two or more review operations, all on this attempt, all cancelled before dispatch.
|
|
482
|
+
const ops = this.store.db
|
|
483
|
+
.prepare('SELECT * FROM operations WHERE session_id=? ORDER BY rowid')
|
|
484
|
+
.all(session);
|
|
485
|
+
if (
|
|
486
|
+
ops.length < 2 ||
|
|
487
|
+
!ops.some((op) => op.id === offered.operation_id) ||
|
|
488
|
+
ops.some(
|
|
489
|
+
(op) =>
|
|
490
|
+
op.kind !== 'review' ||
|
|
491
|
+
op.state !== 'cancelled_before_dispatch' ||
|
|
492
|
+
op.attempt_id !== attempt.id,
|
|
493
|
+
)
|
|
494
|
+
)
|
|
495
|
+
return null;
|
|
496
|
+
const of = (e: Record<string, any>) => JSON.parse(String(e.payload));
|
|
497
|
+
// The invalidations the revocation made: the contiguous block of invalidated approvals and
|
|
498
|
+
// their cancelled operations that ends at the revocation's own cancel request.
|
|
499
|
+
const block = new Set<string>();
|
|
500
|
+
for (let n = events.indexOf(requested) - 1; n >= 0; n--) {
|
|
501
|
+
const e = events[n]!;
|
|
502
|
+
if (e.kind === 'operation.cancelled_before_dispatch') continue;
|
|
503
|
+
if (e.kind !== 'approval.invalidated') break;
|
|
504
|
+
block.add(String(of(e).approval_id));
|
|
505
|
+
}
|
|
506
|
+
const offers = [];
|
|
507
|
+
for (const op of ops) {
|
|
508
|
+
const approvals = this.store.db
|
|
509
|
+
.prepare('SELECT * FROM approvals WHERE operation_id=?')
|
|
510
|
+
.all(String(op.id));
|
|
511
|
+
const approval = approvals[0];
|
|
512
|
+
if (
|
|
513
|
+
approvals.length !== 1 ||
|
|
514
|
+
!approval ||
|
|
515
|
+
approval.session_id !== session ||
|
|
516
|
+
approval.decision !== null ||
|
|
517
|
+
approval.operation_digest !== op.operation_digest
|
|
518
|
+
)
|
|
519
|
+
return null;
|
|
520
|
+
if (approval.state === 'expired') {
|
|
521
|
+
const expired = events.findIndex(
|
|
522
|
+
(e) => e.kind === 'approval.expired' && of(e).approval_id === approval.id,
|
|
523
|
+
);
|
|
524
|
+
if (
|
|
525
|
+
expired < 0 ||
|
|
526
|
+
Number(approval.expires_at) > Date.now() ||
|
|
527
|
+
events[expired + 1]?.kind !== 'operation.cancelled_before_dispatch' ||
|
|
528
|
+
of(events[expired + 1]!).operation_id !== op.id
|
|
529
|
+
)
|
|
530
|
+
return null;
|
|
531
|
+
} else if (approval.state !== 'invalidated' || !block.has(String(approval.id))) return null;
|
|
532
|
+
offers.push({ operation_id: op.id, approval_id: approval.id, state: approval.state });
|
|
533
|
+
}
|
|
534
|
+
// The decision being settled is one that expired unanswered, exactly as it was offered.
|
|
535
|
+
const settled = offers.find((o) => o.approval_id === offered.approval_id);
|
|
536
|
+
const approval = this.store.db
|
|
537
|
+
.prepare('SELECT * FROM approvals WHERE id=?')
|
|
538
|
+
.get(String(offered.approval_id));
|
|
539
|
+
if (
|
|
540
|
+
settled?.state !== 'expired' ||
|
|
541
|
+
!approval ||
|
|
542
|
+
approval.operation_digest !== offered.operation_digest ||
|
|
543
|
+
approval.expires_at !== offered.expires_at
|
|
544
|
+
)
|
|
545
|
+
return null;
|
|
546
|
+
const units = this.noLedgeredEffects(session, task, { operations: true, stationReads: true });
|
|
547
|
+
if (!units) return null;
|
|
548
|
+
return {
|
|
549
|
+
revocation,
|
|
550
|
+
offers,
|
|
551
|
+
reads: this.store.db
|
|
552
|
+
.prepare('SELECT id,input_digest FROM station_calls WHERE session_id=? ORDER BY id')
|
|
553
|
+
.all(session),
|
|
554
|
+
unbound_stopped_units: units,
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
/** An attempt the request-budget control interrupted after its operator MCP calls kept failing:
|
|
558
|
+
* every call is journaled as failed or tool_error, the only station calls are closed reads, and
|
|
559
|
+
* no tool ran after the stop. The failed calls are preserved as the known effects. */
|
|
560
|
+
private requestBudgetStopFailure(
|
|
561
|
+
session: string,
|
|
562
|
+
task: WorkTask,
|
|
563
|
+
runtime: Record<string, any> | undefined,
|
|
564
|
+
attempts: Record<string, any>[],
|
|
565
|
+
) {
|
|
566
|
+
const reason = 'Required control failed: No model request budget remains',
|
|
567
|
+
// Picked by state, never by row order; the count below rejects any other attempt.
|
|
568
|
+
attempt = attempts.find((a) => a.state === 'interrupted');
|
|
569
|
+
if (
|
|
570
|
+
task.state !== 'blocked' ||
|
|
571
|
+
!runtime ||
|
|
572
|
+
runtime.auto_enabled ||
|
|
573
|
+
runtime.reserved_requests ||
|
|
574
|
+
runtime.reserved_tokens ||
|
|
575
|
+
runtime.verification_runs ||
|
|
576
|
+
attempts.length !== 1 ||
|
|
577
|
+
!attempt ||
|
|
578
|
+
attempt.error !== reason ||
|
|
579
|
+
attempt.resolved_by !== null ||
|
|
580
|
+
attempt.continues_attempt_id !== null ||
|
|
581
|
+
attempt.result !== null
|
|
582
|
+
)
|
|
583
|
+
return null;
|
|
584
|
+
const steps = task.steps.filter((s) => s.attempt_id === attempt.id);
|
|
585
|
+
if (
|
|
586
|
+
steps.length !== 1 ||
|
|
587
|
+
steps[0]!.state !== 'interrupted' ||
|
|
588
|
+
task.steps.some((s) => s !== steps[0] && (s.state !== 'pending' || s.attempt_id !== null))
|
|
589
|
+
)
|
|
590
|
+
return null;
|
|
591
|
+
// Stopped by the budget control, not cancelled by anyone.
|
|
592
|
+
if (
|
|
593
|
+
this.store.db
|
|
594
|
+
.prepare('SELECT 1 FROM cancellations WHERE attempt_id=?')
|
|
595
|
+
.get(String(attempt.id))
|
|
596
|
+
)
|
|
597
|
+
return null;
|
|
598
|
+
const events = this.store.db
|
|
599
|
+
.prepare('SELECT seq,kind,payload FROM events WHERE session_id=? ORDER BY seq')
|
|
600
|
+
.all(session);
|
|
601
|
+
const of = (e: Record<string, any>) => JSON.parse(String(e.payload));
|
|
602
|
+
const unhealthy = events.filter((e) => e.kind === 'worker.unhealthy'),
|
|
603
|
+
interrupted = events.filter((e) => e.kind === 'attempt.interrupted');
|
|
604
|
+
if (
|
|
605
|
+
unhealthy.length !== 1 ||
|
|
606
|
+
interrupted.length !== 1 ||
|
|
607
|
+
of(unhealthy[0]!).error !== reason ||
|
|
608
|
+
of(interrupted[0]!).attempt_id !== attempt.id ||
|
|
609
|
+
of(interrupted[0]!).error !== reason ||
|
|
610
|
+
Number(unhealthy[0]!.seq) >= Number(interrupted[0]!.seq)
|
|
611
|
+
)
|
|
612
|
+
return null;
|
|
613
|
+
// Tools: operator MCP (catalog reads, or a journaled failed call) and station reads, every one
|
|
614
|
+
// closed before the stop. Tools may run in parallel, and a call does not name its tool, so each
|
|
615
|
+
// call takes an open operator MCP tool that has none yet; a call with no such tool is refused.
|
|
616
|
+
const calls: Record<string, any>[] = [];
|
|
617
|
+
const open = new Map<string, { tool: string; calls: number }>();
|
|
618
|
+
for (const e of events) {
|
|
619
|
+
if (!['tool.started', 'tool.finished', 'operator_mcp.call'].includes(String(e.kind)))
|
|
620
|
+
continue;
|
|
621
|
+
const payload = of(e);
|
|
622
|
+
if (Number(e.seq) >= Number(unhealthy[0]!.seq) || payload.attempt_id !== attempt.id)
|
|
623
|
+
return null;
|
|
624
|
+
if (e.kind === 'tool.started') {
|
|
625
|
+
if (
|
|
626
|
+
!['salaam_operator_mcp', 'salaam_station', 'salaam_board'].includes(
|
|
627
|
+
String(payload.tool),
|
|
628
|
+
) ||
|
|
629
|
+
typeof payload.tool_call_id !== 'string' ||
|
|
630
|
+
!payload.tool_call_id ||
|
|
631
|
+
payload.parent_tool_call_id
|
|
632
|
+
)
|
|
633
|
+
return null;
|
|
634
|
+
open.set(payload.tool_call_id, { tool: String(payload.tool), calls: 0 });
|
|
635
|
+
} else if (e.kind === 'tool.finished') {
|
|
636
|
+
const started = open.get(String(payload.tool_call_id));
|
|
637
|
+
if (!started || payload.tool !== started.tool) return null;
|
|
638
|
+
open.delete(String(payload.tool_call_id));
|
|
639
|
+
} else {
|
|
640
|
+
const free = [...open.values()].find(
|
|
641
|
+
(t) => t.tool === 'salaam_operator_mcp' && t.calls === 0,
|
|
642
|
+
);
|
|
643
|
+
if (!free || !['failed', 'tool_error'].includes(String(payload.outcome))) return null;
|
|
644
|
+
free.calls++;
|
|
645
|
+
calls.push({ seq: e.seq, ...payload });
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
if (open.size) return null;
|
|
649
|
+
const units = this.noLedgeredEffects(session, task, { stationReads: true });
|
|
650
|
+
if (!units) return null;
|
|
651
|
+
return {
|
|
652
|
+
unhealthy: unhealthy[0],
|
|
653
|
+
interrupted: interrupted[0],
|
|
654
|
+
operator_mcp_calls: calls,
|
|
655
|
+
reads: this.store.db
|
|
656
|
+
.prepare('SELECT id,input_digest FROM station_calls WHERE session_id=? ORDER BY id')
|
|
657
|
+
.all(session),
|
|
658
|
+
unbound_stopped_units: units,
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
/** An attempt the task runtime limit stopped while its approved workspace writes were all
|
|
662
|
+
* refused by the protected filesystem before their mutation boundary. Each write is journaled
|
|
663
|
+
* failed_definite: refused by the station helper after its approval, or (the one still waiting
|
|
664
|
+
* when the stop came) cancelled before dispatch with its approval invalidated. Every other
|
|
665
|
+
* station call is a closed read, every operator MCP call is closed inside its own tool, and no
|
|
666
|
+
* tool started after the stop. The refused writes and the reads are the known effects. */
|
|
667
|
+
private runtimeStopRefusedWritesFailure(
|
|
668
|
+
session: string,
|
|
669
|
+
task: WorkTask,
|
|
670
|
+
runtime: Record<string, any> | undefined,
|
|
671
|
+
attempts: Record<string, any>[],
|
|
672
|
+
) {
|
|
673
|
+
const reason = 'Task runtime limit reached',
|
|
674
|
+
refused = 'Protected filesystem refused before its mutation boundary',
|
|
675
|
+
notSent = 'Workspace apply was cancelled before dispatch',
|
|
676
|
+
attempt = attempts[0];
|
|
677
|
+
if (
|
|
678
|
+
task.state !== 'blocked' ||
|
|
679
|
+
!runtime ||
|
|
680
|
+
runtime.blocked_reason !== reason ||
|
|
681
|
+
runtime.started_at === null ||
|
|
682
|
+
!Number.isFinite(Number(runtime.started_at)) ||
|
|
683
|
+
Date.now() - Number(runtime.started_at) < task.limits.runtime_ms ||
|
|
684
|
+
runtime.auto_enabled ||
|
|
685
|
+
runtime.reserved_requests ||
|
|
686
|
+
runtime.reserved_tokens ||
|
|
687
|
+
runtime.verification_runs ||
|
|
688
|
+
attempts.length !== 1 ||
|
|
689
|
+
!attempt ||
|
|
690
|
+
attempt.state !== 'cancelled' ||
|
|
691
|
+
attempt.error !== 'Attempt aborted' ||
|
|
692
|
+
attempt.resolved_by !== null ||
|
|
693
|
+
attempt.continues_attempt_id !== null ||
|
|
694
|
+
attempt.result !== null
|
|
695
|
+
)
|
|
696
|
+
return null;
|
|
697
|
+
const steps = task.steps.filter((s) => s.attempt_id === attempt.id);
|
|
698
|
+
if (
|
|
699
|
+
steps.length !== 1 ||
|
|
700
|
+
steps[0]!.state !== 'cancelled' ||
|
|
701
|
+
task.steps.some((s) => s !== steps[0] && (s.state !== 'pending' || s.attempt_id !== null))
|
|
702
|
+
)
|
|
703
|
+
return null;
|
|
704
|
+
// Stopped by the runtime limit's own task stop, not by anyone.
|
|
705
|
+
const cancellations = this.store.db
|
|
706
|
+
.prepare('SELECT * FROM cancellations WHERE attempt_id=?')
|
|
707
|
+
.all(String(attempt.id));
|
|
708
|
+
if (cancellations.length !== 1 || cancellations[0]!.requested_by !== 'task-stop') return null;
|
|
709
|
+
const events = this.store.db
|
|
710
|
+
.prepare('SELECT seq,kind,payload FROM events WHERE session_id=? ORDER BY seq')
|
|
711
|
+
.all(session);
|
|
712
|
+
const of = (e: Record<string, any>) => JSON.parse(String(e.payload));
|
|
713
|
+
const blocks = events.filter(
|
|
714
|
+
(e) => e.kind === 'work_task.budget_blocked' && of(e).reason === reason,
|
|
715
|
+
),
|
|
716
|
+
requested = events.filter((e) => e.kind === 'attempt.cancel_requested'),
|
|
717
|
+
cancelled = events.filter((e) => e.kind === 'attempt.cancelled');
|
|
718
|
+
if (
|
|
719
|
+
blocks.length !== 1 ||
|
|
720
|
+
requested.length !== 1 ||
|
|
721
|
+
cancelled.length !== 1 ||
|
|
722
|
+
of(requested[0]!).attempt_id !== attempt.id ||
|
|
723
|
+
of(cancelled[0]!).attempt_id !== attempt.id ||
|
|
724
|
+
!(
|
|
725
|
+
Number(blocks[0]!.seq) < Number(requested[0]!.seq) &&
|
|
726
|
+
Number(requested[0]!.seq) < Number(cancelled[0]!.seq)
|
|
727
|
+
)
|
|
728
|
+
)
|
|
729
|
+
return null;
|
|
730
|
+
// Every operation is a workspace write on this attempt that never crossed its boundary.
|
|
731
|
+
const ops = this.store.db
|
|
732
|
+
.prepare('SELECT * FROM operations WHERE session_id=? ORDER BY rowid')
|
|
733
|
+
.all(session);
|
|
734
|
+
const calls = this.store.db
|
|
735
|
+
.prepare('SELECT * FROM station_calls WHERE session_id=? ORDER BY rowid')
|
|
736
|
+
.all(session);
|
|
737
|
+
if (!ops.length) return null;
|
|
738
|
+
const writes: Record<string, unknown>[] = [];
|
|
739
|
+
for (const op of ops) {
|
|
740
|
+
const approval = this.store.db
|
|
741
|
+
.prepare('SELECT * FROM approvals WHERE operation_id=?')
|
|
742
|
+
.get(String(op.id));
|
|
743
|
+
const linked = calls.filter((c) => c.operation_id === op.id);
|
|
744
|
+
const call = linked[0];
|
|
745
|
+
if (
|
|
746
|
+
op.kind !== 'workspace' ||
|
|
747
|
+
op.attempt_id !== attempt.id ||
|
|
748
|
+
op.state !== 'failed_definite' ||
|
|
749
|
+
linked.length !== 1 ||
|
|
750
|
+
!call ||
|
|
751
|
+
call.kind !== 'patch' ||
|
|
752
|
+
call.attempt_id !== attempt.id ||
|
|
753
|
+
!approval ||
|
|
754
|
+
approval.operation_digest !== op.operation_digest
|
|
755
|
+
)
|
|
756
|
+
return null;
|
|
757
|
+
if (op.error === refused) {
|
|
758
|
+
// Approved, sent, and refused by the helper before it could change anything.
|
|
759
|
+
if (
|
|
760
|
+
approval.state !== 'decided' ||
|
|
761
|
+
approval.decision !== 'approve' ||
|
|
762
|
+
call.state !== 'failed' ||
|
|
763
|
+
call.helper_state !== 'stopped' ||
|
|
764
|
+
!call.result_json ||
|
|
765
|
+
canonical(JSON.parse(String(call.result_json))) !==
|
|
766
|
+
canonical({ code: 'station_files_changed' })
|
|
767
|
+
)
|
|
768
|
+
return null;
|
|
769
|
+
} else if (op.error === notSent) {
|
|
770
|
+
// Still waiting when the stop came: never sent to the helper at all.
|
|
771
|
+
if (
|
|
772
|
+
approval.state !== 'invalidated' ||
|
|
773
|
+
call.state !== 'cancelled' ||
|
|
774
|
+
call.helper_state !== 'none' ||
|
|
775
|
+
call.result_json !== null
|
|
776
|
+
)
|
|
777
|
+
return null;
|
|
778
|
+
} else return null;
|
|
779
|
+
writes.push({
|
|
780
|
+
operation_id: op.id,
|
|
781
|
+
operation_digest: op.operation_digest,
|
|
782
|
+
error: op.error,
|
|
783
|
+
approval: approval.state,
|
|
784
|
+
call_id: call.id,
|
|
785
|
+
call_state: call.state,
|
|
786
|
+
});
|
|
787
|
+
}
|
|
788
|
+
// The remaining station calls are closed reads.
|
|
789
|
+
const reads = calls.filter((c) => c.operation_id === null);
|
|
790
|
+
if (reads.some((c) => c.kind !== 'read' || c.state !== 'returned')) return null;
|
|
791
|
+
// Tools: station calls and operator MCP tools, every one closed, none started after the stop,
|
|
792
|
+
// each operator MCP call inside an open operator MCP tool that has no call yet.
|
|
793
|
+
const mcp: Record<string, any>[] = [];
|
|
794
|
+
const open = new Map<string, { tool: string; calls: number }>();
|
|
795
|
+
for (const e of events) {
|
|
796
|
+
if (!['tool.started', 'tool.finished', 'operator_mcp.call'].includes(String(e.kind)))
|
|
797
|
+
continue;
|
|
798
|
+
const payload = of(e);
|
|
799
|
+
if (Number(e.seq) >= Number(cancelled[0]!.seq) || payload.attempt_id !== attempt.id)
|
|
800
|
+
return null;
|
|
801
|
+
if (e.kind === 'tool.started') {
|
|
802
|
+
if (
|
|
803
|
+
Number(e.seq) >= Number(requested[0]!.seq) ||
|
|
804
|
+
!['salaam_operator_mcp', 'salaam_station', 'salaam_board'].includes(
|
|
805
|
+
String(payload.tool),
|
|
806
|
+
) ||
|
|
807
|
+
typeof payload.tool_call_id !== 'string' ||
|
|
808
|
+
!payload.tool_call_id ||
|
|
809
|
+
payload.parent_tool_call_id
|
|
810
|
+
)
|
|
811
|
+
return null;
|
|
812
|
+
open.set(payload.tool_call_id, { tool: String(payload.tool), calls: 0 });
|
|
813
|
+
} else if (e.kind === 'tool.finished') {
|
|
814
|
+
const started = open.get(String(payload.tool_call_id));
|
|
815
|
+
if (!started || payload.tool !== started.tool) return null;
|
|
816
|
+
open.delete(String(payload.tool_call_id));
|
|
817
|
+
} else {
|
|
818
|
+
const free = [...open.values()].find(
|
|
819
|
+
(t) => t.tool === 'salaam_operator_mcp' && t.calls === 0,
|
|
820
|
+
);
|
|
821
|
+
if (!free || !['returned', 'failed', 'tool_error'].includes(String(payload.outcome)))
|
|
822
|
+
return null;
|
|
823
|
+
free.calls++;
|
|
824
|
+
mcp.push({ seq: e.seq, ...payload });
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
if (open.size) return null;
|
|
828
|
+
const units = this.noLedgeredEffects(session, task, { operations: true, stationCalls: true });
|
|
829
|
+
if (!units) return null;
|
|
830
|
+
return {
|
|
831
|
+
blocked: blocks[0],
|
|
832
|
+
cancel_requested: requested[0],
|
|
833
|
+
cancelled: cancelled[0],
|
|
834
|
+
refused_writes: writes,
|
|
835
|
+
reads: reads.map((c) => ({ id: c.id, input_digest: c.input_digest })),
|
|
836
|
+
operator_mcp_calls: mcp,
|
|
837
|
+
unbound_stopped_units: units,
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
/** An attempt the request-budget control interrupted after its approved work had succeeded:
|
|
841
|
+
* each workspace write approved, applied through exactly one returned patch call and recorded
|
|
842
|
+
* with that call's own receipt, and any review approved and read-only. Every other station call
|
|
843
|
+
* is a closed read or a returned check whose unit stopped with a complete proof, and no tool ran
|
|
844
|
+
* after the stop. The applied writes and the review are the known effects. */
|
|
845
|
+
private requestBudgetStopKnownEffectsFailure(
|
|
846
|
+
session: string,
|
|
847
|
+
binding: unknown,
|
|
848
|
+
task: WorkTask,
|
|
849
|
+
runtime: Record<string, any> | undefined,
|
|
850
|
+
attempts: Record<string, any>[],
|
|
851
|
+
) {
|
|
852
|
+
const reason = 'Required control failed: No model request budget remains',
|
|
853
|
+
attempt = attempts.find((a) => a.state === 'interrupted');
|
|
854
|
+
if (
|
|
855
|
+
task.state !== 'blocked' ||
|
|
856
|
+
!runtime ||
|
|
857
|
+
runtime.auto_enabled ||
|
|
858
|
+
runtime.reserved_requests ||
|
|
859
|
+
runtime.reserved_tokens ||
|
|
860
|
+
runtime.verification_runs ||
|
|
861
|
+
attempts.length !== 1 ||
|
|
862
|
+
!attempt ||
|
|
863
|
+
attempt.error !== reason ||
|
|
864
|
+
attempt.resolved_by !== null ||
|
|
865
|
+
attempt.continues_attempt_id !== null ||
|
|
866
|
+
attempt.result !== null
|
|
867
|
+
)
|
|
868
|
+
return null;
|
|
869
|
+
const steps = task.steps.filter((s) => s.attempt_id === attempt.id);
|
|
870
|
+
if (
|
|
871
|
+
steps.length !== 1 ||
|
|
872
|
+
steps[0]!.state !== 'interrupted' ||
|
|
873
|
+
task.steps.some((s) => s !== steps[0] && (s.state !== 'pending' || s.attempt_id !== null))
|
|
874
|
+
)
|
|
875
|
+
return null;
|
|
876
|
+
if (
|
|
877
|
+
this.store.db
|
|
878
|
+
.prepare('SELECT 1 FROM cancellations WHERE attempt_id=?')
|
|
879
|
+
.get(String(attempt.id))
|
|
880
|
+
)
|
|
881
|
+
return null;
|
|
882
|
+
const events = this.store.db
|
|
883
|
+
.prepare('SELECT seq,kind,payload FROM events WHERE session_id=? ORDER BY seq')
|
|
884
|
+
.all(session);
|
|
885
|
+
const of = (e: Record<string, any>) => JSON.parse(String(e.payload));
|
|
886
|
+
const unhealthy = events.filter((e) => e.kind === 'worker.unhealthy'),
|
|
887
|
+
interrupted = events.filter((e) => e.kind === 'attempt.interrupted');
|
|
888
|
+
if (
|
|
889
|
+
unhealthy.length !== 1 ||
|
|
890
|
+
interrupted.length !== 1 ||
|
|
891
|
+
of(unhealthy[0]!).error !== reason ||
|
|
892
|
+
of(interrupted[0]!).attempt_id !== attempt.id ||
|
|
893
|
+
of(interrupted[0]!).error !== reason ||
|
|
894
|
+
Number(unhealthy[0]!.seq) >= Number(interrupted[0]!.seq)
|
|
895
|
+
)
|
|
896
|
+
return null;
|
|
897
|
+
const ops = this.store.db
|
|
898
|
+
.prepare('SELECT * FROM operations WHERE session_id=? ORDER BY rowid')
|
|
899
|
+
.all(session);
|
|
900
|
+
const calls = this.store.db
|
|
901
|
+
.prepare('SELECT * FROM station_calls WHERE session_id=? ORDER BY rowid')
|
|
902
|
+
.all(session);
|
|
903
|
+
if (!ops.length) return null;
|
|
904
|
+
const root = digest(binding),
|
|
905
|
+
effects: Record<string, unknown>[] = [];
|
|
906
|
+
for (const op of ops) {
|
|
907
|
+
const approval = this.store.db
|
|
908
|
+
.prepare('SELECT * FROM approvals WHERE operation_id=?')
|
|
909
|
+
.get(String(op.id));
|
|
910
|
+
const linked = calls.filter((c) => c.operation_id === op.id);
|
|
911
|
+
if (
|
|
912
|
+
op.attempt_id !== attempt.id ||
|
|
913
|
+
op.state !== 'succeeded' ||
|
|
914
|
+
!approval ||
|
|
915
|
+
approval.state !== 'decided' ||
|
|
916
|
+
approval.decision !== 'approve' ||
|
|
917
|
+
approval.operation_digest !== op.operation_digest ||
|
|
918
|
+
!op.receipt
|
|
919
|
+
)
|
|
920
|
+
return null;
|
|
921
|
+
if (op.kind === 'workspace') {
|
|
922
|
+
const call = linked[0],
|
|
923
|
+
effect = JSON.parse(String(op.receipt));
|
|
924
|
+
if (
|
|
925
|
+
op.resolution !== 'workspace-content-receipt-v1' ||
|
|
926
|
+
// station_calls.operation_id is unique, so a write has at most one call.
|
|
927
|
+
!call ||
|
|
928
|
+
call.kind !== 'patch' ||
|
|
929
|
+
call.state !== 'returned' ||
|
|
930
|
+
call.helper_state !== 'stopped' ||
|
|
931
|
+
call.attempt_id !== attempt.id ||
|
|
932
|
+
!call.result_json ||
|
|
933
|
+
effect.version !== 1 ||
|
|
934
|
+
effect.scope !== 'station-workspace' ||
|
|
935
|
+
effect.call_id !== call.id ||
|
|
936
|
+
effect.root_digest !== root ||
|
|
937
|
+
effect.root_digest !== call.root_digest ||
|
|
938
|
+
effect.configuration_digest !== call.configuration_digest ||
|
|
939
|
+
effect.input_digest !== call.input_digest ||
|
|
940
|
+
canonical(effect.result) !== canonical(JSON.parse(String(call.result_json))) ||
|
|
941
|
+
effect.result?.ok !== true
|
|
942
|
+
)
|
|
943
|
+
return null;
|
|
944
|
+
effects.push({
|
|
945
|
+
operation_id: op.id,
|
|
946
|
+
operation_digest: op.operation_digest,
|
|
947
|
+
call_id: call.id,
|
|
948
|
+
effect,
|
|
949
|
+
});
|
|
950
|
+
} else if (op.kind === 'review') {
|
|
951
|
+
if (
|
|
952
|
+
op.connector !== 'local-review-v1' ||
|
|
953
|
+
op.replay !== 'read-only' ||
|
|
954
|
+
op.resolution !== 'local-review-receipt' ||
|
|
955
|
+
linked.length
|
|
956
|
+
)
|
|
957
|
+
return null;
|
|
958
|
+
effects.push({ operation_id: op.id, operation_digest: op.operation_digest, review: true });
|
|
959
|
+
} else return null;
|
|
960
|
+
}
|
|
961
|
+
// Other station calls: closed reads, or returned checks whose own unit stopped with proof.
|
|
962
|
+
const others = calls.filter((c) => c.operation_id === null);
|
|
963
|
+
if (
|
|
964
|
+
others.some(
|
|
965
|
+
(c) =>
|
|
966
|
+
c.state !== 'returned' ||
|
|
967
|
+
!(c.kind === 'read' || (c.kind === 'check' && c.helper_state === 'stopped')),
|
|
968
|
+
)
|
|
969
|
+
)
|
|
970
|
+
return null;
|
|
971
|
+
const checks = others.filter((c) => c.kind === 'check').map((c) => String(c.id));
|
|
972
|
+
const mcp: Record<string, any>[] = [];
|
|
973
|
+
const open = new Map<string, { tool: string; calls: number }>();
|
|
974
|
+
for (const e of events) {
|
|
975
|
+
if (!['tool.started', 'tool.finished', 'operator_mcp.call'].includes(String(e.kind)))
|
|
976
|
+
continue;
|
|
977
|
+
const payload = of(e);
|
|
978
|
+
if (Number(e.seq) >= Number(unhealthy[0]!.seq) || payload.attempt_id !== attempt.id)
|
|
979
|
+
return null;
|
|
980
|
+
if (e.kind === 'tool.started') {
|
|
981
|
+
if (
|
|
982
|
+
!['salaam_operator_mcp', 'salaam_station', 'salaam_board', 'salaam_review'].includes(
|
|
983
|
+
String(payload.tool),
|
|
984
|
+
) ||
|
|
985
|
+
typeof payload.tool_call_id !== 'string' ||
|
|
986
|
+
!payload.tool_call_id ||
|
|
987
|
+
payload.parent_tool_call_id
|
|
988
|
+
)
|
|
989
|
+
return null;
|
|
990
|
+
open.set(payload.tool_call_id, { tool: String(payload.tool), calls: 0 });
|
|
991
|
+
} else if (e.kind === 'tool.finished') {
|
|
992
|
+
const started = open.get(String(payload.tool_call_id));
|
|
993
|
+
if (!started || payload.tool !== started.tool) return null;
|
|
994
|
+
open.delete(String(payload.tool_call_id));
|
|
995
|
+
} else {
|
|
996
|
+
const free = [...open.values()].find(
|
|
997
|
+
(t) => t.tool === 'salaam_operator_mcp' && t.calls === 0,
|
|
998
|
+
);
|
|
999
|
+
if (!free || !['returned', 'failed', 'tool_error'].includes(String(payload.outcome)))
|
|
1000
|
+
return null;
|
|
1001
|
+
free.calls++;
|
|
1002
|
+
mcp.push({ seq: e.seq, ...payload });
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
if (open.size) return null;
|
|
1006
|
+
const units = this.noLedgeredEffects(session, task, {
|
|
1007
|
+
operations: true,
|
|
1008
|
+
stationCalls: true,
|
|
1009
|
+
checkUnits: checks,
|
|
1010
|
+
});
|
|
1011
|
+
if (!units) return null;
|
|
1012
|
+
return {
|
|
1013
|
+
unhealthy: unhealthy[0],
|
|
1014
|
+
interrupted: interrupted[0],
|
|
1015
|
+
known_effects: effects,
|
|
1016
|
+
station_calls: others.map((c) => ({ id: c.id, kind: c.kind, input_digest: c.input_digest })),
|
|
1017
|
+
operator_mcp_calls: mcp,
|
|
1018
|
+
unbound_stopped_units: units,
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
396
1021
|
/** A disjoint failure profile: the native child was denied before any effect
|
|
397
1022
|
* ledger was created. Stopped resources alone never establish zero effects. */
|
|
398
1023
|
private zeroEffectConcurrencyFailure(
|
|
@@ -499,6 +1124,29 @@ export class AgentPodWork {
|
|
|
499
1124
|
links[0]!.step_id !== task.steps.find((s) => s.attempt_id === attempt.id)?.id
|
|
500
1125
|
)
|
|
501
1126
|
return null;
|
|
1127
|
+
const units = this.noLedgeredEffects(session, task);
|
|
1128
|
+
if (!units) return null;
|
|
1129
|
+
return {
|
|
1130
|
+
block: blocks[0],
|
|
1131
|
+
activity,
|
|
1132
|
+
model: models[0],
|
|
1133
|
+
policy: policies[0],
|
|
1134
|
+
link: links[0],
|
|
1135
|
+
unbound_stopped_units: units,
|
|
1136
|
+
};
|
|
1137
|
+
}
|
|
1138
|
+
/** Every execution/effect ledger, including read calls, is empty for this session and task,
|
|
1139
|
+
* and any station unit is unbound with a complete durable stop proof. Returns those units. */
|
|
1140
|
+
private noLedgeredEffects(
|
|
1141
|
+
session: string,
|
|
1142
|
+
task: WorkTask,
|
|
1143
|
+
checked: {
|
|
1144
|
+
operations?: boolean;
|
|
1145
|
+
stationReads?: boolean;
|
|
1146
|
+
stationCalls?: boolean;
|
|
1147
|
+
checkUnits?: string[];
|
|
1148
|
+
} = {},
|
|
1149
|
+
): Record<string, any>[] | null {
|
|
502
1150
|
// These are all independent execution/effect ledgers, including read calls.
|
|
503
1151
|
for (const table of [
|
|
504
1152
|
'operations',
|
|
@@ -509,6 +1157,19 @@ export class AgentPodWork {
|
|
|
509
1157
|
'local_calls',
|
|
510
1158
|
'command_runs',
|
|
511
1159
|
]) {
|
|
1160
|
+
// A caller that proves its own operations or closed reads checks those itself.
|
|
1161
|
+
if (table === 'operations' && checked.operations) continue;
|
|
1162
|
+
if (table === 'station_calls' && checked.stationCalls) continue;
|
|
1163
|
+
if (
|
|
1164
|
+
table === 'station_calls' &&
|
|
1165
|
+
checked.stationReads &&
|
|
1166
|
+
!this.store.db
|
|
1167
|
+
.prepare(
|
|
1168
|
+
"SELECT 1 FROM station_calls WHERE session_id=? AND NOT (kind='read' AND state='returned')",
|
|
1169
|
+
)
|
|
1170
|
+
.get(session)
|
|
1171
|
+
)
|
|
1172
|
+
continue;
|
|
512
1173
|
if (this.store.db.prepare(`SELECT 1 FROM ${table} WHERE session_id=?`).get(session))
|
|
513
1174
|
return null;
|
|
514
1175
|
}
|
|
@@ -561,9 +1222,13 @@ export class AgentPodWork {
|
|
|
561
1222
|
.prepare('SELECT * FROM station_units WHERE session_id=? OR session_id IS NULL')
|
|
562
1223
|
.all(session);
|
|
563
1224
|
for (const unit of units) {
|
|
1225
|
+
// A proven returned check's own unit belongs to this session; it still needs its stop proof.
|
|
1226
|
+
const ownCheck =
|
|
1227
|
+
unit.session_id === session &&
|
|
1228
|
+
typeof unit.call_id === 'string' &&
|
|
1229
|
+
!!checked.checkUnits?.includes(unit.call_id);
|
|
564
1230
|
if (
|
|
565
|
-
unit.session_id !== null ||
|
|
566
|
-
unit.call_id !== null ||
|
|
1231
|
+
(!ownCheck && (unit.session_id !== null || unit.call_id !== null)) ||
|
|
567
1232
|
unit.state !== 'stopped' ||
|
|
568
1233
|
!unit.stop_json
|
|
569
1234
|
)
|
|
@@ -579,11 +1244,160 @@ export class AgentPodWork {
|
|
|
579
1244
|
)
|
|
580
1245
|
return null;
|
|
581
1246
|
}
|
|
1247
|
+
return units;
|
|
1248
|
+
}
|
|
1249
|
+
/** An attempt the owner's authority revocation cancelled (for example a hub restart) before
|
|
1250
|
+
* any ledgered effect. The only tool activity allowed is closed operator MCP calls, each
|
|
1251
|
+
* journaled with argument and result digests; they are preserved as the known effects. */
|
|
1252
|
+
private revokeCancelledFailure(
|
|
1253
|
+
session: string,
|
|
1254
|
+
task: WorkTask,
|
|
1255
|
+
runtime: Record<string, any> | undefined,
|
|
1256
|
+
attempts: Record<string, any>[],
|
|
1257
|
+
) {
|
|
1258
|
+
const attempt = attempts[0];
|
|
1259
|
+
if (
|
|
1260
|
+
task.state !== 'blocked' ||
|
|
1261
|
+
!runtime ||
|
|
1262
|
+
runtime.auto_enabled ||
|
|
1263
|
+
runtime.reserved_requests ||
|
|
1264
|
+
runtime.reserved_tokens ||
|
|
1265
|
+
runtime.verification_runs ||
|
|
1266
|
+
attempts.length !== 1 ||
|
|
1267
|
+
!attempt ||
|
|
1268
|
+
attempt.state !== 'cancelled' ||
|
|
1269
|
+
attempt.resolved_by !== null ||
|
|
1270
|
+
attempt.continues_attempt_id !== null ||
|
|
1271
|
+
attempt.result !== null ||
|
|
1272
|
+
!attempt.outcome
|
|
1273
|
+
)
|
|
1274
|
+
return null;
|
|
1275
|
+
const outcome = JSON.parse(String(attempt.outcome));
|
|
1276
|
+
if (
|
|
1277
|
+
outcome.version !== 1 ||
|
|
1278
|
+
outcome.stopReason !== 'cancelled' ||
|
|
1279
|
+
outcome.failure?.kind !== 'cancelled' ||
|
|
1280
|
+
outcome.failure.code !== 'attempt_cancelled' ||
|
|
1281
|
+
outcome.failure.retryable !== false ||
|
|
1282
|
+
outcome.failure.sessionId !== session ||
|
|
1283
|
+
outcome.failure.attemptId !== attempt.id
|
|
1284
|
+
)
|
|
1285
|
+
return null;
|
|
1286
|
+
const steps = task.steps.filter((s) => s.attempt_id === attempt.id);
|
|
1287
|
+
if (
|
|
1288
|
+
steps.length !== 1 ||
|
|
1289
|
+
steps[0]!.state !== 'cancelled' ||
|
|
1290
|
+
task.steps.some((s) => s !== steps[0] && (s.state !== 'pending' || s.attempt_id !== null))
|
|
1291
|
+
)
|
|
1292
|
+
return null;
|
|
1293
|
+
const links = this.store.db
|
|
1294
|
+
.prepare(
|
|
1295
|
+
'SELECT b.* FROM task_step_attempts b JOIN task_steps s ON s.id=b.step_id WHERE s.task_id=?',
|
|
1296
|
+
)
|
|
1297
|
+
.all(task.id);
|
|
1298
|
+
if (
|
|
1299
|
+
links.length !== 1 ||
|
|
1300
|
+
links[0]!.attempt_id !== attempt.id ||
|
|
1301
|
+
links[0]!.plan_revision !== task.plan_revision ||
|
|
1302
|
+
links[0]!.step_id !== steps[0]!.id
|
|
1303
|
+
)
|
|
1304
|
+
return null;
|
|
1305
|
+
// The cancellation came from the node's revocation of this session, not a local operator.
|
|
1306
|
+
const cancellations = this.store.db
|
|
1307
|
+
.prepare('SELECT * FROM cancellations WHERE attempt_id=?')
|
|
1308
|
+
.all(String(attempt.id));
|
|
1309
|
+
const revocations = this.store.db
|
|
1310
|
+
.prepare('SELECT * FROM session_revocations WHERE session_id=?')
|
|
1311
|
+
.all(session);
|
|
1312
|
+
if (
|
|
1313
|
+
cancellations.length !== 1 ||
|
|
1314
|
+
cancellations[0]!.requested_by !== 'agentpod-node' ||
|
|
1315
|
+
revocations.length !== 1
|
|
1316
|
+
)
|
|
1317
|
+
return null;
|
|
1318
|
+
const revocation = JSON.parse(String(revocations[0]!.receipt));
|
|
1319
|
+
if (
|
|
1320
|
+
revocation.revoked !== true ||
|
|
1321
|
+
revocation.session_id !== session ||
|
|
1322
|
+
canonical(revocation.session_ids) !== canonical([session]) ||
|
|
1323
|
+
canonical(revocation.attempt_ids) !== canonical([attempt.id]) ||
|
|
1324
|
+
canonical(revocation.unresolved_operations) !== canonical([]) ||
|
|
1325
|
+
typeof revocation.request_id !== 'string' ||
|
|
1326
|
+
!revocation.request_id
|
|
1327
|
+
)
|
|
1328
|
+
return null;
|
|
1329
|
+
const marks = this.store.db
|
|
1330
|
+
.prepare(
|
|
1331
|
+
"SELECT seq,kind,payload,causation_id FROM events WHERE session_id=? AND kind IN ('attempt.cancel_requested','execution.revoked','attempt.cancelled') ORDER BY seq",
|
|
1332
|
+
)
|
|
1333
|
+
.all(session);
|
|
1334
|
+
const only = (kind: string) => {
|
|
1335
|
+
const found = marks.filter((e) => e.kind === kind);
|
|
1336
|
+
return found.length === 1 ? found[0]! : null;
|
|
1337
|
+
};
|
|
1338
|
+
const requested = only('attempt.cancel_requested'),
|
|
1339
|
+
revoked = only('execution.revoked'),
|
|
1340
|
+
cancelled = only('attempt.cancelled');
|
|
1341
|
+
if (
|
|
1342
|
+
marks.length !== 3 ||
|
|
1343
|
+
!requested ||
|
|
1344
|
+
!revoked ||
|
|
1345
|
+
!cancelled ||
|
|
1346
|
+
requested.causation_id !== 'agentpod-revocation' ||
|
|
1347
|
+
JSON.parse(String(requested.payload)).attempt_id !== attempt.id ||
|
|
1348
|
+
revoked.causation_id !== revocation.request_id ||
|
|
1349
|
+
JSON.parse(String(revoked.payload)).owner_session !== session ||
|
|
1350
|
+
JSON.parse(String(cancelled.payload)).attempt_id !== attempt.id ||
|
|
1351
|
+
!(Number(requested.seq) < Number(revoked.seq) && Number(revoked.seq) < Number(cancelled.seq))
|
|
1352
|
+
)
|
|
1353
|
+
return null;
|
|
1354
|
+
// Tool activity: only operator MCP tools, each closed before the revocation, each holding
|
|
1355
|
+
// at most one journaled call (none for a catalog read), and no call outside a tool.
|
|
1356
|
+
const activity = this.store.db
|
|
1357
|
+
.prepare(
|
|
1358
|
+
"SELECT seq,kind,payload FROM events WHERE session_id=? AND kind IN ('tool.started','tool.finished','operator_mcp.call','policy.allowed','policy.denied') ORDER BY seq",
|
|
1359
|
+
)
|
|
1360
|
+
.all(session);
|
|
1361
|
+
const calls: Record<string, any>[] = [];
|
|
1362
|
+
let open: { id: string; calls: number } | null = null;
|
|
1363
|
+
for (const event of activity) {
|
|
1364
|
+
const payload = JSON.parse(String(event.payload));
|
|
1365
|
+
if (Number(event.seq) >= Number(requested.seq)) return null;
|
|
1366
|
+
if (payload.attempt_id !== attempt.id) return null;
|
|
1367
|
+
if (event.kind === 'tool.started') {
|
|
1368
|
+
if (
|
|
1369
|
+
open ||
|
|
1370
|
+
payload.tool !== 'salaam_operator_mcp' ||
|
|
1371
|
+
typeof payload.tool_call_id !== 'string' ||
|
|
1372
|
+
!payload.tool_call_id ||
|
|
1373
|
+
payload.parent_tool_call_id
|
|
1374
|
+
)
|
|
1375
|
+
return null;
|
|
1376
|
+
open = { id: payload.tool_call_id, calls: 0 };
|
|
1377
|
+
} else if (event.kind === 'tool.finished') {
|
|
1378
|
+
if (!open || payload.tool_call_id !== open.id || payload.tool !== 'salaam_operator_mcp')
|
|
1379
|
+
return null;
|
|
1380
|
+
open = null;
|
|
1381
|
+
} else if (event.kind === 'operator_mcp.call') {
|
|
1382
|
+
if (
|
|
1383
|
+
!open ||
|
|
1384
|
+
++open.calls > 1 ||
|
|
1385
|
+
!['returned', 'tool_error', 'failed'].includes(String(payload.outcome))
|
|
1386
|
+
)
|
|
1387
|
+
return null;
|
|
1388
|
+
calls.push({ seq: event.seq, ...payload });
|
|
1389
|
+
} else if (event.kind === 'policy.allowed') {
|
|
1390
|
+
if (!open || payload.tool !== 'salaam_operator_mcp') return null;
|
|
1391
|
+
} else return null;
|
|
1392
|
+
}
|
|
1393
|
+
if (open) return null;
|
|
1394
|
+
const units = this.noLedgeredEffects(session, task);
|
|
1395
|
+
if (!units) return null;
|
|
582
1396
|
return {
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
1397
|
+
revocation,
|
|
1398
|
+
cancellation: cancellations[0],
|
|
1399
|
+
events: marks,
|
|
1400
|
+
operator_mcp_calls: calls,
|
|
587
1401
|
link: links[0],
|
|
588
1402
|
unbound_stopped_units: units,
|
|
589
1403
|
};
|
|
@@ -1567,6 +2381,7 @@ export class AgentPodWork {
|
|
|
1567
2381
|
objective: task.objective,
|
|
1568
2382
|
constraints: task.constraints,
|
|
1569
2383
|
acceptance: task.acceptance,
|
|
2384
|
+
...(task.completion_gate ? { completion_gate: task.completion_gate } : {}),
|
|
1570
2385
|
deliverables: task.deliverables,
|
|
1571
2386
|
limits: task.limits,
|
|
1572
2387
|
steps: task.steps.map(({ key, title, instruction, depends_on, checks }) => ({
|
|
@@ -1602,7 +2417,7 @@ export class AgentPodWork {
|
|
|
1602
2417
|
const bindingArtifact = JSON.parse(String(saved!.response)),
|
|
1603
2418
|
bound = artifact(bindingArtifact, session);
|
|
1604
2419
|
const call = calls.find((c) => c.id === bound.call_id),
|
|
1605
|
-
configured = task.
|
|
2420
|
+
configured = requiredChecks(task).find((c) => c.key === check.check_key);
|
|
1606
2421
|
require(
|
|
1607
2422
|
check.state === 'passed' &&
|
|
1608
2423
|
check.plan_revision === task.plan_revision &&
|
|
@@ -2007,20 +2822,7 @@ export class AgentPodWork {
|
|
|
2007
2822
|
'work_settlement_conflict',
|
|
2008
2823
|
'Exact original work and terminal digest required',
|
|
2009
2824
|
);
|
|
2010
|
-
const stopped = () =>
|
|
2011
|
-
const current = this.authority.inspect(binding);
|
|
2012
|
-
if (
|
|
2013
|
-
!current.revoked ||
|
|
2014
|
-
current.authorized ||
|
|
2015
|
-
!current.owned_execution_stopped ||
|
|
2016
|
-
current.unresolved_operations.length ||
|
|
2017
|
-
current.active_attempts.length
|
|
2018
|
-
)
|
|
2019
|
-
throw new ServiceError(
|
|
2020
|
-
'work_settlement_unsafe',
|
|
2021
|
-
'Revoked stopped root with known effects required',
|
|
2022
|
-
);
|
|
2023
|
-
};
|
|
2825
|
+
const stopped = () => this.assertKnownEffectsStop(binding);
|
|
2024
2826
|
stopped();
|
|
2025
2827
|
return this.store.request(
|
|
2026
2828
|
'agentpod-work-failure',
|
|
@@ -2034,132 +2836,7 @@ export class AgentPodWork {
|
|
|
2034
2836
|
() => {
|
|
2035
2837
|
stopped();
|
|
2036
2838
|
this.assertPlan(task, JSON.parse(String(row.admission_json)));
|
|
2037
|
-
const
|
|
2038
|
-
.prepare('SELECT * FROM task_runtime WHERE task_id=?')
|
|
2039
|
-
.get(task.id);
|
|
2040
|
-
const attempts = this.store.db
|
|
2041
|
-
.prepare('SELECT * FROM attempts WHERE session_id=?')
|
|
2042
|
-
.all(root.session_id);
|
|
2043
|
-
const ops = this.store.db
|
|
2044
|
-
.prepare('SELECT * FROM operations WHERE session_id=?')
|
|
2045
|
-
.all(root.session_id);
|
|
2046
|
-
const calls = this.store.db
|
|
2047
|
-
.prepare("SELECT * FROM station_calls WHERE session_id=? AND kind!='read'")
|
|
2048
|
-
.all(root.session_id);
|
|
2049
|
-
const op = ops[0],
|
|
2050
|
-
attempt = attempts[0],
|
|
2051
|
-
call = calls[0];
|
|
2052
|
-
const approval = op
|
|
2053
|
-
? this.store.db.prepare('SELECT * FROM approvals WHERE operation_id=?').get(String(op.id))
|
|
2054
|
-
: undefined;
|
|
2055
|
-
const effect = op?.receipt ? JSON.parse(String(op.receipt)) : null;
|
|
2056
|
-
const bound = 'Provider payload exceeds declared per-request token bounds';
|
|
2057
|
-
const originalBlock = this.store.db
|
|
2058
|
-
.prepare(
|
|
2059
|
-
"SELECT seq,payload FROM events WHERE session_id=? AND kind='work_task.budget_blocked' AND json_extract(payload,'$.task_id')=? AND json_extract(payload,'$.reason')=? ORDER BY seq LIMIT 1",
|
|
2060
|
-
)
|
|
2061
|
-
.get(root.session_id, task.id, bound);
|
|
2062
|
-
const runtimeExpired =
|
|
2063
|
-
runtime?.blocked_reason === 'Task runtime limit reached' &&
|
|
2064
|
-
runtime.started_at !== null &&
|
|
2065
|
-
Number.isFinite(Number(runtime.started_at)) &&
|
|
2066
|
-
Date.now() - Number(runtime.started_at) >= task.limits.runtime_ms;
|
|
2067
|
-
|
|
2068
|
-
const savedInvalidatedOffer =
|
|
2069
|
-
approval?.state === 'invalidated'
|
|
2070
|
-
? this.store.db
|
|
2071
|
-
.prepare(
|
|
2072
|
-
"SELECT response FROM requests WHERE actor='agentpod-work-decision-offer' AND id=?",
|
|
2073
|
-
)
|
|
2074
|
-
.get(String(approval.id))
|
|
2075
|
-
: undefined;
|
|
2076
|
-
const knownReadOnly =
|
|
2077
|
-
this.historicalKnownReadOnlyFailure(root.binding, task, runtime) ??
|
|
2078
|
-
(ops.length === 1 && op && approval && savedInvalidatedOffer
|
|
2079
|
-
? this.historicalKnownReadOnlyFailure(root.binding, task, runtime, {
|
|
2080
|
-
operation: op,
|
|
2081
|
-
approval,
|
|
2082
|
-
offered: JSON.parse(String(savedInvalidatedOffer.response)),
|
|
2083
|
-
invalidated: true,
|
|
2084
|
-
})
|
|
2085
|
-
: null) ??
|
|
2086
|
-
(ops.length === 1 && op && approval
|
|
2087
|
-
? this.historicalKnownReadOnlyFailure(root.binding, task, runtime, undefined, {
|
|
2088
|
-
operation: op,
|
|
2089
|
-
approval,
|
|
2090
|
-
})
|
|
2091
|
-
: null);
|
|
2092
|
-
const zeroEffect = this.zeroEffectConcurrencyFailure(
|
|
2093
|
-
root.session_id,
|
|
2094
|
-
task,
|
|
2095
|
-
runtime,
|
|
2096
|
-
attempts,
|
|
2097
|
-
);
|
|
2098
|
-
if (
|
|
2099
|
-
!zeroEffect &&
|
|
2100
|
-
!knownReadOnly &&
|
|
2101
|
-
(task.state !== 'blocked' ||
|
|
2102
|
-
!runtime ||
|
|
2103
|
-
!originalBlock ||
|
|
2104
|
-
(runtime.blocked_reason !== bound && !runtimeExpired) ||
|
|
2105
|
-
attempt?.error !== 'Required control failed: ' + bound ||
|
|
2106
|
-
runtime.auto_enabled ||
|
|
2107
|
-
runtime.reserved_requests ||
|
|
2108
|
-
runtime.reserved_tokens ||
|
|
2109
|
-
runtime.verification_runs ||
|
|
2110
|
-
attempts.length !== 1 ||
|
|
2111
|
-
attempt?.state !== 'interrupted' ||
|
|
2112
|
-
attempt.resolved_by !== null ||
|
|
2113
|
-
ops.length !== 1 ||
|
|
2114
|
-
op?.kind !== 'workspace' ||
|
|
2115
|
-
op.state !== 'succeeded' ||
|
|
2116
|
-
op.attempt_id !== attempt.id ||
|
|
2117
|
-
op.resolution !== 'workspace-content-receipt-v1' ||
|
|
2118
|
-
calls.length !== 1 ||
|
|
2119
|
-
call?.kind !== 'patch' ||
|
|
2120
|
-
call.state !== 'returned' ||
|
|
2121
|
-
call.helper_state !== 'stopped' ||
|
|
2122
|
-
call.operation_id !== op.id ||
|
|
2123
|
-
call.attempt_id !== attempt.id ||
|
|
2124
|
-
!call.result_json ||
|
|
2125
|
-
!approval ||
|
|
2126
|
-
approval.state !== 'decided' ||
|
|
2127
|
-
approval.decision !== 'approve' ||
|
|
2128
|
-
approval.operation_digest !== op.operation_digest ||
|
|
2129
|
-
!effect ||
|
|
2130
|
-
effect.version !== 1 ||
|
|
2131
|
-
effect.scope !== 'station-workspace' ||
|
|
2132
|
-
effect.call_id !== call.id ||
|
|
2133
|
-
effect.root_digest !== digest(root.binding) ||
|
|
2134
|
-
effect.root_digest !== call.root_digest ||
|
|
2135
|
-
effect.configuration_digest !== call.configuration_digest ||
|
|
2136
|
-
effect.input_digest !== call.input_digest ||
|
|
2137
|
-
canonical(effect.result) !== canonical(JSON.parse(String(call.result_json))) ||
|
|
2138
|
-
effect.result?.ok !== true ||
|
|
2139
|
-
task.steps.filter((s) => s.attempt_id === attempt.id && s.state === 'interrupted')
|
|
2140
|
-
.length !== 1 ||
|
|
2141
|
-
task.steps.some((s) => !['pending', 'interrupted'].includes(s.state)) ||
|
|
2142
|
-
this.store.db.prepare('SELECT 1 FROM task_checks WHERE task_id=?').get(task.id) ||
|
|
2143
|
-
this.store.db
|
|
2144
|
-
.prepare('SELECT 1 FROM child_tasks WHERE parent_session=?')
|
|
2145
|
-
.get(root.session_id))
|
|
2146
|
-
)
|
|
2147
|
-
throw new ServiceError(
|
|
2148
|
-
'work_settlement_unsafe',
|
|
2149
|
-
'Only exact known approved-patch, zero-effect child denial, or closed historical read-only evidence can retire',
|
|
2150
|
-
);
|
|
2151
|
-
const evidence_digest = digest({
|
|
2152
|
-
binding: root.binding,
|
|
2153
|
-
task_id: task.id,
|
|
2154
|
-
plan_revision: task.plan_revision,
|
|
2155
|
-
blocked_reason: runtime!.blocked_reason,
|
|
2156
|
-
attempt,
|
|
2157
|
-
operation: op ?? null,
|
|
2158
|
-
approval: approval ?? null,
|
|
2159
|
-
call: call ?? null,
|
|
2160
|
-
...(zeroEffect ? { zero_effect_concurrency: zeroEffect } : {}),
|
|
2161
|
-
...(knownReadOnly ? { historical_known_readonly: knownReadOnly } : {}),
|
|
2162
|
-
});
|
|
2839
|
+
const evidence_digest = this.failureEvidence(root, task);
|
|
2163
2840
|
driver.cancelInTransaction(task.id, task.revision, stopped);
|
|
2164
2841
|
const receipt = {
|
|
2165
2842
|
version: 1 as const,
|
|
@@ -2179,6 +2856,453 @@ export class AgentPodWork {
|
|
|
2179
2856
|
},
|
|
2180
2857
|
);
|
|
2181
2858
|
}
|
|
2859
|
+
/** A revoked root whose execution stopped with nothing unresolved or still running. */
|
|
2860
|
+
private assertKnownEffectsStop(binding: unknown): void {
|
|
2861
|
+
const current = this.authority.inspect(binding);
|
|
2862
|
+
if (
|
|
2863
|
+
!current.revoked ||
|
|
2864
|
+
current.authorized ||
|
|
2865
|
+
!current.owned_execution_stopped ||
|
|
2866
|
+
current.unresolved_operations.length ||
|
|
2867
|
+
current.active_attempts.length
|
|
2868
|
+
)
|
|
2869
|
+
throw new ServiceError(
|
|
2870
|
+
'work_settlement_unsafe',
|
|
2871
|
+
'Revoked stopped root with known effects required',
|
|
2872
|
+
);
|
|
2873
|
+
}
|
|
2874
|
+
/** The evidence a known-effects retirement records, or work_settlement_unsafe when the stopped
|
|
2875
|
+
* run is not one of the exact shapes Salaam can prove. Reads only; fail() and status share it. */
|
|
2876
|
+
/** A task the provider rate limit stopped after its retries (RATE_LIMIT_STOP). Everything is
|
|
2877
|
+
* closed and definite: every attempt completed or failed, the one failed step's last attempt
|
|
2878
|
+
* rate-limited and recorded by the driver, every operation succeeded or failed definitely with
|
|
2879
|
+
* no approval pending, every station call settled with its helper stopped, no children. */
|
|
2880
|
+
private rateLimitStopFailure(
|
|
2881
|
+
session: string,
|
|
2882
|
+
task: WorkTask,
|
|
2883
|
+
runtime: Record<string, any> | undefined,
|
|
2884
|
+
attempts: Record<string, any>[],
|
|
2885
|
+
) {
|
|
2886
|
+
if (
|
|
2887
|
+
!['blocked', 'partially_completed'].includes(task.state) ||
|
|
2888
|
+
!runtime ||
|
|
2889
|
+
runtime.blocked_reason !== RATE_LIMIT_STOP ||
|
|
2890
|
+
runtime.auto_enabled ||
|
|
2891
|
+
runtime.reserved_requests ||
|
|
2892
|
+
runtime.reserved_tokens ||
|
|
2893
|
+
attempts.some((a) => !['completed', 'failed'].includes(String(a.state)))
|
|
2894
|
+
)
|
|
2895
|
+
return null;
|
|
2896
|
+
const failed = task.steps.filter((s) => s.state === 'failed');
|
|
2897
|
+
if (
|
|
2898
|
+
failed.length !== 1 ||
|
|
2899
|
+
task.steps.some((s) => !['completed', 'failed', 'pending'].includes(s.state))
|
|
2900
|
+
)
|
|
2901
|
+
return null;
|
|
2902
|
+
const last = String(failed[0]!.attempt_id ?? '');
|
|
2903
|
+
const limits = new RateLimitRetries(this.store);
|
|
2904
|
+
const recorded = this.store.db
|
|
2905
|
+
.prepare(
|
|
2906
|
+
'SELECT attempt_id,failures FROM task_rate_limits WHERE task_id=? AND step_id=? ORDER BY failures',
|
|
2907
|
+
)
|
|
2908
|
+
.all(task.id, failed[0]!.id);
|
|
2909
|
+
if (!limits.rateLimited(last) || !recorded.some((r) => r.attempt_id === last)) return null;
|
|
2910
|
+
const db = this.store.db;
|
|
2911
|
+
const operations = db
|
|
2912
|
+
.prepare('SELECT id,state FROM operations WHERE session_id=? ORDER BY rowid')
|
|
2913
|
+
.all(session);
|
|
2914
|
+
const calls = db
|
|
2915
|
+
.prepare(
|
|
2916
|
+
'SELECT id,kind,state,helper_state FROM station_calls WHERE session_id=? ORDER BY rowid',
|
|
2917
|
+
)
|
|
2918
|
+
.all(session);
|
|
2919
|
+
if (
|
|
2920
|
+
operations.some((o) => !['succeeded', 'failed_definite'].includes(String(o.state))) ||
|
|
2921
|
+
db.prepare("SELECT 1 FROM approvals WHERE session_id=? AND state='pending'").get(session) ||
|
|
2922
|
+
calls.some(
|
|
2923
|
+
(c) =>
|
|
2924
|
+
['prepared', 'dispatching', 'outcome_unknown'].includes(String(c.state)) ||
|
|
2925
|
+
!['none', 'stopped'].includes(String(c.helper_state)),
|
|
2926
|
+
) ||
|
|
2927
|
+
db.prepare('SELECT 1 FROM child_tasks WHERE parent_session=?').get(session)
|
|
2928
|
+
)
|
|
2929
|
+
return null;
|
|
2930
|
+
return {
|
|
2931
|
+
step_id: failed[0]!.id,
|
|
2932
|
+
attempts: attempts.map((a) => ({ id: a.id, state: a.state })),
|
|
2933
|
+
rate_limited: recorded,
|
|
2934
|
+
operations,
|
|
2935
|
+
calls,
|
|
2936
|
+
};
|
|
2937
|
+
}
|
|
2938
|
+
/** A task its verification budget stopped after every step finished (staging run_54648a07).
|
|
2939
|
+
* Everything is closed and definite: every attempt completed or failed, every step completed or
|
|
2940
|
+
* failed, every operation succeeded or failed definitely with no approval pending, every station
|
|
2941
|
+
* call settled with its helper stopped, nothing reserved, no children. */
|
|
2942
|
+
/** Which part of the verification-budget stop shape a run misses: a closed set of codes, so a
|
|
2943
|
+
* refusal can say why without carrying any data. */
|
|
2944
|
+
static readonly VERIFICATION_LIMIT_MISSES = [
|
|
2945
|
+
'vl:task_state',
|
|
2946
|
+
'vl:reason',
|
|
2947
|
+
'vl:auto',
|
|
2948
|
+
'vl:reserved',
|
|
2949
|
+
'vl:attempts',
|
|
2950
|
+
'vl:steps',
|
|
2951
|
+
'vl:operations',
|
|
2952
|
+
'vl:approvals',
|
|
2953
|
+
'vl:calls',
|
|
2954
|
+
'vl:children',
|
|
2955
|
+
] as const;
|
|
2956
|
+
private verificationLimitMiss: (typeof AgentPodWork.VERIFICATION_LIMIT_MISSES)[number] | null =
|
|
2957
|
+
null;
|
|
2958
|
+
private verificationLimitStopFailure(
|
|
2959
|
+
session: string,
|
|
2960
|
+
task: WorkTask,
|
|
2961
|
+
runtime: Record<string, any> | undefined,
|
|
2962
|
+
attempts: Record<string, any>[],
|
|
2963
|
+
) {
|
|
2964
|
+
const miss = (code: (typeof AgentPodWork.VERIFICATION_LIMIT_MISSES)[number]) => {
|
|
2965
|
+
this.verificationLimitMiss = code;
|
|
2966
|
+
return null;
|
|
2967
|
+
};
|
|
2968
|
+
this.verificationLimitMiss = null;
|
|
2969
|
+
// The stop is the verification budget's: the current reason, or the run's first budget block
|
|
2970
|
+
// when its runtime limit overwrote the reason later while it sat parked (staging run_54648a07).
|
|
2971
|
+
const firstBlock = this.store.db
|
|
2972
|
+
.prepare(
|
|
2973
|
+
"SELECT json_extract(payload,'$.reason') AS reason FROM events WHERE session_id=? AND kind='work_task.budget_blocked' AND json_extract(payload,'$.task_id')=? ORDER BY seq LIMIT 1",
|
|
2974
|
+
)
|
|
2975
|
+
.get(session, task.id) as { reason?: string } | undefined;
|
|
2976
|
+
if (
|
|
2977
|
+
!runtime ||
|
|
2978
|
+
!(
|
|
2979
|
+
runtime.blocked_reason === 'Task verification run limit reached' ||
|
|
2980
|
+
(runtime.blocked_reason === 'Task runtime limit reached' &&
|
|
2981
|
+
firstBlock?.reason === 'Task verification run limit reached')
|
|
2982
|
+
)
|
|
2983
|
+
)
|
|
2984
|
+
return miss('vl:reason');
|
|
2985
|
+
if (!['blocked', 'partially_completed'].includes(task.state)) return miss('vl:task_state');
|
|
2986
|
+
if (runtime.auto_enabled) return miss('vl:auto');
|
|
2987
|
+
if (runtime.reserved_requests || runtime.reserved_tokens) return miss('vl:reserved');
|
|
2988
|
+
if (
|
|
2989
|
+
!attempts.length ||
|
|
2990
|
+
attempts.some((a) => !['completed', 'failed'].includes(String(a.state)))
|
|
2991
|
+
)
|
|
2992
|
+
return miss('vl:attempts');
|
|
2993
|
+
if (task.steps.some((s) => !['completed', 'failed'].includes(s.state))) return miss('vl:steps');
|
|
2994
|
+
const db = this.store.db;
|
|
2995
|
+
const operations = db
|
|
2996
|
+
.prepare('SELECT id,state FROM operations WHERE session_id=? ORDER BY rowid')
|
|
2997
|
+
.all(session);
|
|
2998
|
+
const calls = db
|
|
2999
|
+
.prepare(
|
|
3000
|
+
'SELECT id,kind,state,helper_state FROM station_calls WHERE session_id=? ORDER BY rowid',
|
|
3001
|
+
)
|
|
3002
|
+
.all(session);
|
|
3003
|
+
if (operations.some((o) => !['succeeded', 'failed_definite'].includes(String(o.state))))
|
|
3004
|
+
return miss('vl:operations');
|
|
3005
|
+
if (db.prepare("SELECT 1 FROM approvals WHERE session_id=? AND state='pending'").get(session))
|
|
3006
|
+
return miss('vl:approvals');
|
|
3007
|
+
if (
|
|
3008
|
+
calls.some(
|
|
3009
|
+
(c) =>
|
|
3010
|
+
['prepared', 'dispatching', 'outcome_unknown'].includes(String(c.state)) ||
|
|
3011
|
+
!['none', 'stopped'].includes(String(c.helper_state)),
|
|
3012
|
+
)
|
|
3013
|
+
)
|
|
3014
|
+
return miss('vl:calls');
|
|
3015
|
+
if (db.prepare('SELECT 1 FROM child_tasks WHERE parent_session=?').get(session))
|
|
3016
|
+
return miss('vl:children');
|
|
3017
|
+
return {
|
|
3018
|
+
steps: task.steps.map((s) => ({ id: s.id, state: s.state })),
|
|
3019
|
+
attempts: attempts.map((a) => ({ id: a.id, state: a.state })),
|
|
3020
|
+
operations,
|
|
3021
|
+
calls,
|
|
3022
|
+
};
|
|
3023
|
+
}
|
|
3024
|
+
/** A task its own acceptance check stopped (staging card_d74eac4509f1453c): the latest receipt
|
|
3025
|
+
* for the check the stop names failed definitely, and the run is as closed as a
|
|
3026
|
+
* verification-budget stop. The board then gets a failed run with that reason, not a park. */
|
|
3027
|
+
static readonly CHECK_FAILED_MISSES = [
|
|
3028
|
+
'cf:reason',
|
|
3029
|
+
'cf:check',
|
|
3030
|
+
'cf:task_state',
|
|
3031
|
+
'cf:auto',
|
|
3032
|
+
'cf:reserved',
|
|
3033
|
+
'cf:attempts',
|
|
3034
|
+
'cf:steps',
|
|
3035
|
+
'cf:operations',
|
|
3036
|
+
'cf:approvals',
|
|
3037
|
+
'cf:calls',
|
|
3038
|
+
'cf:children',
|
|
3039
|
+
] as const;
|
|
3040
|
+
private checkFailedMiss: (typeof AgentPodWork.CHECK_FAILED_MISSES)[number] | null = null;
|
|
3041
|
+
private checkFailedStopFailure(
|
|
3042
|
+
session: string,
|
|
3043
|
+
task: WorkTask,
|
|
3044
|
+
runtime: Record<string, any> | undefined,
|
|
3045
|
+
attempts: Record<string, any>[],
|
|
3046
|
+
) {
|
|
3047
|
+
const miss = (code: (typeof AgentPodWork.CHECK_FAILED_MISSES)[number]) => {
|
|
3048
|
+
this.checkFailedMiss = code;
|
|
3049
|
+
return null;
|
|
3050
|
+
};
|
|
3051
|
+
this.checkFailedMiss = null;
|
|
3052
|
+
const prefix = 'Verification failed or became stale: ';
|
|
3053
|
+
const reason = String(runtime?.auto_error ?? '');
|
|
3054
|
+
if (!runtime || runtime.blocked_reason || !reason.startsWith(prefix)) return miss('cf:reason');
|
|
3055
|
+
const db = this.store.db;
|
|
3056
|
+
const check = db
|
|
3057
|
+
.prepare(
|
|
3058
|
+
'SELECT id,state FROM task_checks WHERE task_id=? AND check_key=? AND plan_revision=? ORDER BY rowid DESC LIMIT 1',
|
|
3059
|
+
)
|
|
3060
|
+
.get(task.id, reason.slice(prefix.length), task.plan_revision) as
|
|
3061
|
+
{ id: string; state: string } | undefined;
|
|
3062
|
+
if (check?.state !== 'failed') return miss('cf:check');
|
|
3063
|
+
if (!['blocked', 'partially_completed'].includes(task.state)) return miss('cf:task_state');
|
|
3064
|
+
if (runtime.auto_enabled) return miss('cf:auto');
|
|
3065
|
+
if (runtime.reserved_requests || runtime.reserved_tokens) return miss('cf:reserved');
|
|
3066
|
+
if (
|
|
3067
|
+
!attempts.length ||
|
|
3068
|
+
attempts.some((a) => !['completed', 'failed'].includes(String(a.state)))
|
|
3069
|
+
)
|
|
3070
|
+
return miss('cf:attempts');
|
|
3071
|
+
if (task.steps.some((s) => !['completed', 'failed'].includes(s.state))) return miss('cf:steps');
|
|
3072
|
+
const operations = db
|
|
3073
|
+
.prepare('SELECT id,state FROM operations WHERE session_id=? ORDER BY rowid')
|
|
3074
|
+
.all(session);
|
|
3075
|
+
const calls = db
|
|
3076
|
+
.prepare(
|
|
3077
|
+
'SELECT id,kind,state,helper_state FROM station_calls WHERE session_id=? ORDER BY rowid',
|
|
3078
|
+
)
|
|
3079
|
+
.all(session);
|
|
3080
|
+
if (operations.some((o) => !['succeeded', 'failed_definite'].includes(String(o.state))))
|
|
3081
|
+
return miss('cf:operations');
|
|
3082
|
+
if (db.prepare("SELECT 1 FROM approvals WHERE session_id=? AND state='pending'").get(session))
|
|
3083
|
+
return miss('cf:approvals');
|
|
3084
|
+
if (
|
|
3085
|
+
calls.some(
|
|
3086
|
+
(c) =>
|
|
3087
|
+
['prepared', 'dispatching', 'outcome_unknown'].includes(String(c.state)) ||
|
|
3088
|
+
!['none', 'stopped'].includes(String(c.helper_state)),
|
|
3089
|
+
)
|
|
3090
|
+
)
|
|
3091
|
+
return miss('cf:calls');
|
|
3092
|
+
if (db.prepare('SELECT 1 FROM child_tasks WHERE parent_session=?').get(session))
|
|
3093
|
+
return miss('cf:children');
|
|
3094
|
+
return {
|
|
3095
|
+
reason,
|
|
3096
|
+
check: check.id,
|
|
3097
|
+
steps: task.steps.map((s) => ({ id: s.id, state: s.state })),
|
|
3098
|
+
attempts: attempts.map((a) => ({ id: a.id, state: a.state })),
|
|
3099
|
+
operations,
|
|
3100
|
+
calls,
|
|
3101
|
+
};
|
|
3102
|
+
}
|
|
3103
|
+
private failureEvidence(
|
|
3104
|
+
root: { session_id: string; binding: import('./agentpod-protocol.js').AgentPodBinding },
|
|
3105
|
+
task: WorkTask,
|
|
3106
|
+
): string {
|
|
3107
|
+
const runtime = this.store.db
|
|
3108
|
+
.prepare('SELECT * FROM task_runtime WHERE task_id=?')
|
|
3109
|
+
.get(task.id);
|
|
3110
|
+
const attempts = this.store.db
|
|
3111
|
+
.prepare('SELECT * FROM attempts WHERE session_id=?')
|
|
3112
|
+
.all(root.session_id);
|
|
3113
|
+
const ops = this.store.db
|
|
3114
|
+
.prepare('SELECT * FROM operations WHERE session_id=?')
|
|
3115
|
+
.all(root.session_id);
|
|
3116
|
+
const calls = this.store.db
|
|
3117
|
+
.prepare("SELECT * FROM station_calls WHERE session_id=? AND kind!='read'")
|
|
3118
|
+
.all(root.session_id);
|
|
3119
|
+
const op = ops[0],
|
|
3120
|
+
attempt = attempts[0],
|
|
3121
|
+
call = calls[0];
|
|
3122
|
+
const approval = op
|
|
3123
|
+
? this.store.db.prepare('SELECT * FROM approvals WHERE operation_id=?').get(String(op.id))
|
|
3124
|
+
: undefined;
|
|
3125
|
+
const effect = op?.receipt ? JSON.parse(String(op.receipt)) : null;
|
|
3126
|
+
const bound = 'Provider payload exceeds declared per-request token bounds';
|
|
3127
|
+
const originalBlock = this.store.db
|
|
3128
|
+
.prepare(
|
|
3129
|
+
"SELECT seq,payload FROM events WHERE session_id=? AND kind='work_task.budget_blocked' AND json_extract(payload,'$.task_id')=? AND json_extract(payload,'$.reason')=? ORDER BY seq LIMIT 1",
|
|
3130
|
+
)
|
|
3131
|
+
.get(root.session_id, task.id, bound);
|
|
3132
|
+
const runtimeExpired =
|
|
3133
|
+
runtime?.blocked_reason === 'Task runtime limit reached' &&
|
|
3134
|
+
runtime.started_at !== null &&
|
|
3135
|
+
Number.isFinite(Number(runtime.started_at)) &&
|
|
3136
|
+
Date.now() - Number(runtime.started_at) >= task.limits.runtime_ms;
|
|
3137
|
+
|
|
3138
|
+
const savedInvalidatedOffer =
|
|
3139
|
+
approval?.state === 'invalidated'
|
|
3140
|
+
? this.store.db
|
|
3141
|
+
.prepare(
|
|
3142
|
+
"SELECT response FROM requests WHERE actor='agentpod-work-decision-offer' AND id=?",
|
|
3143
|
+
)
|
|
3144
|
+
.get(String(approval.id))
|
|
3145
|
+
: undefined;
|
|
3146
|
+
const knownReadOnly =
|
|
3147
|
+
this.historicalKnownReadOnlyFailure(root.binding, task, runtime) ??
|
|
3148
|
+
(ops.length === 1 && op && approval && savedInvalidatedOffer
|
|
3149
|
+
? this.historicalKnownReadOnlyFailure(root.binding, task, runtime, {
|
|
3150
|
+
operation: op,
|
|
3151
|
+
approval,
|
|
3152
|
+
offered: JSON.parse(String(savedInvalidatedOffer.response)),
|
|
3153
|
+
invalidated: true,
|
|
3154
|
+
})
|
|
3155
|
+
: null) ??
|
|
3156
|
+
(ops.length === 1 && op && approval
|
|
3157
|
+
? this.historicalKnownReadOnlyFailure(root.binding, task, runtime, undefined, {
|
|
3158
|
+
operation: op,
|
|
3159
|
+
approval,
|
|
3160
|
+
})
|
|
3161
|
+
: null);
|
|
3162
|
+
const zeroEffect = this.zeroEffectConcurrencyFailure(root.session_id, task, runtime, attempts);
|
|
3163
|
+
const revokeCancelled = zeroEffect
|
|
3164
|
+
? null
|
|
3165
|
+
: this.revokeCancelledFailure(root.session_id, task, runtime, attempts);
|
|
3166
|
+
const budgetStop =
|
|
3167
|
+
zeroEffect || revokeCancelled
|
|
3168
|
+
? null
|
|
3169
|
+
: this.requestBudgetStopFailure(root.session_id, task, runtime, attempts);
|
|
3170
|
+
const refusedWrites =
|
|
3171
|
+
zeroEffect || revokeCancelled || budgetStop
|
|
3172
|
+
? null
|
|
3173
|
+
: this.runtimeStopRefusedWritesFailure(root.session_id, task, runtime, attempts);
|
|
3174
|
+
const budgetKnown =
|
|
3175
|
+
zeroEffect || revokeCancelled || budgetStop || refusedWrites
|
|
3176
|
+
? null
|
|
3177
|
+
: this.requestBudgetStopKnownEffectsFailure(
|
|
3178
|
+
root.session_id,
|
|
3179
|
+
root.binding,
|
|
3180
|
+
task,
|
|
3181
|
+
runtime,
|
|
3182
|
+
attempts,
|
|
3183
|
+
);
|
|
3184
|
+
const rateLimit =
|
|
3185
|
+
zeroEffect || revokeCancelled || budgetStop || refusedWrites || budgetKnown
|
|
3186
|
+
? null
|
|
3187
|
+
: this.rateLimitStopFailure(root.session_id, task, runtime, attempts);
|
|
3188
|
+
const verificationLimit =
|
|
3189
|
+
zeroEffect || revokeCancelled || budgetStop || refusedWrites || budgetKnown || rateLimit
|
|
3190
|
+
? null
|
|
3191
|
+
: this.verificationLimitStopFailure(root.session_id, task, runtime, attempts);
|
|
3192
|
+
const checkFailed =
|
|
3193
|
+
zeroEffect ||
|
|
3194
|
+
revokeCancelled ||
|
|
3195
|
+
budgetStop ||
|
|
3196
|
+
refusedWrites ||
|
|
3197
|
+
budgetKnown ||
|
|
3198
|
+
rateLimit ||
|
|
3199
|
+
verificationLimit
|
|
3200
|
+
? null
|
|
3201
|
+
: this.checkFailedStopFailure(root.session_id, task, runtime, attempts);
|
|
3202
|
+
if (
|
|
3203
|
+
!zeroEffect &&
|
|
3204
|
+
!revokeCancelled &&
|
|
3205
|
+
!budgetStop &&
|
|
3206
|
+
!refusedWrites &&
|
|
3207
|
+
!rateLimit &&
|
|
3208
|
+
!verificationLimit &&
|
|
3209
|
+
!checkFailed &&
|
|
3210
|
+
!budgetKnown &&
|
|
3211
|
+
!knownReadOnly &&
|
|
3212
|
+
(task.state !== 'blocked' ||
|
|
3213
|
+
!runtime ||
|
|
3214
|
+
!originalBlock ||
|
|
3215
|
+
(runtime.blocked_reason !== bound && !runtimeExpired) ||
|
|
3216
|
+
attempt?.error !== 'Required control failed: ' + bound ||
|
|
3217
|
+
runtime.auto_enabled ||
|
|
3218
|
+
runtime.reserved_requests ||
|
|
3219
|
+
runtime.reserved_tokens ||
|
|
3220
|
+
runtime.verification_runs ||
|
|
3221
|
+
attempts.length !== 1 ||
|
|
3222
|
+
attempt?.state !== 'interrupted' ||
|
|
3223
|
+
attempt.resolved_by !== null ||
|
|
3224
|
+
ops.length !== 1 ||
|
|
3225
|
+
op?.kind !== 'workspace' ||
|
|
3226
|
+
op.state !== 'succeeded' ||
|
|
3227
|
+
op.attempt_id !== attempt.id ||
|
|
3228
|
+
op.resolution !== 'workspace-content-receipt-v1' ||
|
|
3229
|
+
calls.length !== 1 ||
|
|
3230
|
+
call?.kind !== 'patch' ||
|
|
3231
|
+
call.state !== 'returned' ||
|
|
3232
|
+
call.helper_state !== 'stopped' ||
|
|
3233
|
+
call.operation_id !== op.id ||
|
|
3234
|
+
call.attempt_id !== attempt.id ||
|
|
3235
|
+
!call.result_json ||
|
|
3236
|
+
!approval ||
|
|
3237
|
+
approval.state !== 'decided' ||
|
|
3238
|
+
approval.decision !== 'approve' ||
|
|
3239
|
+
approval.operation_digest !== op.operation_digest ||
|
|
3240
|
+
!effect ||
|
|
3241
|
+
effect.version !== 1 ||
|
|
3242
|
+
effect.scope !== 'station-workspace' ||
|
|
3243
|
+
effect.call_id !== call.id ||
|
|
3244
|
+
effect.root_digest !== digest(root.binding) ||
|
|
3245
|
+
effect.root_digest !== call.root_digest ||
|
|
3246
|
+
effect.configuration_digest !== call.configuration_digest ||
|
|
3247
|
+
effect.input_digest !== call.input_digest ||
|
|
3248
|
+
canonical(effect.result) !== canonical(JSON.parse(String(call.result_json))) ||
|
|
3249
|
+
effect.result?.ok !== true ||
|
|
3250
|
+
task.steps.filter((s) => s.attempt_id === attempt.id && s.state === 'interrupted')
|
|
3251
|
+
.length !== 1 ||
|
|
3252
|
+
task.steps.some((s) => !['pending', 'interrupted'].includes(s.state)) ||
|
|
3253
|
+
this.store.db.prepare('SELECT 1 FROM task_checks WHERE task_id=?').get(task.id) ||
|
|
3254
|
+
this.store.db
|
|
3255
|
+
.prepare('SELECT 1 FROM child_tasks WHERE parent_session=?')
|
|
3256
|
+
.get(root.session_id))
|
|
3257
|
+
)
|
|
3258
|
+
throw new ServiceError(
|
|
3259
|
+
'work_settlement_unsafe',
|
|
3260
|
+
this.verificationLimitMiss && this.verificationLimitMiss !== 'vl:reason'
|
|
3261
|
+
? 'Verification-budget stop not provable: ' + this.verificationLimitMiss
|
|
3262
|
+
: this.checkFailedMiss && this.checkFailedMiss !== 'cf:reason'
|
|
3263
|
+
? 'Check-failed stop not provable: ' + this.checkFailedMiss
|
|
3264
|
+
: 'Only exact known approved-patch, zero-effect child denial, revoke-cancelled operator MCP reads, request budget stopped operator MCP reads, runtime-stopped refused workspace writes, request budget stopped known effects, a closed rate-limit, verification-budget or check-failed stop, or closed historical read-only evidence can retire',
|
|
3265
|
+
);
|
|
3266
|
+
const evidence_digest = digest({
|
|
3267
|
+
binding: root.binding,
|
|
3268
|
+
task_id: task.id,
|
|
3269
|
+
plan_revision: task.plan_revision,
|
|
3270
|
+
blocked_reason: runtime!.blocked_reason,
|
|
3271
|
+
attempt,
|
|
3272
|
+
operation: op ?? null,
|
|
3273
|
+
approval: approval ?? null,
|
|
3274
|
+
call: call ?? null,
|
|
3275
|
+
...(zeroEffect ? { zero_effect_concurrency: zeroEffect } : {}),
|
|
3276
|
+
...(revokeCancelled ? { revoke_cancelled: revokeCancelled } : {}),
|
|
3277
|
+
...(budgetStop ? { request_budget_stop: budgetStop } : {}),
|
|
3278
|
+
...(refusedWrites ? { runtime_stop_refused_writes: refusedWrites } : {}),
|
|
3279
|
+
...(budgetKnown ? { request_budget_stop_known_effects: budgetKnown } : {}),
|
|
3280
|
+
...(rateLimit ? { rate_limit_stop: rateLimit } : {}),
|
|
3281
|
+
...(verificationLimit ? { verification_limit_stop: verificationLimit } : {}),
|
|
3282
|
+
...(checkFailed ? { check_failed_stop: checkFailed } : {}),
|
|
3283
|
+
...(knownReadOnly ? { historical_known_readonly: knownReadOnly } : {}),
|
|
3284
|
+
});
|
|
3285
|
+
return evidence_digest;
|
|
3286
|
+
}
|
|
3287
|
+
/** Status's read-only verdict: "known_effects_failure" when fail() would retire this run now. */
|
|
3288
|
+
private settlementReady(
|
|
3289
|
+
binding: unknown,
|
|
3290
|
+
root: { session_id: string; binding: import('./agentpod-protocol.js').AgentPodBinding },
|
|
3291
|
+
row: Record<string, unknown>,
|
|
3292
|
+
task: WorkTask,
|
|
3293
|
+
): 'known_effects_failure' | { refused: string } {
|
|
3294
|
+
try {
|
|
3295
|
+
this.assertKnownEffectsStop(binding);
|
|
3296
|
+
this.assertPlan(task, JSON.parse(String(row.admission_json)));
|
|
3297
|
+
this.failureEvidence(root, task);
|
|
3298
|
+
return 'known_effects_failure';
|
|
3299
|
+
} catch (error) {
|
|
3300
|
+
return {
|
|
3301
|
+
refused:
|
|
3302
|
+
error instanceof ServiceError ? error.code + ': ' + error.message : 'settlement_unproven',
|
|
3303
|
+
};
|
|
3304
|
+
}
|
|
3305
|
+
}
|
|
2182
3306
|
abandon(binding: unknown, request: string, workRequest: string, terminalDigest: string) {
|
|
2183
3307
|
const { root, row, task } = this.owned(binding);
|
|
2184
3308
|
if (workRequest !== row.request_id || !/^[a-f0-9]{64}$/.test(terminalDigest))
|
|
@@ -2351,7 +3475,7 @@ export class AgentPodWork {
|
|
|
2351
3475
|
});
|
|
2352
3476
|
if (
|
|
2353
3477
|
evidence.checks.some((id: unknown) => !checks.some((c) => c.id === id)) ||
|
|
2354
|
-
task.
|
|
3478
|
+
requiredChecks(task).some(
|
|
2355
3479
|
(a) => !checks.some((c) => c.step_id === null && c.check_key === a.key),
|
|
2356
3480
|
) ||
|
|
2357
3481
|
calls.some((c) => c.kind === 'check' && !checkEvidence.some((e) => e.call_id === c.id))
|
|
@@ -2386,6 +3510,90 @@ export class AgentPodWork {
|
|
|
2386
3510
|
},
|
|
2387
3511
|
);
|
|
2388
3512
|
}
|
|
3513
|
+
/** Item 11b: restore a checkpoint's files, operator-approved, only for a run that has ended (its
|
|
3514
|
+
* execution revoked and stopped with nothing unresolved), never while another run is active on
|
|
3515
|
+
* the station, and only exactly as reviewed: the restore digest is recomputed and must match. */
|
|
3516
|
+
async restore(
|
|
3517
|
+
binding: unknown,
|
|
3518
|
+
request: string,
|
|
3519
|
+
checkpoint: string,
|
|
3520
|
+
restoreDigest: string,
|
|
3521
|
+
approvedBy: string,
|
|
3522
|
+
) {
|
|
3523
|
+
const { root, task } = this.owned(binding);
|
|
3524
|
+
const current = this.authority.inspect(binding);
|
|
3525
|
+
if (
|
|
3526
|
+
!current.revoked ||
|
|
3527
|
+
current.authorized ||
|
|
3528
|
+
!current.owned_execution_stopped ||
|
|
3529
|
+
current.unresolved_operations.length ||
|
|
3530
|
+
current.active_attempts.length
|
|
3531
|
+
)
|
|
3532
|
+
throw new ServiceError(
|
|
3533
|
+
'restore_unsafe',
|
|
3534
|
+
'Restore is only for a run that has ended and stopped',
|
|
3535
|
+
);
|
|
3536
|
+
if (
|
|
3537
|
+
this.store.db
|
|
3538
|
+
.prepare("SELECT 1 FROM agentpod_admissions WHERE state!='revoked' AND session_id<>?")
|
|
3539
|
+
.get(root.session_id)
|
|
3540
|
+
)
|
|
3541
|
+
throw new ServiceError('restore_busy', 'Another run is active on this station');
|
|
3542
|
+
const station = this.stationBoundary();
|
|
3543
|
+
if (!station)
|
|
3544
|
+
throw new ServiceError('restore_unavailable', 'Restore needs the station boundary');
|
|
3545
|
+
// A replay of a restore already journaled returns its receipt: the live tree it moved is no
|
|
3546
|
+
// reason to refuse the very approval that moved it.
|
|
3547
|
+
const replay = station.replayed?.(root.session_id, request, restoreDigest) ?? null;
|
|
3548
|
+
if (replay !== null) return replay;
|
|
3549
|
+
const plan = await new StationCheckpoints(this.store).restorePlan(
|
|
3550
|
+
task,
|
|
3551
|
+
checkpoint,
|
|
3552
|
+
await station.live(),
|
|
3553
|
+
);
|
|
3554
|
+
if (plan.restore_digest !== restoreDigest)
|
|
3555
|
+
throw new ServiceError('restore_changed', 'This restore no longer matches what was reviewed');
|
|
3556
|
+
return station.restore(root.session_id, digest(root.binding), plan, request, approvedBy);
|
|
3557
|
+
}
|
|
3558
|
+
/** Item 11: the run's checkpoints, from its own recorded station writes. Read-only. */
|
|
3559
|
+
checkpoints(binding: unknown) {
|
|
3560
|
+
const { task } = this.owned(binding);
|
|
3561
|
+
this.authority.inspect(binding);
|
|
3562
|
+
return {
|
|
3563
|
+
version: 1 as const,
|
|
3564
|
+
task_id: task.id,
|
|
3565
|
+
checkpoints: new StationCheckpoints(this.store).list(task),
|
|
3566
|
+
};
|
|
3567
|
+
}
|
|
3568
|
+
/** Item 11: what restoring a checkpoint would do to each file written after it. Read-only. */
|
|
3569
|
+
async checkpointDiff(binding: unknown, checkpoint: string) {
|
|
3570
|
+
const { task } = this.owned(binding);
|
|
3571
|
+
this.authority.inspect(binding);
|
|
3572
|
+
const checkpoints = new StationCheckpoints(this.store);
|
|
3573
|
+
checkpoints.get(task, checkpoint);
|
|
3574
|
+
const station = this.stationBoundary();
|
|
3575
|
+
if (!station)
|
|
3576
|
+
throw new ServiceError('restore_unavailable', 'A checkpoint diff needs the station boundary');
|
|
3577
|
+
return {
|
|
3578
|
+
version: 1 as const,
|
|
3579
|
+
task_id: task.id,
|
|
3580
|
+
...(await checkpoints.diff(task, checkpoint, await station.live())),
|
|
3581
|
+
};
|
|
3582
|
+
}
|
|
3583
|
+
/** The station boundary, which reads the live workspace and writes restores. */
|
|
3584
|
+
private stationBoundary() {
|
|
3585
|
+
return (
|
|
3586
|
+
this.verification.station as unknown as
|
|
3587
|
+
| {
|
|
3588
|
+
station?: {
|
|
3589
|
+
restore: StationWork['restore'];
|
|
3590
|
+
live: StationWork['live'];
|
|
3591
|
+
replayed?: StationWork['replayed'];
|
|
3592
|
+
};
|
|
3593
|
+
}
|
|
3594
|
+
| undefined
|
|
3595
|
+
)?.station;
|
|
3596
|
+
}
|
|
2389
3597
|
async status(binding: unknown, driver: TaskDriver) {
|
|
2390
3598
|
const { root, row, task } = this.owned(binding);
|
|
2391
3599
|
const view = await driver.viewAsync(task.id);
|
|
@@ -2398,8 +3606,97 @@ export class AgentPodWork {
|
|
|
2398
3606
|
plan_revision: this.tasks.get(task.id).plan_revision,
|
|
2399
3607
|
binding_digest: digest(root.binding),
|
|
2400
3608
|
auto_enabled: !!view.runtime.auto_enabled,
|
|
3609
|
+
activity: this.activity(root.session_id),
|
|
3610
|
+
plan: planUpdate(view),
|
|
3611
|
+
// Board reads the agent is waiting on; the owner serves them (operator ruling 2026-10-28).
|
|
3612
|
+
...(() => {
|
|
3613
|
+
const reads = new BoardReads(this.store).pending(root.session_id);
|
|
3614
|
+
return reads.length ? { board_reads: reads } : {};
|
|
3615
|
+
})(),
|
|
3616
|
+
// The owner may settle a stopped run automatically only on this verdict.
|
|
3617
|
+
...(() => {
|
|
3618
|
+
const ready = this.settlementReady(binding, root, row, task);
|
|
3619
|
+
if (ready === 'known_effects_failure') return { settlement_ready: ready };
|
|
3620
|
+
// Which guard refused, for a stopped run only: the owner logs it, so an unsettled run
|
|
3621
|
+
// says why instead of "cannot prove". Codes and fixed messages only; no data.
|
|
3622
|
+
return ready && this.authority.inspect(binding).revoked
|
|
3623
|
+
? { settlement_refused: ready.refused }
|
|
3624
|
+
: {};
|
|
3625
|
+
})(),
|
|
2401
3626
|
};
|
|
2402
3627
|
}
|
|
3628
|
+
/** The owner's answer to one of this run's board reads. */
|
|
3629
|
+
answerBoardRead(binding: unknown, answer: unknown) {
|
|
3630
|
+
const { root } = this.owned(binding);
|
|
3631
|
+
return new BoardReads(this.store).answer(root.session_id, answer);
|
|
3632
|
+
}
|
|
3633
|
+
/** The latest tool steps, oldest first, for the owner to show as card activity while the work
|
|
3634
|
+
* runs. Observation only: it carries no authority and changes nothing. Input is clipped. */
|
|
3635
|
+
activity(session: string) {
|
|
3636
|
+
const startedOf = this.store.db.prepare(
|
|
3637
|
+
"SELECT json_extract(payload,'$.input') AS input FROM events WHERE session_id=? AND kind='tool.started' AND json_extract(payload,'$.tool_call_id')=? ORDER BY seq DESC LIMIT 1",
|
|
3638
|
+
);
|
|
3639
|
+
const startedInput = (id: unknown) =>
|
|
3640
|
+
typeof id === 'string'
|
|
3641
|
+
? (startedOf.get(session, id) as { input?: unknown } | undefined)?.input
|
|
3642
|
+
: undefined;
|
|
3643
|
+
return this.store.db
|
|
3644
|
+
.prepare(
|
|
3645
|
+
"SELECT seq,kind,payload FROM events WHERE session_id=? AND kind IN ('tool.started','tool.finished','permission.auto-approved','fetch.host_paused') ORDER BY seq DESC LIMIT 64",
|
|
3646
|
+
)
|
|
3647
|
+
.all(session)
|
|
3648
|
+
.reverse()
|
|
3649
|
+
.map((row) => {
|
|
3650
|
+
const p = JSON.parse(String(row.payload)) as Record<string, unknown>;
|
|
3651
|
+
// Approval scopes: a write the run grant answered says so on the card, with the grant.
|
|
3652
|
+
// A fetch host paused by the breaker: one line, and the run carries on.
|
|
3653
|
+
if (row.kind === 'fetch.host_paused')
|
|
3654
|
+
return {
|
|
3655
|
+
seq: Number(row.seq),
|
|
3656
|
+
phase: 'finished' as const,
|
|
3657
|
+
tool_call_id: 'fetch-pause:' + String(row.seq),
|
|
3658
|
+
title: `Paused requests to ${String(p.host)} for ${Math.round(Number(p.pause_ms) / 1000)}s after ${Number(p.failures)} failures`,
|
|
3659
|
+
kind: null,
|
|
3660
|
+
input: null,
|
|
3661
|
+
failed: false,
|
|
3662
|
+
};
|
|
3663
|
+
if (row.kind === 'permission.auto-approved')
|
|
3664
|
+
return {
|
|
3665
|
+
seq: Number(row.seq),
|
|
3666
|
+
phase: 'finished' as const,
|
|
3667
|
+
tool_call_id: 'approval:' + String(p.operation_id ?? ''),
|
|
3668
|
+
title:
|
|
3669
|
+
p.rule === 'run-grant'
|
|
3670
|
+
? 'Approved by the run grant' +
|
|
3671
|
+
(typeof p.granted_by === 'string' ? ` (granted by ${p.granted_by})` : '')
|
|
3672
|
+
: p.rule === 'agent-rule'
|
|
3673
|
+
? "Allowed by the agent's standing rule" +
|
|
3674
|
+
(typeof p.set_by === 'string' ? ` (set by ${p.set_by})` : '')
|
|
3675
|
+
: 'Approved by a standing policy',
|
|
3676
|
+
kind: null,
|
|
3677
|
+
input: null,
|
|
3678
|
+
failed: false,
|
|
3679
|
+
};
|
|
3680
|
+
const started = row.kind === 'tool.started',
|
|
3681
|
+
tool = typeof p.tool === 'string' ? p.tool : '';
|
|
3682
|
+
const input = typeof p.input === 'string' ? p.input.slice(0, 2048) : null;
|
|
3683
|
+
// A finished step carries no input, so it is titled from the call it finishes: the card
|
|
3684
|
+
// read a bare "salaam_station" under every station step (staging card_3a97983751784bf9).
|
|
3685
|
+
const callInput = started ? p.input : startedInput(p.tool_call_id);
|
|
3686
|
+
return {
|
|
3687
|
+
seq: Number(row.seq),
|
|
3688
|
+
phase: started ? ('started' as const) : ('finished' as const),
|
|
3689
|
+
tool_call_id: typeof p.tool_call_id === 'string' ? p.tool_call_id : '',
|
|
3690
|
+
title: tool ? toolTitle(tool, callInput) : 'Tool',
|
|
3691
|
+
kind: started ? String(p.kind ?? toolMetadata(tool, p.input).kind) : null,
|
|
3692
|
+
input: started ? input : null,
|
|
3693
|
+
failed: !started && p.failed === true,
|
|
3694
|
+
// The tool's text output, already redacted and clipped by the worker, so the card shows
|
|
3695
|
+
// what the step returned as Hermes does.
|
|
3696
|
+
...(!started && typeof p.output === 'string' ? { output: p.output.slice(0, 2048) } : {}),
|
|
3697
|
+
};
|
|
3698
|
+
});
|
|
3699
|
+
}
|
|
2403
3700
|
progress(
|
|
2404
3701
|
binding: unknown,
|
|
2405
3702
|
request: string,
|
|
@@ -2423,7 +3720,13 @@ export class AgentPodWork {
|
|
|
2423
3720
|
() => {
|
|
2424
3721
|
this.store.execution.assert(root.session_id);
|
|
2425
3722
|
this.assertPlan(task, JSON.parse(String(row.admission_json)));
|
|
2426
|
-
|
|
3723
|
+
// Exactly the plan's steps, or those plus the capped re-run allowance that funds a
|
|
3724
|
+
// rate-limited retry or a reopened step; steps complete once, so it never adds work.
|
|
3725
|
+
if (
|
|
3726
|
+
(steps !== task.steps.length && steps !== task.steps.length + RERUN_ALLOWANCE) ||
|
|
3727
|
+
steps < 1 ||
|
|
3728
|
+
steps > 32 + RERUN_ALLOWANCE
|
|
3729
|
+
)
|
|
2427
3730
|
throw new ServiceError(
|
|
2428
3731
|
'invalid_limit',
|
|
2429
3732
|
'Progression must bind every admitted step within32',
|