@farmslot/agent-runtime 0.48.0 → 0.48.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/native/claude.d.ts.map +1 -1
- package/dist/native/claude.js +1 -1
- package/dist/native/claude.js.map +1 -1
- package/dist/native/client.d.ts.map +1 -1
- package/dist/native/client.js +12 -1
- package/dist/native/client.js.map +1 -1
- package/dist/native/codex.d.ts.map +1 -1
- package/dist/native/codex.js +2 -1
- package/dist/native/codex.js.map +1 -1
- package/dist/native/conversation.d.ts +1 -1
- package/dist/native/conversation.d.ts.map +1 -1
- package/dist/native/conversation.js +1 -1
- package/dist/native/conversation.js.map +1 -1
- package/dist/native/ipc.d.ts +1 -1
- package/dist/native/ipc.d.ts.map +1 -1
- package/dist/native/ipc.js +1 -1
- package/dist/native/ipc.js.map +1 -1
- package/dist/native/manager.d.ts +1 -1
- package/dist/native/manager.d.ts.map +1 -1
- package/dist/native/manager.js +1 -1
- package/dist/native/manager.js.map +1 -1
- package/dist/native/observation.d.ts +7 -1
- package/dist/native/observation.d.ts.map +1 -1
- package/dist/native/observation.js +24 -7
- package/dist/native/observation.js.map +1 -1
- package/dist/native/pi.d.ts.map +1 -1
- package/dist/native/pi.js +1 -1
- package/dist/native/pi.js.map +1 -1
- package/dist/native/profile-service.d.ts.map +1 -1
- package/dist/native/profile-service.js +1 -1
- package/dist/native/profile-service.js.map +1 -1
- package/dist/native/registry.js +1 -1
- package/dist/native/registry.js.map +1 -1
- package/dist/native/service.d.ts +1 -1
- package/dist/native/service.d.ts.map +1 -1
- package/dist/native/service.js +2 -1
- package/dist/native/service.js.map +1 -1
- package/dist/native/supervisor.js +6 -2
- package/dist/native/supervisor.js.map +1 -1
- package/dist/task-init/required-skills.js +1 -1
- package/dist/task-init/required-skills.js.map +1 -1
- package/dist/task-init/task-document.d.ts.map +1 -1
- package/dist/task-init/task-document.js +1 -1
- package/dist/task-init/task-document.js.map +1 -1
- package/dist/task-init/write.d.ts.map +1 -1
- package/dist/task-init/write.js +12 -2
- package/dist/task-init/write.js.map +1 -1
- package/dist/tool-usage/adapters/claude.d.ts +3 -0
- package/dist/tool-usage/adapters/claude.d.ts.map +1 -0
- package/dist/tool-usage/adapters/claude.js +56 -0
- package/dist/tool-usage/adapters/claude.js.map +1 -0
- package/dist/tool-usage/adapters/codex.d.ts +3 -0
- package/dist/tool-usage/adapters/codex.d.ts.map +1 -0
- package/dist/tool-usage/adapters/codex.js +81 -0
- package/dist/tool-usage/adapters/codex.js.map +1 -0
- package/dist/tool-usage/adapters/cursor.d.ts +3 -0
- package/dist/tool-usage/adapters/cursor.d.ts.map +1 -0
- package/dist/tool-usage/adapters/cursor.js +47 -0
- package/dist/tool-usage/adapters/cursor.js.map +1 -0
- package/dist/tool-usage/adapters/grok.d.ts +3 -0
- package/dist/tool-usage/adapters/grok.d.ts.map +1 -0
- package/dist/tool-usage/adapters/grok.js +55 -0
- package/dist/tool-usage/adapters/grok.js.map +1 -0
- package/dist/tool-usage/adapters/jsonl.d.ts +9 -0
- package/dist/tool-usage/adapters/jsonl.d.ts.map +1 -0
- package/dist/tool-usage/adapters/jsonl.js +40 -0
- package/dist/tool-usage/adapters/jsonl.js.map +1 -0
- package/dist/tool-usage/adapters/native.d.ts +3 -0
- package/dist/tool-usage/adapters/native.d.ts.map +1 -0
- package/dist/tool-usage/adapters/native.js +31 -0
- package/dist/tool-usage/adapters/native.js.map +1 -0
- package/dist/tool-usage/adapters/pi.d.ts +3 -0
- package/dist/tool-usage/adapters/pi.d.ts.map +1 -0
- package/dist/tool-usage/adapters/pi.js +46 -0
- package/dist/tool-usage/adapters/pi.js.map +1 -0
- package/dist/tool-usage/count.d.ts +13 -0
- package/dist/tool-usage/count.d.ts.map +1 -0
- package/dist/tool-usage/count.js +160 -0
- package/dist/tool-usage/count.js.map +1 -0
- package/dist/tool-usage/index.d.ts +19 -0
- package/dist/tool-usage/index.d.ts.map +1 -0
- package/dist/tool-usage/index.js +32 -0
- package/dist/tool-usage/index.js.map +1 -0
- package/dist/tool-usage/ledger.d.ts +11 -0
- package/dist/tool-usage/ledger.d.ts.map +1 -0
- package/dist/tool-usage/ledger.js +29 -0
- package/dist/tool-usage/ledger.js.map +1 -0
- package/dist/tool-usage/stream.d.ts +32 -0
- package/dist/tool-usage/stream.d.ts.map +1 -0
- package/dist/tool-usage/stream.js +17 -0
- package/dist/tool-usage/stream.js.map +1 -0
- package/package.json +10 -4
- package/scripts/check-task-artifact-contract.mjs +36 -0
- package/scripts/checklist-order.cjs +165 -0
- package/scripts/checklist-target.cjs +20 -0
- package/scripts/mark-checklist-step.cjs +285 -23
- package/scripts/proof-artifacts.cjs +258 -0
- package/scripts/subtask-unit.cjs +22 -1
- package/scripts/worker-terminal-contract.cjs +4 -6
|
@@ -17,6 +17,8 @@ const {
|
|
|
17
17
|
parseTaskDirMarkArgs,
|
|
18
18
|
terminalContractInputForChecklist,
|
|
19
19
|
} = require('./checklist-target.cjs');
|
|
20
|
+
const proofArtifacts = require('./proof-artifacts.cjs');
|
|
21
|
+
const checklistOrder = require('./checklist-order.cjs');
|
|
20
22
|
const {
|
|
21
23
|
atomicWrite,
|
|
22
24
|
markStepInLines,
|
|
@@ -76,6 +78,12 @@ function printHelp() {
|
|
|
76
78
|
'Override: farmslot-agent mark <task-dir> --checklist REVIEW.md <step> — optional --signal; signal defaults from checklist name.',
|
|
77
79
|
'Bootstrap: ./mark start — role-owned signal with status running (no checklist box).',
|
|
78
80
|
'Progress: ./mark 1, ./mark 2, ... — checks the box and appends checklistTiming.',
|
|
81
|
+
' Steps are claimed in order: ./mark N is refused while an earlier step is unchecked.',
|
|
82
|
+
' A step whose title carries [OPTIONAL: <when>] can be skipped with ./mark N --skip "<reason>";',
|
|
83
|
+
' the reason is recorded in the ledger. Any other step must be done.',
|
|
84
|
+
' ./mark N --reason "<what you found>" records a note on a step you did do — a judgement call',
|
|
85
|
+
' worth keeping, such as an exit-early step whose check found the bug still reproduces. The step',
|
|
86
|
+
' stays done; only --skip waives one, and the two cannot be given together.',
|
|
79
87
|
'Existing evidence manifests are validated before progress or success marks, with field errors on refusal.',
|
|
80
88
|
'Operator: ./mark wait --who "name" --reason "..."; ./mark resume clears the wait without marking a row.',
|
|
81
89
|
'Child units: ./mark sub start <id> --step N --from <path|inline:text>, then ./mark sub <id> <n> | complete [--report PATH] [--mark-last] | blocked --reason "..." | status.',
|
|
@@ -159,6 +167,16 @@ for (let i = 0; i < rest.length; i += 1) {
|
|
|
159
167
|
i += 1;
|
|
160
168
|
}
|
|
161
169
|
|
|
170
|
+
/**
|
|
171
|
+
* `--skip "<reason>"`: the worker's statement that a step the template declared
|
|
172
|
+
* optional does not apply to this run. Null when not skipping.
|
|
173
|
+
*/
|
|
174
|
+
const skipReason = typeof opts.skip === 'string' && opts.skip.trim() ? opts.skip.trim() : null;
|
|
175
|
+
if (opts.skip !== undefined && skipReason === null) {
|
|
176
|
+
console.error('--skip needs a reason: ./mark N --skip "<why this step does not apply>"');
|
|
177
|
+
process.exit(2);
|
|
178
|
+
}
|
|
179
|
+
|
|
162
180
|
const stepToken = String(stepRaw).toLowerCase();
|
|
163
181
|
const isStartCommand = START_COMMANDS.has(stepToken);
|
|
164
182
|
const isOperatorCommand = OPERATOR_COMMANDS.has(stepToken);
|
|
@@ -171,6 +189,33 @@ if (!isStartCommand && !isOperatorCommand && !terminalCommand) {
|
|
|
171
189
|
|
|
172
190
|
if (isStartCommand && Object.keys(opts).length > 0) usage();
|
|
173
191
|
if (stepToken === 'resume' && Object.keys(opts).length > 0) usage();
|
|
192
|
+
if (skipReason !== null && stepNumber === null) {
|
|
193
|
+
console.error('--skip applies to a numbered step: ./mark N --skip "<reason>"');
|
|
194
|
+
process.exit(2);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* `--reason "<text>"` on a numbered step: a note on the mark, not a skip. The
|
|
199
|
+
* step was done, and this records the judgement behind it — an exit-early step
|
|
200
|
+
* whose check found the bug still reproduces is done by recording that. The
|
|
201
|
+
* terminal and wait commands keep their own `--reason`, which is their
|
|
202
|
+
* disposition rather than a note on a row.
|
|
203
|
+
*/
|
|
204
|
+
let noteReason = null;
|
|
205
|
+
if (stepNumber !== null && opts.reason !== undefined) {
|
|
206
|
+
if (skipReason !== null) {
|
|
207
|
+
console.error(
|
|
208
|
+
'--reason and --skip cannot both be given: a skip already records its reason. ' +
|
|
209
|
+
'Use --skip "<why it does not apply>" to waive the step, or --reason "<what you found>" to note how you did it.',
|
|
210
|
+
);
|
|
211
|
+
process.exit(2);
|
|
212
|
+
}
|
|
213
|
+
noteReason = typeof opts.reason === 'string' ? opts.reason.trim() : '';
|
|
214
|
+
if (!noteReason || noteReason.length > 2048) {
|
|
215
|
+
console.error('--reason needs text (1-2048 characters): ./mark N --reason "<what you found>"');
|
|
216
|
+
process.exit(2);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
174
219
|
if (stepToken === 'wait') {
|
|
175
220
|
if (
|
|
176
221
|
Object.keys(opts).some((key) => !['who', 'reason'].includes(key)) ||
|
|
@@ -334,6 +379,15 @@ function assertWorkerReport(taskDir, taskPath) {
|
|
|
334
379
|
return null;
|
|
335
380
|
}
|
|
336
381
|
|
|
382
|
+
/** True when no step is left unchecked: the worker has run out of steps to do. */
|
|
383
|
+
function checklistFullyChecked(file) {
|
|
384
|
+
try {
|
|
385
|
+
return parseChecklist(fs.readFileSync(file, 'utf8')).items.every((entry) => entry.checked);
|
|
386
|
+
} catch {
|
|
387
|
+
return false; // Unreadable: treat it as work outstanding rather than end a run.
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
337
391
|
function assertChecklistComplete(taskPath, { allowOneUnchecked = false } = {}) {
|
|
338
392
|
const parsed = parseChecklist(fs.readFileSync(taskPath, 'utf8'));
|
|
339
393
|
const unchecked = parsed.items.filter((entry) => !entry.checked);
|
|
@@ -357,6 +411,14 @@ function assertArtifactContract(taskDir, taskPath, contract, terminalCommand) {
|
|
|
357
411
|
const contractPath = terminalContractPath(taskDir, taskPath);
|
|
358
412
|
const args = [ARTIFACT_CONTRACT_SCRIPT, taskDir];
|
|
359
413
|
if (terminalCommand) args.push('--terminal', terminalCommand);
|
|
414
|
+
// The checker must judge the checklist this mark is completing, and read the
|
|
415
|
+
// claims from the signal this mark writes, not from names it re-derives.
|
|
416
|
+
args.push(
|
|
417
|
+
'--proof-checklist',
|
|
418
|
+
path.basename(taskPath),
|
|
419
|
+
'--proof-signal',
|
|
420
|
+
path.basename(signalPath),
|
|
421
|
+
);
|
|
360
422
|
// This explicit flag keeps project opt-ins compatible with older checkers.
|
|
361
423
|
// Current checkers also enforce registered verdicts on terminal complete.
|
|
362
424
|
if (terminalCommand === 'complete' && contract?.acceptance?.require === true) {
|
|
@@ -423,7 +485,17 @@ function assertTerminalPackagedEvidence(taskPath, taskDir, terminalCommand) {
|
|
|
423
485
|
}
|
|
424
486
|
}
|
|
425
487
|
|
|
426
|
-
function buildSignalUpdate(
|
|
488
|
+
function buildSignalUpdate(
|
|
489
|
+
signal,
|
|
490
|
+
terminal,
|
|
491
|
+
target,
|
|
492
|
+
timing,
|
|
493
|
+
events,
|
|
494
|
+
now,
|
|
495
|
+
taskPath,
|
|
496
|
+
_taskDir,
|
|
497
|
+
outOfOrder = [],
|
|
498
|
+
) {
|
|
427
499
|
const sameAttempt =
|
|
428
500
|
!isStartCommand ||
|
|
429
501
|
(signal.waitingForOperator && !process.env.FARMSLOT_SIGNAL_ATTEMPT_ID) ||
|
|
@@ -439,6 +511,7 @@ function buildSignalUpdate(signal, terminal, target, timing, events, now, taskPa
|
|
|
439
511
|
schemaVersion: 1,
|
|
440
512
|
source: timing.source || path.basename(taskPath),
|
|
441
513
|
events,
|
|
514
|
+
...(outOfOrder.length ? { outOfOrder } : {}),
|
|
442
515
|
},
|
|
443
516
|
timestamp: now,
|
|
444
517
|
...operatorWaitHistory(signal, now, { reset: !sameAttempt, closeWait: stepToken !== 'wait' }),
|
|
@@ -500,8 +573,55 @@ function resolveTarget(taskPath, stepNumber, markLast) {
|
|
|
500
573
|
// Also covers `--mark-last`: the parent must not tick a box whose child is
|
|
501
574
|
// still open, whichever way the row was chosen.
|
|
502
575
|
assertStepNotOwnedBySubtask(item.stepNumber);
|
|
576
|
+
const dir = path.dirname(taskPath);
|
|
577
|
+
const checklist = path.basename(taskPath);
|
|
578
|
+
const contract = loadTerminalContract(dir, taskPath);
|
|
579
|
+
if (!item.checked) {
|
|
580
|
+
// Steps are claimed in order. An earlier step still open means this claim
|
|
581
|
+
// would stand on work that has not been done, which is how a trial ends up
|
|
582
|
+
// running changed-file CI before it has a baseline.
|
|
583
|
+
const earlier = checklistOrder.firstUnmarkedEarlierStep(parsed.items, item.stepNumber);
|
|
584
|
+
if (earlier) {
|
|
585
|
+
const optional = checklistOrder.declared(dir, checklist, contract);
|
|
586
|
+
const when = optional.get(earlier.stepNumber);
|
|
587
|
+
console.error(
|
|
588
|
+
`step ${item.stepNumber} cannot be marked while step ${earlier.stepNumber} ` +
|
|
589
|
+
`(${earlier.label}) is unchecked. ` +
|
|
590
|
+
(when
|
|
591
|
+
? `That step is optional (${when}); skip it with ` +
|
|
592
|
+
`${path.join(dir, 'mark')} ${earlier.stepNumber} --skip "<reason>"`
|
|
593
|
+
: 'Do that step first.'),
|
|
594
|
+
);
|
|
595
|
+
process.exit(1);
|
|
596
|
+
}
|
|
597
|
+
if (skipReason !== null) {
|
|
598
|
+
// Only the template grants a skip, and only with a reason on the record.
|
|
599
|
+
const when = checklistOrder.declared(dir, checklist, contract).get(item.stepNumber);
|
|
600
|
+
if (!when) {
|
|
601
|
+
console.error(
|
|
602
|
+
`step ${item.stepNumber} is not declared optional, so it cannot be skipped. ` +
|
|
603
|
+
'Do the step, or stop with ' +
|
|
604
|
+
`${path.join(dir, 'mark')} blocked --reason "<why>".`,
|
|
605
|
+
);
|
|
606
|
+
process.exit(1);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
// A tick is the claim, so a step that declares its proof cannot be checked
|
|
610
|
+
// before that evidence exists. A skipped step claims nothing, so it needs
|
|
611
|
+
// none. Re-marking an already-checked row stays idempotent: completion,
|
|
612
|
+
// not this path, judges evidence that went away.
|
|
613
|
+
if (skipReason === null) {
|
|
614
|
+
const pending = proofArtifacts.pending(dir, checklist, item.stepNumber, contract);
|
|
615
|
+
if (pending) {
|
|
616
|
+
console.error(pending);
|
|
617
|
+
process.exit(1);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
503
621
|
try {
|
|
504
|
-
|
|
622
|
+
// A deterministic step cannot be optional, so a skip never reaches a
|
|
623
|
+
// declared command: this runs for the step that declares one, or nothing.
|
|
624
|
+
deterministicChecks.execute(dir, checklist, item.stepNumber);
|
|
505
625
|
} catch (error) {
|
|
506
626
|
console.error(error.message);
|
|
507
627
|
process.exit(1);
|
|
@@ -524,17 +644,87 @@ function autonomousContract() {
|
|
|
524
644
|
return false; // The contract check below reports an unreadable contract.
|
|
525
645
|
}
|
|
526
646
|
}
|
|
647
|
+
function unattendedContract() {
|
|
648
|
+
try {
|
|
649
|
+
return loadTerminalContract(taskDir, taskPath)?.unattendedCompletion === true;
|
|
650
|
+
} catch {
|
|
651
|
+
return false; // The contract check below reports an unreadable contract.
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/** The refusal text, trimmed to what fits a signal reason. */
|
|
656
|
+
function refusalReason(captured) {
|
|
657
|
+
const text = captured.join('').trim();
|
|
658
|
+
return text.length > 1500 ? `${text.slice(0, 1500)}…` : text;
|
|
659
|
+
}
|
|
660
|
+
|
|
527
661
|
// No operator answers a refused completion in an autonomous run (every blind
|
|
528
662
|
// eval trial): the worker must end it, not wait for a review that never comes.
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
663
|
+
//
|
|
664
|
+
// The stamp is sufficient by itself. It is written only where the run declares
|
|
665
|
+
// `review: false`, which already says nobody attends, and a role checklist's
|
|
666
|
+
// contract carries no `mode` at all — the gateway syncs it without one — so
|
|
667
|
+
// requiring autonomous mode as well meant the stamp it inherits could never be
|
|
668
|
+
// consulted and the stall stayed exactly where it was.
|
|
669
|
+
if (terminalCommand === 'complete' && (autonomousContract() || unattendedContract())) {
|
|
670
|
+
// On an unattended run the gateway stamped `unattendedCompletion`: nobody will
|
|
671
|
+
// read the advice below, so the refusal itself ends the run. The trial is then
|
|
672
|
+
// scored as a refusal with its cause, instead of parking until a human aborts
|
|
673
|
+
// it hours later.
|
|
674
|
+
if (unattendedContract()) {
|
|
675
|
+
const captured = [];
|
|
676
|
+
const inherited = process.stderr.write.bind(process.stderr);
|
|
677
|
+
process.stderr.write = (chunk, ...rest) => {
|
|
678
|
+
captured.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'));
|
|
679
|
+
return inherited(chunk, ...rest);
|
|
680
|
+
};
|
|
681
|
+
process.on('exit', (code) => {
|
|
682
|
+
if (code === 0) return;
|
|
683
|
+
// Read the checklist now, not when this hook was installed: `complete
|
|
684
|
+
// --mark-last` ticks its final row after that point, and every standard
|
|
685
|
+
// template finishes that way. Sampling early left a fully marked run
|
|
686
|
+
// looking unfinished, so an acceptance refusal it could never clear would
|
|
687
|
+
// have parked it — the F6 stall, rebuilt.
|
|
688
|
+
//
|
|
689
|
+
// A refusal ends the run only when every step is marked: the worker has
|
|
690
|
+
// nothing left to try and the trial is scored as a refusal (F6, run
|
|
691
|
+
// 8cb5aea2). With steps outstanding it is the ordinary kind — a premature
|
|
692
|
+
// complete, a missing artifact it can still write — and ending the run
|
|
693
|
+
// there abandons a worker that was still working (GLM eval 4cba5c2f).
|
|
694
|
+
if (!checklistFullyChecked(taskPath)) return;
|
|
695
|
+
const reason = refusalReason(captured);
|
|
696
|
+
const ended = spawnSync(
|
|
697
|
+
process.execPath,
|
|
698
|
+
[
|
|
699
|
+
__filename,
|
|
700
|
+
taskDir,
|
|
701
|
+
'--checklist',
|
|
702
|
+
path.basename(taskPath),
|
|
703
|
+
'blocked',
|
|
704
|
+
'--reason',
|
|
705
|
+
reason || 'completion was refused and this run has no operator',
|
|
706
|
+
],
|
|
707
|
+
{ encoding: 'utf8' },
|
|
536
708
|
);
|
|
537
|
-
|
|
709
|
+
inherited(
|
|
710
|
+
ended.status === 0
|
|
711
|
+
? 'This run has no operator, so the refusal above ended it: the signal is blocked with ' +
|
|
712
|
+
'that reason.\n'
|
|
713
|
+
: `This run has no operator and ending it failed: ${
|
|
714
|
+
ended.stderr || ended.error?.message || `exit ${ended.status}`
|
|
715
|
+
}\n`,
|
|
716
|
+
);
|
|
717
|
+
});
|
|
718
|
+
} else {
|
|
719
|
+
process.on('exit', (code) => {
|
|
720
|
+
if (code !== 0)
|
|
721
|
+
console.error(
|
|
722
|
+
'This autonomous run has no operator: a refused completion reaches no gate or reviewer. ' +
|
|
723
|
+
'Fix the refusal above and mark complete again, or end the run with it: ' +
|
|
724
|
+
`${path.join(taskDir, 'mark')} blocked --reason "<the refusal>"`,
|
|
725
|
+
);
|
|
726
|
+
});
|
|
727
|
+
}
|
|
538
728
|
}
|
|
539
729
|
if (terminalCommand === 'complete') materializeReviewCoverage(taskDir);
|
|
540
730
|
|
|
@@ -635,13 +825,22 @@ if (
|
|
|
635
825
|
assertNoOpenSubtasks(terminalCommand);
|
|
636
826
|
}
|
|
637
827
|
if (
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
828
|
+
// `complete` only. `no-change` is a documented early exit: the fix-bug and
|
|
829
|
+
// review-fix templates tell a worker that finds the bug already fixed or not
|
|
830
|
+
// reproducible to write its reports and call plain `mark no-change`, leaving
|
|
831
|
+
// the implementation, commit and PR rows unchecked. Requiring a full
|
|
832
|
+
// checklist there would refuse the exit the template asks for.
|
|
833
|
+
terminalCommand === 'complete' &&
|
|
641
834
|
!proofMeasurement &&
|
|
642
835
|
!opts['skip-checklist']
|
|
643
836
|
) {
|
|
644
|
-
|
|
837
|
+
// Every success claims the checklist is done, so every success checks it.
|
|
838
|
+
// This was gated on --mark-last, which meant a plain `mark complete` never
|
|
839
|
+
// looked: GLM eval 4cba5c2f completed with nothing marked at all, and the
|
|
840
|
+
// order rule cannot catch that either, since it only reports steps left
|
|
841
|
+
// behind the last checked one and there was no checked one. `--mark-last`
|
|
842
|
+
// still tolerates the row it is about to tick itself.
|
|
843
|
+
assertChecklistComplete(taskPath, { allowOneUnchecked: opts['mark-last'] === true });
|
|
645
844
|
}
|
|
646
845
|
// A successful verdict (no-change writes `complete` too) stays put: `./mark N` on
|
|
647
846
|
// a declared deterministic step validates its receipt against the current tree,
|
|
@@ -659,6 +858,38 @@ if (stepNumber != null && readJson(signalPath).status === 'complete') {
|
|
|
659
858
|
process.exit(1);
|
|
660
859
|
}
|
|
661
860
|
}
|
|
861
|
+
// `start` must not re-open a run that already ended. GLM eval 4cba5c2f marked a
|
|
862
|
+
// terminal blocked signal and then called start, which reset it to running with
|
|
863
|
+
// a fresh attempt id while the gateway had already moved the run to review: the
|
|
864
|
+
// worker went on marking steps nobody was watching.
|
|
865
|
+
//
|
|
866
|
+
// A legitimate re-dispatch never meets this: the gateway removes SIGNAL.json and
|
|
867
|
+
// the role signals before replaying a step (methods/run/replay-step.ts, "cleared
|
|
868
|
+
// worker terminal signals"), and an initial dispatch has no signal yet. So a
|
|
869
|
+
// terminal signal still on disk means the worker is restarting itself inside a
|
|
870
|
+
// run that is over, which is the case to refuse. The guard deliberately does not
|
|
871
|
+
// consult FARMSLOT_SIGNAL_ATTEMPT_ID: only the review terminal sets it
|
|
872
|
+
// (agent-runtime/scripts/review-terminal.cjs), so for a worker run it would
|
|
873
|
+
// never be present and would decide nothing.
|
|
874
|
+
if (isStartCommand) {
|
|
875
|
+
const existing = readJson(signalPath);
|
|
876
|
+
// `blocked` stays resumable: a worker that blocked and is then told to carry
|
|
877
|
+
// on calls start and keeps its checked boxes, which is the contract the
|
|
878
|
+
// harness tests pin. The eval only reached that door because a premature
|
|
879
|
+
// complete had manufactured a blocked signal; with completion refusing an
|
|
880
|
+
// incomplete checklist and a refusal no longer ending the run, that signal
|
|
881
|
+
// never exists. A successful verdict is different: nothing resumes a run that
|
|
882
|
+
// finished, and before this a start on `complete` fell straight through and
|
|
883
|
+
// destroyed the record of a run parked at its human gate.
|
|
884
|
+
if (existing && ['complete', 'no-change'].includes(existing.status)) {
|
|
885
|
+
console.error(
|
|
886
|
+
`signal already terminal: status=${existing.status}; start does not re-open it. ` +
|
|
887
|
+
'This run finished; a re-dispatch clears the signal first. ' +
|
|
888
|
+
`Fix what the refusal reported and mark the run again, or stop with ${path.join(taskDir, 'mark')} blocked --reason "<why>".`,
|
|
889
|
+
);
|
|
890
|
+
process.exit(1);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
662
893
|
const target =
|
|
663
894
|
isStartCommand || isOperatorCommand
|
|
664
895
|
? { stepNumber: null, label: 'started' }
|
|
@@ -702,17 +933,47 @@ const timing =
|
|
|
702
933
|
: { schemaVersion: 1, source: path.basename(taskPath), events: [] };
|
|
703
934
|
const events = Array.isArray(timing.events) ? [...timing.events] : [];
|
|
704
935
|
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
936
|
+
const recordedEvent = events.findIndex(
|
|
937
|
+
(event) =>
|
|
938
|
+
event && (event.stepNumber === target.stepNumber || event.index === target.stepNumber - 1),
|
|
939
|
+
);
|
|
940
|
+
if (!isStartCommand && target.stepNumber != null && recordedEvent === -1) {
|
|
941
|
+
events.push({
|
|
942
|
+
stepNumber: target.stepNumber,
|
|
943
|
+
label: target.label,
|
|
944
|
+
checkedAt: now,
|
|
945
|
+
// Two different claims, two fields. `reason` is why a step was waived and
|
|
946
|
+
// is what the order check quotes back when a skip was not declared; `note`
|
|
947
|
+
// is what the worker observed doing a step it did perform. Sharing one key
|
|
948
|
+
// would let a later note overwrite a waiver's rationale.
|
|
949
|
+
...(skipReason !== null ? { skipped: true, reason: skipReason } : {}),
|
|
950
|
+
...(noteReason !== null ? { note: noteReason } : {}),
|
|
951
|
+
});
|
|
952
|
+
} else if (!isStartCommand && target.stepNumber != null) {
|
|
953
|
+
// Re-marking stays the no-op it has always been, with one exception: a step
|
|
954
|
+
// that declares its proof and whose evidence was rewritten after the first
|
|
955
|
+
// tick. Completion refuses evidence newer than the claim, and this is how the
|
|
956
|
+
// worker re-states the claim over the repaired proof — the sanctioned repair
|
|
957
|
+
// path, not a second tick.
|
|
958
|
+
const stale = proofArtifacts.claimOlderThanEvidence(
|
|
959
|
+
taskDir,
|
|
960
|
+
path.basename(taskPath),
|
|
961
|
+
target.stepNumber,
|
|
962
|
+
loadTerminalContract(taskDir, taskPath),
|
|
963
|
+
path.basename(signalPath),
|
|
964
|
+
);
|
|
965
|
+
// A re-mark restates the claim, so a new note replaces the old one rather
|
|
966
|
+
// than leaving rationale from a statement the worker has moved past. A
|
|
967
|
+
// waiver's own reason is never touched: a step skipped and then rechecked
|
|
968
|
+
// after a rebase keeps why it was waived and gains what the recheck found.
|
|
969
|
+
if (noteReason !== null) events[recordedEvent] = { ...events[recordedEvent], note: noteReason };
|
|
970
|
+
if (stale) events[recordedEvent] = { ...events[recordedEvent], remarkedAt: now };
|
|
714
971
|
}
|
|
715
972
|
|
|
973
|
+
// Ticks recorded after a later-numbered step. Order alone says nothing about the
|
|
974
|
+
// work, so it is kept with the run record and never refuses a mark.
|
|
975
|
+
const outOfOrder = proofArtifacts.outOfOrderMarks(events);
|
|
976
|
+
|
|
716
977
|
if (terminalCommand) {
|
|
717
978
|
assertTerminalPackagedEvidence(taskPath, taskDir, terminalCommand);
|
|
718
979
|
}
|
|
@@ -726,6 +987,7 @@ const next = buildSignalUpdate(
|
|
|
726
987
|
now,
|
|
727
988
|
taskPath,
|
|
728
989
|
taskDir,
|
|
990
|
+
outOfOrder,
|
|
729
991
|
);
|
|
730
992
|
// Re-marking a step that is already done changes nothing: the box was checked
|
|
731
993
|
// and its timing event already exists, so the only difference from the stored
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
// Proof-artifact declarations: the evidence a checklist step claims.
|
|
2
|
+
//
|
|
3
|
+
// ADR-064 froze a step's deterministic command so a checked box could not stand
|
|
4
|
+
// in for an execution. This is the same shape for a proof step's output: a row
|
|
5
|
+
// declares the artifact it produces,
|
|
6
|
+
//
|
|
7
|
+
// - [ ] 9. Retain the failing baseline run [PROOF: artifacts/recipe-baseline-run]
|
|
8
|
+
//
|
|
9
|
+
// and the declaration is frozen into the task's terminal contract at init.
|
|
10
|
+
// `./mark N` refuses the tick while that path is absent, and completion refuses a
|
|
11
|
+
// ticked step whose evidence is missing or was written after the step's last
|
|
12
|
+
// mark. A step without a declaration behaves exactly as before: templates that
|
|
13
|
+
// have not adopted the marker are untouched.
|
|
14
|
+
|
|
15
|
+
const fs = require('node:fs');
|
|
16
|
+
const path = require('node:path');
|
|
17
|
+
|
|
18
|
+
const { enumerateChecklistCheckboxes, signalFileForChecklist } = require('./checklist-target.cjs');
|
|
19
|
+
const { readJson } = require('./mark-io.cjs');
|
|
20
|
+
const checklistOrder = require('./checklist-order.cjs');
|
|
21
|
+
|
|
22
|
+
/** How an absent declared path reads in a refusal. */
|
|
23
|
+
const EVIDENCE_ABSENCE = {
|
|
24
|
+
empty: 'empty',
|
|
25
|
+
outside: 'outside the task directory',
|
|
26
|
+
missing: 'missing',
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const PROOF_REFERENCE = /\[PROOF:\s*([^\]]+?)\s*\]/;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A declared path is task-relative: the worker writes it inside the task dir and
|
|
33
|
+
* the gate reads it from there. An absolute or escaping path would point at
|
|
34
|
+
* something the run does not own, so it is a template error, not a refusal.
|
|
35
|
+
*/
|
|
36
|
+
function normalizeProofPath(raw, checklist, stepNumber) {
|
|
37
|
+
const normalized = String(raw).trim().split(path.sep).join('/');
|
|
38
|
+
const invalid =
|
|
39
|
+
normalized.length === 0 ||
|
|
40
|
+
normalized.startsWith('/') ||
|
|
41
|
+
/^[A-Za-z]:/.test(normalized) ||
|
|
42
|
+
normalized.split('/').includes('..');
|
|
43
|
+
if (invalid)
|
|
44
|
+
throw new Error(
|
|
45
|
+
`Invalid proof declaration in ${checklist} step ${stepNumber}: ` +
|
|
46
|
+
`'${raw}' must be a path inside the task directory`,
|
|
47
|
+
);
|
|
48
|
+
return normalized;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function proofDeclarations(markdown, checklist) {
|
|
52
|
+
return enumerateChecklistCheckboxes(markdown).flatMap((item) => {
|
|
53
|
+
const match = item.rawLabel.match(PROOF_REFERENCE);
|
|
54
|
+
if (!match && item.rawLabel.includes('[PROOF'))
|
|
55
|
+
throw new Error(`Invalid proof declaration in ${checklist} step ${item.stepNumber}`);
|
|
56
|
+
return match
|
|
57
|
+
? [
|
|
58
|
+
{
|
|
59
|
+
checklist,
|
|
60
|
+
stepNumber: item.stepNumber,
|
|
61
|
+
path: normalizeProofPath(match[1], checklist, item.stepNumber),
|
|
62
|
+
},
|
|
63
|
+
]
|
|
64
|
+
: [];
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Freeze the checklist's declarations into the contract, as ADR-064 does for commands. */
|
|
69
|
+
function freezeProofArtifacts(contract, markdown, checklist) {
|
|
70
|
+
const steps = proofDeclarations(markdown, checklist);
|
|
71
|
+
return steps.length ? { ...contract, proofArtifacts: steps } : contract;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The frozen declarations plus any the live checklist carries. A row added after
|
|
76
|
+
* init still gates: the contract is the record, never the waiver.
|
|
77
|
+
*/
|
|
78
|
+
function declaredProofArtifacts(taskDir, checklist, contract) {
|
|
79
|
+
const frozen = (contract?.proofArtifacts ?? []).filter((step) => step.checklist === checklist);
|
|
80
|
+
const file = path.join(taskDir, checklist);
|
|
81
|
+
const live = fs.existsSync(file)
|
|
82
|
+
? proofDeclarations(fs.readFileSync(file, 'utf8'), checklist)
|
|
83
|
+
: [];
|
|
84
|
+
const steps = [...frozen];
|
|
85
|
+
for (const step of live) {
|
|
86
|
+
if (!steps.some((candidate) => candidate.stepNumber === step.stepNumber)) steps.push(step);
|
|
87
|
+
}
|
|
88
|
+
return steps;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Evidence is present when the path exists and holds something: an empty
|
|
93
|
+
* directory is the shape of a retained run that never ran.
|
|
94
|
+
*/
|
|
95
|
+
function proofEvidence(taskDir, relPath) {
|
|
96
|
+
const target = path.join(taskDir, relPath);
|
|
97
|
+
let stat;
|
|
98
|
+
let resolved;
|
|
99
|
+
try {
|
|
100
|
+
// A declared path is evidence only where the task says it is: a symlink can
|
|
101
|
+
// point the gate at a populated directory the run never produced, so the
|
|
102
|
+
// resolved path has to stay inside the task directory.
|
|
103
|
+
resolved = fs.realpathSync(target);
|
|
104
|
+
const root = fs.realpathSync(taskDir);
|
|
105
|
+
if (resolved !== root && !resolved.startsWith(root + path.sep))
|
|
106
|
+
return { present: false, reason: 'outside', mtimeMs: null };
|
|
107
|
+
stat = fs.statSync(resolved);
|
|
108
|
+
} catch (error) {
|
|
109
|
+
if (error.code === 'ENOENT') return { present: false, reason: 'missing', mtimeMs: null };
|
|
110
|
+
throw error;
|
|
111
|
+
}
|
|
112
|
+
if (stat.isDirectory()) {
|
|
113
|
+
// Read the resolved directory and never follow a child link: an inside
|
|
114
|
+
// directory of symlinks must not stand in for files the run never wrote.
|
|
115
|
+
const entries = fs.readdirSync(resolved, { withFileTypes: true });
|
|
116
|
+
const own = entries.filter((entry) => !entry.isSymbolicLink());
|
|
117
|
+
if (own.length === 0) return { present: false, reason: 'empty', mtimeMs: null };
|
|
118
|
+
const newest = own.reduce((latest, entry) => {
|
|
119
|
+
try {
|
|
120
|
+
return Math.max(latest, fs.lstatSync(path.join(resolved, entry.name)).mtimeMs);
|
|
121
|
+
} catch {
|
|
122
|
+
return latest;
|
|
123
|
+
}
|
|
124
|
+
}, stat.mtimeMs);
|
|
125
|
+
return { present: true, reason: null, mtimeMs: newest };
|
|
126
|
+
}
|
|
127
|
+
if (stat.size === 0) return { present: false, reason: 'empty', mtimeMs: null };
|
|
128
|
+
return { present: true, reason: null, mtimeMs: stat.mtimeMs };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* When each step was last marked, from the signal the mark engine writes. The
|
|
133
|
+
* caller passes the signal it selected (`mark --signal`), because a run using
|
|
134
|
+
* that escape hatch keeps its claims there and not in the derived name.
|
|
135
|
+
*/
|
|
136
|
+
function stepMarkTimes(taskDir, checklist, signalFile) {
|
|
137
|
+
const signal = readJson(path.join(taskDir, signalFile ?? signalFileForChecklist(checklist)));
|
|
138
|
+
const events = Array.isArray(signal?.checklistTiming?.events)
|
|
139
|
+
? signal.checklistTiming.events
|
|
140
|
+
: [];
|
|
141
|
+
const times = new Map();
|
|
142
|
+
for (const event of events) {
|
|
143
|
+
if (!event || typeof event.stepNumber !== 'number') continue;
|
|
144
|
+
const stamp = Date.parse(event.remarkedAt ?? event.checkedAt ?? '');
|
|
145
|
+
if (!Number.isNaN(stamp)) times.set(event.stepNumber, stamp);
|
|
146
|
+
}
|
|
147
|
+
return times;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function checkedSteps(taskDir, checklist) {
|
|
151
|
+
const file = path.join(taskDir, checklist);
|
|
152
|
+
if (!fs.existsSync(file)) return new Set();
|
|
153
|
+
return new Set(
|
|
154
|
+
enumerateChecklistCheckboxes(fs.readFileSync(file, 'utf8'))
|
|
155
|
+
.filter((item) => item.checked)
|
|
156
|
+
.map((item) => item.stepNumber),
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Why `./mark N` cannot tick this step yet, or null when it can. Only a declared
|
|
162
|
+
* step can refuse, and only for absent evidence — the tick is the claim, so the
|
|
163
|
+
* artifact has to exist before it.
|
|
164
|
+
*/
|
|
165
|
+
function pendingProofArtifact(taskDir, checklist, stepNumber, contract) {
|
|
166
|
+
const step = declaredProofArtifacts(taskDir, checklist, contract).find(
|
|
167
|
+
(candidate) => candidate.stepNumber === stepNumber,
|
|
168
|
+
);
|
|
169
|
+
if (!step) return null;
|
|
170
|
+
const evidence = proofEvidence(taskDir, step.path);
|
|
171
|
+
if (evidence.present) return null;
|
|
172
|
+
return (
|
|
173
|
+
`step ${stepNumber} declares its proof at ${step.path}, which is ` +
|
|
174
|
+
`${EVIDENCE_ABSENCE[evidence.reason] ?? 'missing'}. ` +
|
|
175
|
+
'Produce the evidence, then mark the step.'
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* The one definition of "the evidence is newer than the claim". A mark and a
|
|
181
|
+
* completion must not decide this differently, so both callers come here. Null
|
|
182
|
+
* when the step declares nothing, has no evidence, or was never marked.
|
|
183
|
+
*/
|
|
184
|
+
function claimOlderThanEvidence(taskDir, checklist, stepNumber, contract, signalFile) {
|
|
185
|
+
const step = declaredProofArtifacts(taskDir, checklist, contract).find(
|
|
186
|
+
(candidate) => candidate.stepNumber === stepNumber,
|
|
187
|
+
);
|
|
188
|
+
if (!step) return null;
|
|
189
|
+
const evidence = proofEvidence(taskDir, step.path);
|
|
190
|
+
const markedAt = stepMarkTimes(taskDir, checklist, signalFile).get(stepNumber);
|
|
191
|
+
if (!evidence.present || evidence.mtimeMs == null || markedAt == null) return null;
|
|
192
|
+
return evidence.mtimeMs > markedAt ? { step, evidence, markedAt } : null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Completion issues for declared steps: a ticked step needs its evidence, and
|
|
197
|
+
* evidence written after the step's last mark means the claim came first. An
|
|
198
|
+
* unticked step is simply unfinished work and not an issue here.
|
|
199
|
+
*/
|
|
200
|
+
function proofArtifactIssues(taskDir, checklist, contract, signalFile) {
|
|
201
|
+
const ticked = checkedSteps(taskDir, checklist);
|
|
202
|
+
// A skipped step claims the work did not apply, so it owes no evidence — but
|
|
203
|
+
// only where the frozen template granted the skip. A `skipped: true` ledger
|
|
204
|
+
// row on a step the contract never declared optional waives nothing, and the
|
|
205
|
+
// order check refuses it separately.
|
|
206
|
+
const skipped = checklistOrder.authorizedSkips(taskDir, checklist, contract, signalFile);
|
|
207
|
+
const issues = [];
|
|
208
|
+
for (const step of declaredProofArtifacts(taskDir, checklist, contract)) {
|
|
209
|
+
if (!ticked.has(step.stepNumber) || skipped.has(step.stepNumber)) continue;
|
|
210
|
+
const evidence = proofEvidence(taskDir, step.path);
|
|
211
|
+
if (!evidence.present) {
|
|
212
|
+
issues.push(
|
|
213
|
+
`${checklist} step ${step.stepNumber} is checked but its declared proof ${step.path} is ` +
|
|
214
|
+
`${EVIDENCE_ABSENCE[evidence.reason] ?? 'missing'}`,
|
|
215
|
+
);
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
const late = claimOlderThanEvidence(taskDir, checklist, step.stepNumber, contract, signalFile);
|
|
219
|
+
if (late) {
|
|
220
|
+
issues.push(
|
|
221
|
+
`${checklist} step ${step.stepNumber}: its declared proof ${step.path} was written ` +
|
|
222
|
+
`${new Date(late.evidence.mtimeMs).toISOString()}, after the step was marked ` +
|
|
223
|
+
`${new Date(late.markedAt).toISOString()}; mark the step again now the evidence is current`,
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
return issues;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Steps marked after a later-numbered step, recorded for the run record only.
|
|
232
|
+
* Order alone proves nothing about the work, so it never refuses anything.
|
|
233
|
+
*/
|
|
234
|
+
function outOfOrderMarks(events) {
|
|
235
|
+
const ordered = (Array.isArray(events) ? events : [])
|
|
236
|
+
.filter((event) => event && typeof event.stepNumber === 'number' && event.checkedAt)
|
|
237
|
+
.map((event) => ({ stepNumber: event.stepNumber, checkedAt: event.checkedAt }));
|
|
238
|
+
const anomalies = [];
|
|
239
|
+
let highest = 0;
|
|
240
|
+
for (const event of ordered) {
|
|
241
|
+
if (event.stepNumber < highest)
|
|
242
|
+
anomalies.push({ stepNumber: event.stepNumber, checkedAt: event.checkedAt, after: highest });
|
|
243
|
+
highest = Math.max(highest, event.stepNumber);
|
|
244
|
+
}
|
|
245
|
+
return anomalies;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
module.exports = {
|
|
249
|
+
PROOF_REFERENCE,
|
|
250
|
+
declarations: proofDeclarations,
|
|
251
|
+
freeze: freezeProofArtifacts,
|
|
252
|
+
declared: declaredProofArtifacts,
|
|
253
|
+
evidence: proofEvidence,
|
|
254
|
+
pending: pendingProofArtifact,
|
|
255
|
+
issues: proofArtifactIssues,
|
|
256
|
+
claimOlderThanEvidence,
|
|
257
|
+
outOfOrderMarks,
|
|
258
|
+
};
|