omp-conductor 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +12 -8
- package/package.json +1 -1
- package/schema/config.schema.json +40 -1
- package/src/admission.ts +263 -44
- package/src/ask.ts +39 -3
- package/src/availability.ts +27 -1
- package/src/backups.ts +2 -2
- package/src/briefs/orchestrator.md +1 -0
- package/src/briefs/worker.md +38 -19
- package/src/command-help.ts +8 -1
- package/src/command-manifest.ts +5 -2
- package/src/commands/arm.ts +6 -3
- package/src/commands/message.ts +32 -4
- package/src/commands/watch.ts +62 -3
- package/src/config-schema.ts +53 -0
- package/src/config.ts +97 -1
- package/src/daemon.ts +1479 -1483
- package/src/decisions.ts +51 -6
- package/src/depends-on.ts +261 -1
- package/src/diff-flags.ts +350 -0
- package/src/digest-schedule.ts +37 -0
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +71 -15
- package/src/fleet.ts +189 -34
- package/src/gitops.ts +103 -24
- package/src/graph-health.ts +20 -7
- package/src/graph.ts +313 -68
- package/src/lifecycle.ts +43 -7
- package/src/omp.ts +42 -0
- package/src/orchestrator-tick.ts +430 -162
- package/src/release-policy.ts +177 -5
- package/src/routing.ts +11 -3
- package/src/session-host.ts +16 -0
- package/src/settlement.ts +1728 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +91 -30
- package/src/setup-wizard.ts +1257 -78
- package/src/setup.ts +153 -6
- package/src/status-render.ts +36 -4
- package/src/store.ts +411 -17
- package/src/tracker/github.ts +607 -12
- package/src/types.ts +331 -5
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +270 -13
- package/src/worker.ts +239 -6
- package/src/worktree.ts +115 -8
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/worker.ts
CHANGED
|
@@ -30,6 +30,77 @@ const GITHUB_PR_URL_PATTERN = /https:\/\/github\.com\/([^/\s]+\/[^/\s]+)\/pull\/
|
|
|
30
30
|
/** `{{KEY}}` placeholders in a brief template. */
|
|
31
31
|
const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* The structured settlement a worker yields at the end of a run (#540).
|
|
35
|
+
*
|
|
36
|
+
* Fields decided here, not per project. The daemon stays the authority on
|
|
37
|
+
* green — it verifies the PR itself — so this is the worker's *claim*,
|
|
38
|
+
* cross-checked exactly as the prose report it replaces: a `green` claim
|
|
39
|
+
* must carry the PR URL and the head SHA the worker actually watched go
|
|
40
|
+
* green, and anything less fails closed.
|
|
41
|
+
*/
|
|
42
|
+
export interface WorkerSettlement {
|
|
43
|
+
status: "green" | "blocked" | "failed";
|
|
44
|
+
/** The run's pull request URL. Required with `status: "green"`. */
|
|
45
|
+
prUrl?: string;
|
|
46
|
+
/** The branch the PR was pushed from. */
|
|
47
|
+
branch?: string;
|
|
48
|
+
/** The 40-hex head SHA the green CI verdict was observed at. Required with `status: "green"`. */
|
|
49
|
+
headSha?: string;
|
|
50
|
+
/** What changed and why — the narrative a reviewer reads. */
|
|
51
|
+
summary: string;
|
|
52
|
+
/** What a decision or credential this run needed (blocked runs). */
|
|
53
|
+
blockers?: string[];
|
|
54
|
+
/** Commands run as evidence, exactly as executed. */
|
|
55
|
+
proof?: string[];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The JSON Schema form of {@link WorkerSettlement}, handed to the harness as
|
|
60
|
+
* the session's `outputSchema` (#540).
|
|
61
|
+
*
|
|
62
|
+
* Used two ways, and both must stay honest to the same contract:
|
|
63
|
+
* `requireYieldTool` puts the `yield` tool in front of the worker with this
|
|
64
|
+
* schema as its description, and `outputSchemaMode: "permissive"` means a
|
|
65
|
+
* violation is a fallback, never a lost report — the worker's text path must
|
|
66
|
+
* still settle a run whose yield does not parse.
|
|
67
|
+
*/
|
|
68
|
+
export const WORKER_SETTLEMENT_SCHEMA = {
|
|
69
|
+
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
70
|
+
type: "object",
|
|
71
|
+
description:
|
|
72
|
+
"The run's settlement: what this worker claims it achieved. The daemon verifies the PR itself; " +
|
|
73
|
+
"this is the claim, cross-checked like the settlement report it replaces.",
|
|
74
|
+
additionalProperties: false,
|
|
75
|
+
required: ["status", "summary"],
|
|
76
|
+
properties: {
|
|
77
|
+
status: {
|
|
78
|
+
type: "string",
|
|
79
|
+
enum: ["green", "blocked", "failed"],
|
|
80
|
+
description:
|
|
81
|
+
"green: pushed and the checks you watched are green. blocked: a decision, credential or " +
|
|
82
|
+
"repo fact is missing. failed: the run could not complete.",
|
|
83
|
+
},
|
|
84
|
+
prUrl: { type: "string", description: "The run's pull request URL — required with status green." },
|
|
85
|
+
branch: { type: "string", description: "The branch the PR was pushed from." },
|
|
86
|
+
headSha: {
|
|
87
|
+
type: "string",
|
|
88
|
+
description: "The 40-character head SHA you observed green — required with status green.",
|
|
89
|
+
},
|
|
90
|
+
summary: { type: "string", description: "One short paragraph: what changed and why." },
|
|
91
|
+
blockers: {
|
|
92
|
+
type: "array",
|
|
93
|
+
items: { type: "string" },
|
|
94
|
+
description: "What is missing or uncertain, one item per blocker (blocked runs).",
|
|
95
|
+
},
|
|
96
|
+
proof: {
|
|
97
|
+
type: "array",
|
|
98
|
+
items: { type: "string" },
|
|
99
|
+
description: "The commands you ran as evidence, exactly as executed.",
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
} as const;
|
|
103
|
+
|
|
33
104
|
function scheduleWallClock(callback: () => void, delayMs: number): () => void {
|
|
34
105
|
const timer = setTimeout(callback, delayMs);
|
|
35
106
|
return () => clearTimeout(timer);
|
|
@@ -74,6 +145,32 @@ export const ORPHAN_RESUME_PROMPT =
|
|
|
74
145
|
"then continue exactly where you left off: re-check the outcome of your last action before repeating it, and keep " +
|
|
75
146
|
"working your original brief to the same report contract.";
|
|
76
147
|
|
|
148
|
+
/**
|
|
149
|
+
* What a review-revision worker is told instead of re-sending its brief (#677).
|
|
150
|
+
*
|
|
151
|
+
* Same rule as {@link ORPHAN_RESUME_PROMPT}: the original brief is already in
|
|
152
|
+
* the resumed transcript, and re-sending it is how a resumed worker re-does
|
|
153
|
+
* work it already did. What makes this a revision rather than a resume is the
|
|
154
|
+
* deliverable contract: the PR stays open, the branch stays the run's own, and
|
|
155
|
+
* the session ends only when the same final-report shape says the revised head
|
|
156
|
+
* is green. One literal so tests can pin it.
|
|
157
|
+
*/
|
|
158
|
+
export const REVIEW_REVISION_PROMPT =
|
|
159
|
+
"The orchestrator reviewed your green pull request and found blocking findings. Continue this same session " +
|
|
160
|
+
"on this same run: same branch, same pull request — do not close or reopen it, and do not open another. " +
|
|
161
|
+
"Address exactly the findings below, push with conductor_push, verify the checks with conductor_pr_status, " +
|
|
162
|
+
"and finish with the same final-report contract as before (yield your settlement: status green, prUrl, headSha). " +
|
|
163
|
+
"Your original brief is already in this transcript; redo only what the findings implicate.";
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The full opening prompt for one review revision round: the pinned continuation
|
|
167
|
+
* text plus the orchestrator's findings verbatim. Exported so a unit test can
|
|
168
|
+
* pin the exact handoff without standing up a session.
|
|
169
|
+
*/
|
|
170
|
+
export function renderReviewRevisionPrompt(findings: string, round: number): string {
|
|
171
|
+
return `${REVIEW_REVISION_PROMPT}\n\nReview round ${round} — blocking findings:\n\n${findings}`;
|
|
172
|
+
}
|
|
173
|
+
|
|
77
174
|
export interface WorkerOpts {
|
|
78
175
|
brief: string;
|
|
79
176
|
cwd: string;
|
|
@@ -179,6 +276,13 @@ export interface WorkerResult {
|
|
|
179
276
|
headSha?: string;
|
|
180
277
|
turns: number;
|
|
181
278
|
spendUsd: number;
|
|
279
|
+
/**
|
|
280
|
+
* The run's settlement text. Today this is the worker's own last words; a
|
|
281
|
+
* run that yielded a structured {@link WorkerSettlement} instead carries the
|
|
282
|
+
* canonical rendering of that parsed object (#540) — the narrative lives in
|
|
283
|
+
* its `summary`, transcribed here field for field rather than regexed out of
|
|
284
|
+
* prose.
|
|
285
|
+
*/
|
|
182
286
|
report: string;
|
|
183
287
|
/** In-session HTTP 429 responses the session recorded (stopReason "error",
|
|
184
288
|
* errorStatus 429), counted as the messages streamed in. A healthy run
|
|
@@ -251,12 +355,34 @@ export function renderBrief(template: string, vars: Record<string, string>): str
|
|
|
251
355
|
* must carry both a PR URL and the head SHA observed after CI; the daemon then
|
|
252
356
|
* asks the tracker to verify those facts independently. Missing or malformed
|
|
253
357
|
* evidence fails closed.
|
|
358
|
+
*
|
|
359
|
+
* A parsed {@link WorkerSettlement} wins over the text whenever it is present
|
|
360
|
+
* (#540): the schema is the contract now, and a misleadingly regular-looking
|
|
361
|
+
* prose block must not out-vote the object the worker actually yielded. The
|
|
362
|
+
* text path is unchanged underneath, so a run that never yields settles
|
|
363
|
+
* exactly as it did before.
|
|
254
364
|
*/
|
|
255
|
-
export function deriveResult(
|
|
365
|
+
export function deriveResult(
|
|
366
|
+
report: string,
|
|
367
|
+
repoSlug?: string,
|
|
368
|
+
structured?: WorkerSettlement,
|
|
369
|
+
): {
|
|
256
370
|
state: RunState;
|
|
257
371
|
prUrl?: string;
|
|
258
372
|
headSha?: string;
|
|
259
373
|
} {
|
|
374
|
+
if (structured !== undefined && (structured.status === "green" || structured.status === "blocked" || structured.status === "failed")) {
|
|
375
|
+
const prUrl = structured.prUrl;
|
|
376
|
+
const headSha = structured.headSha?.toLowerCase();
|
|
377
|
+
if (structured.status === "green" && prUrl !== undefined && headSha !== undefined) {
|
|
378
|
+
return { state: "pushed-green", prUrl, headSha };
|
|
379
|
+
}
|
|
380
|
+
return {
|
|
381
|
+
state: structured.status === "blocked" ? "blocked" : "failed",
|
|
382
|
+
...(prUrl === undefined ? {} : { prUrl }),
|
|
383
|
+
...(headSha === undefined ? {} : { headSha }),
|
|
384
|
+
};
|
|
385
|
+
}
|
|
260
386
|
const structuredPrUrl = PR_URL_PATTERN.exec(report)?.[1];
|
|
261
387
|
const headSha = HEAD_SHA_PATTERN.exec(report)?.[1]?.toLowerCase();
|
|
262
388
|
if (PUSHED_GREEN_PATTERN.test(report) && structuredPrUrl !== undefined && headSha !== undefined) {
|
|
@@ -279,6 +405,81 @@ export function deriveResult(report: string, repoSlug?: string): {
|
|
|
279
405
|
};
|
|
280
406
|
}
|
|
281
407
|
|
|
408
|
+
/**
|
|
409
|
+
* Parse a worker's structured settlement out of one assistant message's
|
|
410
|
+
* content (#540).
|
|
411
|
+
*
|
|
412
|
+
* The worker calls the `yield` tool with `{ result: { data: <settlement> } }`
|
|
413
|
+
* and the harness records that call as a `toolCall` content block on the
|
|
414
|
+
* assistant message — the same content `reportText` flattens. Parsed
|
|
415
|
+
* permissively on purpose: `outputSchemaMode` is permissive, so a yield the
|
|
416
|
+
* schema rejected must still fall back to the text path rather than vanish.
|
|
417
|
+
* The acceptance bar mirrors the schema's own `required`: a recognizable
|
|
418
|
+
* `status` and a string `summary`; every other field is adopted when it has
|
|
419
|
+
* the right type and dropped otherwise. Exported so the run loop's precedence
|
|
420
|
+
* (yield over prose) is pinned by a unit test.
|
|
421
|
+
*/
|
|
422
|
+
export function structuredSettlement(content: unknown): WorkerSettlement | undefined {
|
|
423
|
+
if (!Array.isArray(content)) return undefined;
|
|
424
|
+
let parsed: WorkerSettlement | undefined;
|
|
425
|
+
for (const block of content) {
|
|
426
|
+
if (field(block, "type") !== "toolCall") continue;
|
|
427
|
+
if (field(block, "name") !== "yield") continue;
|
|
428
|
+
const args = field(block, "arguments");
|
|
429
|
+
if (args === null || typeof args !== "object") continue;
|
|
430
|
+
const result = field(args, "result");
|
|
431
|
+
if (result === null || typeof result !== "object") continue;
|
|
432
|
+
const data = field(result, "data");
|
|
433
|
+
if (data === null || typeof data !== "object" || Array.isArray(data)) continue;
|
|
434
|
+
const record = data as Record<string, unknown>;
|
|
435
|
+
const status = record.status;
|
|
436
|
+
if (status !== "green" && status !== "blocked" && status !== "failed") continue;
|
|
437
|
+
const summary = record.summary;
|
|
438
|
+
if (typeof summary !== "string") continue;
|
|
439
|
+
const settlement: WorkerSettlement = { status, summary };
|
|
440
|
+
for (const key of ["prUrl", "branch", "headSha"] as const) {
|
|
441
|
+
const value = record[key];
|
|
442
|
+
if (typeof value === "string" && value !== "") settlement[key] = value;
|
|
443
|
+
}
|
|
444
|
+
for (const key of ["blockers", "proof"] as const) {
|
|
445
|
+
const value = record[key];
|
|
446
|
+
if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
|
|
447
|
+
settlement[key] = value;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
parsed = settlement;
|
|
451
|
+
}
|
|
452
|
+
return parsed;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* Render a parsed {@link WorkerSettlement} as the run's stored report (#540).
|
|
457
|
+
*
|
|
458
|
+
* A worker that yields structured output may write little or no prose after
|
|
459
|
+
* the yield call, and the prose shape is no longer the contract — so the
|
|
460
|
+
* report the store keeps for such a run is this canonical rendering of the
|
|
461
|
+
* parsed object itself: the same fields a reviewer would have had to regex
|
|
462
|
+
* out of prose, derived from the object, field for field. The daemon's own
|
|
463
|
+
* postfixes — the PR-diff `changed:` line and the reliability sentence — are
|
|
464
|
+
* appended to it exactly as they are to a prose report.
|
|
465
|
+
*/
|
|
466
|
+
export function renderSettlement(settlement: WorkerSettlement): string {
|
|
467
|
+
const lines: string[] = [`status: ${settlement.status}`];
|
|
468
|
+
if (settlement.prUrl !== undefined) lines.push(`pr: ${settlement.prUrl}`);
|
|
469
|
+
if (settlement.branch !== undefined) lines.push(`branch: ${settlement.branch}`);
|
|
470
|
+
if (settlement.headSha !== undefined) lines.push(`head: ${settlement.headSha}`);
|
|
471
|
+
if (settlement.summary !== "") lines.push("", settlement.summary);
|
|
472
|
+
if (settlement.blockers !== undefined && settlement.blockers.length > 0) {
|
|
473
|
+
lines.push("", "blockers:");
|
|
474
|
+
for (const blocker of settlement.blockers) lines.push(` - ${blocker}`);
|
|
475
|
+
}
|
|
476
|
+
if (settlement.proof !== undefined && settlement.proof.length > 0) {
|
|
477
|
+
lines.push("", "proof:");
|
|
478
|
+
for (const item of settlement.proof) lines.push(` - ${item}`);
|
|
479
|
+
}
|
|
480
|
+
return lines.join("\n");
|
|
481
|
+
}
|
|
482
|
+
|
|
282
483
|
/**
|
|
283
484
|
* Did this report state a verdict at all?
|
|
284
485
|
*
|
|
@@ -354,6 +555,13 @@ export async function runWorker(
|
|
|
354
555
|
...(o.verbSocketPath === undefined ? {} : { verbSocketPath: o.verbSocketPath }),
|
|
355
556
|
...(o.onChildLog === undefined ? {} : { onChildLog: o.onChildLog }),
|
|
356
557
|
...(o.maySpawn === undefined ? {} : { maySpawn: o.maySpawn }),
|
|
558
|
+
// The structured settlement contract (#540): the worker's `yield` tool
|
|
559
|
+
// validates its `data` payload against this schema, permissively — an
|
|
560
|
+
// invalid or absent yield falls back to the text path, never a lost
|
|
561
|
+
// report. Always-on for workers: one schema, and no per-project shape.
|
|
562
|
+
outputSchema: WORKER_SETTLEMENT_SCHEMA,
|
|
563
|
+
outputSchemaMode: "permissive",
|
|
564
|
+
requireYieldTool: true,
|
|
357
565
|
});
|
|
358
566
|
} catch (err) {
|
|
359
567
|
// The pre-spawn gate closed (#374): a daemon stop landed while the session
|
|
@@ -447,6 +655,10 @@ export async function runWorker(
|
|
|
447
655
|
let spendUsd = 0;
|
|
448
656
|
let provider429Count = 0;
|
|
449
657
|
let report = "";
|
|
658
|
+
// The newest structured settlement the worker yielded (#540). A terminal
|
|
659
|
+
// `yield` ends the run, so anything parsed here is the run's own claim, and
|
|
660
|
+
// any later message is stale-polling chatter, never a newer verdict.
|
|
661
|
+
let structured: WorkerSettlement | undefined;
|
|
450
662
|
// Which model/provider actually wrote the newest assistant message. Last
|
|
451
663
|
// assistant message wins: that is the durable answer even for a run that
|
|
452
664
|
// never failed over (#535 slice 1, read off the message field which is where
|
|
@@ -603,15 +815,22 @@ export async function runWorker(
|
|
|
603
815
|
session.on("message_end", (event) => {
|
|
604
816
|
const message = field(event, "message");
|
|
605
817
|
if (field(message, "role") !== "assistant") return;
|
|
818
|
+
const content = field(message, "content");
|
|
606
819
|
// Keep the newest non-empty assistant text: whatever the worker said last
|
|
607
820
|
// is its report, whether it finished cleanly or was cut off.
|
|
608
|
-
const text = reportText(
|
|
821
|
+
const text = reportText(content);
|
|
609
822
|
if (text !== "") {
|
|
610
823
|
report = text;
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
824
|
+
}
|
|
825
|
+
// The structured settlement (#540): the `yield` tool call rides the same
|
|
826
|
+
// message's content as a `toolCall` block. Newest yield wins; the
|
|
827
|
+
// derivation itself prefers it over prose, so a yield and a misleadingly
|
|
828
|
+
// regular-looking text block on the same message agree in its favour.
|
|
829
|
+
const yielded = structuredSettlement(content);
|
|
830
|
+
if (yielded !== undefined) structured = yielded;
|
|
831
|
+
const stated = deriveResult(text, o.repoSlug, yielded);
|
|
832
|
+
if (stated.state === "pushed-green" && stated.prUrl !== undefined && stated.headSha !== undefined) {
|
|
833
|
+
claim = { prUrl: stated.prUrl, headSha: stated.headSha };
|
|
615
834
|
}
|
|
616
835
|
|
|
617
836
|
// Real cost lives on assistant messages as `usage.cost.total` (live hermes
|
|
@@ -776,6 +995,20 @@ export async function runWorker(
|
|
|
776
995
|
}));
|
|
777
996
|
}
|
|
778
997
|
|
|
998
|
+
// A structured yield is the run's settlement: it wins over the last text —
|
|
999
|
+
// the worker may have written nothing after the yield call, and the stored
|
|
1000
|
+
// report is the canonical rendering of the parsed object, not prose the
|
|
1001
|
+
// regex has to recover (#540).
|
|
1002
|
+
if (structured !== undefined) {
|
|
1003
|
+
return withMetrics(withSessionFacts({
|
|
1004
|
+
...deriveResult(report, o.repoSlug, structured),
|
|
1005
|
+
turns,
|
|
1006
|
+
spendUsd,
|
|
1007
|
+
provider429Count,
|
|
1008
|
+
report: renderSettlement(structured),
|
|
1009
|
+
}));
|
|
1010
|
+
}
|
|
1011
|
+
|
|
779
1012
|
// An explicit later verdict always wins: a worker that pushed green and then
|
|
780
1013
|
// stopped to ask a question means the question. The earlier claim is only
|
|
781
1014
|
// restored when the last thing said was not a verdict at all.
|
package/src/worktree.ts
CHANGED
|
@@ -759,12 +759,87 @@ export async function removeWorktree(
|
|
|
759
759
|
|
|
760
760
|
export type RetainedWorktreeCleanup =
|
|
761
761
|
| { kind: "removed" }
|
|
762
|
-
| {
|
|
762
|
+
| {
|
|
763
|
+
kind: "retained";
|
|
764
|
+
reason: "dirty" | "unpushed" | "unknown" | "quarantined";
|
|
765
|
+
detail: string;
|
|
766
|
+
};
|
|
767
|
+
|
|
768
|
+
/**
|
|
769
|
+
* Repairs a run repository whose `objects/info/alternates` names a path that no
|
|
770
|
+
* longer exists, re-pointing the dangling entry at this project's current
|
|
771
|
+
* mirror. Worktrees created before mirrors moved into per-project roots keep
|
|
772
|
+
* the old flat path (`<home>/mirrors/<repo>.git`); the store they borrow from
|
|
773
|
+
* is gone, so any fetch against them dies on unresolved deltas (#728). The
|
|
774
|
+
* repair belongs here — in the code that discovers the stale entry — rather
|
|
775
|
+
* than in a one-off host edit, so the next layout migration heals itself too.
|
|
776
|
+
*
|
|
777
|
+
* Only missing entries are rewritten: a valid alternate is left alone and a
|
|
778
|
+
* worktree with no alternates file is already sound. When the store cannot be
|
|
779
|
+
* made sound — the git dir is unresolvable, the file cannot be read or
|
|
780
|
+
* written, or a missing path survives the rewrite — the caller must not fetch
|
|
781
|
+
* into the broken state; a quarantine detail is returned instead.
|
|
782
|
+
*/
|
|
783
|
+
async function repairAlternates(
|
|
784
|
+
worktreePath: string,
|
|
785
|
+
mirrorPath: string,
|
|
786
|
+
): Promise<{ kind: "ok" } | { kind: "quarantine"; detail: string }> {
|
|
787
|
+
const common = await runGit(["rev-parse", "--git-common-dir"], worktreePath);
|
|
788
|
+
if (common.code !== 0) {
|
|
789
|
+
return {
|
|
790
|
+
kind: "quarantine",
|
|
791
|
+
detail: `cannot resolve the git dir: ${common.stderr.trim() || common.stdout.trim() || "no output"}`,
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
const raw = common.stdout.trim();
|
|
795
|
+
if (raw === "") {
|
|
796
|
+
return { kind: "quarantine", detail: "cannot resolve the git dir: git printed nothing" };
|
|
797
|
+
}
|
|
798
|
+
// `--git-common-dir` is relative for linked worktrees; resolve against the
|
|
799
|
+
// tree so bare-mirror layouts and plain clones both land on the alternates.
|
|
800
|
+
const commonDir = raw.startsWith("/") ? raw : join(worktreePath, raw);
|
|
801
|
+
const alternates = join(commonDir, "objects", "info", "alternates");
|
|
802
|
+
if (!existsSync(alternates)) return { kind: "ok" };
|
|
803
|
+
|
|
804
|
+
const current = join(mirrorPath, "objects");
|
|
805
|
+
let content: string;
|
|
806
|
+
try {
|
|
807
|
+
content = readFileSync(alternates, "utf8");
|
|
808
|
+
} catch (err) {
|
|
809
|
+
return { kind: "quarantine", detail: `${alternates} cannot be read: ${err instanceof Error ? err.message : String(err)}` };
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
let changed = false;
|
|
813
|
+
const rewritten = content.split("\n").map((line) => {
|
|
814
|
+
const candidate = line.trim();
|
|
815
|
+
if (candidate === "" || candidate === current || existsSync(candidate)) return line;
|
|
816
|
+
changed = true;
|
|
817
|
+
return current;
|
|
818
|
+
});
|
|
819
|
+
if (changed) {
|
|
820
|
+
try {
|
|
821
|
+
writeFileSync(alternates, rewritten.join("\n"));
|
|
822
|
+
} catch (err) {
|
|
823
|
+
return { kind: "quarantine", detail: `${alternates} cannot be rewritten: ${err instanceof Error ? err.message : String(err)}` };
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
const dangling = rewritten.map((line) => line.trim()).filter((path) => path !== "" && !existsSync(path));
|
|
828
|
+
if (dangling.length > 0) {
|
|
829
|
+
return {
|
|
830
|
+
kind: "quarantine",
|
|
831
|
+
detail: `${alternates} still names missing path(s): ${dangling.join(", ")}`,
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
return { kind: "ok" };
|
|
835
|
+
}
|
|
763
836
|
|
|
764
837
|
/**
|
|
765
838
|
* Reap a terminal run's tree and local mirror branch without deleting the only
|
|
766
839
|
* copy of work. Tracker state is proved by the caller; this function proves the
|
|
767
|
-
* local half after refreshing remote refs. Any ambiguity retains everything
|
|
840
|
+
* local half after refreshing remote refs. Any ambiguity retains everything;
|
|
841
|
+
* a remote read that failed is ambiguity, never a pushed/unpushed claim, and
|
|
842
|
+
* the unpushed verdicts below are reachable only from reads that succeeded.
|
|
768
843
|
*/
|
|
769
844
|
export async function cleanupRetainedWorktree(
|
|
770
845
|
mirrorPath: string,
|
|
@@ -779,6 +854,20 @@ export async function cleanupRetainedWorktree(
|
|
|
779
854
|
|
|
780
855
|
try {
|
|
781
856
|
if (existsSync(worktreePath)) {
|
|
857
|
+
// A run repo created before mirrors moved into per-project roots may
|
|
858
|
+
// borrow objects from a path that no longer exists. Fetching into it is
|
|
859
|
+
// a guaranteed failure, so repair the alternates first — and when the
|
|
860
|
+
// store cannot be made sound, quarantine the tree instead of fetching
|
|
861
|
+
// into the broken state.
|
|
862
|
+
const repair = await repairAlternates(worktreePath, mirrorPath);
|
|
863
|
+
if (repair.kind === "quarantine") {
|
|
864
|
+
return {
|
|
865
|
+
kind: "retained",
|
|
866
|
+
reason: "quarantined",
|
|
867
|
+
detail: `worktree quarantined: ${repair.detail}`,
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
|
|
782
871
|
const dirty = await git(["status", "--porcelain"], worktreePath);
|
|
783
872
|
if (dirty !== "") {
|
|
784
873
|
return { kind: "retained", reason: "dirty", detail: "worktree has uncommitted changes" };
|
|
@@ -794,8 +883,18 @@ export async function cleanupRetainedWorktree(
|
|
|
794
883
|
}
|
|
795
884
|
|
|
796
885
|
// A deleted remote branch can make a pushed commit look local-only until
|
|
797
|
-
// the default branch is fetched.
|
|
798
|
-
|
|
886
|
+
// the default branch is fetched. A failed fetch is ambiguity, never
|
|
887
|
+
// permission: without fresh remote refs the unpushed verdicts below are
|
|
888
|
+
// unreachable and the run is retained on an explicit unknown.
|
|
889
|
+
try {
|
|
890
|
+
await git(["fetch", "--prune", "origin"], mirrorPath);
|
|
891
|
+
} catch (err) {
|
|
892
|
+
return {
|
|
893
|
+
kind: "retained",
|
|
894
|
+
reason: "unknown",
|
|
895
|
+
detail: `remote refs could not be refreshed: ${err instanceof Error ? err.message : String(err)}`,
|
|
896
|
+
};
|
|
897
|
+
}
|
|
799
898
|
|
|
800
899
|
const ref = `refs/heads/${branch}`;
|
|
801
900
|
|
|
@@ -806,10 +905,18 @@ export async function cleanupRetainedWorktree(
|
|
|
806
905
|
// data loss #121 exists to prevent, reintroduced by the move to per-run
|
|
807
906
|
// repositories, so it is checked where the objects actually are.
|
|
808
907
|
if (existsSync(worktreePath)) {
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
908
|
+
try {
|
|
909
|
+
await git(
|
|
910
|
+
["fetch", "--no-tags", mirrorPath, "+refs/remotes/origin/*:refs/remotes/origin/*"],
|
|
911
|
+
worktreePath,
|
|
912
|
+
);
|
|
913
|
+
} catch (err) {
|
|
914
|
+
return {
|
|
915
|
+
kind: "retained",
|
|
916
|
+
reason: "unknown",
|
|
917
|
+
detail: `the run repository's remote refs could not be read: ${err instanceof Error ? err.message : String(err)}`,
|
|
918
|
+
};
|
|
919
|
+
}
|
|
813
920
|
const runUnique = await git(["rev-list", ref, "--not", "--remotes"], worktreePath);
|
|
814
921
|
if (runUnique !== "") {
|
|
815
922
|
return {
|
|
@@ -84,6 +84,10 @@ JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
|
|
|
84
84
|
VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
|
|
85
85
|
VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
|
|
86
86
|
PROJECT=${RECOVER_PROJECT:-}
|
|
87
|
+
# The bound on the post-restore re-arm. `arm` with a claim-only proof performs
|
|
88
|
+
# no Telegram send or wait, so it completes in milliseconds; the timeout exists
|
|
89
|
+
# so a misbehaving conductor can never hang this bounded playbook (#613).
|
|
90
|
+
ARM_TIMEOUT_S=${RECOVER_ARM_TIMEOUT_S:-30}
|
|
87
91
|
|
|
88
92
|
# --------------------------------------------------------------------------
|
|
89
93
|
# output
|
|
@@ -298,6 +302,55 @@ choose_action() { # <failed-unit>
|
|
|
298
302
|
printf 'NONE|\n'
|
|
299
303
|
}
|
|
300
304
|
|
|
305
|
+
# --------------------------------------------------------------------------
|
|
306
|
+
# re-arm — the one follow-on after a successful restoration (#613)
|
|
307
|
+
# --------------------------------------------------------------------------
|
|
308
|
+
|
|
309
|
+
# Whether the scoped project opted into claim-only arming. The unit renders
|
|
310
|
+
# RECOVER_PROJECT only for an unambiguous single-project host (#510/#530), and
|
|
311
|
+
# "proof" is a value unique to the project's "arm" block, so two exact-key
|
|
312
|
+
# greps decide it on the machine-written config. Any unreadable, hand-formatted
|
|
313
|
+
# or legacy shape (no "arm" key at all) reads as challenge — the explicit human
|
|
314
|
+
# gate — which is the fail-safe direction: recovery never arms a project that
|
|
315
|
+
# did not opt into claim-only.
|
|
316
|
+
project_arms_claim_only() { # <config-path>
|
|
317
|
+
local path=${1:-$CONFIG_PATH}
|
|
318
|
+
[[ -f $path ]] || return 1
|
|
319
|
+
grep -qE '"arm"[[:space:]]*:' "$path" || return 1
|
|
320
|
+
grep -qE '"proof"[[:space:]]*:[[:space:]]*"claim-only"' "$path"
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
# Re-arm the scoped project through the ordinary `arm` path, unattended only
|
|
324
|
+
# when the project opted into claim-only proof: that proof performs no Telegram
|
|
325
|
+
# send or wait, so the command completes headless. A challenge project is never
|
|
326
|
+
# poked — its gate is a human answering the challenge — and the caller's
|
|
327
|
+
# escalation says so plainly instead. Prints exactly one outcome line (the
|
|
328
|
+
# progress line goes to stderr, so the captured return value is the outcome).
|
|
329
|
+
rearm_if_claim_only() {
|
|
330
|
+
if [[ -z $PROJECT ]]; then
|
|
331
|
+
printf 'no scoped project (RECOVER_PROJECT unset) — nothing to re-arm\n'
|
|
332
|
+
return 0
|
|
333
|
+
fi
|
|
334
|
+
if ! project_arms_claim_only "$CONFIG_PATH"; then
|
|
335
|
+
printf 'project %s keeps the challenge proof — arming stays an explicit human gate\n' "$PROJECT"
|
|
336
|
+
return 0
|
|
337
|
+
fi
|
|
338
|
+
log "re-arming $PROJECT via the ordinary arm path (claim-only proof)"
|
|
339
|
+
if dry_run; then
|
|
340
|
+
printf 'dry run: would run %s arm --project %s (claim-only, bounded by %ss)\n' "$CONDUCTOR" "$PROJECT" "$ARM_TIMEOUT_S"
|
|
341
|
+
return 0
|
|
342
|
+
fi
|
|
343
|
+
local out code
|
|
344
|
+
out=$(timeout "$ARM_TIMEOUT_S" "$CONDUCTOR" arm --project "$PROJECT" 2>&1)
|
|
345
|
+
code=$?
|
|
346
|
+
if (( code == 0 )); then
|
|
347
|
+
printf 're-armed %s via the claim-only proof\n' "$PROJECT"
|
|
348
|
+
return 0
|
|
349
|
+
fi
|
|
350
|
+
printf 're-arm of %s failed (exit %s): %s\n' "$PROJECT" "$code" "$out"
|
|
351
|
+
return 0
|
|
352
|
+
}
|
|
353
|
+
|
|
301
354
|
# --------------------------------------------------------------------------
|
|
302
355
|
# main
|
|
303
356
|
# --------------------------------------------------------------------------
|
|
@@ -318,6 +371,13 @@ run_recovery() {
|
|
|
318
371
|
IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
|
|
319
372
|
say "plan: failed unit $failed; action: $action_name"
|
|
320
373
|
fi
|
|
374
|
+
if [[ -z $PROJECT ]]; then
|
|
375
|
+
say "plan: re-arm — none (RECOVER_PROJECT unset)"
|
|
376
|
+
elif project_arms_claim_only "$CONFIG_PATH"; then
|
|
377
|
+
say "plan: re-arm — $CONDUCTOR arm --project $PROJECT (claim-only proof, bounded by ${ARM_TIMEOUT_S}s)"
|
|
378
|
+
else
|
|
379
|
+
say "plan: re-arm — none ($PROJECT keeps the challenge proof; arming is an explicit human gate)"
|
|
380
|
+
fi
|
|
321
381
|
say "plan: escalation — $(escalation_text "$failed" "${action_name:-none}" "dry run" "DRY-RUN")"
|
|
322
382
|
return 0
|
|
323
383
|
fi
|
|
@@ -411,6 +471,19 @@ run_recovery() {
|
|
|
411
471
|
bump_failed_attempts
|
|
412
472
|
fi
|
|
413
473
|
|
|
474
|
+
# The one follow-on a verified restoration earns (#613): re-arm the scoped
|
|
475
|
+
# project through the ordinary `arm` command. Claim-only completes headless;
|
|
476
|
+
# a challenge project stays an explicit human gate and the escalation below
|
|
477
|
+
# says so plainly. Only after `verify` passed — writing an arm marker for a
|
|
478
|
+
# fleet that is still down is the improvisation this playbook never does.
|
|
479
|
+
local rearm_line="not attempted (recovery did not verify the fleet restored)"
|
|
480
|
+
if (( ok == 0 )); then
|
|
481
|
+
rearm_line=$(rearm_if_claim_only)
|
|
482
|
+
fi
|
|
483
|
+
say "re-arm: $rearm_line"
|
|
484
|
+
append_evidence "$ev" "re-arm: $rearm_line"
|
|
485
|
+
outcome="$outcome; re-arm: $rearm_line"
|
|
486
|
+
|
|
414
487
|
if escalate "$failed" "$action_name" "$outcome" "$ev"; then
|
|
415
488
|
if (( ok == 0 )); then
|
|
416
489
|
return 0
|
|
@@ -97,6 +97,9 @@ case "$1" in
|
|
|
97
97
|
--version)
|
|
98
98
|
echo "0.15.11"
|
|
99
99
|
;;
|
|
100
|
+
arm)
|
|
101
|
+
echo "armed demo via fixture"
|
|
102
|
+
;;
|
|
100
103
|
report)
|
|
101
104
|
echo "report 485-abc queued for demo (tier2)"
|
|
102
105
|
;;
|
|
@@ -147,6 +150,12 @@ marker() { # <case-dir> -> attempts count or 0
|
|
|
147
150
|
fi
|
|
148
151
|
}
|
|
149
152
|
|
|
153
|
+
# The minimal machine-written config shape the re-arm decision greps: the arm
|
|
154
|
+
# block with the chosen proof, as `saveConfig` would write it.
|
|
155
|
+
write_project_config() { # <case-dir> <proof>
|
|
156
|
+
printf '{\n "version": 2,\n "projects": [\n { "name": "demo", "arm": { "proof": "%s" } }\n ]\n}\n' "$2" >"$1/state/config.json"
|
|
157
|
+
}
|
|
158
|
+
|
|
150
159
|
run_recover() { # <case-dir> [env assignments…]
|
|
151
160
|
local d="$1"
|
|
152
161
|
shift
|
|
@@ -300,6 +309,58 @@ check 'ROLLBACK escalates as well' \
|
|
|
300
309
|
check 'the pre-upgrade snapshot survives the rollback for the operator' \
|
|
301
310
|
'{"preUpgrade":true}' "$(cat "$d/state/backups/config"/config.json.pre-upgrade-*)"
|
|
302
311
|
|
|
312
|
+
# ---------------------------------------------------------------------------
|
|
313
|
+
# case — the post-restore re-arm (#613): a verified restoration re-arms a
|
|
314
|
+
# claim-only project through the ordinary arm command; a challenge project is
|
|
315
|
+
# never poked and the escalation says the human gate is explicit.
|
|
316
|
+
# ---------------------------------------------------------------------------
|
|
317
|
+
|
|
318
|
+
d=$(newcase rearm-claim-only)
|
|
319
|
+
write_project_config "$d" claim-only
|
|
320
|
+
unit_state "$d" herdr-fleet.service failed
|
|
321
|
+
unit_state "$d" omp-conductor.service active
|
|
322
|
+
code=$(run_with_heal "$d" herdr-fleet.service)
|
|
323
|
+
check 'claim-only re-arm exits 0 when the unit heals' '0' "$code"
|
|
324
|
+
check 'claim-only recovery re-arms via the ordinary arm path' \
|
|
325
|
+
'omp-conductor arm --project demo' "$(grep '^omp-conductor arm' "$d/calls")"
|
|
326
|
+
check 'the escalation names the claim-only re-arm' \
|
|
327
|
+
're-armed demo via the claim-only proof' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
|
|
328
|
+
check 'the evidence bundle records the re-arm' \
|
|
329
|
+
're-arm: re-armed demo via the claim-only proof' "$(cat "$d/state/recovery"/evidence-*)"
|
|
330
|
+
|
|
331
|
+
d=$(newcase rearm-challenge)
|
|
332
|
+
write_project_config "$d" challenge
|
|
333
|
+
unit_state "$d" herdr-fleet.service failed
|
|
334
|
+
unit_state "$d" omp-conductor.service active
|
|
335
|
+
code=$(run_with_heal "$d" herdr-fleet.service)
|
|
336
|
+
check 'a challenge project exits 0 when the unit heals' '0' "$code"
|
|
337
|
+
check 'a challenge project is never poked with arm' \
|
|
338
|
+
'' "$(grep '^omp-conductor arm' "$d/calls" || true)"
|
|
339
|
+
check 'the escalation says the challenge gate is explicit' \
|
|
340
|
+
'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
|
|
341
|
+
|
|
342
|
+
# A project whose config predates the key reads as challenge: no "arm" block.
|
|
343
|
+
d=$(newcase rearm-legacy-config)
|
|
344
|
+
unit_state "$d" herdr-fleet.service failed
|
|
345
|
+
unit_state "$d" omp-conductor.service active
|
|
346
|
+
code=$(run_with_heal "$d" herdr-fleet.service)
|
|
347
|
+
check 'a legacy config without arm.proof is never poked with arm' \
|
|
348
|
+
'' "$(grep '^omp-conductor arm' "$d/calls" || true)"
|
|
349
|
+
check 'a legacy config keeps the human gate in the escalation' \
|
|
350
|
+
'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
|
|
351
|
+
|
|
352
|
+
# RESTART_PEER never verifies a restoration, so even a claim-only project must
|
|
353
|
+
# not be re-armed while the daemon is still down.
|
|
354
|
+
d=$(newcase rearm-unverified)
|
|
355
|
+
write_project_config "$d" claim-only
|
|
356
|
+
unit_state "$d" omp-conductor.service failed
|
|
357
|
+
unit_state "$d" herdr-fleet.service active
|
|
358
|
+
code=$(run_recover "$d")
|
|
359
|
+
check 'an unverified recovery never re-arms' \
|
|
360
|
+
're-arm: not attempted' "$(grep 're-arm:' "$d/out" | head -n 1)"
|
|
361
|
+
check 'an unverified recovery calls no arm' \
|
|
362
|
+
'' "$(grep '^omp-conductor arm' "$d/calls" || true)"
|
|
363
|
+
|
|
303
364
|
# ---------------------------------------------------------------------------
|
|
304
365
|
# case RESTART_PEER and the bound: attempts cap at two, then escalate-only.
|
|
305
366
|
# A daemon crash with no upgrade evidence must never get a blind daemon start.
|