omp-conductor 0.18.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/REFERENCE.md +60 -10
- package/agents/to-spec.md +90 -0
- package/package.json +2 -1
- package/schema/config.schema.json +29 -0
- package/src/admission.ts +204 -75
- package/src/ask.ts +268 -7
- package/src/board.ts +17 -3
- package/src/briefs/orchestrator.md +42 -14
- package/src/briefs/to-spec.md +84 -0
- package/src/briefs/worker.md +2 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +11 -0
- package/src/command-manifest.ts +22 -0
- package/src/commands/context.ts +1 -0
- package/src/commands/drain.ts +176 -0
- package/src/commands/extend.ts +6 -10
- package/src/commands/status.ts +5 -1
- package/src/commands/watch.ts +50 -2
- package/src/commands/worker.ts +9 -10
- package/src/config-schema.ts +24 -0
- package/src/config.ts +42 -1
- package/src/daemon.ts +965 -36
- package/src/dashboard/app.js +4 -1
- package/src/dashboard/server.ts +5 -2
- package/src/decisions.ts +235 -17
- package/src/diff-flags.ts +75 -1
- package/src/doctor.ts +52 -0
- package/src/escalate.ts +9 -3
- package/src/failure-class.ts +28 -2
- package/src/fleet.ts +146 -22
- package/src/gitops.ts +188 -81
- package/src/graph-health.ts +35 -1
- package/src/graph.ts +66 -1
- package/src/harness-loader.ts +59 -0
- package/src/host.ts +567 -2
- package/src/lifecycle.ts +122 -1
- package/src/omp.ts +227 -20
- package/src/orchestrator-tick.ts +1386 -15
- package/src/orchestrator.ts +12 -0
- package/src/privileged.ts +1 -4
- package/src/release-policy.ts +503 -9
- package/src/session-host.ts +99 -5
- package/src/settlement.ts +69 -17
- package/src/setup-host.ts +1205 -6
- package/src/setup-install.ts +28 -0
- package/src/setup-wizard.ts +13 -2
- package/src/setup.ts +29 -13
- package/src/shell.ts +15 -0
- package/src/status-render.ts +78 -11
- package/src/store.ts +443 -42
- package/src/to-spec.ts +387 -0
- package/src/tracker/github.ts +104 -14
- package/src/types.ts +343 -13
- package/src/upgrade-verify.ts +209 -2
- package/src/upgrade.ts +175 -1
- package/src/verbs/protocol.ts +39 -0
- package/src/verbs/server.ts +730 -56
- package/src/verbs/socket.ts +24 -5
- package/src/worker.ts +25 -2
- package/src/worktree.ts +29 -12
package/src/dashboard/app.js
CHANGED
|
@@ -35,6 +35,7 @@ const STATE_TEXT = {
|
|
|
35
35
|
ok: "daemon up",
|
|
36
36
|
stopped: "daemon down",
|
|
37
37
|
unreachable: "daemon down — not answering /healthz",
|
|
38
|
+
unresponsive: "daemon up — healthz timed out",
|
|
38
39
|
"other-project": "daemon down — serves another project",
|
|
39
40
|
};
|
|
40
41
|
|
|
@@ -239,7 +240,9 @@ async function openProject(name) {
|
|
|
239
240
|
projectDaemon.textContent = daemonLine(row);
|
|
240
241
|
projectDaemon.classList.toggle(
|
|
241
242
|
"state-degraded",
|
|
242
|
-
row.daemon.state === "unreachable" ||
|
|
243
|
+
row.daemon.state === "unreachable" ||
|
|
244
|
+
row.daemon.state === "unresponsive" ||
|
|
245
|
+
row.daemon.state === "other-project",
|
|
243
246
|
);
|
|
244
247
|
}
|
|
245
248
|
await refreshProject("board");
|
package/src/dashboard/server.ts
CHANGED
|
@@ -114,7 +114,7 @@ export interface DashboardProjectView {
|
|
|
114
114
|
repo: string;
|
|
115
115
|
daemon: {
|
|
116
116
|
/** The shared verdict every viewer uses (#379). */
|
|
117
|
-
state: "ok" | "stopped" | "unreachable" | "other-project";
|
|
117
|
+
state: "ok" | "stopped" | "unreachable" | "unresponsive" | "other-project";
|
|
118
118
|
/** The living daemon's port when a record exists; null when there is none. */
|
|
119
119
|
port: number | null;
|
|
120
120
|
/** The raw `/healthz` body, parsed, when this project's daemon answered it. */
|
|
@@ -210,7 +210,10 @@ export async function fleetOverview(): Promise<FleetOverviewRow[]> {
|
|
|
210
210
|
liveWorkers: snap.liveWorkers,
|
|
211
211
|
spendTodayUsd: snap.spendTodayUsd,
|
|
212
212
|
baseHealth: snap.baseHealth,
|
|
213
|
-
degraded:
|
|
213
|
+
degraded:
|
|
214
|
+
view.daemon.state === "unreachable" ||
|
|
215
|
+
view.daemon.state === "unresponsive" ||
|
|
216
|
+
view.daemon.state === "other-project",
|
|
214
217
|
};
|
|
215
218
|
});
|
|
216
219
|
}
|
package/src/decisions.ts
CHANGED
|
@@ -12,13 +12,14 @@
|
|
|
12
12
|
* open row is rendered into a tick prompt.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import type { DecisionRecord, Store, Tracker } from "./types.ts";
|
|
15
|
+
import type { DecisionRecord, RunRecord, Store, Tracker } from "./types.ts";
|
|
16
|
+
import { shellQuote } from "./shell.ts";
|
|
16
17
|
import { fetchRateLimit, RATE_LIMIT_COOLDOWN_MS } from "./tracker/github.ts";
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* A precondition whose truth this package can check on its own.
|
|
20
21
|
*
|
|
21
|
-
* Exactly
|
|
22
|
+
* Exactly seven kinds, deliberately. Each one is a question the tracker or npm
|
|
22
23
|
* already answers, so the row moves from "parked" to "act on this" without a
|
|
23
24
|
* human re-reading it. Anything richer — a label appearing, a workflow going
|
|
24
25
|
* green — is a follow-on issue rather than a grammar nobody validated.
|
|
@@ -28,6 +29,7 @@ export type DecisionCondition =
|
|
|
28
29
|
| { kind: "issue-closed"; issue: number }
|
|
29
30
|
| { kind: "npm-version"; spec: string }
|
|
30
31
|
| { kind: "pr-checks-green"; url: string }
|
|
32
|
+
| { kind: "pr-review-ready"; url: string }
|
|
31
33
|
| { kind: "pr-mergeable"; url: string }
|
|
32
34
|
| { kind: "rate-limit-reset" };
|
|
33
35
|
|
|
@@ -42,6 +44,64 @@ const NPM_SPEC = /^(@[^/@\s]+\/)?[^@\s]+@[^\s]+$/;
|
|
|
42
44
|
*/
|
|
43
45
|
const GREEN_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
|
|
44
46
|
|
|
47
|
+
/**
|
|
48
|
+
* The run states a review revision may start from (#795) — the single
|
|
49
|
+
* definition shared by `conductor_pr_review` and the `pr-review-ready` watch,
|
|
50
|
+
* so the verb's gate and the condition can never name different sets.
|
|
51
|
+
*
|
|
52
|
+
* A revision round resumes the exact run whose row owns the PR, so the
|
|
53
|
+
* revisable states are exactly the terminal runs that pushed one: a settled
|
|
54
|
+
* `pushed-green` row, or a `failed` / `killed` row — a run that capped or
|
|
55
|
+
* failed *after* pushing a green PR. The PR is the durable artefact, the
|
|
56
|
+
* exact-head green verification is the gate on "green at the reviewed SHA",
|
|
57
|
+
* and a terminal row proves no worker is in flight, so findings are returned
|
|
58
|
+
* without the close-PR → unblock → continuation dance.
|
|
59
|
+
*
|
|
60
|
+
* Closed on purpose: a live row (`running` / `claimed`) is already doing its
|
|
61
|
+
* own work, a `pushed-pending` PR is not green yet, and a `blocked` /
|
|
62
|
+
* `orphaned` / `stopped` / `merged` row is not work returned for revision.
|
|
63
|
+
*/
|
|
64
|
+
export const REVISABLE_RUN_STATES: Record<string, true> = {
|
|
65
|
+
"pushed-green": true,
|
|
66
|
+
failed: true,
|
|
67
|
+
killed: true,
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* How far back a PR's project-owned run may be looked up. PR verbs and the
|
|
72
|
+
* `pr-review-ready` watch share this bound: a pull request older than it is
|
|
73
|
+
* not one an orchestrator is mid-flight on.
|
|
74
|
+
*/
|
|
75
|
+
export const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* One predicate for "a run-owned pull request can be returned for revision",
|
|
79
|
+
* read by both `conductor_pr_review` and the `pr-review-ready` watch so they
|
|
80
|
+
* cannot drift.
|
|
81
|
+
*
|
|
82
|
+
* Ownership is the same newest-first resolution the review verb uses: the
|
|
83
|
+
* newest attempt of this project that recorded the PR within the
|
|
84
|
+
* recent-history window (`runsForProjectPr`), never the issue's newest row —
|
|
85
|
+
* a requeued continuation must not hide the PR its predecessor opened
|
|
86
|
+
* (#434), and a newer live owner hides an older settled one. That older
|
|
87
|
+
* settled row is exactly the shape dogfood #844 started from: PR #838 was
|
|
88
|
+
* green while its newest owner was still `running`, and a watch keyed only to
|
|
89
|
+
* checks woke the orchestrator before `conductor_pr_review` was actionable.
|
|
90
|
+
*
|
|
91
|
+
* `no-owner` and `not-revisable` both fail closed: a review can never act, so
|
|
92
|
+
* a watch must not wake, even when the checks are green.
|
|
93
|
+
*/
|
|
94
|
+
export type PrReviewReadiness =
|
|
95
|
+
| { kind: "ready"; run: RunRecord }
|
|
96
|
+
| { kind: "no-owner" }
|
|
97
|
+
| { kind: "not-revisable"; run: RunRecord };
|
|
98
|
+
|
|
99
|
+
export function prReviewReadiness(store: Store, project: string, prUrl: string, now: number): PrReviewReadiness {
|
|
100
|
+
const run = store.runsForProjectPr(project, prUrl, now - PR_LOOKUP_WINDOW_MS)[0];
|
|
101
|
+
if (run === undefined) return { kind: "no-owner" };
|
|
102
|
+
return REVISABLE_RUN_STATES[run.state] === true ? { kind: "ready", run } : { kind: "not-revisable", run };
|
|
103
|
+
}
|
|
104
|
+
|
|
45
105
|
/**
|
|
46
106
|
* Parse a raw condition, or `undefined` when it is not one of the three forms.
|
|
47
107
|
*
|
|
@@ -58,7 +118,7 @@ export function parseCondition(raw: string): DecisionCondition | undefined {
|
|
|
58
118
|
const rest = text.slice(at + 1).trim();
|
|
59
119
|
if (rest.length === 0) return undefined;
|
|
60
120
|
|
|
61
|
-
if (kind === "pr-merged" || kind === "pr-checks-green" || kind === "pr-mergeable") {
|
|
121
|
+
if (kind === "pr-merged" || kind === "pr-checks-green" || kind === "pr-review-ready" || kind === "pr-mergeable") {
|
|
62
122
|
return rest.startsWith("https://") ? { kind, url: rest } : undefined;
|
|
63
123
|
}
|
|
64
124
|
if (kind === "issue-closed") {
|
|
@@ -77,12 +137,13 @@ export function parseCondition(raw: string): DecisionCondition | undefined {
|
|
|
77
137
|
return undefined;
|
|
78
138
|
}
|
|
79
139
|
|
|
80
|
-
/** The
|
|
140
|
+
/** The seven accepted forms, for a refusal that can be acted on in one turn. */
|
|
81
141
|
export const CONDITION_FORMS = [
|
|
82
142
|
"pr-merged:https://github.com/owner/repo/pull/123",
|
|
83
143
|
"issue-closed:123",
|
|
84
144
|
"npm-version:omp-conductor@0.4.3",
|
|
85
145
|
"pr-checks-green:https://github.com/owner/repo/pull/123",
|
|
146
|
+
"pr-review-ready:https://github.com/owner/repo/pull/123",
|
|
86
147
|
"pr-mergeable:https://github.com/owner/repo/pull/123",
|
|
87
148
|
"rate-limit-reset:github",
|
|
88
149
|
] as const;
|
|
@@ -154,7 +215,8 @@ export async function probeRateLimitReset(runGh: RateLimitGh = ghRateLimit): Pro
|
|
|
154
215
|
}
|
|
155
216
|
|
|
156
217
|
/**
|
|
157
|
-
* Check every open decision that carries a condition
|
|
218
|
+
* Check every open decision that carries a condition; a met row is only
|
|
219
|
+
* revisited when its condition can stop holding (`pr-checks-green`, #808).
|
|
158
220
|
*
|
|
159
221
|
* Errors are swallowed per row on purpose: this runs fire-and-forget beside a
|
|
160
222
|
* tick, and one deleted PR or one flaky `gh` call must not stop the rest of the
|
|
@@ -171,6 +233,16 @@ export async function probeRateLimitReset(runGh: RateLimitGh = ghRateLimit): Pro
|
|
|
171
233
|
* was waiting and why it stopped survives. Operator questions are untouched:
|
|
172
234
|
* they answer to a human and expire on the seven-day clock.
|
|
173
235
|
*
|
|
236
|
+
* A met `pr-checks-green` / `pr-review-ready` row is the exception to "met is
|
|
237
|
+
* final" (#808): both are verdicts *about a commit*, so the met state is bound
|
|
238
|
+
* to the head it was observed at and re-read here. A head change drops the met
|
|
239
|
+
* state (and with it the binding) and the row re-evaluates against the live
|
|
240
|
+
* head — it cannot render `[CONDITION MET]` for a head whose checks are still
|
|
241
|
+
* running or have gone red. A row met before head binding existed has no
|
|
242
|
+
* binding to compare and is preserved as met as-is: nothing is guessed for it,
|
|
243
|
+
* and it is never withdrawn with the never-met reason. Every other condition
|
|
244
|
+
* is monotone or human-answered and keeps its permanent met transition.
|
|
245
|
+
*
|
|
174
246
|
* Returns the rows that just became met, so the caller can log what changed
|
|
175
247
|
* rather than a count.
|
|
176
248
|
*/
|
|
@@ -183,10 +255,53 @@ export async function evaluateDecisionConditions(
|
|
|
183
255
|
): Promise<DecisionRecord[]> {
|
|
184
256
|
const met: DecisionRecord[] = [];
|
|
185
257
|
for (const decision of store.openDecisions(project)) {
|
|
186
|
-
if (decision.condition === undefined
|
|
258
|
+
if (decision.condition === undefined) continue;
|
|
187
259
|
const condition = parseCondition(decision.condition);
|
|
188
260
|
if (condition === undefined) continue;
|
|
261
|
+
if (decision.conditionMetAt !== undefined) {
|
|
262
|
+
// A met row is normally final: the digest's "act on this now" is the
|
|
263
|
+
// whole point, and flapping a verdict would reopen a question the
|
|
264
|
+
// orchestrator may already be acting on. The two head-bound conditions
|
|
265
|
+
// — `pr-checks-green` and `pr-review-ready` — are the ones whose
|
|
266
|
+
// observation can stop holding: they are about a commit, and a PR head
|
|
267
|
+
// change makes the old verdict stale.
|
|
268
|
+
if (condition.kind !== "pr-checks-green" && condition.kind !== "pr-review-ready") continue;
|
|
269
|
+
// A row met before head binding existed has no binding to compare and no
|
|
270
|
+
// way to prove a head moved. It is preserved as met rather than cleared
|
|
271
|
+
// and re-verified: a clear would let the settled-PR path below withdraw
|
|
272
|
+
// it as "before its checks were seen", which the durable ledger itself
|
|
273
|
+
// contradicts (review round 2, #808). No head is ever guessed for it.
|
|
274
|
+
if (decision.conditionHead === undefined) continue;
|
|
275
|
+
let head: string | undefined;
|
|
276
|
+
try {
|
|
277
|
+
head = await tracker.prHead(condition.url);
|
|
278
|
+
} catch {
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
// An unreadable head is not proof a head changed: flip the row only on a
|
|
282
|
+
// definitive observation, or it flaps on every flaky `gh` call.
|
|
283
|
+
if (head === undefined) continue;
|
|
284
|
+
if (decision.conditionHead === head) {
|
|
285
|
+
// A stable head is the whole story for `pr-checks-green` — checks are
|
|
286
|
+
// a verdict about a commit, and the commit did not move. For
|
|
287
|
+
// `pr-review-ready` it is only half: the row also gates on which run
|
|
288
|
+
// owns the PR, and a review revision resumes the owner to live
|
|
289
|
+
// (`running` / `claimed`) without moving the head. So a met
|
|
290
|
+
// review-ready row falls through to the branch below, whose shared
|
|
291
|
+
// readiness gate clears it the moment the verb can no longer accept —
|
|
292
|
+
// the repeated wake this condition exists to prevent (#844 review 1).
|
|
293
|
+
if (condition.kind === "pr-checks-green") continue;
|
|
294
|
+
} else {
|
|
295
|
+
// The head moved — or the row predates head binding — so the old met
|
|
296
|
+
// observation is no longer the PR's state. Clear it and let the pass
|
|
297
|
+
// below judge the live head: still green, the row re-marks with the
|
|
298
|
+
// new binding and wakes the orchestrator; pending or red, it stays
|
|
299
|
+
// pending until it is not.
|
|
300
|
+
store.clearDecisionConditionMet(decision.id);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
189
303
|
let satisfied = false;
|
|
304
|
+
let satisfiedHead: string | undefined;
|
|
190
305
|
try {
|
|
191
306
|
if (condition.kind === "pr-merged") {
|
|
192
307
|
const state = await tracker.prState(condition.url);
|
|
@@ -204,21 +319,115 @@ export async function evaluateDecisionConditions(
|
|
|
204
319
|
} else if (condition.kind === "pr-checks-green") {
|
|
205
320
|
// A watch whose PR settled before its checks were seen green cannot
|
|
206
321
|
// ever be met: the checks surface does not answer for a merged or
|
|
207
|
-
// closed PR, so the row would render `pending` forever (#664).
|
|
322
|
+
// closed PR, so the row would render `pending` forever (#664). A row
|
|
323
|
+
// that was already met — then had its head move and its PR settle
|
|
324
|
+
// before the new head's checks — did see its checks green at the
|
|
325
|
+
// earlier head, and its withdrawal must record that, or the durable
|
|
326
|
+
// ledger gains a fact it contradicts (review round 2, #808).
|
|
208
327
|
if (decision.kind === "watch") {
|
|
209
328
|
const state = await tracker.prState(condition.url);
|
|
210
329
|
if (state === "merged" || state === "closed") {
|
|
211
|
-
|
|
330
|
+
const seenGreen = decision.conditionMetAt !== undefined;
|
|
331
|
+
store.resolveDecision(
|
|
332
|
+
decision.id,
|
|
333
|
+
"withdrawn",
|
|
334
|
+
seenGreen ? `PR ${state} after its checks were seen green` : `PR ${state} before its checks were seen`,
|
|
335
|
+
now(),
|
|
336
|
+
);
|
|
212
337
|
continue;
|
|
213
338
|
}
|
|
214
339
|
}
|
|
340
|
+
// The head anchor is read BEFORE the checks: a verdict is only ever
|
|
341
|
+
// bound to a head that did not move while the checks were read, and
|
|
342
|
+
// the anchor-first order is what closes the push race in both
|
|
343
|
+
// directions. Reading the checks first would let a push that landed
|
|
344
|
+
// after the checks answered bind the old head's green verdict to the
|
|
345
|
+
// new, untested commit — both confirming head reads would agree on a
|
|
346
|
+
// head the checks were never read for (review round 1, #808).
|
|
347
|
+
const before = await tracker.prHead(condition.url);
|
|
215
348
|
// The same conclusion values the daemon's failure classifier treats as
|
|
216
349
|
// a green verdict (`success` / `neutral`, lowercased): a non-empty list
|
|
217
350
|
// in which every check is terminally successful and none is failing or
|
|
218
351
|
// pending (#189).
|
|
219
352
|
const checks = await tracker.checkConclusions(condition.url);
|
|
220
|
-
|
|
221
|
-
|
|
353
|
+
if (
|
|
354
|
+
before !== undefined &&
|
|
355
|
+
checks.length > 0 &&
|
|
356
|
+
checks.every((c) => GREEN_CHECK_STATES[c.state.trim().toLowerCase()] === true)
|
|
357
|
+
) {
|
|
358
|
+
// Confirm the anchor still holds after the checks read: a push that
|
|
359
|
+
// landed anywhere inside the window surfaces as a mismatch, so the
|
|
360
|
+
// green list is bound only to a head it was actually read for. A
|
|
361
|
+
// list without a readable, stable anchor is not yet met — an
|
|
362
|
+
// unbound verdict is exactly the staleness this fixes, and the next
|
|
363
|
+
// tick can ask again for free.
|
|
364
|
+
const after = await tracker.prHead(condition.url);
|
|
365
|
+
if (before === after) {
|
|
366
|
+
satisfied = true;
|
|
367
|
+
satisfiedHead = before;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
} else if (condition.kind === "pr-review-ready") {
|
|
371
|
+
// A watch whose PR settled before it was seen review-ready can never
|
|
372
|
+
// be met: neither the checks surface nor a revisable run answers for
|
|
373
|
+
// a merged or closed PR, so the row would render `pending` forever
|
|
374
|
+
// (#664). A row that was already met — then had its head move and its
|
|
375
|
+
// PR settle before the new head's checks — did see its PR ready at
|
|
376
|
+
// the earlier head, and its withdrawal must record that, the same
|
|
377
|
+
// history-preserving convention as pr-checks-green (review round 2,
|
|
378
|
+
// #808).
|
|
379
|
+
if (decision.kind === "watch") {
|
|
380
|
+
const state = await tracker.prState(condition.url);
|
|
381
|
+
if (state === "merged" || state === "closed") {
|
|
382
|
+
const seenReady = decision.conditionMetAt !== undefined;
|
|
383
|
+
store.resolveDecision(
|
|
384
|
+
decision.id,
|
|
385
|
+
"withdrawn",
|
|
386
|
+
seenReady ? `PR ${state} after it was review-ready` : `PR ${state} before it was review-ready`,
|
|
387
|
+
now(),
|
|
388
|
+
);
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
// The store-owned half of review readiness, shared with the verb
|
|
393
|
+
// (#844): the newest project-owned run that recorded this PR must be
|
|
394
|
+
// in a revisable state (`pushed-green` / `failed` / `killed`). A PR
|
|
395
|
+
// with no owner, or whose newest owner is live (`claimed` / `running`
|
|
396
|
+
// — the original worker or an earlier review revision) or otherwise
|
|
397
|
+
// non-revisable, stays pending even while its checks are green: the
|
|
398
|
+
// watch must not wake the orchestrator before `conductor_pr_review`
|
|
399
|
+
// can legally act.
|
|
400
|
+
const readiness = prReviewReadiness(store, project, condition.url, now());
|
|
401
|
+
if (readiness.kind !== "ready") {
|
|
402
|
+
// A met row whose owner stopped being revisable — the revision
|
|
403
|
+
// dispatched and the owner is live again (`running` / `claimed`)
|
|
404
|
+
// or the owner ended non-revisable — returns to pending on the
|
|
405
|
+
// same tick: the watch exists precisely so the orchestrator is
|
|
406
|
+
// not woken while `conductor_pr_review` would refuse. The clear
|
|
407
|
+
// is idempotent, and a revision that settles at the same head
|
|
408
|
+
// re-marks the row on a later tick.
|
|
409
|
+
if (decision.conditionMetAt !== undefined) store.clearDecisionConditionMet(decision.id);
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
// The tracker half, with the same fail-closed exact-head anchor as
|
|
413
|
+
// pr-checks-green (#808): the current head must be readable, every
|
|
414
|
+
// current check terminally green, and the head must not move while
|
|
415
|
+
// the checks were read. An unreadable or unstable head is not yet
|
|
416
|
+
// ready — never a verdict fabricated from a head the checks were not
|
|
417
|
+
// read for.
|
|
418
|
+
const before = await tracker.prHead(condition.url);
|
|
419
|
+
const checks = await tracker.checkConclusions(condition.url);
|
|
420
|
+
if (
|
|
421
|
+
before !== undefined &&
|
|
422
|
+
checks.length > 0 &&
|
|
423
|
+
checks.every((c) => GREEN_CHECK_STATES[c.state.trim().toLowerCase()] === true)
|
|
424
|
+
) {
|
|
425
|
+
const after = await tracker.prHead(condition.url);
|
|
426
|
+
if (before === after) {
|
|
427
|
+
satisfied = true;
|
|
428
|
+
satisfiedHead = before;
|
|
429
|
+
}
|
|
430
|
+
}
|
|
222
431
|
} else if (condition.kind === "pr-mergeable") {
|
|
223
432
|
// A watch whose PR settled before it was seen mergeable is the same
|
|
224
433
|
// unobservable case as checks: the mergeability literal exists only
|
|
@@ -248,8 +457,12 @@ export async function evaluateDecisionConditions(
|
|
|
248
457
|
continue;
|
|
249
458
|
}
|
|
250
459
|
if (!satisfied) continue;
|
|
251
|
-
if (store.markDecisionConditionMet(decision.id, now())) {
|
|
252
|
-
met.push({
|
|
460
|
+
if (store.markDecisionConditionMet(decision.id, now(), satisfiedHead)) {
|
|
461
|
+
met.push({
|
|
462
|
+
...decision,
|
|
463
|
+
conditionMetAt: now(),
|
|
464
|
+
...(satisfiedHead === undefined ? {} : { conditionHead: satisfiedHead }),
|
|
465
|
+
});
|
|
253
466
|
}
|
|
254
467
|
}
|
|
255
468
|
return met;
|
|
@@ -280,8 +493,10 @@ function age(since: number, now: number): string {
|
|
|
280
493
|
* GitHub's checks is never mistaken for a fleet that is waiting on its
|
|
281
494
|
* operator. A met watch still surfaces to the orchestrator with its note and
|
|
282
495
|
* the same `[CONDITION MET]` flag a met question gets. Every watch line also
|
|
283
|
-
* names the command that closes it (`watch withdraw <id>`),
|
|
284
|
-
* surface that shows a watch is where the reader learns how to end
|
|
496
|
+
* names the command that closes it (`watch withdraw <id> --project <project>`),
|
|
497
|
+
* because the surface that shows a watch is where the reader learns how to end
|
|
498
|
+
* one (#664). The row's own project is always named — a digest is per-project,
|
|
499
|
+
* and a bare `watch withdraw <id>` is ambiguous on a host with several (#810).
|
|
285
500
|
*/
|
|
286
501
|
export function formatDecisionDigest(open: readonly DecisionRecord[], now = Date.now()): string {
|
|
287
502
|
const questions = open.filter((d) => d.kind !== "watch");
|
|
@@ -300,9 +515,12 @@ export function formatDecisionDigest(open: readonly DecisionRecord[], now = Date
|
|
|
300
515
|
`Watches (${watches.length}) — conditions the orchestrator set for itself; no operator action needed:`,
|
|
301
516
|
);
|
|
302
517
|
for (const d of watches) {
|
|
303
|
-
// Each line names the verb that closes it,
|
|
304
|
-
//
|
|
305
|
-
|
|
518
|
+
// Each line names the verb that closes it, qualified by the row's stored
|
|
519
|
+
// project so a copied command runs verbatim on a multi-project host
|
|
520
|
+
// (#810). The rows are project-scoped by construction; the stored
|
|
521
|
+
// project is the one the command must name, shell-quoted because a
|
|
522
|
+
// project name may contain characters a shell would read.
|
|
523
|
+
lines.push(decisionLine(d, now, `watch withdraw ${d.id} --project ${shellQuote(d.project)}`));
|
|
306
524
|
}
|
|
307
525
|
}
|
|
308
526
|
return lines.join("\n");
|
package/src/diff-flags.ts
CHANGED
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
* repository.
|
|
38
38
|
*/
|
|
39
39
|
|
|
40
|
-
import type { PrDiff, PrDiffFile, SettlementFlag } from "./types.ts";
|
|
40
|
+
import type { FileLane, PrDiff, PrDiffFile, SettlementFlag } from "./types.ts";
|
|
41
41
|
|
|
42
42
|
// ------------------------------------------------------------------ diff parse
|
|
43
43
|
|
|
@@ -504,6 +504,15 @@ export interface SettlementAudit {
|
|
|
504
504
|
* had none or it could not be read. Absent means no claimed command can be
|
|
505
505
|
* checked — same silence, for the same reason. */
|
|
506
506
|
transcript?: string;
|
|
507
|
+
/**
|
|
508
|
+
* The effective file lane admission resolved for this run at dispatch
|
|
509
|
+
* (`effectiveLane` — a pre-dispatch comment declaration supersedes an older
|
|
510
|
+
* body one) and persisted on the row (#744/#758): the exact snapshot the
|
|
511
|
+
* gate enforced and the worker brief rendered. Absent means the issue was
|
|
512
|
+
* admitted with no declaration (fail open) — never "empty lane" — and no
|
|
513
|
+
* diff can then be outside it.
|
|
514
|
+
*/
|
|
515
|
+
lane?: FileLane;
|
|
507
516
|
}
|
|
508
517
|
|
|
509
518
|
/**
|
|
@@ -522,6 +531,7 @@ export function analyseSettlement(audit: SettlementAudit): SettlementFlag[] {
|
|
|
522
531
|
const flags: SettlementFlag[] = [];
|
|
523
532
|
detectWeakening(audit, flags);
|
|
524
533
|
detectClaimedProof(audit, flags);
|
|
534
|
+
detectLaneEscape(audit, flags);
|
|
525
535
|
return flags;
|
|
526
536
|
}
|
|
527
537
|
|
|
@@ -538,6 +548,70 @@ export const UNREADABLE_TREE_FLAG: SettlementFlag = {
|
|
|
538
548
|
"the settlement could not read the PR's diff, so no `changed:` file list could be derived",
|
|
539
549
|
};
|
|
540
550
|
|
|
551
|
+
// ---------------------------------------------------------- declared file lane
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* Whether a diff path counts as inside the declared lane: an explicitly
|
|
555
|
+
* declared path, or the co-located test of one — `foo.ts` vouches for
|
|
556
|
+
* `foo.test.ts`, which is the "obviously intended" case. Other test shapes
|
|
557
|
+
* (`.spec.ts`, pytest's `test_` prefix) are not vouched for: the rule is the
|
|
558
|
+
* shape the fleet actually uses, and a lane that wants a differently-shaped
|
|
559
|
+
* sibling declares it. Deliberately one-directional: a lane that declares a
|
|
560
|
+
* *test* file does not vouch for its source, because declaring the test alone
|
|
561
|
+
* is a narrower promise and widening it silently is exactly what this flag
|
|
562
|
+
* exists to name. A containing directory never vouches for its contents
|
|
563
|
+
* either — the lane grammar names files, and a lane that means "everything
|
|
564
|
+
* under `src/`" fails open exactly as an undeclared one would if it cannot
|
|
565
|
+
* name them.
|
|
566
|
+
*/
|
|
567
|
+
function withinLane(path: string, declared: readonly string[]): boolean {
|
|
568
|
+
if (declared.includes(path)) return true;
|
|
569
|
+
for (const d of declared) {
|
|
570
|
+
if (coLocatedTest(d) === path) return true;
|
|
571
|
+
}
|
|
572
|
+
return false;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/** `omp/src/verbs/server.ts` → `omp/src/verbs/server.test.ts`; undefined when
|
|
576
|
+
* the declared path has no file extension to splice before, since the
|
|
577
|
+
* co-located-test shape is defined by an extension. */
|
|
578
|
+
function coLocatedTest(declared: string): string | undefined {
|
|
579
|
+
const dot = declared.lastIndexOf(".");
|
|
580
|
+
const slash = declared.lastIndexOf("/");
|
|
581
|
+
if (dot <= slash + 1) return undefined;
|
|
582
|
+
return `${declared.slice(0, dot)}.test${declared.slice(dot)}`;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* A diff that escapes its issue's declared file lane (#739).
|
|
587
|
+
*
|
|
588
|
+
* The lane is the effective declaration admission resolved at dispatch —
|
|
589
|
+
* `effectiveLane(body, comments)`, so a pre-dispatch comment beats an older
|
|
590
|
+
* body declaration — and the flag uses that resolved snapshot, never a re-parse
|
|
591
|
+
* of the body. The finding names every delivered file outside it, which is the
|
|
592
|
+
* part a reviewer is worst placed to notice: the diff's own file list is the
|
|
593
|
+
* only surface that shows the escape, and reading PR file lists by hand is
|
|
594
|
+
* exactly what nothing else in the loop does.
|
|
595
|
+
*
|
|
596
|
+
* Fail-open, like admission: an issue with no lane declaration has nothing to
|
|
597
|
+
* escape, so no flag — a flag on every undeclared run would be noise within a
|
|
598
|
+
* day, worse than no flag. Advisory like every other flag here: a widened lane
|
|
599
|
+
* is often legitimate, and the orchestrator is the judge this exists to brief.
|
|
600
|
+
*/
|
|
601
|
+
function detectLaneEscape(audit: SettlementAudit, flags: SettlementFlag[]): void {
|
|
602
|
+
const lane = audit.lane;
|
|
603
|
+
if (lane === undefined || lane.files.length === 0) return;
|
|
604
|
+
const outside = audit.diff.files
|
|
605
|
+
.map((f) => f.path)
|
|
606
|
+
.filter((path) => !withinLane(path, lane.files));
|
|
607
|
+
if (outside.length === 0) return;
|
|
608
|
+
flags.push({
|
|
609
|
+
kind: "lane-escape",
|
|
610
|
+
file: "(lane)",
|
|
611
|
+
detail: `PR diff touches files outside the declared file lane: ${outside.join(", ")}`,
|
|
612
|
+
});
|
|
613
|
+
}
|
|
614
|
+
|
|
541
615
|
function detectWeakening(audit: SettlementAudit, flags: SettlementFlag[]): void {
|
|
542
616
|
// A test file that left one path and arrived at another is a move, not a
|
|
543
617
|
// deletion. `status: renamed` covers the renames git detected; the basename
|
package/src/doctor.ts
CHANGED
|
@@ -85,7 +85,10 @@ import {
|
|
|
85
85
|
SYSTEMD_UNIT_DIR,
|
|
86
86
|
tickCwdForProject,
|
|
87
87
|
totalConfiguredWorkers,
|
|
88
|
+
workerAclHealth,
|
|
89
|
+
type WorkerAclHealth,
|
|
88
90
|
} from "./setup-host.ts";
|
|
91
|
+
import { WORKER_ACCOUNT } from "./host.ts";
|
|
89
92
|
import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
|
|
90
93
|
import { DB_SNAPSHOT_STEM, dbPath, LIVE_STATES } from "./store.ts";
|
|
91
94
|
import { telegramReportSend, type ReportSend } from "./reports.ts";
|
|
@@ -142,6 +145,10 @@ const INCIDENTS = {
|
|
|
142
145
|
spend:
|
|
143
146
|
"spend telemetry was once absent, so the USD cap never fired — $0.00 spend is not proof of no spend",
|
|
144
147
|
ghauth: "gh auth expired under a live daemon and every tracker call failed silently",
|
|
148
|
+
workerAcl:
|
|
149
|
+
"the #835 incident: a setup host granted the worker's path ACLs before restarting the fleet, an OMP startup chmod'd " +
|
|
150
|
+
"the agent config dir back to 0700 and rewrote the ACL mask, and the next two admitted workers died on EACCES " +
|
|
151
|
+
"before connecting — the named ACL entry was still there, only its effective permissions were gone",
|
|
145
152
|
} as const;
|
|
146
153
|
|
|
147
154
|
// ------------------------------------------------------------------ dependencies
|
|
@@ -268,6 +275,13 @@ export interface DoctorDeps {
|
|
|
268
275
|
tickAgentName?: (project: ProjectConfig) => string | undefined;
|
|
269
276
|
/** Clock, so a run is deterministic in tests. */
|
|
270
277
|
now?: () => number;
|
|
278
|
+
/** The linked worker config paths' effective-ACL verdict for the dedicated
|
|
279
|
+
* worker account (#835): `checkable` paths judged, `missing` the ones that
|
|
280
|
+
* do not currently grant the worker's needed effective access. Read-only
|
|
281
|
+
* and injectable so no test ever touches the host's ACLs; the production
|
|
282
|
+
* wiring is the identity plan's own getfacl probe, so `doctor` and
|
|
283
|
+
* `setup host` cannot disagree about an ACL. */
|
|
284
|
+
workerAclHealth?: () => WorkerAclHealth;
|
|
271
285
|
/** The one opt-in side effect: send one self-identified Telegram probe. */
|
|
272
286
|
probeTelegram?: boolean;
|
|
273
287
|
}
|
|
@@ -902,6 +916,37 @@ function ownershipProbe(probes: Probes): Finding {
|
|
|
902
916
|
return failFinding("systemd-ownership", summary, fix);
|
|
903
917
|
}
|
|
904
918
|
|
|
919
|
+
/**
|
|
920
|
+
* The dedicated worker account's path ACLs (#835): every linked worker config
|
|
921
|
+
* path must grant the worker its *effective* search/read permissions. The
|
|
922
|
+
* installed OMP harness chmods its agent config dir back to 0700 on every
|
|
923
|
+
* open, and a chmod rewrites the ACL mask — the named entry survives as
|
|
924
|
+
* `user:omp-worker:--x #effective:---` while the worker's access silently
|
|
925
|
+
* vanishes, which is exactly the incident this finding exists to catch
|
|
926
|
+
* (`${INCIDENTS.workerAcl}`). The verdict comes from the identity plan's own
|
|
927
|
+
* probe through the read-only {@link DoctorDeps.workerAclHealth} seam, so a
|
|
928
|
+
* pass here is a pass the plan agrees with and a fail names the paths whose
|
|
929
|
+
* mask is stripping the grant.
|
|
930
|
+
*/
|
|
931
|
+
function workerAclProbeFinding(probes: Probes): Finding {
|
|
932
|
+
const health = probes.workerAclHealth();
|
|
933
|
+
if (health.checkable === 0) {
|
|
934
|
+
return passFinding("worker-acl", "no linked worker config paths on this host yet — nothing to check");
|
|
935
|
+
}
|
|
936
|
+
if (health.missing.length === 0) {
|
|
937
|
+
return passFinding(
|
|
938
|
+
"worker-acl",
|
|
939
|
+
`${health.checkable} linked worker config path(s) grant ${WORKER_ACCOUNT} effective search/read access`,
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
return failFinding(
|
|
943
|
+
"worker-acl",
|
|
944
|
+
`the ${WORKER_ACCOUNT} account's ACL is not effective on ${health.missing.join(", ")} — a named entry the ACL mask strips is ` +
|
|
945
|
+
"unreachable by the worker, and the next admitted worker dies on EACCES before connecting",
|
|
946
|
+
"re-run `omp-conductor setup host`: the transaction re-applies the ACL grants (mask included) after the final fleet restart",
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
|
|
905
950
|
/** IANA timezone in reporting config — an invalid zone silently mis-schedules
|
|
906
951
|
* the availability window and the daily digest (#273). */
|
|
907
952
|
function timezoneProbe(project: ProjectConfig | undefined): Finding {
|
|
@@ -1645,6 +1690,10 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
1645
1690
|
findings.push(unitProbe(probes, projects[0], cfg));
|
|
1646
1691
|
findings.push(recoveryProbe(probes));
|
|
1647
1692
|
findings.push(ownershipProbe(probes));
|
|
1693
|
+
// The worker's path ACLs are a host-wide fact like the unit ownership: one
|
|
1694
|
+
// shared daemon account, one shared worker identity, one shared set of
|
|
1695
|
+
// linked config paths. A finding once, never once per project.
|
|
1696
|
+
findings.push(workerAclProbeFinding(probes));
|
|
1648
1697
|
// #541 seam checks, host-global: the live herdr config and the plugin's
|
|
1649
1698
|
// config.env are single files on the host, not per-project facts.
|
|
1650
1699
|
findings.push(herdrResumeProbe(probes));
|
|
@@ -1688,6 +1737,9 @@ export function defaultProbes(): Probes {
|
|
|
1688
1737
|
readUnit: defaultReadUnit,
|
|
1689
1738
|
stat: defaultStat,
|
|
1690
1739
|
uidOf: defaultUidOf,
|
|
1740
|
+
// The same effective-ACL read the identity plan plans with — doctor and
|
|
1741
|
+
// setup cannot disagree about a worker grant (#835).
|
|
1742
|
+
workerAclHealth: () => workerAclHealth(join(homedir(), ".omp", "agent")),
|
|
1691
1743
|
dbIntegrity: defaultDbIntegrity,
|
|
1692
1744
|
snapshotDirState: defaultSnapshotDirState,
|
|
1693
1745
|
sessionRootState: defaultSessionRootState,
|
package/src/escalate.ts
CHANGED
|
@@ -403,11 +403,17 @@ function errText(e: unknown): string {
|
|
|
403
403
|
*
|
|
404
404
|
* Exported for the report outbox (#123), which pages over the same bot and must
|
|
405
405
|
* resolve the token the same way. Two readers of one `.env` is fine; two
|
|
406
|
-
* *implementations* of the parse below is how they drift.
|
|
406
|
+
* *implementations* of the parse below is how they drift. The state dir is the
|
|
407
|
+
* shared resolution both the token read and the interactive ask surface (#722)
|
|
408
|
+
* need, so it is one function rather than a second copy of the override.
|
|
407
409
|
*/
|
|
408
|
-
export function
|
|
410
|
+
export function telegramStateDir(): string {
|
|
409
411
|
const override = process.env.OMP_TELEGRAM_STATE_DIR?.trim();
|
|
410
|
-
|
|
412
|
+
return override ? override : join(homedir(), ".omp", "agent", "telegram");
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
export function readTelegramToken(): string | undefined {
|
|
416
|
+
const dir = telegramStateDir();
|
|
411
417
|
let raw: string;
|
|
412
418
|
try {
|
|
413
419
|
raw = readFileSync(join(dir, ".env"), "utf8");
|