omp-conductor 0.15.10 → 0.15.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +273 -2544
- package/REFERENCE.md +2680 -0
- package/package.json +3 -2
- package/schema/config.schema.json +11 -23
- package/src/arm-challenge.ts +112 -0
- package/src/ask.ts +434 -0
- package/src/board.ts +81 -15
- package/src/brief-upgrade.ts +114 -8
- package/src/briefs/orchestrator.md +113 -34
- package/src/briefs/policy.md +33 -8
- package/src/briefs/worker.md +18 -9
- package/src/chain-check.ts +1 -1
- package/src/check-trailing-newlines.ts +82 -0
- package/src/cli.ts +225 -1406
- package/src/commands/arm.ts +21 -0
- package/src/commands/board.ts +23 -0
- package/src/commands/brief-upgrade.ts +186 -0
- package/src/commands/context.ts +150 -0
- package/src/commands/daemon.ts +71 -0
- package/src/commands/dashboard.ts +74 -0
- package/src/commands/decision.ts +103 -0
- package/src/commands/disarm.ts +21 -0
- package/src/commands/doctor.ts +100 -0
- package/src/commands/event.ts +62 -0
- package/src/commands/extend.ts +64 -0
- package/src/commands/friction.ts +56 -0
- package/src/commands/help.ts +9 -0
- package/src/commands/hold.ts +26 -0
- package/src/commands/intake.ts +155 -0
- package/src/commands/ledger.ts +69 -0
- package/src/commands/message.ts +96 -0
- package/src/commands/report.ts +206 -0
- package/src/commands/restart.ts +76 -0
- package/src/commands/resume.ts +58 -0
- package/src/commands/setup.ts +143 -0
- package/src/commands/start.ts +23 -0
- package/src/commands/stats.ts +131 -0
- package/src/commands/status.ts +48 -0
- package/src/commands/stop.ts +51 -0
- package/src/commands/tail.ts +109 -0
- package/src/commands/unblock.ts +39 -0
- package/src/commands/upgrade-install.ts +31 -0
- package/src/commands/upgrade-rollback.ts +32 -0
- package/src/commands/upgrade.ts +25 -0
- package/src/commands/verb.ts +83 -0
- package/src/commands/version.ts +30 -0
- package/src/commands/worker.ts +100 -0
- package/src/config-schema.ts +47 -1
- package/src/config.ts +45 -4
- package/src/daemon.ts +972 -101
- package/src/dashboard/app.js +459 -0
- package/src/dashboard/index.html +61 -0
- package/src/dashboard/server.ts +481 -0
- package/src/dashboard/style.css +348 -0
- package/src/decisions.ts +39 -14
- package/src/diff-flags.ts +131 -241
- package/src/doctor.ts +932 -0
- package/src/escalate.ts +2 -2
- package/src/failure-class.ts +66 -3
- package/src/fleet.ts +58 -1
- package/src/gitops.ts +157 -0
- package/src/graph-health.ts +1 -1
- package/src/label-projection.ts +1 -1
- package/src/lifecycle.ts +198 -2
- package/src/model-fallback.ts +177 -0
- package/src/notices.ts +9 -0
- package/src/omp.ts +93 -13
- package/src/orchestrator-tick.ts +414 -20
- package/src/orchestrator.ts +4 -4
- package/src/privileged.ts +10 -0
- package/src/release-policy.ts +342 -30
- package/src/reports.ts +19 -5
- package/src/session-host.ts +11 -5
- package/src/setup-host.ts +663 -25
- package/src/setup-install.ts +292 -28
- package/src/setup-wizard.ts +255 -74
- package/src/setup.ts +156 -35
- package/src/stats.ts +331 -0
- package/src/store.ts +219 -22
- package/src/tracker/github.ts +74 -4
- package/src/types.ts +215 -31
- package/src/unblock.ts +55 -11
- package/src/upgrade-journal.ts +220 -0
- package/src/upgrade-verify.ts +506 -0
- package/src/upgrade.ts +385 -58
- package/src/verbs/actions.ts +73 -1
- package/src/verbs/protocol.ts +29 -4
- package/src/verbs/server.ts +183 -20
- package/src/worker.ts +3 -3
- package/systemd/omp-conductor-recover.sh +433 -0
- package/systemd/omp-conductor.service.example +14 -3
- package/systemd/recover-unit-test.sh +428 -0
package/src/escalate.ts
CHANGED
|
@@ -20,12 +20,12 @@
|
|
|
20
20
|
* those strings end up in daemon logs and, on the fallback path, in a public
|
|
21
21
|
* issue comment.
|
|
22
22
|
*/
|
|
23
|
-
import { createHash } from "node:crypto";
|
|
24
23
|
import { readFileSync } from "node:fs";
|
|
25
24
|
import { homedir } from "node:os";
|
|
26
25
|
import { join } from "node:path";
|
|
27
26
|
|
|
28
27
|
import { availabilityOpen, interruptDisposition, type InterruptDisposition } from "./availability.ts";
|
|
28
|
+
import { heldNoticeId } from "./notices.ts";
|
|
29
29
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
30
30
|
import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
|
|
31
31
|
|
|
@@ -240,7 +240,7 @@ export function createEscalator(
|
|
|
240
240
|
}
|
|
241
241
|
if (disposition !== "interrupt") {
|
|
242
242
|
store.addHeldNotice({
|
|
243
|
-
id:
|
|
243
|
+
id: heldNoticeId(key),
|
|
244
244
|
project: p.name,
|
|
245
245
|
category,
|
|
246
246
|
summary: e.summary,
|
package/src/failure-class.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* green test per class" possible without a daemon, a tracker or a network.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import type { FailureClass, RecoveryAction, RunRecord } from "./types.ts";
|
|
18
|
+
import type { Caps, FailureClass, RecoveryAction, RunRecord } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
/** Facts the caller fetched, each only for the rows that need it. */
|
|
21
21
|
export interface ClassifyFacts {
|
|
@@ -222,7 +222,32 @@ export function dispatchInfra(
|
|
|
222
222
|
// front of the thrown message, so a bare /^git / would never match a real row.
|
|
223
223
|
const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s;
|
|
224
224
|
|
|
225
|
-
|
|
225
|
+
/** The prefix the draining/restarting process writes to a run it killed. */
|
|
226
|
+
const ADMIN_RESTART_MARKER = "admin restart:";
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* The actor a draining restart recorded on a run it killed, or `undefined`
|
|
230
|
+
* when the row carries no such attribution. Read from `lastError` for the
|
|
231
|
+
* same reason {@link dispatchInfra} is read from it: it is the one surface a
|
|
232
|
+
* run's own row already persists, and the restarting process writes it the
|
|
233
|
+
* way `recordOperatorStop` writes "operator stopped: …". Recognised as an
|
|
234
|
+
* administrative kill, never a charging failure, because nothing the worker
|
|
235
|
+
* did ended it — #512's gap was that a killed session left no attribution.
|
|
236
|
+
*/
|
|
237
|
+
export function adminRestartAttribution(lastError: string | undefined): string | undefined {
|
|
238
|
+
if (lastError === undefined) return undefined;
|
|
239
|
+
const marker = lastError.indexOf(ADMIN_RESTART_MARKER);
|
|
240
|
+
if (marker === -1) return undefined;
|
|
241
|
+
const firstLine = lastError.slice(marker + ADMIN_RESTART_MARKER.length).trim().split("\n")[0];
|
|
242
|
+
if (firstLine === undefined || firstLine.trim() === "") return undefined;
|
|
243
|
+
return firstLine.trim();
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export function classifyRun(
|
|
247
|
+
run: RunRecord,
|
|
248
|
+
facts: ClassifyFacts,
|
|
249
|
+
caps?: Pick<Caps, "workerWallClockMs">,
|
|
250
|
+
): Classification {
|
|
226
251
|
const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined;
|
|
227
252
|
const providerError =
|
|
228
253
|
facts.sessionError ?? (run.lastError === undefined ? undefined : { message: run.lastError });
|
|
@@ -296,6 +321,22 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
296
321
|
}
|
|
297
322
|
}
|
|
298
323
|
|
|
324
|
+
// A run the draining/restarting process attributed before it restarted the
|
|
325
|
+
// session host: killed under its own ceilings by an administrative action,
|
|
326
|
+
// not by anything the worker did. Costs no budget and requeues, naming the
|
|
327
|
+
// actor (#512). A killed session used to leave no attribution and fell
|
|
328
|
+
// through to `unknown`, charging a failure for an operator's restart.
|
|
329
|
+
if (run.state === "failed" || run.state === "killed") {
|
|
330
|
+
const actor = adminRestartAttribution(run.lastError);
|
|
331
|
+
if (actor !== undefined) {
|
|
332
|
+
return {
|
|
333
|
+
cls: "admin-kill",
|
|
334
|
+
recovery: "requeue",
|
|
335
|
+
evidence: `killed by an administrative restart (${actor}) — an operator's restart, not the run's work`,
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
299
340
|
if (run.state === "blocked") {
|
|
300
341
|
return {
|
|
301
342
|
cls: "question",
|
|
@@ -337,7 +378,29 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
|
|
|
337
378
|
evidence: `turns ${run.turns}/${run.maxTurns}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
|
|
338
379
|
};
|
|
339
380
|
}
|
|
340
|
-
//
|
|
381
|
+
// A run killed at the wall-clock cap is a cap kill too, even though its
|
|
382
|
+
// turns ceiling never tripped — the harness would have kept burning a
|
|
383
|
+
// worker slot until the ticks ran out (#490). The same split as the turns
|
|
384
|
+
// path: artifacts mean the next attempt resumes from real work; nothing to
|
|
385
|
+
// show means a human re-scopes, naming the time and spend consumed.
|
|
386
|
+
const wallClockCap = caps?.workerWallClockMs;
|
|
387
|
+
const wallClockElapsed =
|
|
388
|
+
run.endedAt === undefined ? undefined : run.endedAt - run.startedAt;
|
|
389
|
+
if (wallClockCap !== undefined && wallClockElapsed !== undefined && wallClockElapsed >= wallClockCap) {
|
|
390
|
+
const clock = `${Math.round(wallClockElapsed / 60_000)}m of ${Math.round(wallClockCap / 60_000)}m`;
|
|
391
|
+
return hasArtifacts
|
|
392
|
+
? {
|
|
393
|
+
cls: "wall-clock-cap-progress",
|
|
394
|
+
recovery: "continue",
|
|
395
|
+
evidence: `wall clock ${clock} with work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
|
|
396
|
+
}
|
|
397
|
+
: {
|
|
398
|
+
cls: "wall-clock-cap-spinning",
|
|
399
|
+
recovery: "escalate",
|
|
400
|
+
evidence: `wall clock ${clock}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
// Below its own ceilings, so nothing this worker did ended it: a daemon
|
|
341
404
|
// restart, a drain, an operator. Costs no budget and needs no human.
|
|
342
405
|
return {
|
|
343
406
|
cls: "admin-kill",
|
package/src/fleet.ts
CHANGED
|
@@ -29,6 +29,7 @@ import { homedir } from "node:os";
|
|
|
29
29
|
import { dirname, join } from "node:path";
|
|
30
30
|
import { formatZonedMinute } from "./availability.ts";
|
|
31
31
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
32
|
+
import { clearArmChallenge, recordArmChallenge } from "./arm-challenge.ts";
|
|
32
33
|
import { resolveProjectTopicId, sendTelegram } from "./escalate.ts";
|
|
33
34
|
import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
34
35
|
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
@@ -250,6 +251,18 @@ function armGate(projectName?: string): { path: string; named?: string } {
|
|
|
250
251
|
return named === undefined ? { path } : { path, named };
|
|
251
252
|
}
|
|
252
253
|
|
|
254
|
+
/**
|
|
255
|
+
* This project's current tick-arm verdict — the marker the heartbeat reads,
|
|
256
|
+
* returned with the path that carries it so a verb can report what actually
|
|
257
|
+
* remains. Read-only: the same decision `disarm` takes, so control and status
|
|
258
|
+
* about one fleet cannot disagree (a verb telling the operator to re-arm a
|
|
259
|
+
* fleet that is already armed is the defect #375 removes).
|
|
260
|
+
*/
|
|
261
|
+
export function armState(projectName?: string): { path: string; armed: boolean } {
|
|
262
|
+
const { path, named } = armGate(projectName);
|
|
263
|
+
return { path, armed: resolveArmState(path, named).armed };
|
|
264
|
+
}
|
|
265
|
+
|
|
253
266
|
/**
|
|
254
267
|
* Clears the arm gate for this project — and `wasArmed` is the gate the
|
|
255
268
|
* heartbeat reads, not merely one file's presence.
|
|
@@ -362,23 +375,34 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
362
375
|
}
|
|
363
376
|
|
|
364
377
|
const send = deps.sendChallenge ?? sendTelegramMessage;
|
|
378
|
+
const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
|
|
365
379
|
// Read before the send, not after: a transcript untouched since this instant
|
|
366
380
|
// cannot contain the reply, and that is what the waiter filters on.
|
|
367
381
|
const sentAt = (deps.now ?? Date.now)();
|
|
382
|
+
// The orchestrator must be able to classify the reply as an *active* arming
|
|
383
|
+
// proof for as long as this handshake is live, so the authenticated record
|
|
384
|
+
// (hash + expiry, never the code) is written before the challenge goes out
|
|
385
|
+
// and cleared the moment this end settles (#415).
|
|
386
|
+
recordArmChallenge(named, code, sentAt + timeoutMs);
|
|
368
387
|
try {
|
|
369
388
|
await send(token, channel.owner, text, topicId);
|
|
370
389
|
} catch (err) {
|
|
390
|
+
// The challenge never went out, so it must not linger as a classifiable
|
|
391
|
+
// proof either.
|
|
392
|
+
clearArmChallenge(named);
|
|
371
393
|
throw new Error(
|
|
372
394
|
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
373
395
|
);
|
|
374
396
|
}
|
|
375
397
|
|
|
376
|
-
const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
|
|
377
398
|
const injected = deps.waitForUserTurn;
|
|
378
399
|
const scan: SessionScan = injected
|
|
379
400
|
? { seen: await injected(dir, code, sentAt, timeoutMs), scanned: [], ignored: [] }
|
|
380
401
|
: await waitForChallengeInSessions(dir, code, sentAt, timeoutMs, deps);
|
|
381
402
|
if (!scan.seen) {
|
|
403
|
+
// The proof missed its window: a lookalike reply must not stay classifiable
|
|
404
|
+
// after arming gave up, so the record is cleared before the failure lands.
|
|
405
|
+
clearArmChallenge(named);
|
|
382
406
|
const listing = [
|
|
383
407
|
`session dir: ${dir}`,
|
|
384
408
|
...scan.scanned.map((f) => ` watched: ${f}`),
|
|
@@ -402,6 +426,10 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
402
426
|
// marker that re-arms the fleet, and turn into a meaningless legacy warning
|
|
403
427
|
// the moment a second project is configured.
|
|
404
428
|
if (arm.legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
|
|
429
|
+
// The transcript proof landed and this project is armed: the record has done
|
|
430
|
+
// its job. A stale record would also keep a later unsolicited lookalike alive
|
|
431
|
+
// longer than the fresh challenge it was cut for.
|
|
432
|
+
clearArmChallenge(named);
|
|
405
433
|
return { path, alreadyArmed, owner: channel.owner, challenge: code };
|
|
406
434
|
}
|
|
407
435
|
|
|
@@ -1393,6 +1421,7 @@ export function formatFleetStatus(
|
|
|
1393
1421
|
decisions: string | undefined = undefined,
|
|
1394
1422
|
failureClasses: string | undefined = undefined,
|
|
1395
1423
|
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1424
|
+
intake: string | undefined = undefined,
|
|
1396
1425
|
): string {
|
|
1397
1426
|
const tickLine =
|
|
1398
1427
|
layers.ticksDetail === undefined
|
|
@@ -1473,6 +1502,7 @@ export function formatFleetStatus(
|
|
|
1473
1502
|
...(brief === undefined ? [] : [brief]),
|
|
1474
1503
|
...(decisions === undefined ? [] : [decisions]),
|
|
1475
1504
|
...(failureClasses === undefined ? [] : [failureClasses]),
|
|
1505
|
+
...(intake === undefined ? [] : [intake]),
|
|
1476
1506
|
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
1477
1507
|
daemonBlock,
|
|
1478
1508
|
"",
|
|
@@ -1656,6 +1686,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1656
1686
|
decisionStatusLine(project.name),
|
|
1657
1687
|
failureClassBlock(project.name),
|
|
1658
1688
|
workerPhases,
|
|
1689
|
+
intakeStatusLine(project.name),
|
|
1659
1690
|
);
|
|
1660
1691
|
}
|
|
1661
1692
|
|
|
@@ -1734,6 +1765,32 @@ function decisionStatusLine(projectName: string): string | undefined {
|
|
|
1734
1765
|
}
|
|
1735
1766
|
}
|
|
1736
1767
|
|
|
1768
|
+
/**
|
|
1769
|
+
* One line naming how many raw ideas are still pending (#299), or nothing when
|
|
1770
|
+
* there are none.
|
|
1771
|
+
*
|
|
1772
|
+
* Deliberately absent at zero, unlike the decision line: an empty intake is the
|
|
1773
|
+
* common state of a healthy fleet, and a row that prints "0 pending" every
|
|
1774
|
+
* status would be permanent noise. Only a nonzero backlog is news, exactly like
|
|
1775
|
+
* {@link failureClassBlock}. Omitted on any throw, like the other optional rows.
|
|
1776
|
+
* Exported so the conditional is testable against a real store.
|
|
1777
|
+
*/
|
|
1778
|
+
export function intakeStatusLine(projectName: string): string | undefined {
|
|
1779
|
+
const path = dbPath();
|
|
1780
|
+
if (!existsSync(path)) return undefined;
|
|
1781
|
+
let store: Store | undefined;
|
|
1782
|
+
try {
|
|
1783
|
+
store = openStore(path);
|
|
1784
|
+
const pending = store.pendingIntake(projectName);
|
|
1785
|
+
if (pending.length === 0) return undefined;
|
|
1786
|
+
return `intake ${pending.length} pending idea(s) — omp-conductor intake list`;
|
|
1787
|
+
} catch {
|
|
1788
|
+
return undefined;
|
|
1789
|
+
} finally {
|
|
1790
|
+
store?.close();
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1737
1794
|
// ---------------------------------------------------------------------------
|
|
1738
1795
|
// arm proof helpers
|
|
1739
1796
|
// ---------------------------------------------------------------------------
|
package/src/gitops.ts
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
* branch cannot disagree.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
+
import { existsSync } from "node:fs";
|
|
20
21
|
import { join } from "node:path";
|
|
21
22
|
|
|
22
23
|
import { parseChainSource, type ChainEntry } from "./chain-check.ts";
|
|
@@ -321,3 +322,159 @@ export async function readBaseChain(
|
|
|
321
322
|
return { ok: false, stderr: err instanceof Error ? err.message : String(err) };
|
|
322
323
|
}
|
|
323
324
|
}
|
|
325
|
+
|
|
326
|
+
// ------------------------------------------- the shared-host critical-base guard
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* The stale-base admission verdict (#428): whether a preserved continuation
|
|
330
|
+
* branch may be reattached at all, once a project marks a base commit as
|
|
331
|
+
* critical. Fail closed — a branch that cannot be *proven* to contain every
|
|
332
|
+
* marker is refused.
|
|
333
|
+
*/
|
|
334
|
+
export type CriticalBaseVerdict =
|
|
335
|
+
/** No preserved branch for this issue in the mirror — a clean first attempt. */
|
|
336
|
+
| { state: "no-branch" }
|
|
337
|
+
/** The reattach source contains every configured marker. */
|
|
338
|
+
| { state: "fresh" }
|
|
339
|
+
/** The reattach source predates a marker and cannot be safely advanced to it,
|
|
340
|
+
* so the dispatcher must not reattach it. `range` names the base commits
|
|
341
|
+
* the branch has not absorbed, for the hold's recovery wording. */
|
|
342
|
+
| { state: "stale"; marker: string; range: string[] }
|
|
343
|
+
/** The marker could not be verified at all (unresolvable, fetch failed). */
|
|
344
|
+
| { state: "unknown"; error: string };
|
|
345
|
+
|
|
346
|
+
/** The stubborn-session view of the probe, injectable for deterministic tests. */
|
|
347
|
+
export type CriticalBaseProbe = (
|
|
348
|
+
repo: RepoTarget,
|
|
349
|
+
markers: readonly string[],
|
|
350
|
+
branch: string,
|
|
351
|
+
) => Promise<CriticalBaseVerdict>;
|
|
352
|
+
|
|
353
|
+
/** The bounded sample of base commits a held branch is missing. */
|
|
354
|
+
const CRITICAL_BASE_RANGE_MAX = 8;
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Answer whether the preserved branch {@link addRunRepo} would reattach
|
|
358
|
+
* contains every configured critical-base marker (#428). A base safety fix
|
|
359
|
+
* protects only branches forked after it landed; a continuation forked before
|
|
360
|
+
* it still carries the dangerous code, and re-running the lifecycle suite it
|
|
361
|
+
* retains on a shared host is what SIGTERMed the production daemon.
|
|
362
|
+
*
|
|
363
|
+
* The reattach source is the mirror's `refs/heads/<branch>`. When it lacks a
|
|
364
|
+
* marker but the live remote branch now carries it and the reattach source is
|
|
365
|
+
* that live head's ancestor, the operator advanced the branch on GitHub (e.g.
|
|
366
|
+
* merged base into the PR branch): the fast-forward is folded into the mirror
|
|
367
|
+
* — safe and work-preserving, the same reconcile `pushRunBranch` performs —
|
|
368
|
+
* and the marker is accepted. Judged against the reattach source, never the
|
|
369
|
+
* live head alone, so a stale local copy cannot smuggle pre-fix code back in.
|
|
370
|
+
*/
|
|
371
|
+
export async function probeCriticalBase(
|
|
372
|
+
project: Pick<ProjectConfig, "mirrorRoot">,
|
|
373
|
+
repo: RepoTarget,
|
|
374
|
+
branch: string,
|
|
375
|
+
markers: readonly string[],
|
|
376
|
+
exec: Exec = spawnCaptured,
|
|
377
|
+
): Promise<CriticalBaseVerdict> {
|
|
378
|
+
const mirror = mirrorPath(project, repo);
|
|
379
|
+
// A mirror is where preserved branches live; without one there is nothing to
|
|
380
|
+
// reattach. Judged "no branch" rather than "unknown" so a fleet that has not
|
|
381
|
+
// yet created a mirror is not deadlocked at admission.
|
|
382
|
+
if (!existsSync(mirror)) return { state: "no-branch" };
|
|
383
|
+
const branchRef = `refs/heads/${branch}`;
|
|
384
|
+
const env = credentialedEnv();
|
|
385
|
+
|
|
386
|
+
const present = await exec(
|
|
387
|
+
["git", "--git-dir", mirror, "show-ref", "--verify", "--quiet", branchRef],
|
|
388
|
+
{ env },
|
|
389
|
+
);
|
|
390
|
+
if (present.code !== 0) return { state: "no-branch" };
|
|
391
|
+
|
|
392
|
+
// The configured marker is a commit that landed on base, so base has to be
|
|
393
|
+
// resolvable to prove anything about the branch against it. Fetch today's
|
|
394
|
+
// base tip into the mirror's tracking refs first, so the judgement is against
|
|
395
|
+
// the current base rather than whatever a run last left behind.
|
|
396
|
+
const baseRef = `refs/remotes/origin/${repo.defaultBranch}`;
|
|
397
|
+
const fetched = await exec(
|
|
398
|
+
["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${repo.defaultBranch}:${baseRef}`],
|
|
399
|
+
{ env },
|
|
400
|
+
);
|
|
401
|
+
if (fetched.code !== 0) {
|
|
402
|
+
return {
|
|
403
|
+
state: "unknown",
|
|
404
|
+
error: scrubUserinfo(fetched.stderr.trim() || fetched.stdout.trim() || `git fetch exited ${String(fetched.code)}`),
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
for (const marker of markers) {
|
|
409
|
+
const resolved = await exec(
|
|
410
|
+
["git", "--git-dir", mirror, "rev-parse", "--verify", `${marker}^{commit}`],
|
|
411
|
+
{ env },
|
|
412
|
+
);
|
|
413
|
+
if (resolved.code !== 0) {
|
|
414
|
+
return {
|
|
415
|
+
state: "unknown",
|
|
416
|
+
error: `critical-base marker "${marker}" is not resolvable in the ${repo.name} mirror`,
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
const markerSha = resolved.stdout.trim();
|
|
420
|
+
if (markerSha === "") {
|
|
421
|
+
return { state: "unknown", error: `critical-base marker "${marker}" resolved to no commit` };
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
const localHas = await exec(
|
|
425
|
+
["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, branchRef],
|
|
426
|
+
{ env },
|
|
427
|
+
);
|
|
428
|
+
if (localHas.code === 0) continue;
|
|
429
|
+
|
|
430
|
+
// The reattach source lacks the marker. If the live branch now carries it
|
|
431
|
+
// and the reattach source is its ancestor, the operator advanced the branch
|
|
432
|
+
// on GitHub (e.g. merged base into the PR branch): fold that fast-forward
|
|
433
|
+
// into the reattach source — safe, work-preserving — and accept the marker.
|
|
434
|
+
const tracked = `refs/remotes/origin/${branch}`;
|
|
435
|
+
const liveFetched = await exec(
|
|
436
|
+
["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${branch}:${tracked}`],
|
|
437
|
+
{ env },
|
|
438
|
+
);
|
|
439
|
+
if (liveFetched.code !== 0) {
|
|
440
|
+
return {
|
|
441
|
+
state: "unknown",
|
|
442
|
+
error: scrubUserinfo(liveFetched.stderr.trim() || liveFetched.stdout.trim() || `git fetch ${branch} exited ${String(liveFetched.code)}`),
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
const liveHas = await exec(
|
|
446
|
+
["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, tracked],
|
|
447
|
+
{ env },
|
|
448
|
+
);
|
|
449
|
+
if (liveHas.code === 0) {
|
|
450
|
+
const localIsAncestor = await exec(
|
|
451
|
+
["git", "--git-dir", mirror, "merge-base", "--is-ancestor", branchRef, tracked],
|
|
452
|
+
{ env },
|
|
453
|
+
);
|
|
454
|
+
if (localIsAncestor.code === 0) {
|
|
455
|
+
await exec(["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+${branchRef}:${branchRef}`], { env });
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// Predates the marker and cannot be safely advanced to it. Name the base
|
|
461
|
+
// commits the branch has not absorbed so the hold's recovery is concrete.
|
|
462
|
+
const range: string[] = [];
|
|
463
|
+
const missing = await exec(
|
|
464
|
+
["git", "--git-dir", mirror, "log", "--oneline", "--format=%h", `${branchRef}..${baseRef}`],
|
|
465
|
+
{ env },
|
|
466
|
+
);
|
|
467
|
+
if (missing.code === 0) {
|
|
468
|
+
range.push(
|
|
469
|
+
...missing.stdout
|
|
470
|
+
.trim()
|
|
471
|
+
.split("\n")
|
|
472
|
+
.filter((line) => line.length > 0)
|
|
473
|
+
.slice(0, CRITICAL_BASE_RANGE_MAX),
|
|
474
|
+
);
|
|
475
|
+
}
|
|
476
|
+
return { state: "stale", marker: markerSha, range };
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
return { state: "fresh" };
|
|
480
|
+
}
|
package/src/graph-health.ts
CHANGED
|
@@ -264,7 +264,7 @@ export async function probeCodeGraph(
|
|
|
264
264
|
const repos: CodeGraphRepoHealth[] = configured.map((repo) => {
|
|
265
265
|
const clone: CheckState = deps.exists(repo.graphProject) ? "present" : "missing";
|
|
266
266
|
const index: CheckState = roots === undefined ? "unknown" : roots.has(repo.graphProject) ? "present" : "missing";
|
|
267
|
-
if (clone === "missing") reasons.push(`${repo.name}:
|
|
267
|
+
if (clone === "missing") reasons.push(`${repo.name}: no code-graph clone at ${repo.graphProject}; run \`omp-conductor setup graph\``);
|
|
268
268
|
if (index === "missing") reasons.push(`${repo.name}: no indexed project exactly matches ${repo.graphProject}`);
|
|
269
269
|
return { name: repo.name, path: repo.graphProject, clone, index };
|
|
270
270
|
});
|
package/src/label-projection.ts
CHANGED
package/src/lifecycle.ts
CHANGED
|
@@ -66,6 +66,18 @@ const LOG_TAIL_BYTES = 4_096;
|
|
|
66
66
|
export function runtimeDir(): string {
|
|
67
67
|
const override = process.env["OMP_CONDUCTOR_RUNTIME_DIR"];
|
|
68
68
|
if (override !== undefined && override.length > 0) return override;
|
|
69
|
+
if (process.env["NODE_ENV"] === "test") {
|
|
70
|
+
// A test process must never read the host-global runtime dir: doing so
|
|
71
|
+
// names the real installed daemon's pid, and a test `stop` that inherits
|
|
72
|
+
// that record would signal — and stop — production's own unit (#399).
|
|
73
|
+
// Fail closed: a lifecycle test that omitted OMP_CONDUCTOR_RUNTIME_DIR
|
|
74
|
+
// (the runtime half of the test lifecycle seam) gets a loud error rather
|
|
75
|
+
// than a silent read of the operator's pidfile.
|
|
76
|
+
throw new Error(
|
|
77
|
+
"OMP_CONDUCTOR_RUNTIME_DIR must be set under NODE_ENV=test — " +
|
|
78
|
+
"refusing to read the host runtime directory (~/.omp/run/daemons/omp-conductor) from a test process",
|
|
79
|
+
);
|
|
80
|
+
}
|
|
69
81
|
return join(homedir(), ".omp", "run", "daemons", "omp-conductor");
|
|
70
82
|
}
|
|
71
83
|
|
|
@@ -178,6 +190,152 @@ export function livingDaemon(): DaemonRecord | undefined {
|
|
|
178
190
|
return undefined;
|
|
179
191
|
}
|
|
180
192
|
|
|
193
|
+
/**
|
|
194
|
+
* The `daemon --once` lease file, one per process. Created `O_EXCL` so two
|
|
195
|
+
* simultaneous `--once` runs cannot both win ownership, and it names the
|
|
196
|
+
* holder's pid so a stale file (holder died) is removable rather than a
|
|
197
|
+
* permanent lock.
|
|
198
|
+
*
|
|
199
|
+
* The pidfile (`daemon.json`) is what makes this process *visible* as the owner
|
|
200
|
+
* for the duration of the tick (`status`, `worker stop`, a concurrent `daemon`
|
|
201
|
+
* start, orphan reconciliation all probe it); the lease file is what makes
|
|
202
|
+
* ownership *exclusive* — it is the thing that lets two simultaneous `--once`
|
|
203
|
+
* processes serialise, which the pidfile alone cannot (each would just
|
|
204
|
+
* overwrite it). They are deliberately two files: the lease is never the
|
|
205
|
+
* daemon record, so a short-lived drill can never impersonate or clear the
|
|
206
|
+
* managed daemon's record by virtue of holding a lock.
|
|
207
|
+
*/
|
|
208
|
+
function onceLeasePath(): string {
|
|
209
|
+
return join(runtimeDir(), "once.lock");
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
interface OnceLeaseHolder {
|
|
213
|
+
pid: number;
|
|
214
|
+
startedAt: number;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Parses the lease file; garbage is the same answer as absence — "not held". */
|
|
218
|
+
function readOnceLeaseHolder(): OnceLeaseHolder | undefined {
|
|
219
|
+
let parsed: unknown;
|
|
220
|
+
try {
|
|
221
|
+
parsed = JSON.parse(readFileSync(onceLeasePath(), "utf8"));
|
|
222
|
+
} catch {
|
|
223
|
+
return undefined;
|
|
224
|
+
}
|
|
225
|
+
if (parsed === null || typeof parsed !== "object") return undefined;
|
|
226
|
+
const pid = Reflect.get(parsed, "pid");
|
|
227
|
+
const startedAt = Reflect.get(parsed, "startedAt");
|
|
228
|
+
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 1) return undefined;
|
|
229
|
+
if (typeof startedAt !== "number" || !Number.isFinite(startedAt)) return undefined;
|
|
230
|
+
return { pid, startedAt };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Bounded attempts to clear a stale lease and re-acquire it. */
|
|
234
|
+
const ONCE_LEASE_RETRIES = 3;
|
|
235
|
+
|
|
236
|
+
/** A once tick has no log file of its own; the record's field is informational. */
|
|
237
|
+
const ONCE_RECORD_LOG = "<once tick — no daemon log>";
|
|
238
|
+
|
|
239
|
+
export interface OnceLease {
|
|
240
|
+
/** The owned record that makes this process the visible daemon for the tick. */
|
|
241
|
+
record: DaemonRecord;
|
|
242
|
+
/**
|
|
243
|
+
* Drops exclusivity — but only while it is still ours. A manager that
|
|
244
|
+
* claimed the record mid-tick must be left alone: clearing it would clear
|
|
245
|
+
* somebody else's daemon.
|
|
246
|
+
*/
|
|
247
|
+
release(): void;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export type OnceLeaseAcquire = { ok: true; lease: OnceLease } | { ok: false; pid: number };
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Acquires the exclusive daemon/state ownership a `--once` tick must hold for
|
|
254
|
+
* its whole lifecycle (#431). A drill is still a dispatcher — it settles rows,
|
|
255
|
+
* projects labels, admits and launches workers — so it takes the same
|
|
256
|
+
* ownership the long-running dispatcher holds via the pidfile, and refuses
|
|
257
|
+
* (having done nothing) while a live daemon owns the record or another live
|
|
258
|
+
* `--once` holds the lease.
|
|
259
|
+
*
|
|
260
|
+
* Returns `{ ok: false, pid }` — having performed no mutation — when another
|
|
261
|
+
* live process owns the project/state. A stale lease and a stale record whose
|
|
262
|
+
* holder died are cleared and retried. On success the process both holds the
|
|
263
|
+
* exclusive lease file and has published itself as the daemon owner, so every
|
|
264
|
+
* ownership probe (`livingDaemon`, a `daemon` start, a second `--once`) sees
|
|
265
|
+
* this process for the whole tick.
|
|
266
|
+
*/
|
|
267
|
+
export function acquireOnceLease(
|
|
268
|
+
o: { port?: number; project?: string; logFile?: string } = {},
|
|
269
|
+
): OnceLeaseAcquire {
|
|
270
|
+
// A live dispatcher already owns the shared record. Never run beside it.
|
|
271
|
+
const daemon = livingDaemon();
|
|
272
|
+
if (daemon !== undefined && daemon.pid !== process.pid) {
|
|
273
|
+
return { ok: false, pid: daemon.pid };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const dir = ensureRuntimeDir();
|
|
277
|
+
const path = onceLeasePath();
|
|
278
|
+
|
|
279
|
+
for (let attempt = 0; attempt < ONCE_LEASE_RETRIES; attempt++) {
|
|
280
|
+
let created = false;
|
|
281
|
+
try {
|
|
282
|
+
const fd = openSync(path, "wx", 0o600);
|
|
283
|
+
writeFileSync(fd, `${JSON.stringify({ pid: process.pid, startedAt: Date.now() })}\n`);
|
|
284
|
+
closeSync(fd);
|
|
285
|
+
created = true;
|
|
286
|
+
} catch (err) {
|
|
287
|
+
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
|
|
288
|
+
const holder = readOnceLeaseHolder();
|
|
289
|
+
if (holder !== undefined && isAlive(holder.pid)) {
|
|
290
|
+
// A live holder — another `--once` (or a process still finishing one).
|
|
291
|
+
return { ok: false, pid: holder.pid };
|
|
292
|
+
}
|
|
293
|
+
// Stale or unreadable holder: clear it and try once more.
|
|
294
|
+
rmSync(path, { force: true });
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
if (!created) continue;
|
|
298
|
+
|
|
299
|
+
const record: DaemonRecord = {
|
|
300
|
+
pid: process.pid,
|
|
301
|
+
port: o.port ?? DEFAULT_PORT,
|
|
302
|
+
startedAt: Date.now(),
|
|
303
|
+
logFile: o.logFile ?? ONCE_RECORD_LOG,
|
|
304
|
+
...(o.project === undefined ? {} : { project: o.project }),
|
|
305
|
+
};
|
|
306
|
+
try {
|
|
307
|
+
writeRecord(record);
|
|
308
|
+
} catch (err) {
|
|
309
|
+
rmSync(path, { force: true });
|
|
310
|
+
throw err;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// Confirm we still own the shared record: a manager may have claimed it in
|
|
314
|
+
// the instant between our lease create and our record write.
|
|
315
|
+
const current = readRecord();
|
|
316
|
+
if (current !== undefined && current.pid === process.pid) {
|
|
317
|
+
return {
|
|
318
|
+
ok: true,
|
|
319
|
+
lease: {
|
|
320
|
+
record,
|
|
321
|
+
release: () => {
|
|
322
|
+
const owned = readRecord();
|
|
323
|
+
if (owned !== undefined && owned.pid === process.pid) clearRecord();
|
|
324
|
+
const holder = readOnceLeaseHolder();
|
|
325
|
+
if (holder !== undefined && holder.pid === process.pid) {
|
|
326
|
+
rmSync(path, { force: true });
|
|
327
|
+
}
|
|
328
|
+
},
|
|
329
|
+
},
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
rmSync(path, { force: true });
|
|
333
|
+
return { ok: false, pid: current?.pid ?? process.pid };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
throw new Error("could not acquire the daemon lease — a stale holder kept reappearing");
|
|
337
|
+
}
|
|
338
|
+
|
|
181
339
|
/**
|
|
182
340
|
* Probes the daemon's own health endpoint. Never throws: a refused connection,
|
|
183
341
|
* a DNS-less host, a hung socket and a 500 are all just "not healthy", and the
|
|
@@ -892,8 +1050,17 @@ async function terminate(pid: number, timeoutMs: number): Promise<boolean> {
|
|
|
892
1050
|
return !isAlive(pid);
|
|
893
1051
|
}
|
|
894
1052
|
|
|
895
|
-
/**
|
|
896
|
-
|
|
1053
|
+
/** Whether a raw signal should be delivered to a pid. */
|
|
1054
|
+
export type SignalFn = (pid: number, sig: NodeJS.Signals) => boolean;
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* The default signal sender. Production reaches the real `kill`. Under
|
|
1058
|
+
* NODE_ENV=test it dispatches to the fail-closed refusal instead: a test
|
|
1059
|
+
* process must never signal a pid it did not explicitly authorise, and the
|
|
1060
|
+
* default authorises nothing (#399).
|
|
1061
|
+
*/
|
|
1062
|
+
function defaultSignal(pid: number, sig: NodeJS.Signals): boolean {
|
|
1063
|
+
if (process.env["NODE_ENV"] === "test") return testSignal(pid, sig);
|
|
897
1064
|
try {
|
|
898
1065
|
process.kill(pid, sig);
|
|
899
1066
|
return true;
|
|
@@ -902,6 +1069,35 @@ function signal(pid: number, sig: NodeJS.Signals): boolean {
|
|
|
902
1069
|
}
|
|
903
1070
|
}
|
|
904
1071
|
|
|
1072
|
+
/**
|
|
1073
|
+
* Child-process CLI tests cannot inject {@link setSignalForTest}. Under
|
|
1074
|
+
* NODE_ENV=test they therefore get a refusal: a test process may only signal
|
|
1075
|
+
* a pid its own code explicitly authorised, and a child has authorised none.
|
|
1076
|
+
* Without this, a child `stop` that reads the operator's real daemon.json
|
|
1077
|
+
* (an *inherited* runtime record) would SIGTERM the installed daemon — the
|
|
1078
|
+
* #399 incident.
|
|
1079
|
+
*/
|
|
1080
|
+
function testSignal(_pid: number, _sig: NodeJS.Signals): boolean {
|
|
1081
|
+
return false;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
let signalFn: SignalFn = defaultSignal;
|
|
1085
|
+
|
|
1086
|
+
/**
|
|
1087
|
+
* Test-only: replace the raw signal sender. Pass `undefined` to restore.
|
|
1088
|
+
* A test that intentionally spawned a helper may authorise signals to exactly
|
|
1089
|
+
* that helper pid; every other pid is refused. Ownership is never inferred
|
|
1090
|
+
* from writing a record — a pidfile fixture authorises nothing (#399).
|
|
1091
|
+
*/
|
|
1092
|
+
export function setSignalForTest(fn: SignalFn | undefined): void {
|
|
1093
|
+
signalFn = fn ?? defaultSignal;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/** Sends a signal, treating "already gone" as success. Returns false if the send failed. */
|
|
1097
|
+
function signal(pid: number, sig: NodeJS.Signals): boolean {
|
|
1098
|
+
return signalFn(pid, sig);
|
|
1099
|
+
}
|
|
1100
|
+
|
|
905
1101
|
/**
|
|
906
1102
|
* The tail of the daemon log, formatted for an error message. Reads the last
|
|
907
1103
|
* few KB by offset rather than the whole file, because the log is append-only
|