omp-conductor 0.19.7 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +10 -1
- package/agents/to-spec.md +76 -9
- package/package.json +1 -1
- package/schema/config.schema.json +4 -0
- package/src/arm-challenge.ts +204 -85
- package/src/ask.ts +130 -615
- package/src/board.ts +7 -1
- package/src/brief-upgrade.ts +24 -0
- package/src/briefs/console.md +253 -0
- package/src/briefs/correction.md +203 -0
- package/src/briefs/orchestrator.md +167 -97
- package/src/briefs/policy.md +19 -16
- package/src/briefs/to-spec.md +76 -9
- package/src/briefs/worker.md +50 -16
- package/src/cli.ts +4 -0
- package/src/command-manifest.ts +54 -8
- package/src/commands/arm.ts +113 -49
- package/src/commands/console.ts +70 -0
- package/src/commands/context.ts +2 -0
- package/src/commands/epic.ts +132 -0
- package/src/commands/extend.ts +9 -1
- package/src/commands/intake.ts +44 -14
- package/src/commands/stats.ts +19 -4
- package/src/commands/worker.ts +9 -1
- package/src/config-schema.ts +13 -0
- package/src/config.ts +27 -0
- package/src/daemon/ack.ts +159 -0
- package/src/daemon/admission-pass.ts +135 -0
- package/src/daemon/brief.ts +461 -0
- package/src/daemon/deps.ts +539 -0
- package/src/daemon/dispatch.ts +1779 -0
- package/src/daemon/drain.ts +185 -0
- package/src/daemon/groom-pass.ts +412 -0
- package/src/daemon/http.ts +417 -0
- package/src/daemon/integrity.ts +108 -0
- package/src/daemon/panes.ts +180 -0
- package/src/daemon/review.ts +1888 -0
- package/src/daemon/runtime.ts +736 -0
- package/src/daemon/settle-pass.ts +589 -0
- package/src/daemon/supervision.ts +438 -0
- package/src/daemon/tick.ts +968 -0
- package/src/daemon/views.ts +751 -0
- package/src/daemon.ts +105 -7923
- package/src/dashboard/app.js +58 -0
- package/src/dashboard/controls.ts +22 -3
- package/src/dashboard/server.ts +4 -0
- package/src/diff-flags.ts +24 -3
- package/src/failure-class.ts +75 -1
- package/src/fleet.ts +290 -164
- package/src/groom.ts +461 -0
- package/src/http-token.ts +142 -0
- package/src/knowledge.ts +229 -0
- package/src/mining.ts +316 -0
- package/src/orchestrator-tick.ts +428 -1681
- package/src/ready-gate.ts +267 -0
- package/src/settlement.ts +72 -6
- package/src/setup-host.ts +32 -9
- package/src/setup-wizard.ts +55 -7
- package/src/setup.ts +229 -3
- package/src/stats.ts +257 -2
- package/src/status-render.ts +158 -7
- package/src/store.ts +604 -26
- package/src/to-spec.ts +194 -21
- package/src/tracker/github.ts +50 -0
- package/src/types.ts +416 -15
- package/src/verbs/protocol.ts +28 -0
- package/src/verbs/server.ts +330 -39
- package/src/wake.ts +19 -2
- package/src/worker.ts +456 -1
package/src/commands/extend.ts
CHANGED
|
@@ -6,22 +6,30 @@
|
|
|
6
6
|
* same run-control target `worker` uses, so a live systemd-run daemon
|
|
7
7
|
* stays reachable when its pidfile is missing (#811) and the refusal
|
|
8
8
|
* answers cannot drift between the two verbs.
|
|
9
|
+
*
|
|
10
|
+
* The turn-limit endpoint is mutating, so it now carries the daemon's bearer
|
|
11
|
+
* token (Phase 4). The token is read, never minted: the daemon mints it at
|
|
12
|
+
* start, so an absent file means no daemon has run here — the honest refusal
|
|
13
|
+
* is that message, not an unauthenticated request that comes back a bare 401.
|
|
9
14
|
*/
|
|
10
15
|
|
|
11
16
|
import type { CommandContext } from "./context.ts";
|
|
12
17
|
import { findProject, loadConfig } from "../config.ts";
|
|
13
18
|
import { requireDaemonControl } from "../lifecycle.ts";
|
|
19
|
+
import { httpAuthHeader, missingHttpTokenMessage } from "../http-token.ts";
|
|
14
20
|
|
|
15
21
|
export async function extendCommand(ctx: CommandContext): Promise<void> {
|
|
16
22
|
const issue = ctx.issueArg("extend", ctx.argv[1]);
|
|
17
23
|
const maxTurns = ctx.turnsFlag();
|
|
18
24
|
const project = findProject(loadConfig(), ctx.projectFlag);
|
|
19
25
|
const daemon = await requireDaemonControl(project.name);
|
|
26
|
+
const auth = httpAuthHeader();
|
|
27
|
+
if (auth === undefined) throw new Error(missingHttpTokenMessage());
|
|
20
28
|
const response = await fetch(
|
|
21
29
|
`http://127.0.0.1:${daemon.port}/runs/${issue}/turn-limit`,
|
|
22
30
|
{
|
|
23
31
|
method: "PUT",
|
|
24
|
-
headers: { "content-type": "application/json" },
|
|
32
|
+
headers: { "content-type": "application/json", ...auth },
|
|
25
33
|
body: JSON.stringify({ project: project.name, maxTurns }),
|
|
26
34
|
},
|
|
27
35
|
);
|
package/src/commands/intake.ts
CHANGED
|
@@ -12,9 +12,15 @@ import type { CommandContext } from "./context.ts";
|
|
|
12
12
|
import { findProject, loadConfig } from "../config.ts";
|
|
13
13
|
import { dbPath, openStore } from "../store.ts";
|
|
14
14
|
|
|
15
|
-
/** Flags the intake surface understands
|
|
16
|
-
* {@link CommandContext.projectFlag}; the value
|
|
17
|
-
|
|
15
|
+
/** Flags the intake surface understands, and whether each takes a value.
|
|
16
|
+
* `--project` is consumed by {@link CommandContext.projectFlag}; the value
|
|
17
|
+
* token stays in argv, so the arg scan has to step over it. `--json` is a
|
|
18
|
+
* toggle and must NOT swallow the token after it. */
|
|
19
|
+
const INTAKE_FLAGS: Record<string, "value" | "toggle"> = {
|
|
20
|
+
"--project": "value",
|
|
21
|
+
"--issue": "value",
|
|
22
|
+
"--json": "toggle",
|
|
23
|
+
};
|
|
18
24
|
|
|
19
25
|
/** Every remaining token after the subverb must be one of the known flags
|
|
20
26
|
* (or its value), otherwise the operator typo'd something that would be
|
|
@@ -23,10 +29,12 @@ function assertKnownArgs(ctx: CommandContext, from: number): void {
|
|
|
23
29
|
for (let i = from; i < ctx.argv.length; i++) {
|
|
24
30
|
const token = ctx.argv[i];
|
|
25
31
|
if (token === undefined) continue;
|
|
26
|
-
|
|
32
|
+
const kind = INTAKE_FLAGS[token];
|
|
33
|
+
if (kind === "value") {
|
|
27
34
|
i += 1;
|
|
28
35
|
continue;
|
|
29
36
|
}
|
|
37
|
+
if (kind === "toggle") continue;
|
|
30
38
|
if (token.startsWith("--project=")) continue;
|
|
31
39
|
process.stderr.write(`omp-conductor: intake: unexpected argument "${token}"\n`);
|
|
32
40
|
process.exit(2);
|
|
@@ -45,17 +53,36 @@ export async function intakeCommand(ctx: CommandContext): Promise<void> {
|
|
|
45
53
|
if (sub === "list") {
|
|
46
54
|
assertKnownArgs(ctx, 2);
|
|
47
55
|
const pending = store.pendingIntake(project.name);
|
|
48
|
-
|
|
56
|
+
const now = Date.now();
|
|
57
|
+
// Age at capture granularity: minutes for a fresh idea, hours once it
|
|
58
|
+
// has been waiting a while — the unit the operator actually cares about.
|
|
59
|
+
const items = pending.map((item) => {
|
|
60
|
+
const minutes = Math.max(0, Math.round((now - item.createdAt) / 60_000));
|
|
61
|
+
return {
|
|
62
|
+
id: item.id,
|
|
63
|
+
age: minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h`,
|
|
64
|
+
// null, not omitted, so a consumer can tell "operator's own idea"
|
|
65
|
+
// from "this build predates provenance" — the key is always there.
|
|
66
|
+
source: item.source ?? null,
|
|
67
|
+
text: item.text,
|
|
68
|
+
};
|
|
69
|
+
});
|
|
70
|
+
if (ctx.argv.includes("--json")) {
|
|
71
|
+
process.stdout.write(`${JSON.stringify({ project: project.name, items }, null, 2)}\n`);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
if (items.length === 0) {
|
|
49
75
|
process.stdout.write("no pending ideas\n");
|
|
50
76
|
return;
|
|
51
77
|
}
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
78
|
+
for (const item of items) {
|
|
79
|
+
// The source column exists so an operator reading this list can tell a
|
|
80
|
+
// machine-filed signal from something they thought of themselves —
|
|
81
|
+
// they are answerable for one and merely informed of the other. A dash
|
|
82
|
+
// marks their own idea; anything else names the signal that filed it.
|
|
83
|
+
process.stdout.write(
|
|
84
|
+
`${item.id} ${item.age.padStart(5)} ${item.source ?? "-"} ${item.text}\n`,
|
|
85
|
+
);
|
|
59
86
|
}
|
|
60
87
|
return;
|
|
61
88
|
}
|
|
@@ -95,7 +122,7 @@ export async function intakeCommand(ctx: CommandContext): Promise<void> {
|
|
|
95
122
|
return;
|
|
96
123
|
}
|
|
97
124
|
|
|
98
|
-
if (sub !== undefined && sub.startsWith("--") && INTAKE_FLAGS[sub]
|
|
125
|
+
if (sub !== undefined && sub.startsWith("--") && INTAKE_FLAGS[sub] === undefined) {
|
|
99
126
|
process.stderr.write(`omp-conductor: intake: unexpected argument "${sub}"\n`);
|
|
100
127
|
process.exit(2);
|
|
101
128
|
}
|
|
@@ -106,7 +133,10 @@ export async function intakeCommand(ctx: CommandContext): Promise<void> {
|
|
|
106
133
|
for (let i = 1; i < ctx.argv.length; i++) {
|
|
107
134
|
const token = ctx.argv[i];
|
|
108
135
|
if (token === undefined) continue;
|
|
109
|
-
|
|
136
|
+
// Only value flags step over their value here. `--json` is a list-only
|
|
137
|
+
// toggle, so on the capture path it falls through to the rejection below
|
|
138
|
+
// rather than being silently ignored on a verb that cannot honour it.
|
|
139
|
+
if (INTAKE_FLAGS[token] === "value") {
|
|
110
140
|
i += 1;
|
|
111
141
|
continue;
|
|
112
142
|
}
|
package/src/commands/stats.ts
CHANGED
|
@@ -32,21 +32,35 @@ merge settlement), attempts per merged issue, metered spend per merged issue,
|
|
|
32
32
|
the failure-class breakdown of everything that did not merge, and the tracked
|
|
33
33
|
GitHub API calls consumed over the window's days.
|
|
34
34
|
|
|
35
|
+
It then attributes that spend: worker runs grouped by the model actually
|
|
36
|
+
billed (runs, merges, metered spend, $/merge — runs that never observed a
|
|
37
|
+
resolved model group under "(unresolved)" rather than being guessed into a
|
|
38
|
+
bucket), and the daemon's own sessions per role (to-spec grooming and review
|
|
39
|
+
adjudication, with turns and metered spend). The orchestrator's tick session is
|
|
40
|
+
listed as unmetered: no usage events are observable for it, so its cost is
|
|
41
|
+
unknown and no figure includes it.
|
|
42
|
+
|
|
35
43
|
Continuation chains collapse into one journey: an issue that took six attempts
|
|
36
44
|
is one merged outcome with six runs and one lead time. Runs whose spend reads
|
|
37
45
|
$0.00 — harness telemetry absent (README Limitations) — are counted separately
|
|
38
|
-
as unmetered and never averaged into cost as if they were free.
|
|
46
|
+
as unmetered and never averaged into cost as if they were free. A daemon
|
|
47
|
+
session whose provider reported no cost is counted the same way, and never
|
|
48
|
+
printed as $0.00.
|
|
39
49
|
|
|
40
50
|
flags:
|
|
41
51
|
--since 7d | 30d | YYYY-MM-DD window start (default 7d)
|
|
42
52
|
--json print the stable report shape {project,
|
|
43
|
-
window, ghCalls, empty, total, repos
|
|
44
|
-
|
|
53
|
+
window, ghCalls, empty, total, repos,
|
|
54
|
+
models, sessions, groomSpendUsd,
|
|
55
|
+
groomUnmeteredSessions, unmeteredRoles} —
|
|
56
|
+
keys never move, so a consumer can rely on it
|
|
45
57
|
--project NAME the project to report (required only when
|
|
46
58
|
the config has more than one)
|
|
47
59
|
|
|
48
60
|
When nothing settled in the window the report says so explicitly ("empty:
|
|
49
|
-
true" in --json) rather than printing zeros that could read as measurements
|
|
61
|
+
true" in --json) rather than printing zeros that could read as measurements.
|
|
62
|
+
Daemon session spend is still reported there: a dry queue is exactly when
|
|
63
|
+
grooming runs, and that spend bought no outcome.`;
|
|
50
64
|
|
|
51
65
|
export async function statsCommand(ctx: CommandContext): Promise<void> {
|
|
52
66
|
// Help first: parsing stops before any config read or store open.
|
|
@@ -101,6 +115,7 @@ export async function statsCommand(ctx: CommandContext): Promise<void> {
|
|
|
101
115
|
window,
|
|
102
116
|
ghCalls: store.ghCallsBetween(window.sinceDay, window.untilDay),
|
|
103
117
|
runs: store.statsRuns(project.name, window.sinceEpochMs),
|
|
118
|
+
sessions: store.sessionSpendSince(project.name, window.sinceEpochMs),
|
|
104
119
|
});
|
|
105
120
|
if (ctx.argv.includes("--json")) process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
106
121
|
else {
|
package/src/commands/worker.ts
CHANGED
|
@@ -4,11 +4,17 @@
|
|
|
4
4
|
* Moved out of cli.ts's switch by the per-verb module split (#462);
|
|
5
5
|
* the daemon discovery resolves through {@link requireDaemonControl} so a
|
|
6
6
|
* live systemd-run daemon stays reachable when its pidfile is missing (#811).
|
|
7
|
+
*
|
|
8
|
+
* Run control is the most consequential mutating route on the daemon port — it
|
|
9
|
+
* can terminally settle a live run — so it carries the daemon's bearer token
|
|
10
|
+
* (Phase 4). The token is read, never minted: an absent file means no daemon
|
|
11
|
+
* has run under this home, and saying so is more useful than a bare 401.
|
|
7
12
|
*/
|
|
8
13
|
|
|
9
14
|
import type { CommandContext } from "./context.ts";
|
|
10
15
|
import { findProject, loadConfig } from "../config.ts";
|
|
11
16
|
import { requireDaemonControl } from "../lifecycle.ts";
|
|
17
|
+
import { httpAuthHeader, missingHttpTokenMessage } from "../http-token.ts";
|
|
12
18
|
|
|
13
19
|
export async function workerCommand(ctx: CommandContext): Promise<void> {
|
|
14
20
|
const sub = ctx.argv[1];
|
|
@@ -34,11 +40,13 @@ const project = findProject(loadConfig(), ctx.projectFlag);
|
|
|
34
40
|
// because the runtime directory was wiped (#811). The refusal chain is
|
|
35
41
|
// shared with `extend` through requireDaemonControl.
|
|
36
42
|
const daemon = await requireDaemonControl(project.name);
|
|
43
|
+
const auth = httpAuthHeader();
|
|
44
|
+
if (auth === undefined) throw new Error(missingHttpTokenMessage());
|
|
37
45
|
const response = await fetch(
|
|
38
46
|
`http://127.0.0.1:${daemon.port}/runs/${issue}/${sub}`,
|
|
39
47
|
{
|
|
40
48
|
method: "PUT",
|
|
41
|
-
headers: { "content-type": "application/json" },
|
|
49
|
+
headers: { "content-type": "application/json", ...auth },
|
|
42
50
|
body: JSON.stringify({
|
|
43
51
|
project: project.name,
|
|
44
52
|
source: "cli",
|
package/src/config-schema.ts
CHANGED
|
@@ -404,6 +404,19 @@ const projectSchema = z
|
|
|
404
404
|
// authored line instead of silently degrading the fleet's grooming duty
|
|
405
405
|
// to the default.
|
|
406
406
|
groomBelow: z.union([z.number().int().min(1), z.literal("always")]).optional(),
|
|
407
|
+
// The daemon's own grooming-scout role (#1041). Validated with exactly the
|
|
408
|
+
// regex and message the review adjudicator is: one role-name grammar for
|
|
409
|
+
// every role token conductor stores, so an operator who mistypes a
|
|
410
|
+
// provider/model here hears the same authored line either place. Optional
|
|
411
|
+
// rather than `.default()`ed — a config that never answered stays
|
|
412
|
+
// unwritten and reads as DEFAULT_GROOM_ROLE, exactly like `groomBelow`.
|
|
413
|
+
groomRole: z
|
|
414
|
+
.string()
|
|
415
|
+
.regex(
|
|
416
|
+
REVIEW_ADJUDICATOR_RE,
|
|
417
|
+
"must name one OMP model role — a single role token like \"task\" (never a provider/model, which omp owns)",
|
|
418
|
+
)
|
|
419
|
+
.optional(),
|
|
407
420
|
stateLabels: stateLabelsSchema.optional(),
|
|
408
421
|
routing: routingSchema.optional(),
|
|
409
422
|
caps: capsSchema.optional(),
|
package/src/config.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
DEFAULT_ARM_PROOF,
|
|
26
26
|
DEFAULT_AUTHORITY,
|
|
27
27
|
DEFAULT_CAPS,
|
|
28
|
+
DEFAULT_GROOM_ROLE,
|
|
28
29
|
DEFAULT_PROJECT_POLICY,
|
|
29
30
|
DEFAULT_REPORT_POLICY,
|
|
30
31
|
DEFAULT_REPORT_SCOPE,
|
|
@@ -387,6 +388,24 @@ export function resolveReview(p: ProjectConfig): ReviewPolicy {
|
|
|
387
388
|
};
|
|
388
389
|
}
|
|
389
390
|
|
|
391
|
+
/**
|
|
392
|
+
* The OMP model role this project's daemon-owned grooming scouts run under
|
|
393
|
+
* (#1041).
|
|
394
|
+
*
|
|
395
|
+
* One reader for the same reason {@link resolveReview} is one: the daemon
|
|
396
|
+
* launches these sessions unattended, so "which role did it spend under" must
|
|
397
|
+
* have exactly one answer, and a project that never chose gets
|
|
398
|
+
* {@link DEFAULT_GROOM_ROLE} — `task`, the general session role every install
|
|
399
|
+
* can launch without pinning a provider. Deliberately not folded into
|
|
400
|
+
* `review.adjudicator`: grooming judgement and review adjudication are
|
|
401
|
+
* separate operator choices, and sharing the key would silently move one when
|
|
402
|
+
* the other is retuned.
|
|
403
|
+
*/
|
|
404
|
+
export function resolveGroomRole(p: ProjectConfig): string {
|
|
405
|
+
const configured = p.groomRole;
|
|
406
|
+
return configured !== undefined && configured.trim() !== "" ? configured.trim() : DEFAULT_GROOM_ROLE;
|
|
407
|
+
}
|
|
408
|
+
|
|
390
409
|
/**
|
|
391
410
|
* A policy with no array shared with its source.
|
|
392
411
|
*
|
|
@@ -1263,6 +1282,13 @@ function finalizeProject(
|
|
|
1263
1282
|
const workerModel =
|
|
1264
1283
|
typeof rawWorkerModel === "string" && rawWorkerModel.trim() !== "" ? rawWorkerModel : undefined;
|
|
1265
1284
|
|
|
1285
|
+
// The daemon's grooming-scout role (#1041), a hint like `workerModel`: zod
|
|
1286
|
+
// already enforced the role-token grammar, so this only narrows the type and
|
|
1287
|
+
// drops a blank. Absent stays absent — the daemon reads it as
|
|
1288
|
+
// DEFAULT_GROOM_ROLE — so no config gains a key its operator never chose.
|
|
1289
|
+
const rawGroomRole = p["groomRole"];
|
|
1290
|
+
const groomRole = typeof rawGroomRole === "string" && rawGroomRole !== "" ? rawGroomRole : undefined;
|
|
1291
|
+
|
|
1266
1292
|
// The failover chain is a hint like workerModel, not a ceiling: an unusable
|
|
1267
1293
|
// entry or a malformed threshold is dropped rather than failing the load,
|
|
1268
1294
|
// and absent/empty `modelFallbacks` keeps today's dispatch byte for byte.
|
|
@@ -1367,6 +1393,7 @@ function finalizeProject(
|
|
|
1367
1393
|
tracker: { kind: "github", repo: trackerRepo },
|
|
1368
1394
|
queueLabel: p["queueLabel"] as string,
|
|
1369
1395
|
...(groomBelow === undefined ? {} : { groomBelow }),
|
|
1396
|
+
...(groomRole === undefined ? {} : { groomRole }),
|
|
1370
1397
|
stateLabels: {
|
|
1371
1398
|
inProgress: pickString(stateLabels?.["inProgress"], DEFAULT_STATE_LABELS.inProgress),
|
|
1372
1399
|
blocked: pickString(stateLabels?.["blocked"], DEFAULT_STATE_LABELS.blocked),
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The admission acknowledgement fence: proof that a *running* daemon, not just
|
|
3
|
+
* a daemon-shaped process, has seen the current install and admitted work under
|
|
4
|
+
* it.
|
|
5
|
+
*
|
|
6
|
+
* Setup and upgrade both need that proof and neither can get it from the store,
|
|
7
|
+
* because a daemon that never reached its first tick would leave the last
|
|
8
|
+
* daemon's rows sitting there looking like an answer. So the record is written
|
|
9
|
+
* by the tick itself, stamped with the generation of the process that wrote it,
|
|
10
|
+
* and `wakeDaemon` is here beside it as the one nudge that shortens the wait for
|
|
11
|
+
* it to appear.
|
|
12
|
+
*
|
|
13
|
+
* Kept out of `admission.ts` (the top-level gate module) and out of
|
|
14
|
+
* `admission-pass.ts` (the tick's use of it) on purpose: this is neither a gate
|
|
15
|
+
* nor a pass, it is the handshake either side of a restart reads.
|
|
16
|
+
*/
|
|
17
|
+
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { stateDir } from "../config.ts";
|
|
20
|
+
import { httpAuthHeader } from "../http-token.ts";
|
|
21
|
+
import { SYSTEMD_UNIT, livingDaemon, probeUnit } from "../lifecycle.ts";
|
|
22
|
+
import { pauseInstance } from "../pause.ts";
|
|
23
|
+
|
|
24
|
+
// acknowledgement (#651, review #3)
|
|
25
|
+
//
|
|
26
|
+
// The setup-fence's "acknowledgement" must come from the daemon itself, not a
|
|
27
|
+
// second synchronous worker count: a tick that passed its pause gate before
|
|
28
|
+
// the fence landed can sit in awaited tracker/routing/admission work and claim
|
|
29
|
+
// after the count and before setup mutates. Two places make that impossible
|
|
30
|
+
// and record it durably:
|
|
31
|
+
//
|
|
32
|
+
// - the tick's pause gate re-acknowledges every held pass (the daemon has
|
|
33
|
+
// reached its admission boundary and claims nothing);
|
|
34
|
+
// - the claim itself re-checks the pause immediately before creating the run
|
|
35
|
+
// row, so a tick already past its gate when the fence landed refuses at
|
|
36
|
+
// the claim — and writes the acknowledgement — instead of admitting.
|
|
37
|
+
//
|
|
38
|
+
// The acknowledgement file names the exact pause instance observed and the
|
|
39
|
+
// daemon generation that observed it, so the setup barrier can prove the
|
|
40
|
+
// acknowledged fence is the fence it froze and the acknowledgement belongs to
|
|
41
|
+
// the daemon it began with.
|
|
42
|
+
|
|
43
|
+
/** The durable admission acknowledgement a daemon writes when it observes a
|
|
44
|
+
* pause fence at an admission boundary. */
|
|
45
|
+
export interface AdmissionAckRecord {
|
|
46
|
+
/** The pause instance observed (source token, reason, creation instant). */
|
|
47
|
+
pause: { source: string; reason?: string; since: number };
|
|
48
|
+
/** The daemon generation that observed it — {@link daemonGeneration}. */
|
|
49
|
+
daemon: string;
|
|
50
|
+
/** When the daemon observed the fence. */
|
|
51
|
+
observedAt: number;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The acknowledgement file path. One per host: the fence is host-global. */
|
|
55
|
+
export function admissionAckPath(): string {
|
|
56
|
+
return join(stateDir(), "admission-ack.json");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The generation identity of the running daemon, or `undefined` when nothing
|
|
61
|
+
* provably runs. The live pidfile record wins (its pid and boot instant
|
|
62
|
+
* identify the exact instance, #377); a record-less ACTIVE unit is still a
|
|
63
|
+
* running daemon and its MainPID is the generation that lets a fence spot a
|
|
64
|
+
* supervisor restart (#651 review #2). Shared by the ack writer and the setup
|
|
65
|
+
* barrier's identity so both sides of the acknowledgement name the same
|
|
66
|
+
* instance by the same rule.
|
|
67
|
+
*/
|
|
68
|
+
export function daemonGeneration(): string | undefined {
|
|
69
|
+
const daemon = livingDaemon();
|
|
70
|
+
if (daemon !== undefined) return `${daemon.pid}@${daemon.startedAt}`;
|
|
71
|
+
const ownership = probeUnit(SYSTEMD_UNIT);
|
|
72
|
+
return ownership.kind === "active" ? `systemd:${ownership.pid}` : undefined;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Reads the current admission acknowledgement, if one is readable. A corrupt
|
|
76
|
+
* or unreadable file is absence — the barrier fails closed on the absence. */
|
|
77
|
+
export function readAdmissionAck(): AdmissionAckRecord | undefined {
|
|
78
|
+
let parsed: unknown;
|
|
79
|
+
try {
|
|
80
|
+
parsed = JSON.parse(readFileSync(admissionAckPath(), "utf8"));
|
|
81
|
+
} catch {
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
|
84
|
+
if (parsed === null || typeof parsed !== "object") return undefined;
|
|
85
|
+
const r = parsed as Record<string, unknown>;
|
|
86
|
+
const pause = r["pause"];
|
|
87
|
+
const daemon = r["daemon"];
|
|
88
|
+
const observedAt = r["observedAt"];
|
|
89
|
+
if (pause === null || typeof pause !== "object") return undefined;
|
|
90
|
+
const p = pause as Record<string, unknown>;
|
|
91
|
+
const source = p["source"];
|
|
92
|
+
const since = p["since"];
|
|
93
|
+
const reason = p["reason"];
|
|
94
|
+
if (typeof source !== "string" || source.length === 0) return undefined;
|
|
95
|
+
if (typeof since !== "number" || !Number.isFinite(since)) return undefined;
|
|
96
|
+
if (reason !== undefined && typeof reason !== "string") return undefined;
|
|
97
|
+
if (typeof daemon !== "string" || daemon.length === 0) return undefined;
|
|
98
|
+
if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return undefined;
|
|
99
|
+
return { pause: { source, since, ...(reason === undefined ? {} : { reason }) }, daemon, observedAt };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Writes the admission acknowledgement for the pause the daemon just observed.
|
|
104
|
+
* The observed instance is the most restrictive fence in force: a host-global
|
|
105
|
+
* sentinel gates every project and is the pause a host-wide transaction (setup,
|
|
106
|
+
* an all-projects hold) froze, so it wins over a project-scoped sentinel;
|
|
107
|
+
* otherwise the project's own sentinel is the effective one for the daemon's
|
|
108
|
+
* claims, matching {@link pauseInstance}. Recording the project sentinel while
|
|
109
|
+
* a global fence is also in force would name a narrower, older hold instead of
|
|
110
|
+
* the fence that actually gates the host — and a barrier that froze the global
|
|
111
|
+
* sentinel could then never match its own acknowledgement (#651 review #4). A
|
|
112
|
+
* no-op when no pause is readable: there is nothing to acknowledge. The
|
|
113
|
+
* generation is the writing daemon's own (the record it owns, or its
|
|
114
|
+
* supervised MainPID).
|
|
115
|
+
*/
|
|
116
|
+
export function writeAdmissionAck(project: string): void {
|
|
117
|
+
// `pauseInstance()` reads only the global sentinel; while it is in force it
|
|
118
|
+
// is the fence that gates every project, and the durable record must name it
|
|
119
|
+
// rather than a per-project hold that predates it.
|
|
120
|
+
const pause = pauseInstance() ?? pauseInstance(project);
|
|
121
|
+
if (pause === undefined) return;
|
|
122
|
+
const generation = daemonGeneration();
|
|
123
|
+
if (generation === undefined) return;
|
|
124
|
+
const path = admissionAckPath();
|
|
125
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
126
|
+
const record: AdmissionAckRecord = { pause, daemon: generation, observedAt: Date.now() };
|
|
127
|
+
const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
|
|
128
|
+
try {
|
|
129
|
+
writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
|
|
130
|
+
renameSync(tmp, path);
|
|
131
|
+
} catch (err) {
|
|
132
|
+
rmSync(tmp, { force: true });
|
|
133
|
+
throw err;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Wake the daemon's dispatch loop to prompt a pass (best-effort; used by the
|
|
138
|
+
* setup barrier so a live daemon acknowledges the fence without waiting for
|
|
139
|
+
* its next five-minute tick). A refused/failed wake just lengthens the wait;
|
|
140
|
+
* the barrier's deadline is what bounds it.
|
|
141
|
+
*
|
|
142
|
+
* `POST /wake` is authenticated (Phase 4), so this reads the token the daemon
|
|
143
|
+
* minted at start. An absent token means no daemon has ever started under this
|
|
144
|
+
* `$OMP_CONDUCTOR_HOME`, so there is nothing to wake: skip the request rather
|
|
145
|
+
* than spend a round trip earning a 401. Still void, still best-effort — the
|
|
146
|
+
* contract is unchanged, only the reason a wake can be a no-op is wider. */
|
|
147
|
+
export async function wakeDaemon(port: number): Promise<void> {
|
|
148
|
+
const auth = httpAuthHeader();
|
|
149
|
+
if (auth === undefined) return;
|
|
150
|
+
try {
|
|
151
|
+
await fetch(`http://127.0.0.1:${port}/wake`, {
|
|
152
|
+
method: "POST",
|
|
153
|
+
headers: auth,
|
|
154
|
+
signal: AbortSignal.timeout(1_500),
|
|
155
|
+
});
|
|
156
|
+
} catch {
|
|
157
|
+
// Best-effort by contract: the caller's bounded wait is the backstop.
|
|
158
|
+
}
|
|
159
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The dispatcher's side of admission: how many slots there are, what to do with
|
|
3
|
+
* the candidates that got one, and how to describe the ones that did not.
|
|
4
|
+
*
|
|
5
|
+
* The gates themselves are `admission.ts` at the top level — pure, storeless,
|
|
6
|
+
* and shared with everything that needs to ask "would this be admitted". This
|
|
7
|
+
* module is deliberately the other half: it holds no rule, only the concurrency
|
|
8
|
+
* primitive every pass launches through (`WorkerPool`) and the two summaries the
|
|
9
|
+
* tick logs and the digest read. Hence the name — it is the *pass*, not the
|
|
10
|
+
* policy, and the two must not be one file or the policy becomes untestable
|
|
11
|
+
* without a pool.
|
|
12
|
+
*/
|
|
13
|
+
import type { Admission, AdmissionHold } from "../admission.ts";
|
|
14
|
+
import type { AdmissionHoldReason, DispatchSummary } from "../types.ts";
|
|
15
|
+
|
|
16
|
+
export const HOLD_SAMPLE_SIZE = 5;
|
|
17
|
+
export const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
18
|
+
"parent-lookup-error",
|
|
19
|
+
"open-pr-lookup-error",
|
|
20
|
+
"issue-state-lookup-error",
|
|
21
|
+
"critical-base-verify-error",
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
/** Groups transient decisions into the bounded record exposed by status. */
|
|
25
|
+
export function summarizeDispatch(
|
|
26
|
+
ready: number,
|
|
27
|
+
routed: number,
|
|
28
|
+
claimed: number,
|
|
29
|
+
admitted: number,
|
|
30
|
+
holds: readonly AdmissionHold[],
|
|
31
|
+
completedAt = Date.now(),
|
|
32
|
+
settled = 0,
|
|
33
|
+
parked = 0,
|
|
34
|
+
): DispatchSummary {
|
|
35
|
+
const groups = new Map<AdmissionHoldReason, { count: number; issues: number[]; details: string[] }>();
|
|
36
|
+
for (const hold of holds) {
|
|
37
|
+
const group = groups.get(hold.reason) ?? { count: 0, issues: [], details: [] };
|
|
38
|
+
group.count += 1;
|
|
39
|
+
if (group.issues.length < HOLD_SAMPLE_SIZE) {
|
|
40
|
+
// `issues` and `details` share the sample: the detail is only kept when
|
|
41
|
+
// the issue it explains is, so the arrays stay index-aligned.
|
|
42
|
+
group.issues.push(hold.issue);
|
|
43
|
+
if (hold.detail !== undefined) group.details.push(hold.detail);
|
|
44
|
+
}
|
|
45
|
+
groups.set(hold.reason, group);
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
completedAt,
|
|
49
|
+
ready,
|
|
50
|
+
routed,
|
|
51
|
+
claimed,
|
|
52
|
+
admitted,
|
|
53
|
+
degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
|
|
54
|
+
holds: [...groups]
|
|
55
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
56
|
+
.map(([reason, group]) => ({
|
|
57
|
+
reason,
|
|
58
|
+
count: group.count,
|
|
59
|
+
issues: group.issues,
|
|
60
|
+
...(group.details.length === 0 ? {} : { details: group.details }),
|
|
61
|
+
})),
|
|
62
|
+
settled,
|
|
63
|
+
// Omitted at zero so a pass with nothing parked keeps the pre-#507 record
|
|
64
|
+
// shape byte for byte — old persisted rows lack the key and readers use
|
|
65
|
+
// `?? 0` either way.
|
|
66
|
+
...(parked === 0 ? {} : { parked }),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The record of a held pass (#497): the loop ran — settling and reconciling
|
|
72
|
+
* above the pause gate — but routing never happened and nothing was admitted.
|
|
73
|
+
* `ready`/`routed`/`claimed` stay 0 because they were never computed; the
|
|
74
|
+
* `paused` flag is what keeps a reader from reading them as an empty queue.
|
|
75
|
+
*/
|
|
76
|
+
export function summarizeHeldPass(settled: number, completedAt = Date.now()): DispatchSummary {
|
|
77
|
+
return {
|
|
78
|
+
completedAt,
|
|
79
|
+
ready: 0,
|
|
80
|
+
routed: 0,
|
|
81
|
+
claimed: 0,
|
|
82
|
+
admitted: 0,
|
|
83
|
+
degraded: false,
|
|
84
|
+
holds: [],
|
|
85
|
+
settled,
|
|
86
|
+
paused: true,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface WorkerPool {
|
|
91
|
+
launch(work: Promise<void>): void;
|
|
92
|
+
activeCount(): number;
|
|
93
|
+
drain(): Promise<void>;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Keeps background workers alive without making the five-minute tick await them.
|
|
98
|
+
*
|
|
99
|
+
* `onSettled` fires as each worker leaves the pool, whichever way it ended
|
|
100
|
+
* (#878): that is the instant a slot frees, and without it queued work waited
|
|
101
|
+
* for the next scheduled pass — up to five minutes of idle capacity that reads,
|
|
102
|
+
* from outside, exactly like a stalled queue. It only *prompts* a pass; every
|
|
103
|
+
* hold, drain, lane, budget and routing gate is re-evaluated by that pass as
|
|
104
|
+
* usual.
|
|
105
|
+
*/
|
|
106
|
+
export function createWorkerPool(onSettled?: () => void): WorkerPool {
|
|
107
|
+
const active = new Set<Promise<void>>();
|
|
108
|
+
return {
|
|
109
|
+
launch(work) {
|
|
110
|
+
active.add(work);
|
|
111
|
+
const settled = (): void => {
|
|
112
|
+
active.delete(work);
|
|
113
|
+
onSettled?.();
|
|
114
|
+
};
|
|
115
|
+
void work.then(settled, settled);
|
|
116
|
+
},
|
|
117
|
+
activeCount: () => active.size,
|
|
118
|
+
async drain() {
|
|
119
|
+
await Promise.allSettled(active);
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** `--once` awaits workers; the resident daemon registers them for shutdown. */
|
|
125
|
+
export async function dispatchAdmissions(
|
|
126
|
+
admitted: readonly Admission[],
|
|
127
|
+
run: (admission: Admission) => Promise<void>,
|
|
128
|
+
pool?: WorkerPool,
|
|
129
|
+
): Promise<void> {
|
|
130
|
+
if (pool !== undefined) {
|
|
131
|
+
for (const admission of admitted) pool.launch(run(admission));
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
await Promise.allSettled(admitted.map(run));
|
|
135
|
+
}
|