omp-conductor 0.20.0 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/schema/config.schema.json +24 -0
- package/src/admission.ts +58 -14
- package/src/arm-challenge.ts +54 -3
- package/src/briefs/console.md +10 -5
- package/src/commands/arm.ts +7 -5
- package/src/commands/companion.ts +52 -16
- package/src/commands/drain.ts +12 -5
- package/src/commands/worker.ts +5 -0
- package/src/config-schema.ts +8 -0
- package/src/config.ts +17 -2
- package/src/daemon/drain.ts +33 -0
- package/src/daemon/groom-pass.ts +16 -6
- package/src/daemon/http.ts +28 -0
- package/src/daemon/review.ts +11 -11
- package/src/daemon/runtime.ts +55 -3
- package/src/daemon/settle-pass.ts +19 -2
- package/src/daemon/supervision.ts +144 -1
- package/src/daemon/tick.ts +31 -14
- package/src/daemon/views.ts +17 -0
- package/src/daemon.ts +4 -3
- package/src/decisions.ts +20 -9
- package/src/diff-flags.ts +204 -28
- package/src/doctor.ts +27 -9
- package/src/escalate.ts +187 -15
- package/src/failure-class.ts +358 -5
- package/src/fleet.ts +55 -25
- package/src/graph-health.ts +17 -1
- package/src/groom.ts +11 -0
- package/src/orchestrator-tick.ts +302 -24
- package/src/reports.ts +4 -1
- package/src/settlement.ts +133 -12
- package/src/status-render.ts +38 -9
- package/src/store.ts +64 -15
- package/src/to-spec.ts +285 -24
- package/src/tracker/github.ts +132 -10
- package/src/types.ts +49 -3
- package/src/verbs/server.ts +16 -12
- package/src/worker.ts +149 -35
package/src/failure-class.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* green test per class" possible without a daemon, a tracker or a network.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import type
|
|
18
|
+
import { DEFAULT_CAPS, type Caps, type FailureClass, type RecoveryAction, type RunRecord } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
/** Facts the caller fetched, each only for the rows that need it.
|
|
21
21
|
*
|
|
@@ -34,6 +34,14 @@ export interface ClassifyFacts {
|
|
|
34
34
|
* reachable. Lets the table tell an infrastructure outage (#177) from a
|
|
35
35
|
* deterministic test failure by the log's own words. */
|
|
36
36
|
failingLog?: string;
|
|
37
|
+
/** The PR's changed-file list, derived at the settle call site from the
|
|
38
|
+
* diff the settlement already fetches (#1059). Lets the table tell a
|
|
39
|
+
* Compose dependency-startup failure no PR could have caused from one a
|
|
40
|
+
* PR's own diff did — a PR that touches no container configuration cannot
|
|
41
|
+
* have broken `docker compose up`. Absent means the diff could not be
|
|
42
|
+
* read (or was cut short), which is "could not tell": the conditional
|
|
43
|
+
* signature below then stays silent and the attempt is charged. */
|
|
44
|
+
changedFiles?: string[];
|
|
37
45
|
}
|
|
38
46
|
|
|
39
47
|
export interface Classification {
|
|
@@ -194,6 +202,45 @@ export const SPINNING_CAP_CLASSES: readonly FailureClass[] = [
|
|
|
194
202
|
"wall-clock-cap-spinning",
|
|
195
203
|
];
|
|
196
204
|
|
|
205
|
+
/** How much of the wall-clock ceiling the default stall window may take (#1086). */
|
|
206
|
+
export const WALL_CLOCK_STALL_SHARE = 3;
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* The transcript-silence window a stalled run is settled at (#1086): the
|
|
210
|
+
* configured `workerStallSilenceMs`, or a third of the wall-clock ceiling when
|
|
211
|
+
* it is null or unconfigured. Shared by the daemon's progress watch (which
|
|
212
|
+
* settles at this threshold) and the classifier (which recognises a row killed
|
|
213
|
+
* at it), so the two can never disagree about what a stall is.
|
|
214
|
+
*/
|
|
215
|
+
export function stallSilenceMs(
|
|
216
|
+
caps:
|
|
217
|
+
| {
|
|
218
|
+
workerStallSilenceMs?: number | null;
|
|
219
|
+
workerWallClockMs: number;
|
|
220
|
+
}
|
|
221
|
+
| undefined,
|
|
222
|
+
): number {
|
|
223
|
+
const wallClock = caps?.workerWallClockMs ?? DEFAULT_CAPS.workerWallClockMs;
|
|
224
|
+
return caps?.workerStallSilenceMs ?? Math.floor(wallClock / WALL_CLOCK_STALL_SHARE);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* How long a run sat silent before its stall kill, from the row's own facts
|
|
229
|
+
* (#1086). Defined only when the progress watch had observed the run AND the
|
|
230
|
+
* silence reached the configured window — the exact predicate that keeps every
|
|
231
|
+
* other killed-under-ceiling row (a drain, an operator stop) reading as
|
|
232
|
+
* `admin-kill`.
|
|
233
|
+
*/
|
|
234
|
+
export function stallSilence(
|
|
235
|
+
run: RunRecord,
|
|
236
|
+
caps: { workerStallSilenceMs?: number | null; workerWallClockMs: number } | undefined,
|
|
237
|
+
): number | undefined {
|
|
238
|
+
if (run.lastProgressAt === undefined || run.endedAt === undefined) return undefined;
|
|
239
|
+
const silent = run.endedAt - run.lastProgressAt;
|
|
240
|
+
if (silent < stallSilenceMs(caps)) return undefined;
|
|
241
|
+
return silent;
|
|
242
|
+
}
|
|
243
|
+
|
|
197
244
|
/**
|
|
198
245
|
* A stable fingerprint of the infrastructure signature list (#638). The
|
|
199
246
|
* historical reconciliation persists a per-project review cursor stamped with
|
|
@@ -208,6 +255,62 @@ export function infraSignatureVersion(): string {
|
|
|
208
255
|
return INFRA_LOG_SIGNATURES.join("|");
|
|
209
256
|
}
|
|
210
257
|
|
|
258
|
+
/**
|
|
259
|
+
* Compose's own sentence for a dependency that failed its healthcheck during
|
|
260
|
+
* `docker compose up`: "dependency failed to start: container
|
|
261
|
+
* chad-postgres-1 is unhealthy" (#1059). Matched with its trailing context,
|
|
262
|
+
* never a bare `is unhealthy` — that phrase is common enough in application
|
|
263
|
+
* logs to be unsafe on its own. Shared with the settle call site so the diff
|
|
264
|
+
* fetch and the classifier can never disagree about which log sentence needs
|
|
265
|
+
* the changed-file list.
|
|
266
|
+
*/
|
|
267
|
+
export const COMPOSE_DEPENDENCY_STARTUP_SIGNATURE =
|
|
268
|
+
"dependency failed to start: container ";
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* A changed path that configures the containers Compose jobs run, or the
|
|
272
|
+
* workflow that invokes them: a compose file — `docker-compose*.ya?ml`,
|
|
273
|
+
* Compose v2's default `compose.ya?ml`, or any `*.ya?ml` directly inside a
|
|
274
|
+
* `compose/` directory (#1071) — anywhere in the tree, a `Dockerfile*`, or
|
|
275
|
+
* anything under `.github/workflows/`. A PR touching any of these can have
|
|
276
|
+
* broken `docker compose up` itself — that is what turns the signature below
|
|
277
|
+
* from infrastructure into a charged, deterministic failure. Matched
|
|
278
|
+
* case-insensitively: the price of missing a container file is waiving a
|
|
279
|
+
* genuine attempt, while a false positive (a path merely *named* like one)
|
|
280
|
+
* only charges one.
|
|
281
|
+
*/
|
|
282
|
+
const CONTAINER_CONFIG_PATH =
|
|
283
|
+
/(?:^|\/)(?:docker-compose[^/]*\.ya?ml|compose\.ya?ml|compose\/[^/]+\.ya?ml|Dockerfile[^/]*)$|(?:^|\/)\.github\/workflows\//i;
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Evidence that a failed check's log names a Compose dependency-startup
|
|
287
|
+
* failure, or `undefined` when the log does not name one or the PR's own diff
|
|
288
|
+
* could have caused it (#1059).
|
|
289
|
+
*
|
|
290
|
+
* Deliberately conditional on the PR's changed-file list, and deliberately
|
|
291
|
+
* left out of {@link INFRA_LOG_SIGNATURES}: a job that runs `docker compose`
|
|
292
|
+
* can be broken by the diff itself — a bad `docker-compose.yml` in a PR
|
|
293
|
+
* produces this exact sentence — so the blanket signature would waive the
|
|
294
|
+
* implementation attempt for a real defect. Only when the diff touches no
|
|
295
|
+
* compose file, no Dockerfile and no workflow is the fault one the PR cannot
|
|
296
|
+
* have introduced, and the recovery (`rerun-checks`) is bounded by the strike
|
|
297
|
+
* cap either way.
|
|
298
|
+
*
|
|
299
|
+
* Fails closed: `undefined` changedFiles is "could not read the diff", never
|
|
300
|
+
* "clean", and stays silent — an unknown or truncated diff must not waive an
|
|
301
|
+
* attempt.
|
|
302
|
+
*/
|
|
303
|
+
export function composeDependencyStartup(
|
|
304
|
+
log: string,
|
|
305
|
+
changedFiles: string[] | undefined,
|
|
306
|
+
): string | undefined {
|
|
307
|
+
if (changedFiles === undefined) return undefined;
|
|
308
|
+
const lower = log.toLowerCase();
|
|
309
|
+
if (!lower.includes(COMPOSE_DEPENDENCY_STARTUP_SIGNATURE)) return undefined;
|
|
310
|
+
if (changedFiles.some((path) => CONTAINER_CONFIG_PATH.test(path))) return undefined;
|
|
311
|
+
return COMPOSE_DEPENDENCY_STARTUP_SIGNATURE.trim();
|
|
312
|
+
}
|
|
313
|
+
|
|
211
314
|
/** Lowercased check state — `gh pr checks` has emitted both `failure` and
|
|
212
315
|
* `FAILURE` across versions, and the classifier's callers must agree on one
|
|
213
316
|
* spelling so log selection and classification see the same set of checks. */
|
|
@@ -601,10 +704,163 @@ export function noVerdictExit(run: RunRecord): string | undefined {
|
|
|
601
704
|
return `the session ended without delivering a settlement verdict; last words: "${lastWords.slice(0, 80)}"`;
|
|
602
705
|
}
|
|
603
706
|
|
|
707
|
+
/** A reliable PR reference inside a blocker's prose: a `PR #N`/
|
|
708
|
+
* `pull request #N` mention or a GitHub `…/pull/N` URL. Named once so the
|
|
709
|
+
* evidence's "the next act is an observation of this PR" claim and the code
|
|
710
|
+
* that finds the PR stay one pattern (#1068). */
|
|
711
|
+
const PR_REFERENCE_PATTERN =
|
|
712
|
+
/(?:PR|pull request)\s+#?\s*\d+|github\.com\/[^\s/]+\/[^\s/]+\/pull\/\d+/i;
|
|
713
|
+
|
|
714
|
+
/**
|
|
715
|
+
* The `blockers:` list of a blocked settlement's stored report, in order
|
|
716
|
+
* (#1068). The worker's structured yield renders one item per blocker as
|
|
717
|
+
* ` - <item>` under a `blockers:` heading, and prose settlements that use the
|
|
718
|
+
* same heading parse the same way. `undefined` when the report carries no
|
|
719
|
+
* such section — a report that names no blockers and a missing report read
|
|
720
|
+
* identically, which is honest: the worker did not name a condition to
|
|
721
|
+
* observe.
|
|
722
|
+
*/
|
|
723
|
+
function blockedSettlementBlockers(report: string | undefined): string[] | undefined {
|
|
724
|
+
if (report === undefined) return undefined;
|
|
725
|
+
const blockers: string[] = [];
|
|
726
|
+
let inBlockers = false;
|
|
727
|
+
for (const line of report.split("\n")) {
|
|
728
|
+
if (inBlockers) {
|
|
729
|
+
const item = /^ {2,}-\s+(.*)$/.exec(line);
|
|
730
|
+
if (item === null) break;
|
|
731
|
+
blockers.push(item[1]!.trim());
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
if (/^blockers:\s*$/i.test(line)) inBlockers = true;
|
|
735
|
+
}
|
|
736
|
+
return blockers.length === 0 ? undefined : blockers;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
/**
|
|
740
|
+
* An explicit question in the blocked row's own words, or `undefined` when the
|
|
741
|
+
* row names none (#1068). The worker's question travels in the pre-existing
|
|
742
|
+
* `lastError` slot, or as a `blockers:` item that ends interrogative; a
|
|
743
|
+
* blocker that names a condition to observe ("PR #1067 checks still
|
|
744
|
+
* pending…") is not a question. Quoted verbatim into the escalation evidence,
|
|
745
|
+
* because the orchestrator answers the worker's own words rather than a
|
|
746
|
+
* paraphrase.
|
|
747
|
+
*/
|
|
748
|
+
function blockedQuestion(run: RunRecord): string | undefined {
|
|
749
|
+
if (run.lastError !== undefined && run.lastError.trim() !== "") return run.lastError;
|
|
750
|
+
return blockedSettlementBlockers(run.report)?.find((b) => /[??]\s*$/.test(b));
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
/**
|
|
754
|
+
* The report's first substantive line: skips the `status:`/`pr:`/`branch:`/
|
|
755
|
+
* `head:` headers and blank lines a structured settlement puts on top, and
|
|
756
|
+
* bounds the quote so a prose dump cannot balloon an evidence line. `undefined`
|
|
757
|
+
* when the report is empty or carries nothing but scaffold.
|
|
758
|
+
*/
|
|
759
|
+
function firstReportContentLine(report: string | undefined): string | undefined {
|
|
760
|
+
if (report === undefined) return undefined;
|
|
761
|
+
for (const raw of report.split("\n")) {
|
|
762
|
+
const line = raw.trim();
|
|
763
|
+
if (line === "") continue;
|
|
764
|
+
if (/^(?:status|pr|branch|head):\s*\S/.test(line)) continue;
|
|
765
|
+
return line.length > 200 ? `${line.slice(0, 200)}…` : line;
|
|
766
|
+
}
|
|
767
|
+
return undefined;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
/**
|
|
771
|
+
* What the wall-clock ceiling actually buys on this fleet right now (#1063).
|
|
772
|
+
*
|
|
773
|
+
* `caps.workerMaxTurns` and `caps.workerWallClockMs` are independent config
|
|
774
|
+
* values, but they meet at an implicit pace: 0.5 minutes per turn at the
|
|
775
|
+
* shipped defaults. A model slower than that can never reach the turn ceiling,
|
|
776
|
+
* so the budget a groomer sized a slice against is fiction — the run dies of
|
|
777
|
+
* the clock first. This observation is derived from *completed* runs (never
|
|
778
|
+
* from the configured numbers, whose ratio is a constant and tells nobody
|
|
779
|
+
* anything), so `status` and the settlement report can name the real number.
|
|
780
|
+
*/
|
|
781
|
+
export interface ObservedTurnBudget {
|
|
782
|
+
/** Aggregate elapsed minutes ÷ aggregate turns across the sample. */
|
|
783
|
+
minutesPerTurn: number;
|
|
784
|
+
/** Completed runs the figure was derived from. */
|
|
785
|
+
sampleSize: number;
|
|
786
|
+
/** ⌊wall clock ÷ minutesPerTurn⌋ — what the clock buys at that pace. */
|
|
787
|
+
effectiveTurns: number;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
/** How many completed runs the effective-budget sample spans. */
|
|
791
|
+
export const EFFECTIVE_BUDGET_SAMPLE_RUNS = 20;
|
|
792
|
+
|
|
793
|
+
/** A wall-clock kill below this share of its turn ceiling is a latency
|
|
794
|
+
* verdict; at or above it, both ceilings were nearly exhausted and size is
|
|
795
|
+
* the honest reading (#1063). */
|
|
796
|
+
export const WALL_CLOCK_SIZE_VERDICT_SHARE = 0.7;
|
|
797
|
+
|
|
798
|
+
/** Strips omp selector suffixes (`:thinking`, `:max`) so only the model path
|
|
799
|
+
* is compared. */
|
|
800
|
+
function modelBase(selector: string): string {
|
|
801
|
+
const colon = selector.indexOf(":");
|
|
802
|
+
return colon === -1 ? selector : selector.slice(0, colon);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* Whether one model attribution names the same model as the configured
|
|
807
|
+
* selector, compared as path suffixes: config says
|
|
808
|
+
* `openrouter/stealth/ox-alpha:max` while the transcript's resolved model
|
|
809
|
+
* reads `stealth/ox-alpha`, and strict equality would empty every sample.
|
|
810
|
+
* Role aliases (`@slow`) have no concrete spelling to resolve to from here, so
|
|
811
|
+
* they match themselves only — no sample, rather than an invented one.
|
|
812
|
+
*/
|
|
813
|
+
function sameModel(configured: string, observed: string): boolean {
|
|
814
|
+
if (configured.startsWith("@") || observed.startsWith("@")) return configured === observed;
|
|
815
|
+
const c = modelBase(configured);
|
|
816
|
+
const o = modelBase(observed);
|
|
817
|
+
return c === o || c.endsWith(`/${o}`) || o.endsWith(`/${c}`);
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/**
|
|
821
|
+
* Derives {@link ObservedTurnBudget} from completed-run rows, newest-first
|
|
822
|
+
* callers' ordering irrelevant. Rows without turns or elapsed time carry no
|
|
823
|
+
* pace and are skipped; when a configured worker model is named, rows written
|
|
824
|
+
* by another model are skipped with them (`resolvedModel` preferred — what
|
|
825
|
+
* actually wrote the messages — falling back to the dispatch record).
|
|
826
|
+
* `undefined` when nothing qualifying remains: absence is the honest answer,
|
|
827
|
+
* never a guess from the configured constants.
|
|
828
|
+
*/
|
|
829
|
+
export function observeTurnBudget(
|
|
830
|
+
samples: readonly Pick<RunRecord, "id" | "turns" | "startedAt" | "endedAt" | "model" | "resolvedModel">[],
|
|
831
|
+
caps: Pick<Caps, "workerWallClockMs">,
|
|
832
|
+
configuredModel?: string,
|
|
833
|
+
excludeRunId?: string,
|
|
834
|
+
): ObservedTurnBudget | undefined {
|
|
835
|
+
let minutes = 0;
|
|
836
|
+
let turns = 0;
|
|
837
|
+
let used = 0;
|
|
838
|
+
for (const s of samples) {
|
|
839
|
+
if (s.id === excludeRunId) continue;
|
|
840
|
+
if (s.turns <= 0 || s.endedAt === undefined || s.endedAt <= s.startedAt) continue;
|
|
841
|
+
if (configuredModel !== undefined) {
|
|
842
|
+
const attribution = s.resolvedModel ?? s.model;
|
|
843
|
+
if (attribution === undefined || !sameModel(configuredModel, attribution)) continue;
|
|
844
|
+
}
|
|
845
|
+
minutes += (s.endedAt - s.startedAt) / 60_000;
|
|
846
|
+
turns += s.turns;
|
|
847
|
+
used += 1;
|
|
848
|
+
}
|
|
849
|
+
if (used === 0 || turns <= 0) return undefined;
|
|
850
|
+
const minutesPerTurn = minutes / turns;
|
|
851
|
+
if (!Number.isFinite(minutesPerTurn) || minutesPerTurn <= 0) return undefined;
|
|
852
|
+
return {
|
|
853
|
+
minutesPerTurn,
|
|
854
|
+
sampleSize: used,
|
|
855
|
+
effectiveTurns: Math.floor(caps.workerWallClockMs / 60_000 / minutesPerTurn),
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
|
|
604
859
|
export function classifyRun(
|
|
605
860
|
run: RunRecord,
|
|
606
861
|
facts: ClassifyFacts,
|
|
607
|
-
caps?: Pick<Caps, "workerWallClockMs"
|
|
862
|
+
caps?: Pick<Caps, "workerWallClockMs"> & { workerStallSilenceMs?: number | null },
|
|
863
|
+
observedBudget?: ObservedTurnBudget,
|
|
608
864
|
): Classification {
|
|
609
865
|
const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined;
|
|
610
866
|
const providerError =
|
|
@@ -744,10 +1000,44 @@ export function classifyRun(
|
|
|
744
1000
|
}
|
|
745
1001
|
|
|
746
1002
|
if (run.state === "blocked") {
|
|
1003
|
+
// `blocked` is overloaded (#1068). A worker that finishes its work and
|
|
1004
|
+
// stops because it is waiting on an observable condition — almost always
|
|
1005
|
+
// its PR's checks — records that condition as `blockers:` in its
|
|
1006
|
+
// settlement report; a worker that stops to ask records an actual
|
|
1007
|
+
// question. Only the second shape is a human escalation: the first names
|
|
1008
|
+
// the next act (observe, then the sweep settles the row the way it
|
|
1009
|
+
// settles `settlement-stuck` when the PR resolves), so escalating it
|
|
1010
|
+
// wakes the orchestrator for a question that does not exist and the act
|
|
1011
|
+
// that *is* required appears nowhere. The evidence never claims the
|
|
1012
|
+
// worker "left no report" — it quotes the question or the first blocker,
|
|
1013
|
+
// and when the report genuinely is empty it says which fields are.
|
|
1014
|
+
const question = blockedQuestion(run);
|
|
1015
|
+
if (question !== undefined) {
|
|
1016
|
+
return { cls: "question", recovery: "escalate", evidence: question };
|
|
1017
|
+
}
|
|
1018
|
+
const blockers = blockedSettlementBlockers(run.report);
|
|
1019
|
+
if (blockers !== undefined) {
|
|
1020
|
+
// Prefer the blocker that names a PR: the reader's next act is an
|
|
1021
|
+
// observation of that PR and a merge, and the evidence must say so.
|
|
1022
|
+
const named = blockers.find((b) => PR_REFERENCE_PATTERN.test(b)) ?? blockers[0]!;
|
|
1023
|
+
return {
|
|
1024
|
+
cls: "awaiting-observation",
|
|
1025
|
+
recovery: "observe",
|
|
1026
|
+
evidence: `waiting on ${named}`,
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
const reportEmpty = run.report === undefined || run.report.trim() === "";
|
|
1030
|
+
const firstContent = firstReportContentLine(run.report);
|
|
747
1031
|
return {
|
|
748
1032
|
cls: "question",
|
|
749
1033
|
recovery: "escalate",
|
|
750
|
-
evidence:
|
|
1034
|
+
evidence: reportEmpty
|
|
1035
|
+
? `the worker stopped without a run report (run.report is empty${
|
|
1036
|
+
run.lastError === undefined || run.lastError.trim() === "" ? "; lastError is empty too" : ""
|
|
1037
|
+
})`
|
|
1038
|
+
: firstContent === undefined
|
|
1039
|
+
? "the worker stopped without asking a question or naming a blocker (run.report is non-empty but has no readable content)"
|
|
1040
|
+
: `the worker stopped without asking a question or naming a blocker — ${firstContent}`,
|
|
751
1041
|
};
|
|
752
1042
|
}
|
|
753
1043
|
|
|
@@ -771,6 +1061,30 @@ export function classifyRun(
|
|
|
771
1061
|
}
|
|
772
1062
|
|
|
773
1063
|
if (run.state === "killed") {
|
|
1064
|
+
// A stall the daemon settled (#1086), recognised from the row's own facts
|
|
1065
|
+
// — the watch observed a transcript write instant, and the silence to the
|
|
1066
|
+
// kill reached the configured window. Checked before the cap branches on
|
|
1067
|
+
// purpose: a hung session's turns never move, so this row is under its
|
|
1068
|
+
// ceilings, but even if a config change made it *reach* one mid-hang, the
|
|
1069
|
+
// silence is the operative fact about why it died. The same split as the
|
|
1070
|
+
// caps: artifacts continue, nothing shows escalates.
|
|
1071
|
+
const silentMs = stallSilence(run, caps);
|
|
1072
|
+
if (silentMs !== undefined) {
|
|
1073
|
+
const minutes = Math.round(silentMs / 60_000);
|
|
1074
|
+
const silent =
|
|
1075
|
+
minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`;
|
|
1076
|
+
return hasArtifacts
|
|
1077
|
+
? {
|
|
1078
|
+
cls: "progress-stall",
|
|
1079
|
+
recovery: "continue",
|
|
1080
|
+
evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}; the session never came back — work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
|
|
1081
|
+
}
|
|
1082
|
+
: {
|
|
1083
|
+
cls: "progress-stall",
|
|
1084
|
+
recovery: "escalate",
|
|
1085
|
+
evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
774
1088
|
if (run.turns >= run.maxTurns) {
|
|
775
1089
|
return hasArtifacts
|
|
776
1090
|
? {
|
|
@@ -789,21 +1103,40 @@ export function classifyRun(
|
|
|
789
1103
|
// worker slot until the ticks ran out (#490). The same split as the turns
|
|
790
1104
|
// path: artifacts mean the next attempt resumes from real work; nothing to
|
|
791
1105
|
// show means a human re-scopes, naming the time and spend consumed.
|
|
1106
|
+
//
|
|
1107
|
+
// #1063: which ceiling was ever reachable? The two caps meet at an implied
|
|
1108
|
+
// pace (wall clock ÷ max turns), so a run killed at the clock far under
|
|
1109
|
+
// its turn budget died of *latency* — re-attempting it as an oversized
|
|
1110
|
+
// slice spends a continuation on the wrong diagnosis. The evidence names
|
|
1111
|
+
// the run's own pace, the latency/size verdict the share of the turn
|
|
1112
|
+
// ceiling implies, and — when the caller supplied one — the effective
|
|
1113
|
+
// turn budget observed across completed runs on this model.
|
|
792
1114
|
const wallClockCap = caps?.workerWallClockMs;
|
|
793
1115
|
const wallClockElapsed =
|
|
794
1116
|
run.endedAt === undefined ? undefined : run.endedAt - run.startedAt;
|
|
795
1117
|
if (wallClockCap !== undefined && wallClockElapsed !== undefined && wallClockElapsed >= wallClockCap) {
|
|
796
1118
|
const clock = `${Math.round(wallClockElapsed / 60_000)}m of ${Math.round(wallClockCap / 60_000)}m`;
|
|
1119
|
+
const pace =
|
|
1120
|
+
run.turns > 0 ? ` at ${((wallClockElapsed / 60_000) / run.turns).toFixed(2)} min/turn` : "";
|
|
1121
|
+
const share = run.maxTurns > 0 ? run.turns / run.maxTurns : 1;
|
|
1122
|
+
const verdict =
|
|
1123
|
+
share < WALL_CLOCK_SIZE_VERDICT_SHARE
|
|
1124
|
+
? `latency verdict: ${run.turns}/${run.maxTurns} turns before the clock — the slice fits, the model is slow`
|
|
1125
|
+
: `size verdict: ${run.turns}/${run.maxTurns} turns — both ceilings nearly exhausted`;
|
|
1126
|
+
const observed =
|
|
1127
|
+
observedBudget === undefined
|
|
1128
|
+
? ""
|
|
1129
|
+
: `; effective ~${observedBudget.effectiveTurns} turns at ${observedBudget.minutesPerTurn.toFixed(2)} min/turn (last ${observedBudget.sampleSize} runs)`;
|
|
797
1130
|
return hasArtifacts
|
|
798
1131
|
? {
|
|
799
1132
|
cls: "wall-clock-cap-progress",
|
|
800
1133
|
recovery: "continue",
|
|
801
|
-
evidence: `wall clock ${clock}
|
|
1134
|
+
evidence: `wall clock ${clock}${pace} — ${verdict}${observed}; work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
|
|
802
1135
|
}
|
|
803
1136
|
: {
|
|
804
1137
|
cls: "wall-clock-cap-spinning",
|
|
805
1138
|
recovery: "escalate",
|
|
806
|
-
evidence: `wall clock ${clock}
|
|
1139
|
+
evidence: `wall clock ${clock}${pace} — ${verdict}${observed}; no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
|
|
807
1140
|
};
|
|
808
1141
|
}
|
|
809
1142
|
// Below its own ceilings, so nothing this worker did ended it: a daemon
|
|
@@ -854,6 +1187,26 @@ export function classifyRun(
|
|
|
854
1187
|
evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${signature}"`,
|
|
855
1188
|
};
|
|
856
1189
|
}
|
|
1190
|
+
// A Compose dependency-startup failure is infra only when the PR's
|
|
1191
|
+
// diff could not have caused it (#1059): a job that runs `docker
|
|
1192
|
+
// compose` is broken by a PR that breaks its own compose file, and
|
|
1193
|
+
// that attempt is genuinely spent. The settlement hands the
|
|
1194
|
+
// classifier the PR's changed-file list, so "did this diff touch any
|
|
1195
|
+
// container configuration" is a fact, not a guess from the log alone.
|
|
1196
|
+
// Absent list (or a log without the sentence) stays silent and the
|
|
1197
|
+
// row reads `ci-deterministic`.
|
|
1198
|
+
const compose = composeDependencyStartup(facts.failingLog, facts.changedFiles);
|
|
1199
|
+
if (compose !== undefined) {
|
|
1200
|
+
const check = checks.find((c) => normalise(c.state) === "failure");
|
|
1201
|
+
return {
|
|
1202
|
+
cls: "ci-infra",
|
|
1203
|
+
recovery: "rerun-checks",
|
|
1204
|
+
// Both halves, so the evidence says *which* sentence matched and
|
|
1205
|
+
// *why* it was not the diff's fault — the changed-file list
|
|
1206
|
+
// showed no container configuration.
|
|
1207
|
+
evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${compose}" — the PR diff touches no container configuration, so the PR cannot have failed this check`,
|
|
1208
|
+
};
|
|
1209
|
+
}
|
|
857
1210
|
}
|
|
858
1211
|
const failing = unresolved.filter((c) => normalise(c.state) === "failure");
|
|
859
1212
|
if (failing.length > 0) {
|
package/src/fleet.ts
CHANGED
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
lockPidAlive,
|
|
38
38
|
pidAlive,
|
|
39
39
|
readTelegramDmOwner,
|
|
40
|
+
readTelegramMisroutes,
|
|
40
41
|
readTelegramPollState,
|
|
41
42
|
resolveClaimedSessionFile,
|
|
42
43
|
resolveProjectTopicId,
|
|
@@ -64,6 +65,7 @@ export {
|
|
|
64
65
|
consumeDrain,
|
|
65
66
|
createDrain,
|
|
66
67
|
drainPath,
|
|
68
|
+
markDrained,
|
|
67
69
|
readDrain,
|
|
68
70
|
type CreateDrainOptions,
|
|
69
71
|
type DrainProblem,
|
|
@@ -103,6 +105,7 @@ import {
|
|
|
103
105
|
resolveArmState,
|
|
104
106
|
TICK_CONFIG_FILE,
|
|
105
107
|
tickConfigMatchesProject,
|
|
108
|
+
writeArmedMarker,
|
|
106
109
|
type ArmState,
|
|
107
110
|
type TickConfig,
|
|
108
111
|
type TickConfigResult,
|
|
@@ -308,9 +311,10 @@ export interface ArmMarkersWritten {
|
|
|
308
311
|
* A challenge filed and sent, with nothing armed yet.
|
|
309
312
|
*
|
|
310
313
|
* Arming used to block here for up to five minutes on an in-session
|
|
311
|
-
* acknowledgement.
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
+
* acknowledgement. Nothing waits now: the reply is consumed mechanically by
|
|
315
|
+
* the session whose topic the challenge went to (the orchestrator pane,
|
|
316
|
+
* #1061), or by `omp-conductor arm --reply` on a console host whose DM the
|
|
317
|
+
* operator answered in. This is the send half's receipt.
|
|
314
318
|
*/
|
|
315
319
|
export interface ArmChallengeSent {
|
|
316
320
|
outcome: "challenge-sent";
|
|
@@ -433,9 +437,12 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
433
437
|
// escalations already do (#318), following the bridge's current claim when the
|
|
434
438
|
// pinned id has gone stale (#407). Missing project config keeps flat-chat 0.13.
|
|
435
439
|
let sendTopic: number | undefined;
|
|
440
|
+
let sendProject: { name: string; workspaceRoot?: string } | undefined;
|
|
436
441
|
if (named !== undefined) {
|
|
437
442
|
try {
|
|
438
|
-
|
|
443
|
+
const cfg = findProject(loadConfig(), named);
|
|
444
|
+
sendTopic = resolveProjectTopicId(cfg, deps.pidAlive ?? pidAlive);
|
|
445
|
+
sendProject = { name: cfg.name, workspaceRoot: cfg.workspaceRoot };
|
|
439
446
|
} catch {
|
|
440
447
|
/* no project config */
|
|
441
448
|
}
|
|
@@ -508,7 +515,7 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
508
515
|
owner: channel.owner,
|
|
509
516
|
});
|
|
510
517
|
try {
|
|
511
|
-
await send(token, channel.owner, text, sendTopic);
|
|
518
|
+
await send(token, channel.owner, text, sendTopic, sendProject);
|
|
512
519
|
} catch (err) {
|
|
513
520
|
// The challenge never went out, so its transaction must not linger as a
|
|
514
521
|
// classifiable proof either.
|
|
@@ -566,7 +573,8 @@ function armReplyCommand(projectName?: string): string {
|
|
|
566
573
|
* proof cannot weaken it.
|
|
567
574
|
*
|
|
568
575
|
* Like {@link armTicks}, this returns as soon as the challenge is filed and
|
|
569
|
-
* sent:
|
|
576
|
+
* sent: the reply settles it mechanically on the orchestrator pane's own
|
|
577
|
+
* topic, or through `omp-conductor arm --reply` from the console.
|
|
570
578
|
*/
|
|
571
579
|
export async function armFleet(
|
|
572
580
|
projectNames: readonly string[],
|
|
@@ -625,9 +633,12 @@ export async function armFleet(
|
|
|
625
633
|
// operator is reading, and the reply step reaches the record from any project
|
|
626
634
|
// because it is fleet-wide rather than topic-scoped.
|
|
627
635
|
let sendTopic: number | undefined;
|
|
636
|
+
let sendProject: { name: string; workspaceRoot?: string } | undefined;
|
|
628
637
|
for (const target of targets) {
|
|
629
638
|
try {
|
|
630
|
-
|
|
639
|
+
const cfg = findProject(loadConfig(), target.project);
|
|
640
|
+
sendTopic = resolveProjectTopicId(cfg);
|
|
641
|
+
sendProject = { name: cfg.name, workspaceRoot: cfg.workspaceRoot };
|
|
631
642
|
} catch {
|
|
632
643
|
continue; /* no project config — try the next */
|
|
633
644
|
}
|
|
@@ -661,7 +672,7 @@ export async function armFleet(
|
|
|
661
672
|
owner: channel.owner,
|
|
662
673
|
});
|
|
663
674
|
try {
|
|
664
|
-
await send(token, channel.owner, text, sendTopic);
|
|
675
|
+
await send(token, channel.owner, text, sendTopic, sendProject);
|
|
665
676
|
} catch (err) {
|
|
666
677
|
clearArmTransaction(FLEET_ARM_KEY, challengeId);
|
|
667
678
|
throw new Error(
|
|
@@ -709,10 +720,12 @@ export type ArmReplyResult = ArmReplyAccepted | ArmReplyRefused;
|
|
|
709
720
|
* The verification half of the ceremony: classify the operator's verbatim
|
|
710
721
|
* message and, on a match, arm exactly the projects the challenge recorded.
|
|
711
722
|
*
|
|
712
|
-
* This
|
|
713
|
-
*
|
|
714
|
-
*
|
|
715
|
-
*
|
|
723
|
+
* This is the host CLI half, and it is one of two consumers of the same
|
|
724
|
+
* durable record: the orchestrator session's inbound path settles a reply
|
|
725
|
+
* sent to the project topic mechanically (#1061), while this command remains
|
|
726
|
+
* the console's route for a reply that landed in the operator DM. No session
|
|
727
|
+
* waits for anything: the challenge is durable state, so the halves are
|
|
728
|
+
* ordinary commands that can run minutes apart in different processes.
|
|
716
729
|
*
|
|
717
730
|
* The security properties are the send half's, unchanged. Only the project's
|
|
718
731
|
* own record and the fleet record are read (no other project's ceremony can be
|
|
@@ -3084,6 +3097,9 @@ export type FleetStatusReport = StatusSnapshot & {
|
|
|
3084
3097
|
/** The project-scoped durable to-spec grooming lifecycle as status lines
|
|
3085
3098
|
* (#809), or nothing when there is nothing to report. */
|
|
3086
3099
|
grooming: string | undefined;
|
|
3100
|
+
/** The sends this project's stale pin already delivered to the flat chat
|
|
3101
|
+
* (#1094), or undefined when the pin is healthy or freshly repaired. */
|
|
3102
|
+
topicMisroute: string | undefined;
|
|
3087
3103
|
lastStop: DaemonStop | undefined;
|
|
3088
3104
|
siblings: { project: string; live: number }[];
|
|
3089
3105
|
};
|
|
@@ -3182,6 +3198,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
|
|
|
3182
3198
|
workerPhases,
|
|
3183
3199
|
intake: intakeStatusLine(project.name),
|
|
3184
3200
|
grooming,
|
|
3201
|
+
topicMisroute: telegramMisrouteStatusLine(project),
|
|
3185
3202
|
lastStop,
|
|
3186
3203
|
siblings,
|
|
3187
3204
|
};
|
|
@@ -3203,6 +3220,7 @@ export function renderFleetStatusReport(report: FleetStatusReport): string {
|
|
|
3203
3220
|
report.lastStop,
|
|
3204
3221
|
report.siblings,
|
|
3205
3222
|
report.grooming,
|
|
3223
|
+
report.topicMisroute,
|
|
3206
3224
|
);
|
|
3207
3225
|
}
|
|
3208
3226
|
|
|
@@ -3210,6 +3228,26 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
3210
3228
|
return renderFleetStatusReport(await collectFleetStatus(projectName));
|
|
3211
3229
|
}
|
|
3212
3230
|
|
|
3231
|
+
/**
|
|
3232
|
+
* One status line naming the sends this project's stale pin already delivered
|
|
3233
|
+
* to the flat chat (#1094) — the send-side evidence behind doctor's topic-pin
|
|
3234
|
+
* warning. Rows match the *current* pin id, so re-pinning retires the row
|
|
3235
|
+
* without anyone deleting history.
|
|
3236
|
+
*/
|
|
3237
|
+
function telegramMisrouteStatusLine(project: ProjectConfig): string | undefined {
|
|
3238
|
+
const pinned = project.escalation.telegramTopicId;
|
|
3239
|
+
if (pinned === undefined) return undefined;
|
|
3240
|
+
const rows = readTelegramMisroutes().filter(
|
|
3241
|
+
(row) => row.project === project.name && row.staleTopicId === pinned,
|
|
3242
|
+
);
|
|
3243
|
+
if (rows.length === 0) return undefined;
|
|
3244
|
+
const last = new Date(rows[rows.length - 1]!.at).toISOString();
|
|
3245
|
+
return (
|
|
3246
|
+
`misroute ${rows.length} send(s) went to the flat chat because pinned topic ${pinned} was stale — ` +
|
|
3247
|
+
`last ${last}; re-pin escalation.telegramTopicId to clear`
|
|
3248
|
+
);
|
|
3249
|
+
}
|
|
3250
|
+
|
|
3213
3251
|
/**
|
|
3214
3252
|
* One line naming the brief layout, or nothing when it cannot be read.
|
|
3215
3253
|
*
|
|
@@ -3552,18 +3590,6 @@ function armClock(ms: number): string {
|
|
|
3552
3590
|
return minutes === 0 ? `${seconds}s` : `${minutes}m${String(seconds).padStart(2, "0")}s`;
|
|
3553
3591
|
}
|
|
3554
3592
|
|
|
3555
|
-
/**
|
|
3556
|
-
* The one armed-marker write both proofs share: same content, same mode, and
|
|
3557
|
-
* the same restamp of the pre-per-project shared marker the heartbeat still
|
|
3558
|
-
* honours — a project that just armed must not leave the bare marker around to
|
|
3559
|
-
* re-arm future fleets through `disarm` (#316).
|
|
3560
|
-
*/
|
|
3561
|
-
function writeArmedMarker(path: string, owner: string, arm: ArmState): void {
|
|
3562
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
3563
|
-
writeFileSync(path, `armed ${new Date().toISOString()} owner=${owner}\n`, { mode: 0o600 });
|
|
3564
|
-
if (arm.legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
|
|
3565
|
-
}
|
|
3566
|
-
|
|
3567
3593
|
/**
|
|
3568
3594
|
* The session surface the claim-only verdict judges: the tick-cwd-derived
|
|
3569
3595
|
* session directory plus, when the live claim's file lives elsewhere in the
|
|
@@ -3619,9 +3645,13 @@ async function sendTelegramMessage(
|
|
|
3619
3645
|
owner: string,
|
|
3620
3646
|
text: string,
|
|
3621
3647
|
topicId?: number,
|
|
3648
|
+
project?: { name: string; workspaceRoot?: string },
|
|
3622
3649
|
): Promise<void> {
|
|
3623
3650
|
// Shared transport: stale-topic retry + message_thread_id live in one place.
|
|
3624
|
-
|
|
3651
|
+
// The project pair rides along (#1094), so a challenge whose pinned topic is
|
|
3652
|
+
// refused follows the project's live claim or lands labelled flat like every
|
|
3653
|
+
// other send on this seam.
|
|
3654
|
+
await sendTelegram(token, owner, text, { topicId, ...(project === undefined ? {} : { project }) });
|
|
3625
3655
|
}
|
|
3626
3656
|
|
|
3627
3657
|
// ---------------------------------------------------------------------------
|