omp-conductor 0.18.0 → 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/REFERENCE.md +61 -11
- package/agents/to-spec.md +94 -0
- package/package.json +2 -1
- package/schema/config.schema.json +35 -1
- package/src/admission.ts +204 -75
- package/src/arm-challenge.ts +250 -57
- package/src/ask.ts +268 -7
- package/src/board.ts +17 -3
- package/src/briefs/orchestrator.md +62 -21
- package/src/briefs/to-spec.md +88 -0
- package/src/briefs/worker.md +2 -1
- package/src/cli.ts +124 -1
- package/src/command-help.ts +11 -0
- package/src/command-manifest.ts +38 -5
- package/src/commands/arm.ts +1 -1
- package/src/commands/context.ts +1 -0
- package/src/commands/drain.ts +176 -0
- package/src/commands/extend.ts +6 -10
- package/src/commands/intake.ts +4 -19
- package/src/commands/status.ts +5 -1
- package/src/commands/watch.ts +51 -16
- package/src/commands/worker.ts +9 -10
- package/src/config-schema.ts +43 -6
- package/src/config.ts +65 -9
- package/src/daemon.ts +879 -41
- package/src/dashboard/app.js +4 -1
- package/src/dashboard/server.ts +5 -2
- package/src/decisions.ts +243 -17
- package/src/diff-flags.ts +75 -1
- package/src/doctor.ts +60 -82
- package/src/escalate.ts +31 -14
- package/src/failure-class.ts +28 -2
- package/src/fleet.ts +239 -240
- package/src/gitops.ts +188 -81
- package/src/graph-health.ts +35 -1
- package/src/graph.ts +66 -1
- package/src/harness-loader.ts +59 -0
- package/src/host.ts +242 -2
- package/src/lifecycle.ts +122 -1
- package/src/omp-settings.ts +19 -0
- package/src/omp.ts +183 -21
- package/src/orchestrator-tick.ts +1591 -32
- package/src/orchestrator.ts +12 -0
- package/src/privileged.ts +1 -4
- package/src/release-policy.ts +503 -9
- package/src/session-host.ts +65 -6
- package/src/settlement.ts +69 -17
- package/src/setup-host.ts +1225 -9
- package/src/setup-install.ts +28 -0
- package/src/setup-wizard.ts +154 -3
- package/src/setup.ts +83 -17
- package/src/shell.ts +15 -0
- package/src/status-render.ts +216 -12
- package/src/store.ts +443 -42
- package/src/to-spec.ts +408 -0
- package/src/tracker/github.ts +104 -14
- package/src/types.ts +405 -19
- package/src/upgrade-verify.ts +209 -2
- package/src/upgrade.ts +175 -1
- package/src/verbs/protocol.ts +39 -0
- package/src/verbs/server.ts +765 -56
- package/src/verbs/socket.ts +24 -5
- package/src/worker.ts +12 -2
- package/src/worktree.ts +29 -12
package/src/daemon.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
|
-
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { chmodSync, closeSync, constants, existsSync, fchownSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, writeFileSync, type Dirent } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
13
|
import {
|
|
14
14
|
configPath,
|
|
@@ -36,12 +36,14 @@ import {
|
|
|
36
36
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
37
37
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
38
38
|
import { graphHint } from "./graph.ts";
|
|
39
|
+
import { hostConstraintsNotice } from "./host.ts";
|
|
39
40
|
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
40
41
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
41
42
|
import { runDoctor } from "./doctor.ts";
|
|
42
43
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
43
44
|
import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
44
|
-
import {
|
|
45
|
+
import { processStartTimeMs } from "./upgrade-verify.ts";
|
|
46
|
+
import { fleetLayers, herdrPaneOmpStarts, resolveHerdrSession } from "./fleet.ts";
|
|
45
47
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
46
48
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
47
49
|
import {
|
|
@@ -66,11 +68,13 @@ import type { Routed, UnroutableReason } from "./routing.ts";
|
|
|
66
68
|
import {
|
|
67
69
|
admitCandidates,
|
|
68
70
|
effectiveLane,
|
|
71
|
+
effectiveModel,
|
|
69
72
|
hasContinuationBudget,
|
|
70
73
|
hasFailedAttemptBudget,
|
|
71
74
|
laneEcho,
|
|
72
75
|
} from "./admission.ts";
|
|
73
|
-
import type { Admission, AdmissionHold
|
|
76
|
+
import type { Admission, AdmissionHold } from "./admission.ts";
|
|
77
|
+
import type { EffectiveModel, FileLane } from "./types.ts";
|
|
74
78
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
75
79
|
import {
|
|
76
80
|
adoptSalvagedPrs,
|
|
@@ -107,7 +111,7 @@ import {
|
|
|
107
111
|
snapshotDb,
|
|
108
112
|
utcDay,
|
|
109
113
|
} from "./store.ts";
|
|
110
|
-
import { GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
114
|
+
import { GhPrMissingError, GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
111
115
|
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
112
116
|
import type {
|
|
113
117
|
BaseFreeze,
|
|
@@ -118,6 +122,7 @@ import type {
|
|
|
118
122
|
DispatchSummary,
|
|
119
123
|
DigestBacklog,
|
|
120
124
|
Escalation,
|
|
125
|
+
InterruptCategory,
|
|
121
126
|
IssueComment,
|
|
122
127
|
IssueSnapshot,
|
|
123
128
|
MergedPrInfo,
|
|
@@ -125,13 +130,16 @@ import type {
|
|
|
125
130
|
ReleaseShape,
|
|
126
131
|
OrchestratorIncident,
|
|
127
132
|
PrState,
|
|
133
|
+
PrVerification,
|
|
128
134
|
ProjectConfig,
|
|
129
135
|
ReadyIssue,
|
|
136
|
+
ReportScope,
|
|
130
137
|
ReportingPolicy,
|
|
131
138
|
RepoTarget,
|
|
132
139
|
ReportRecord,
|
|
133
140
|
ResolvedGrants,
|
|
134
141
|
FailureClass,
|
|
142
|
+
HostConstraints,
|
|
135
143
|
RecoveryAction,
|
|
136
144
|
ReviewPolicy,
|
|
137
145
|
ReviewRevisionOutcome,
|
|
@@ -170,6 +178,7 @@ import {
|
|
|
170
178
|
import { githubVerbActions } from "./verbs/actions.ts";
|
|
171
179
|
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
172
180
|
import {
|
|
181
|
+
isHeadMismatch,
|
|
173
182
|
listenVerbChannel,
|
|
174
183
|
type VerbActions,
|
|
175
184
|
type VerbDeps,
|
|
@@ -244,6 +253,10 @@ export interface DrainSignal {
|
|
|
244
253
|
interface Deps {
|
|
245
254
|
project: ProjectConfig;
|
|
246
255
|
caps: Caps;
|
|
256
|
+
/** The typed host-constraints block (#721), re-resolved at the tick
|
|
257
|
+
* boundary like `project`/`caps`, and rendered into every brief the tick
|
|
258
|
+
* dispatches. Optional so tests that never exercise it can omit it. */
|
|
259
|
+
host?: HostConstraints;
|
|
247
260
|
tracker: Tracker;
|
|
248
261
|
store: Store;
|
|
249
262
|
/**
|
|
@@ -572,6 +585,29 @@ export function pauseProvenance(
|
|
|
572
585
|
}
|
|
573
586
|
}
|
|
574
587
|
|
|
588
|
+
/**
|
|
589
|
+
* One pause sentinel FILE read as an instance identity: who set it, why, and
|
|
590
|
+
* the creation instant, all from that exact file. An unreadable or malformed
|
|
591
|
+
* file is undefined — the caller may treat it as absence.
|
|
592
|
+
*/
|
|
593
|
+
function pauseInstanceAt(
|
|
594
|
+
path: string,
|
|
595
|
+
): { source: string; reason?: string; since: number } | undefined {
|
|
596
|
+
try {
|
|
597
|
+
const [line1, line2] = readFileSync(path, "utf8").split("\n");
|
|
598
|
+
const since = Date.parse(line1?.trim() ?? "");
|
|
599
|
+
if (!Number.isFinite(since)) return undefined;
|
|
600
|
+
if (line2 === undefined) return undefined;
|
|
601
|
+
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
|
|
602
|
+
if (match === null) return undefined;
|
|
603
|
+
const source = match[1]!;
|
|
604
|
+
const reason = match[2];
|
|
605
|
+
return { source, since, ...(reason === undefined ? {} : { reason }) };
|
|
606
|
+
} catch {
|
|
607
|
+
return undefined;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
575
611
|
/**
|
|
576
612
|
* One pause sentinel read as a single identity: who set it, why, and the
|
|
577
613
|
* creation instant, all from the SAME file that was selected. Unlike pairing
|
|
@@ -591,19 +627,34 @@ export function pauseInstance(
|
|
|
591
627
|
: [pausedPath(project), pausedPath()];
|
|
592
628
|
const path = paths.find((candidate) => existsSync(candidate));
|
|
593
629
|
if (path === undefined) return undefined;
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
630
|
+
return pauseInstanceAt(path);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Compare-and-clear one pause sentinel (#780 review): remove `path` only while
|
|
635
|
+
* it still holds exactly the `expected` instance — same source, same reason,
|
|
636
|
+
* same creation instant — as read by {@link pauseInstance}. A hold or pause
|
|
637
|
+
* that replaced or recreated the sentinel between the read and the clear is a
|
|
638
|
+
* newer instance (writes always re-stamp `since`), so it is never destroyed:
|
|
639
|
+
* returning false keeps the newer fence in force. Scoped strictly to one
|
|
640
|
+
* caller-provided path, so auto-expiry can clear the per-project spend-cap
|
|
641
|
+
* sentinel without ever touching the legacy global sentinel.
|
|
642
|
+
*/
|
|
643
|
+
export function clearPauseIfUnchanged(
|
|
644
|
+
path: string,
|
|
645
|
+
expected: { source: string; reason?: string; since: number },
|
|
646
|
+
): boolean {
|
|
647
|
+
const current = pauseInstanceAt(path);
|
|
648
|
+
if (current === undefined) return false;
|
|
649
|
+
if (
|
|
650
|
+
current.source !== expected.source ||
|
|
651
|
+
current.since !== expected.since ||
|
|
652
|
+
current.reason !== expected.reason
|
|
653
|
+
) {
|
|
654
|
+
return false;
|
|
606
655
|
}
|
|
656
|
+
rmSync(path, { force: true });
|
|
657
|
+
return true;
|
|
607
658
|
}
|
|
608
659
|
|
|
609
660
|
/**
|
|
@@ -640,6 +691,173 @@ export function setPaused(
|
|
|
640
691
|
}
|
|
641
692
|
}
|
|
642
693
|
|
|
694
|
+
// ------------------------------------------------------------------- drain
|
|
695
|
+
// (#484 slice 1) A project drain is a durable, self-expiring admission fence:
|
|
696
|
+
// the same boundary as a pause — settlement above it, nothing claimed below —
|
|
697
|
+
// but recorded with an absolute deadline, so an orchestrator crash can never
|
|
698
|
+
// strand admission. The record is a JSON file under the state directory (like
|
|
699
|
+
// the pause sentinel), scoped to exactly one configured project, and replaced
|
|
700
|
+
// atomically on create. Human CLI wording and release-verb coupling are later
|
|
701
|
+
// #484 children.
|
|
702
|
+
|
|
703
|
+
/** The persisted shape of one project drain. Every field is validated when a
|
|
704
|
+
* record is read — a record that cannot be trusted is never a fence. */
|
|
705
|
+
export interface DrainRecord {
|
|
706
|
+
/** The configured project this drain fences; must match the reader. */
|
|
707
|
+
project: string;
|
|
708
|
+
/** ISO instant the drain intent was recorded. */
|
|
709
|
+
createdAt: string;
|
|
710
|
+
/** Absolute ISO deadline: admission resumes automatically at or after it. */
|
|
711
|
+
expiresAt: string;
|
|
712
|
+
/** Purpose recorded at creation (release window, maintenance…). */
|
|
713
|
+
reason?: string;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/** Create-time options for {@link createDrain}. */
|
|
717
|
+
export interface CreateDrainOptions {
|
|
718
|
+
/** Absolute epoch-ms deadline — a drain must always expire on its own. */
|
|
719
|
+
expiresAt: number;
|
|
720
|
+
/** Purpose, persisted on the record and shown in structured status. */
|
|
721
|
+
reason?: string;
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/** Why a drain record could not be trusted, named deterministically. */
|
|
725
|
+
export type DrainProblem =
|
|
726
|
+
| "unparseable-json"
|
|
727
|
+
| "invalid-record"
|
|
728
|
+
| "invalid-project"
|
|
729
|
+
| "invalid-created-at"
|
|
730
|
+
| "invalid-expires-at"
|
|
731
|
+
| "expiry-not-future"
|
|
732
|
+
| "invalid-reason";
|
|
733
|
+
|
|
734
|
+
/** The verdict of one drain read. `error` is the malformed-record case: the
|
|
735
|
+
* caller may fail its pass closed, and the dispatch-side
|
|
736
|
+
* {@link consumeDrain} removes the record in the same transition, so it can
|
|
737
|
+
* never become a permanent drain. */
|
|
738
|
+
export type DrainVerdict =
|
|
739
|
+
| { kind: "active"; drain: DrainRecord }
|
|
740
|
+
| { kind: "inactive" }
|
|
741
|
+
| { kind: "error"; problem: DrainProblem };
|
|
742
|
+
|
|
743
|
+
/** Where this project's drain record lives. Project-scoped by construction:
|
|
744
|
+
* a record present at one project's path never fences another project. */
|
|
745
|
+
export function drainPath(project: string): string {
|
|
746
|
+
return join(stateDir(), `drain-${project}.json`);
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/**
|
|
750
|
+
* Records a bounded drain intent for `project`, replacing any prior drain of
|
|
751
|
+
* the same project atomically (tmp + rename, exactly like the admission ack).
|
|
752
|
+
* The record is a file, so it survives orchestrator and daemon loss; the
|
|
753
|
+
* absolute `expiresAt` is what stops it from ever stranding admission.
|
|
754
|
+
*/
|
|
755
|
+
export function createDrain(
|
|
756
|
+
project: string,
|
|
757
|
+
opts: CreateDrainOptions,
|
|
758
|
+
now = Date.now(),
|
|
759
|
+
): void {
|
|
760
|
+
if (project === "") {
|
|
761
|
+
throw new Error("drain project must not be empty");
|
|
762
|
+
}
|
|
763
|
+
if (!Number.isFinite(opts.expiresAt) || opts.expiresAt <= now) {
|
|
764
|
+
throw new Error("drain expiresAt must be a finite epoch-ms timestamp in the future");
|
|
765
|
+
}
|
|
766
|
+
if (opts.reason !== undefined && typeof opts.reason !== "string") {
|
|
767
|
+
throw new Error("drain reason must be a string");
|
|
768
|
+
}
|
|
769
|
+
const record: DrainRecord = {
|
|
770
|
+
project,
|
|
771
|
+
createdAt: new Date(now).toISOString(),
|
|
772
|
+
expiresAt: new Date(opts.expiresAt).toISOString(),
|
|
773
|
+
...(opts.reason === undefined ? {} : { reason: opts.reason }),
|
|
774
|
+
};
|
|
775
|
+
const path = drainPath(project);
|
|
776
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
777
|
+
const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
|
|
778
|
+
writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`);
|
|
779
|
+
renameSync(tmp, path);
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/**
|
|
783
|
+
* One drained state read with no side effects — this never touches the record
|
|
784
|
+
* on disk. That purity is what lets observational surfaces live off it: a
|
|
785
|
+
* status/dashboard/health read must not consume a malformed record before a
|
|
786
|
+
* dispatch pass fails closed on it, or the pass would read absent and admit.
|
|
787
|
+
* The dispatch side performs the actual cleanup transition through
|
|
788
|
+
* {@link consumeDrain}. Callers that only need to *see* the state (including
|
|
789
|
+
* the claim path, which refuses but must not unbind its own pass) use this.
|
|
790
|
+
*/
|
|
791
|
+
export function readDrain(project: string, now = Date.now()): DrainVerdict {
|
|
792
|
+
const path = drainPath(project);
|
|
793
|
+
if (!existsSync(path)) return { kind: "inactive" };
|
|
794
|
+
let raw: unknown;
|
|
795
|
+
try {
|
|
796
|
+
raw = JSON.parse(readFileSync(path, "utf8"));
|
|
797
|
+
} catch {
|
|
798
|
+
return { kind: "error", problem: "unparseable-json" };
|
|
799
|
+
}
|
|
800
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
801
|
+
return { kind: "error", problem: "invalid-record" };
|
|
802
|
+
}
|
|
803
|
+
const rec = raw as Record<string, unknown>;
|
|
804
|
+
if (typeof rec["project"] !== "string" || rec["project"] === "") {
|
|
805
|
+
return { kind: "error", problem: "invalid-project" };
|
|
806
|
+
}
|
|
807
|
+
if (typeof rec["createdAt"] !== "string" || Number.isNaN(Date.parse(rec["createdAt"]))) {
|
|
808
|
+
return { kind: "error", problem: "invalid-created-at" };
|
|
809
|
+
}
|
|
810
|
+
if (typeof rec["expiresAt"] !== "string" || Number.isNaN(Date.parse(rec["expiresAt"]))) {
|
|
811
|
+
return { kind: "error", problem: "invalid-expires-at" };
|
|
812
|
+
}
|
|
813
|
+
if (Date.parse(rec["expiresAt"] as string) <= Date.parse(rec["createdAt"] as string)) {
|
|
814
|
+
return { kind: "error", problem: "expiry-not-future" };
|
|
815
|
+
}
|
|
816
|
+
const reason = rec["reason"];
|
|
817
|
+
if (reason !== undefined && typeof reason !== "string") {
|
|
818
|
+
return { kind: "error", problem: "invalid-reason" };
|
|
819
|
+
}
|
|
820
|
+
if (rec["project"] !== project) {
|
|
821
|
+
// A record persisted at this project's path but naming another project is
|
|
822
|
+
// either a copy or a rename mishap; it fences nobody (that project's drain
|
|
823
|
+
// lives at its own path).
|
|
824
|
+
return { kind: "inactive" };
|
|
825
|
+
}
|
|
826
|
+
if (Date.parse(rec["expiresAt"] as string) <= now) {
|
|
827
|
+
return { kind: "inactive" };
|
|
828
|
+
}
|
|
829
|
+
const drain: DrainRecord = {
|
|
830
|
+
project: rec["project"],
|
|
831
|
+
createdAt: rec["createdAt"],
|
|
832
|
+
expiresAt: rec["expiresAt"],
|
|
833
|
+
...(reason === undefined ? {} : { reason }),
|
|
834
|
+
};
|
|
835
|
+
return { kind: "active", drain };
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
/**
|
|
839
|
+
* The dispatch-side drain read: the same verdict as {@link readDrain}, and the
|
|
840
|
+
* one place a stale or untrustworthy record is cleared as a side effect —
|
|
841
|
+
* expired and wrong-project records are removed (bounded stale-state cleanup
|
|
842
|
+
* on the next pass), and a malformed record is removed in the very transition
|
|
843
|
+
* that fails the pass closed, so it can never become an unbounded permanent
|
|
844
|
+
* drain. Only dispatch callers use this: an observational read here would let
|
|
845
|
+
* a status/health reader consume the malformed marker before the pass that
|
|
846
|
+
* must fail closed on it ever ran.
|
|
847
|
+
*/
|
|
848
|
+
export function consumeDrain(project: string, now = Date.now()): DrainVerdict {
|
|
849
|
+
const verdict = readDrain(project, now);
|
|
850
|
+
if (verdict.kind !== "active") {
|
|
851
|
+
rmSync(drainPath(project), { force: true });
|
|
852
|
+
}
|
|
853
|
+
return verdict;
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/** Removes this project's drain, idempotently — a second cancel is a no-op. */
|
|
857
|
+
export function cancelDrain(project: string): void {
|
|
858
|
+
rmSync(drainPath(project), { force: true });
|
|
859
|
+
}
|
|
860
|
+
|
|
643
861
|
// ----------------------------------------------------------------- admission
|
|
644
862
|
// acknowledgement (#651, review #3)
|
|
645
863
|
//
|
|
@@ -1023,6 +1241,22 @@ function laneBlock(lane: FileLane | undefined): string {
|
|
|
1023
1241
|
return ["## File lane (as parsed)", "", body, "", ""].join("\n");
|
|
1024
1242
|
}
|
|
1025
1243
|
|
|
1244
|
+
/**
|
|
1245
|
+
* The declared model as a brief section (#535): the selector the dispatch
|
|
1246
|
+
* will launch on — the same parse admission carried, never a second read —
|
|
1247
|
+
* or the explicit fail-open note, so the worker reads the run's model on the
|
|
1248
|
+
* brief itself rather than inferring it. Deliberately only the selector,
|
|
1249
|
+
* never the source line: the declaration's prose already renders in the body
|
|
1250
|
+
* or Discussion.
|
|
1251
|
+
*/
|
|
1252
|
+
function modelBlock(model: EffectiveModel | undefined): string {
|
|
1253
|
+
const body =
|
|
1254
|
+
model === undefined
|
|
1255
|
+
? "_no model declared — the project's workerModel (or harness default) is in effect_"
|
|
1256
|
+
: `\`${model.model}\``;
|
|
1257
|
+
return ["## Model (as parsed)", "", body, "", ""].join("\n");
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1026
1260
|
/**
|
|
1027
1261
|
* What an orphan-resumed worker is told about the file lane on top of the
|
|
1028
1262
|
* continuation notice (#608). The original brief already in the transcript
|
|
@@ -1100,6 +1334,22 @@ export async function buildBrief(
|
|
|
1100
1334
|
* resolved from the rendered thread itself.
|
|
1101
1335
|
*/
|
|
1102
1336
|
lane?: FileLane;
|
|
1337
|
+
/**
|
|
1338
|
+
* The effective `Model:` declaration admission resolved for this
|
|
1339
|
+
* candidate (#535). When carried, the brief echoes exactly this
|
|
1340
|
+
* selector — the same value dispatch launches on — instead of
|
|
1341
|
+
* recomputing from the dispatch-time comment read, so the run's model is
|
|
1342
|
+
* one value on every surface. Absent (unit-level callers), the model is
|
|
1343
|
+
* resolved from the rendered thread itself.
|
|
1344
|
+
*/
|
|
1345
|
+
model?: EffectiveModel;
|
|
1346
|
+
/**
|
|
1347
|
+
* The typed host-constraints block the brief renders (#721), re-read with
|
|
1348
|
+
* the config at the tick boundary so an operator edit applies on the next
|
|
1349
|
+
* dispatch. Absent renders no host-constraints section at all — the
|
|
1350
|
+
* brief is byte-for-byte what it always was.
|
|
1351
|
+
*/
|
|
1352
|
+
host?: HostConstraints;
|
|
1103
1353
|
} = {},
|
|
1104
1354
|
): Promise<string> {
|
|
1105
1355
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
@@ -1149,6 +1399,11 @@ export async function buildBrief(
|
|
|
1149
1399
|
// itself. Both the discussion renderer and the parsed-lane section draw on
|
|
1150
1400
|
// the same value, so the brief shows one lane on every surface (#608, #724).
|
|
1151
1401
|
const lane = opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments));
|
|
1402
|
+
// The effective model declaration admission resolved (or resolves) for this
|
|
1403
|
+
// candidate: the carried admission snapshot when dispatch has one, else the
|
|
1404
|
+
// thread itself. Dispatch launches on the same value, so the brief shows
|
|
1405
|
+
// one model on every surface (#535).
|
|
1406
|
+
const model = opts.model ?? (comments === "unread" ? undefined : effectiveModel(r.issue.body, comments));
|
|
1152
1407
|
return renderBrief(template, {
|
|
1153
1408
|
ISSUE_NUMBER: String(r.issue.number),
|
|
1154
1409
|
ISSUE_TITLE: r.issue.title,
|
|
@@ -1159,12 +1414,18 @@ export async function buildBrief(
|
|
|
1159
1414
|
ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
|
|
1160
1415
|
ISSUE_COMMENTS: renderDiscussion(comments, lane),
|
|
1161
1416
|
FILE_LANE: laneBlock(lane),
|
|
1417
|
+
MODEL: modelBlock(model),
|
|
1162
1418
|
GATES: gatesBlock(r.repo),
|
|
1163
1419
|
// The guarded shell suites and their sanctioned parse-only check, derived
|
|
1164
1420
|
// from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
|
|
1165
1421
|
// (#687). Non-empty whenever the guard has paths, so the placeholder line
|
|
1166
1422
|
// always renders to a line.
|
|
1167
1423
|
SHARED_HOST_NOTICE: sharedHostBriefNotice(),
|
|
1424
|
+
// The typed host-constraints paragraph (#721): derived cores/RAM folded
|
|
1425
|
+
// into the operator's description, the non-interactive PATH, and the
|
|
1426
|
+
// routed repo's convention. Empty when the config names none — no
|
|
1427
|
+
// section, no placeholder text.
|
|
1428
|
+
HOST_CONSTRAINTS: hostConstraintsNotice(opts.host, repoSlug(r.repo)),
|
|
1168
1429
|
// The brief's code-graph paragraph: the exact `project` key for a
|
|
1169
1430
|
// configured repo, or an explicit "no graph" statement for an
|
|
1170
1431
|
// unconfigured one — never silence, because a worker that knows there is
|
|
@@ -1359,17 +1620,147 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1359
1620
|
};
|
|
1360
1621
|
}
|
|
1361
1622
|
|
|
1623
|
+
/**
|
|
1624
|
+
* The daemon half of a worker's `pushed-green` claim (#85's contact with
|
|
1625
|
+
* reality; #782's publication attribution).
|
|
1626
|
+
*
|
|
1627
|
+
* A yield is a transport, not proof of GitHub side effects: `claim` is
|
|
1628
|
+
* caller-supplied text and the worker adapter turns it into a green result
|
|
1629
|
+
* without the daemon ever having seen a mediated publication verb for it — the
|
|
1630
|
+
* shape that let the #777 incident guess a non-existent PR. The tracker read
|
|
1631
|
+
* at the end proves the PR is open and green at the exact head; the evidence
|
|
1632
|
+
* gate here proves that PR is *this run's own mediated work* first.
|
|
1633
|
+
*
|
|
1634
|
+
* Two things are asked, in cost order:
|
|
1635
|
+
*
|
|
1636
|
+
* - the claimed URL must equal the one this run's row records, which is never
|
|
1637
|
+
* a worker-supplied string at the moment a live run is verified —
|
|
1638
|
+
* conductor_pr_create (create or adoption) writes it on the run, the daemon
|
|
1639
|
+
* seeds it at claim from a terminal predecessor on the same branch (#434),
|
|
1640
|
+
* and orchestrator-only recovery writes it for settled runs;
|
|
1641
|
+
* - and the claimed head must have a mediated publisher: an allowed
|
|
1642
|
+
* `conductor_push` on this exact run that published it on this run's branch
|
|
1643
|
+
* (the verb only ever pushes `refs/heads/<branch>`), an allowed
|
|
1644
|
+
* `conductor_pr_create` on this run for this exact PR, or — for a run that
|
|
1645
|
+
* published nothing new — the live tip of this run's own branch.
|
|
1646
|
+
*
|
|
1647
|
+
* That last path is the one the row cannot supply. It used to be served by
|
|
1648
|
+
* comparing the claim against `run.prUrl`/`run.headSha`, and both are
|
|
1649
|
+
* worker-tainted upstream: terminal settlement writes a worker's reported pair
|
|
1650
|
+
* onto the row *before* any verification, and continuation inheritance
|
|
1651
|
+
* validates only the predecessor's metadata and open state — so a turn-capped
|
|
1652
|
+
* attempt that reported someone else's real green PR had that pair inherited
|
|
1653
|
+
* and re-presented as its own evidence. Asking the tracker for the branch tip
|
|
1654
|
+
* removes the worker from the loop entirely: `refs/heads/<branch>` is a ref
|
|
1655
|
+
* only a mediated `conductor_push` or a mediated `conductor_pr_update_branch`
|
|
1656
|
+
* can move, so its live commit is daemon provenance no reported string can
|
|
1657
|
+
* forge. It is also why a legitimate mediated base-branch update now passes:
|
|
1658
|
+
* that server-side merge creates a head no `conductor_push` ever published and
|
|
1659
|
+
* the row still carries the older one, which the recorded-pair test rejected.
|
|
1660
|
+
*
|
|
1661
|
+
* Binding to the *exact current run* is what makes an old attempt, a different
|
|
1662
|
+
* branch, or #806's orchestrator-only settled-run recovery invisible here: the
|
|
1663
|
+
* ledger query is run-scoped by `runId`, and recoveries store no runId at all.
|
|
1664
|
+
* That query asks for the run's complete history rather than the ledger's
|
|
1665
|
+
* newest rows — publication evidence sits at the *start* of a run, and a
|
|
1666
|
+
* review-revision round or a burst of refused mutations pushed it past the
|
|
1667
|
+
* default page, turning a verified push into a definitive false failure.
|
|
1668
|
+
*
|
|
1669
|
+
* A claim that fails this gate is a definitive failure — never a retryable
|
|
1670
|
+
* `pushed-pending` — because a guessed URL is not something a later tick is
|
|
1671
|
+
* waiting on. A claim the gate could not *read* is the opposite: an unreadable
|
|
1672
|
+
* branch tip is #781's transient outage, so it stays retryable rather than
|
|
1673
|
+
* burning an attempt on a flaky read.
|
|
1674
|
+
*/
|
|
1362
1675
|
export async function verifyPushedGreenClaim(
|
|
1363
|
-
tracker: Pick<Tracker, "verifyPr">,
|
|
1676
|
+
tracker: Pick<Tracker, "verifyPr" | "branchHead">,
|
|
1364
1677
|
claim: Pick<WorkerResult, "prUrl" | "headSha">,
|
|
1678
|
+
publication: {
|
|
1679
|
+
project: string;
|
|
1680
|
+
issue: number;
|
|
1681
|
+
runId: string;
|
|
1682
|
+
/** The branch conductor routed this run onto. `conductor_push` publishes
|
|
1683
|
+
* exactly `refs/heads/<branch>` and refuses any other ref. */
|
|
1684
|
+
branch: string;
|
|
1685
|
+
/** The identity `tracker.branchHead` reads the live tip with: the
|
|
1686
|
+
* canonical `owner/repo` when the routed clone URL carries one, else the
|
|
1687
|
+
* routed repository name. A tracker that cannot resolve it answers
|
|
1688
|
+
* undefined, which stays retryable rather than definitive. */
|
|
1689
|
+
repo: string;
|
|
1690
|
+
store: Pick<Store, "verbLedger" | "getRun">;
|
|
1691
|
+
},
|
|
1365
1692
|
): Promise<{
|
|
1366
1693
|
state: "pushed-green" | "pushed-pending" | "failed";
|
|
1367
1694
|
reason?: string;
|
|
1368
1695
|
}> {
|
|
1369
|
-
|
|
1696
|
+
const { prUrl, headSha } = claim;
|
|
1697
|
+
if (prUrl === undefined || headSha === undefined) {
|
|
1370
1698
|
return { state: "failed", reason: "Worker did not report a PR URL and observed head SHA" };
|
|
1371
1699
|
}
|
|
1372
|
-
|
|
1700
|
+
// Exact current run, never "some publication for the issue": the query is
|
|
1701
|
+
// scoped to this runId plus project/issue, so a previous attempt's verbs and
|
|
1702
|
+
// the orchestrator's recovery verbs are invisible here. Unbounded on
|
|
1703
|
+
// purpose — see the note above about evidence ageing off the newest page.
|
|
1704
|
+
const ledger = publication.store.verbLedger(publication.project, {
|
|
1705
|
+
runId: publication.runId,
|
|
1706
|
+
issue: publication.issue,
|
|
1707
|
+
limit: Number.MAX_SAFE_INTEGER,
|
|
1708
|
+
});
|
|
1709
|
+
const run = publication.store.getRun(publication.runId);
|
|
1710
|
+
const branchRef = `refs/heads/${publication.branch}`;
|
|
1711
|
+
if (run?.prUrl !== prUrl) {
|
|
1712
|
+
return {
|
|
1713
|
+
state: "failed",
|
|
1714
|
+
reason:
|
|
1715
|
+
"Pushed-green claim has no mediated publication evidence: the claimed PR is not this run's " +
|
|
1716
|
+
"recorded PR",
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
const pushedThisHead = ledger.some(
|
|
1720
|
+
(entry) =>
|
|
1721
|
+
entry.decision === "allowed" &&
|
|
1722
|
+
entry.verb === "conductor_push" &&
|
|
1723
|
+
entry.sha === headSha &&
|
|
1724
|
+
entry.detail.includes(branchRef),
|
|
1725
|
+
);
|
|
1726
|
+
// Bound to the claimed PR, not merely to "a create happened": both allowed
|
|
1727
|
+
// details name the URL they produced (`opened <url> …`, `adopted <url> …`),
|
|
1728
|
+
// so an unrelated create on this run cannot vouch for another PR.
|
|
1729
|
+
const createdHere = ledger.some(
|
|
1730
|
+
(entry) =>
|
|
1731
|
+
entry.decision === "allowed" &&
|
|
1732
|
+
entry.verb === "conductor_pr_create" &&
|
|
1733
|
+
entry.detail.includes(prUrl),
|
|
1734
|
+
);
|
|
1735
|
+
|
|
1736
|
+
if (!pushedThisHead && !createdHere) {
|
|
1737
|
+
// Nothing this run published carries the claimed head, so the only
|
|
1738
|
+
// remaining evidence is the branch itself: a continuation or review round
|
|
1739
|
+
// that pushed nothing, or a head a mediated base-branch update produced.
|
|
1740
|
+
let tip: string | undefined;
|
|
1741
|
+
try {
|
|
1742
|
+
tip = await tracker.branchHead(publication.repo, publication.branch);
|
|
1743
|
+
} catch {
|
|
1744
|
+
tip = undefined;
|
|
1745
|
+
}
|
|
1746
|
+
if (tip === undefined) {
|
|
1747
|
+
return {
|
|
1748
|
+
state: "pushed-pending",
|
|
1749
|
+
reason: `Live head of ${branchRef} unavailable; retrying`,
|
|
1750
|
+
};
|
|
1751
|
+
}
|
|
1752
|
+
if (tip !== headSha) {
|
|
1753
|
+
return {
|
|
1754
|
+
state: "failed",
|
|
1755
|
+
reason:
|
|
1756
|
+
"Pushed-green claim has no mediated publication evidence: this run's conductor_push / " +
|
|
1757
|
+
`conductor_pr_create ledger does not cover the claimed head, and ${branchRef} is at ${tip}, ` +
|
|
1758
|
+
"not the claimed head",
|
|
1759
|
+
};
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
const verification = await tracker.verifyPr(prUrl, headSha);
|
|
1373
1764
|
if (verification === undefined) {
|
|
1374
1765
|
return { state: "pushed-pending", reason: "GitHub PR verification unavailable; retrying" };
|
|
1375
1766
|
}
|
|
@@ -1528,6 +1919,7 @@ export async function handleIssue(
|
|
|
1528
1919
|
r: Routed,
|
|
1529
1920
|
attempt: number,
|
|
1530
1921
|
admittedLane?: FileLane,
|
|
1922
|
+
admittedModel?: EffectiveModel,
|
|
1531
1923
|
): Promise<void> {
|
|
1532
1924
|
const { project, caps, tracker, store } = d;
|
|
1533
1925
|
const issue = r.issue.number;
|
|
@@ -1689,6 +2081,24 @@ export async function handleIssue(
|
|
|
1689
2081
|
return;
|
|
1690
2082
|
}
|
|
1691
2083
|
|
|
2084
|
+
// The claim-side of the project drain fence (#484): the tick's gate sits
|
|
2085
|
+
// above routing, so a drain created after that gate can still land
|
|
2086
|
+
// mid-pass — the claim re-checks and refuses while a fresh drain is in
|
|
2087
|
+
// force. An invalid record fails closed the same way. The read here is
|
|
2088
|
+
// observational on purpose: the claim refuses but must not consume the
|
|
2089
|
+
// marker, or the first refused claim would unbind the rest of its own
|
|
2090
|
+
// pass; the tick's consumeDrain removes it on the next pass, so no claim
|
|
2091
|
+
// path can ever be blocked permanently by it.
|
|
2092
|
+
const claimDrain = readDrain(d.project.name);
|
|
2093
|
+
if (claimDrain.kind === "active" || claimDrain.kind === "error") {
|
|
2094
|
+
log(
|
|
2095
|
+
claimDrain.kind === "error"
|
|
2096
|
+
? `#${issue} not claimed: drain record invalid (${claimDrain.problem})`
|
|
2097
|
+
: `#${issue} not claimed: project drain in effect`,
|
|
2098
|
+
);
|
|
2099
|
+
return;
|
|
2100
|
+
}
|
|
2101
|
+
|
|
1692
2102
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
1693
2103
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
1694
2104
|
// are local, written before any network call, and the startup orphan
|
|
@@ -1716,13 +2126,21 @@ export async function handleIssue(
|
|
|
1716
2126
|
// unconfigured project — and today's dispatch is byte for byte what it
|
|
1717
2127
|
// has always been.
|
|
1718
2128
|
const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
|
|
2129
|
+
// A `Model:` declaration admission resolved for this candidate (#535) is
|
|
2130
|
+
// this issue's workerModel: the orchestrator names a tier when it
|
|
2131
|
+
// promotes, and dispatch launches on that selector exactly as if the
|
|
2132
|
+
// project's `workerModel` were the declared value. Absent a declaration,
|
|
2133
|
+
// today's `project.workerModel` is unchanged, and the failover chain
|
|
2134
|
+
// (#286) keeps its semantics in both cases — it is the same resolution,
|
|
2135
|
+
// one different primary.
|
|
2136
|
+
const declaredModel = admittedModel?.model;
|
|
1719
2137
|
const choice = resolveDispatchModel({
|
|
1720
|
-
workerModel: project.workerModel,
|
|
2138
|
+
workerModel: declaredModel ?? project.workerModel,
|
|
1721
2139
|
modelFallbacks: project.modelFallbacks,
|
|
1722
2140
|
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
1723
2141
|
streak: chainFacts.streak,
|
|
1724
2142
|
});
|
|
1725
|
-
const clause = fallbackClause(choice, chainFacts, project.workerModel);
|
|
2143
|
+
const clause = fallbackClause(choice, chainFacts, declaredModel ?? project.workerModel);
|
|
1726
2144
|
|
|
1727
2145
|
// #536: an orphan-clean requeue resumes the interrupted session instead of
|
|
1728
2146
|
// re-reading the repo from zero. Decided here, before this attempt's row
|
|
@@ -1768,6 +2186,13 @@ export async function handleIssue(
|
|
|
1768
2186
|
// above fired. `undefined` for a fresh dispatch — the store maps that to
|
|
1769
2187
|
// NULL, so a fresh row simply never carries the field.
|
|
1770
2188
|
resumedFromRunId: resuming?.id,
|
|
2189
|
+
// #744: the file-lane declaration admission resolved for this candidate
|
|
2190
|
+
// is persisted on the row, so lane occupancy survives across dispatch
|
|
2191
|
+
// passes — a later pass knows what this run *intends* to touch, not only
|
|
2192
|
+
// what it has touched so far. This is the exact `Admission.lane` the gate
|
|
2193
|
+
// enforced and the brief rendered, never a re-parse. Absent for a run
|
|
2194
|
+
// with no declaration (fail open), exactly as it was admitted.
|
|
2195
|
+
lane: admittedLane,
|
|
1771
2196
|
});
|
|
1772
2197
|
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
1773
2198
|
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
@@ -1806,6 +2231,7 @@ export async function handleIssue(
|
|
|
1806
2231
|
if (await settleStopBeforeSession()) return;
|
|
1807
2232
|
if (await settleDrainBeforeSession()) return;
|
|
1808
2233
|
|
|
2234
|
+
|
|
1809
2235
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
1810
2236
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
1811
2237
|
// to be cleared first. Both helpers are pure path math, and removeWorktree
|
|
@@ -1886,7 +2312,9 @@ export async function handleIssue(
|
|
|
1886
2312
|
runRepoPath: worktreePath,
|
|
1887
2313
|
branch,
|
|
1888
2314
|
},
|
|
1889
|
-
{
|
|
2315
|
+
{
|
|
2316
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
2317
|
+
},
|
|
1890
2318
|
);
|
|
1891
2319
|
if (await settleStopBeforeSession()) return;
|
|
1892
2320
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -1943,6 +2371,8 @@ export async function handleIssue(
|
|
|
1943
2371
|
: {}),
|
|
1944
2372
|
comments,
|
|
1945
2373
|
lane: admittedLane,
|
|
2374
|
+
model: admittedModel,
|
|
2375
|
+
host: d.host,
|
|
1946
2376
|
});
|
|
1947
2377
|
}
|
|
1948
2378
|
if (await settleStopBeforeSession()) return;
|
|
@@ -1950,6 +2380,7 @@ export async function handleIssue(
|
|
|
1950
2380
|
|
|
1951
2381
|
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
1952
2382
|
|
|
2383
|
+
|
|
1953
2384
|
let result: WorkerResult;
|
|
1954
2385
|
try {
|
|
1955
2386
|
result = await runWorker({
|
|
@@ -2036,7 +2467,14 @@ export async function handleIssue(
|
|
|
2036
2467
|
|
|
2037
2468
|
const verified: { state: RunState; reason?: string } =
|
|
2038
2469
|
result.state === "pushed-green"
|
|
2039
|
-
? await verifyPushedGreenClaim(tracker, result
|
|
2470
|
+
? await verifyPushedGreenClaim(tracker, result, {
|
|
2471
|
+
project: project.name,
|
|
2472
|
+
issue,
|
|
2473
|
+
runId,
|
|
2474
|
+
branch,
|
|
2475
|
+
repo: repoSlug ?? r.repo.name,
|
|
2476
|
+
store,
|
|
2477
|
+
})
|
|
2040
2478
|
: { state: result.state };
|
|
2041
2479
|
const state = verified.state;
|
|
2042
2480
|
|
|
@@ -2061,6 +2499,12 @@ export async function handleIssue(
|
|
|
2061
2499
|
// The claimed-proof check compares the PR's Verified commands
|
|
2062
2500
|
// against what this run's session actually recorded.
|
|
2063
2501
|
sessionFile: result.sessionFile,
|
|
2502
|
+
// The effective file lane admission resolved for this run at
|
|
2503
|
+
// dispatch — the same value the brief rendered, pre-dispatch
|
|
2504
|
+
// comment declarations included (#608, #744). The audit flags a
|
|
2505
|
+
// diff that escapes it, so a widened lane is named on evidence
|
|
2506
|
+
// rather than found by reading the PR's file list by hand (#739).
|
|
2507
|
+
lane: admittedLane,
|
|
2064
2508
|
})
|
|
2065
2509
|
: undefined;
|
|
2066
2510
|
if (result.state === "pushed-green" && audit?.truncated) {
|
|
@@ -2170,6 +2614,10 @@ export async function handleIssue(
|
|
|
2170
2614
|
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
2171
2615
|
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
2172
2616
|
sessionFile: result.sessionFile,
|
|
2617
|
+
// The code-graph session observation (#726): what the run's own session
|
|
2618
|
+
// registry held at start, persisted with the rest of the run's facts.
|
|
2619
|
+
// Absent only when the session surface did not record one.
|
|
2620
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
2173
2621
|
// Every terminal state persists the worker's report — with the `changed:`
|
|
2174
2622
|
// file list derived from the PR's diff where one could be read — not
|
|
2175
2623
|
// just a green push: a stopped attempt's partial report is still part of
|
|
@@ -2441,6 +2889,91 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
2441
2889
|
);
|
|
2442
2890
|
break;
|
|
2443
2891
|
}
|
|
2892
|
+
// A capped/failed run has no pushed-green settle sweep keeping its row
|
|
2893
|
+
// honest (#795 review round 1): nothing transitions a `failed` / `killed`
|
|
2894
|
+
// row when its PR merges or closes, so a PR that changed after the verb
|
|
2895
|
+
// recorded the round would otherwise be claimed and a worker resumed
|
|
2896
|
+
// against a dead PR. The settled-green origin keeps its own interlock
|
|
2897
|
+
// (the settle sweep flips the row and the claim below refuses it), so
|
|
2898
|
+
// only terminal-origin rounds re-read the reviewed PR fact here: the
|
|
2899
|
+
// round dispatches only while the PR is still open at the exact reviewed
|
|
2900
|
+
// head.
|
|
2901
|
+
//
|
|
2902
|
+
// The skip decision is decisive-fact only (review round 2): a definitively
|
|
2903
|
+
// missing PR (`GhPrMissingError` — a corroborated 404, #779), a PR that
|
|
2904
|
+
// is definitively not open, and a head that has definitively MOVED are
|
|
2905
|
+
// settled skipped, exactly like any other row that moved. Everything
|
|
2906
|
+
// transient or unreadable — a bare 404 that could not be corroborated, an
|
|
2907
|
+
// undefined answer, a pending check run, a same-head red check that a
|
|
2908
|
+
// rerun may clear, a throwing tracker — stays PENDING for the next tick:
|
|
2909
|
+
// settling it would permanently discard the orchestrator's findings and
|
|
2910
|
+
// consume the round.
|
|
2911
|
+
const origin = d.store.getRun(revision.runId)?.state;
|
|
2912
|
+
if (origin === "failed" || origin === "killed") {
|
|
2913
|
+
let prState: PrState | undefined;
|
|
2914
|
+
try {
|
|
2915
|
+
prState = await d.tracker.prState(revision.prUrl);
|
|
2916
|
+
} catch (err) {
|
|
2917
|
+
if (err instanceof GhPrMissingError) {
|
|
2918
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2919
|
+
log(
|
|
2920
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} does not exist — the reviewed PR is gone`,
|
|
2921
|
+
);
|
|
2922
|
+
continue;
|
|
2923
|
+
}
|
|
2924
|
+
log(
|
|
2925
|
+
`#${revision.issue} review round ${revision.round} held: PR state re-check failed (${errText(err)}) — retrying next tick`,
|
|
2926
|
+
);
|
|
2927
|
+
continue;
|
|
2928
|
+
}
|
|
2929
|
+
if (prState === undefined) {
|
|
2930
|
+
log(
|
|
2931
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} state could not be re-read — retrying next tick`,
|
|
2932
|
+
);
|
|
2933
|
+
continue;
|
|
2934
|
+
}
|
|
2935
|
+
if (prState !== "open") {
|
|
2936
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2937
|
+
log(
|
|
2938
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} is ${prState}, not open — a review round resumes an open PR only`,
|
|
2939
|
+
);
|
|
2940
|
+
continue;
|
|
2941
|
+
}
|
|
2942
|
+
let verification: PrVerification | undefined;
|
|
2943
|
+
try {
|
|
2944
|
+
verification = await d.tracker.verifyPr(revision.prUrl, revision.headSha);
|
|
2945
|
+
} catch (err) {
|
|
2946
|
+
log(
|
|
2947
|
+
`#${revision.issue} review round ${revision.round} held: the reviewed-head check failed (${errText(err)}) — retrying next tick`,
|
|
2948
|
+
);
|
|
2949
|
+
continue;
|
|
2950
|
+
}
|
|
2951
|
+
if (verification === undefined || verification.status === "pending") {
|
|
2952
|
+
log(
|
|
2953
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
2954
|
+
`${verification?.status ?? "unverifiable"} — retrying next tick`,
|
|
2955
|
+
);
|
|
2956
|
+
continue;
|
|
2957
|
+
}
|
|
2958
|
+
if (verification.status === "failed" && isHeadMismatch(verification.reason)) {
|
|
2959
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2960
|
+
log(
|
|
2961
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} no longer stands at ` +
|
|
2962
|
+
`${revision.headSha} (${verification.reason}) — the reviewed head moved`,
|
|
2963
|
+
);
|
|
2964
|
+
continue;
|
|
2965
|
+
}
|
|
2966
|
+
// Anything else — a same-head red check a rerun may clear, an unknown
|
|
2967
|
+
// verdict — is not definitive: the round stays pending for the next
|
|
2968
|
+
// tick rather than discarding the findings.
|
|
2969
|
+
if (verification.status !== "green") {
|
|
2970
|
+
log(
|
|
2971
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
2972
|
+
`${verification.status} — retrying next tick`,
|
|
2973
|
+
);
|
|
2974
|
+
continue;
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2444
2977
|
if (!d.store.claimRunForReview(revision.runId)) {
|
|
2445
2978
|
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2446
2979
|
log(
|
|
@@ -2601,18 +3134,77 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2601
3134
|
// and shutdown fences cover the whole wake window, exactly as they do for a
|
|
2602
3135
|
// fresh claim in `handleIssue` (#374). The original run's entries were
|
|
2603
3136
|
// closed at its settle, so reopening by issue is safe.
|
|
3137
|
+
//
|
|
3138
|
+
// #747: the round gets its own bounded allowance instead of the original
|
|
3139
|
+
// attempt's remainder. The resumed session continues the run's own counter
|
|
3140
|
+
// — the attempt's spent turns travel with the session — so a ceiling of
|
|
3141
|
+
// `run.maxTurns` would leave the round only `run.maxTurns - run.turns`,
|
|
3142
|
+
// and returning a PR from a near-cap run could not finish. Open at the
|
|
3143
|
+
// run's ceiling, then raise it by the round's own allowance — the run's
|
|
3144
|
+
// ceiling again, capped by the project's hard ceiling — through the same
|
|
3145
|
+
// override path `omp-conductor extend` uses, so the raise is persisted on
|
|
3146
|
+
// the row (the run's budget stays explainable in status) while `turns`
|
|
3147
|
+
// keeps accumulating the run's true cost: nothing is reset.
|
|
2604
3148
|
turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
|
|
3149
|
+
const revisionAllowance = Math.min(run.maxTurns, caps.workerMaxTurnsCeiling);
|
|
3150
|
+
const revisionCeiling = run.turns + revisionAllowance;
|
|
3151
|
+
const revisionRaise = d.turnLimits.extend(project.name, issue, revisionCeiling);
|
|
3152
|
+
if (revisionRaise.kind === "extended") {
|
|
3153
|
+
log(
|
|
3154
|
+
`#${issue} review round ${revision.round} turn ceiling ${run.maxTurns} → ${revisionCeiling} ` +
|
|
3155
|
+
`(+${revisionAllowance} round allowance on ${run.turns} spent)`,
|
|
3156
|
+
);
|
|
3157
|
+
}
|
|
2605
3158
|
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
2606
3159
|
if (await settleStopBeforeSession()) return;
|
|
2607
3160
|
if (await settleDrainBeforeSession()) return;
|
|
2608
3161
|
|
|
2609
3162
|
try {
|
|
2610
3163
|
// Reattach the run's own branch at the same per-issue path the run used:
|
|
2611
|
-
//
|
|
2612
|
-
//
|
|
2613
|
-
//
|
|
2614
|
-
//
|
|
3164
|
+
// a `pushed-green` settle removed the worktree, so provisioning is the
|
|
3165
|
+
// same continuation reattach as a normal re-claim. A capped/failed run's
|
|
3166
|
+
// tree was RETAINED at its terminal settle (`tree: "keep"`, #795) — the
|
|
3167
|
+
// reviewed PR is green, so the branch already holds the work and the kept
|
|
3168
|
+
// tree holds nothing the branch does not, unless its salvage failed. Clear
|
|
3169
|
+
// it the way restart recovery does (#692): re-attempt the salvage through
|
|
3170
|
+
// `tree: "remove"`, and refuse the removal if that still fails — the tree
|
|
3171
|
+
// is then the only copy of work and is never destroyed. A provisioning
|
|
3172
|
+
// failure restores the row — the PR is still green and still open, and
|
|
3173
|
+
// only the wake failed — and says so.
|
|
2615
3174
|
try {
|
|
3175
|
+
if (run.worktree !== "" && existsSync(run.worktree)) {
|
|
3176
|
+
const keptSettlement = await settleWorktree({
|
|
3177
|
+
issue: run.issue,
|
|
3178
|
+
attempt: run.attempt,
|
|
3179
|
+
ending: `review round ${revision.round} resume`,
|
|
3180
|
+
worktree: run.worktree,
|
|
3181
|
+
branch: run.branch,
|
|
3182
|
+
publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
|
|
3183
|
+
tree: "remove",
|
|
3184
|
+
mirrorPath,
|
|
3185
|
+
});
|
|
3186
|
+
if (keptSettlement.retained) {
|
|
3187
|
+
log(
|
|
3188
|
+
`#${issue} review round ${revision.round} retained worktree ${run.worktree}: ` +
|
|
3189
|
+
"its salvage failed, so the tree is the only copy of work and will not be removed",
|
|
3190
|
+
);
|
|
3191
|
+
store.updateRun(runId, {
|
|
3192
|
+
state: "pushed-green",
|
|
3193
|
+
endedAt: Date.now(),
|
|
3194
|
+
lastError: `review round ${revision.round} could not clear the run's retained worktree: ${keptSettlement.lines.join(" ")}`,
|
|
3195
|
+
});
|
|
3196
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3197
|
+
await safeEscalate(d, {
|
|
3198
|
+
tier: 1,
|
|
3199
|
+
project: project.name,
|
|
3200
|
+
issue,
|
|
3201
|
+
runId,
|
|
3202
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched — the run's retained worktree is the only copy of work`,
|
|
3203
|
+
detail: [revision.prUrl, "", ...keptSettlement.lines].join("\n"),
|
|
3204
|
+
});
|
|
3205
|
+
return;
|
|
3206
|
+
}
|
|
3207
|
+
}
|
|
2616
3208
|
const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
|
|
2617
3209
|
worktreePath = provisioned.path;
|
|
2618
3210
|
runRepo = { repo, runRepoPath: worktreePath, branch };
|
|
@@ -2680,7 +3272,9 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2680
3272
|
runRepoPath: worktreePath,
|
|
2681
3273
|
branch,
|
|
2682
3274
|
},
|
|
2683
|
-
{
|
|
3275
|
+
{
|
|
3276
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
3277
|
+
},
|
|
2684
3278
|
);
|
|
2685
3279
|
if (await settleStopBeforeSession()) return;
|
|
2686
3280
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -2695,6 +3289,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2695
3289
|
|
|
2696
3290
|
log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
|
|
2697
3291
|
|
|
3292
|
+
|
|
2698
3293
|
let result: WorkerResult;
|
|
2699
3294
|
try {
|
|
2700
3295
|
result = await runWorker({
|
|
@@ -2754,7 +3349,14 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2754
3349
|
|
|
2755
3350
|
const verified: { state: RunState; reason?: string } =
|
|
2756
3351
|
result.state === "pushed-green"
|
|
2757
|
-
? await verifyPushedGreenClaim(tracker, result
|
|
3352
|
+
? await verifyPushedGreenClaim(tracker, result, {
|
|
3353
|
+
project: project.name,
|
|
3354
|
+
issue,
|
|
3355
|
+
runId,
|
|
3356
|
+
branch,
|
|
3357
|
+
repo: repoSlug ?? repo.name,
|
|
3358
|
+
store,
|
|
3359
|
+
})
|
|
2758
3360
|
: { state: result.state };
|
|
2759
3361
|
const state = verified.state;
|
|
2760
3362
|
|
|
@@ -2813,6 +3415,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2813
3415
|
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
2814
3416
|
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
2815
3417
|
sessionFile: result.sessionFile,
|
|
3418
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
2816
3419
|
report: finalReport,
|
|
2817
3420
|
...settlement?.patch,
|
|
2818
3421
|
};
|
|
@@ -3021,8 +3624,12 @@ export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRe
|
|
|
3021
3624
|
}
|
|
3022
3625
|
// A revision the previous daemon never claimed: the run is still a
|
|
3023
3626
|
// settled green row and the round is still pending, so the ordinary
|
|
3024
|
-
// dispatch pass wakes it on the next tick untouched.
|
|
3025
|
-
|
|
3627
|
+
// dispatch pass wakes it on the next tick untouched. Same for a
|
|
3628
|
+
// capped/failed run whose review round is pending (#795): a `failed` /
|
|
3629
|
+
// `killed` row is terminal too — no process died on this round, the
|
|
3630
|
+
// revision was never claimed, and the ordinary dispatch pass claims it
|
|
3631
|
+
// exactly like a pending round on a pushed-green row.
|
|
3632
|
+
if (run.state === "pushed-green" || run.state === "failed" || run.state === "killed") continue;
|
|
3026
3633
|
// The run a previous daemon claimed for this round and died on: the
|
|
3027
3634
|
// orphan sweep just marked it `orphaned` (salvaging the tree to the
|
|
3028
3635
|
// branch), so restore it to the reviewable state the verb recorded.
|
|
@@ -3609,6 +4216,7 @@ export function summarizeDispatch(
|
|
|
3609
4216
|
holds: readonly AdmissionHold[],
|
|
3610
4217
|
completedAt = Date.now(),
|
|
3611
4218
|
settled = 0,
|
|
4219
|
+
parked = 0,
|
|
3612
4220
|
): DispatchSummary {
|
|
3613
4221
|
const groups = new Map<AdmissionHoldReason, { count: number; issues: number[]; details: string[] }>();
|
|
3614
4222
|
for (const hold of holds) {
|
|
@@ -3638,6 +4246,10 @@ export function summarizeDispatch(
|
|
|
3638
4246
|
...(group.details.length === 0 ? {} : { details: group.details }),
|
|
3639
4247
|
})),
|
|
3640
4248
|
settled,
|
|
4249
|
+
// Omitted at zero so a pass with nothing parked keeps the pre-#507 record
|
|
4250
|
+
// shape byte for byte — old persisted rows lack the key and readers use
|
|
4251
|
+
// `?? 0` either way.
|
|
4252
|
+
...(parked === 0 ? {} : { parked }),
|
|
3641
4253
|
};
|
|
3642
4254
|
}
|
|
3643
4255
|
|
|
@@ -3770,6 +4382,13 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
|
|
|
3770
4382
|
);
|
|
3771
4383
|
return launched.ok === true ? { ok: true, unit: launched.unit } : { ok: false, stderr: launched.stderr };
|
|
3772
4384
|
},
|
|
4385
|
+
herdrSession: resolveHerdrSession(process.env),
|
|
4386
|
+
// The durable pane probe for the post-restart verifier (#832): every omp
|
|
4387
|
+
// process the fleet pane currently claims by herdr, resolved to its start
|
|
4388
|
+
// time — the evidence an external (pane-owned) orchestrator actually
|
|
4389
|
+
// restarted after the install began, through the same shared probe as the
|
|
4390
|
+
// in-process upgrade.
|
|
4391
|
+
probePaneOmp: (session) => herdrPaneOmpStarts(runCommand, session, processStartTimeMs),
|
|
3773
4392
|
log,
|
|
3774
4393
|
now: () => Date.now(),
|
|
3775
4394
|
};
|
|
@@ -3870,6 +4489,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3870
4489
|
}
|
|
3871
4490
|
d.project = fresh;
|
|
3872
4491
|
d.caps = freshCaps;
|
|
4492
|
+
d.host = cfg.host;
|
|
3873
4493
|
d.deliveryPolicyValid = true;
|
|
3874
4494
|
} catch (err) {
|
|
3875
4495
|
log(`config reload failed (${errText(err)}) — retaining boot values but blocking autonomous delivery`);
|
|
@@ -4030,6 +4650,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4030
4650
|
log(`decision condition pass failed: ${errText(err)}`);
|
|
4031
4651
|
});
|
|
4032
4652
|
|
|
4653
|
+
// A spend-cap pause is self-expiring (#780). The spend gate below persists a
|
|
4654
|
+
// pause once today's spend reaches the cap, but once that latch is on disk
|
|
4655
|
+
// every later pass returns at the pause gate and never reaches the spend
|
|
4656
|
+
// check again — so the fleet stayed stopped after the rolling window reset
|
|
4657
|
+
// until an operator ran `resume`. The same measurement that closed the gate
|
|
4658
|
+
// reopens it: clear a per-project spend-cap pause when the current
|
|
4659
|
+
// rolling-window spend is below the cap (or no cap is configured at all —
|
|
4660
|
+
// the gate that justified the pause is gone). Two guards keep the clear from
|
|
4661
|
+
// ever overriding another stop: only the exact instance that was read is
|
|
4662
|
+
// removed (a hold that replaced the sentinel meanwhile is untouched, review
|
|
4663
|
+
// #780), and only the per-project sentinel is ever considered — the legacy
|
|
4664
|
+
// global sentinel, which may carry an operator/integrity/setup hold, is
|
|
4665
|
+
// never removed, so the pause gate below still observes it.
|
|
4666
|
+
const spendPausePath = pausedPath(d.project.name);
|
|
4667
|
+
const spendPause = pauseInstanceAt(spendPausePath);
|
|
4668
|
+
if (spendPause?.source === "spend-cap") {
|
|
4669
|
+
const spent = d.store.spendSince(d.project.name, startOfToday());
|
|
4670
|
+
if (
|
|
4671
|
+
(d.caps.dailySpendUsd === null || spent < d.caps.dailySpendUsd) &&
|
|
4672
|
+
clearPauseIfUnchanged(spendPausePath, spendPause)
|
|
4673
|
+
) {
|
|
4674
|
+
log(
|
|
4675
|
+
`spend-cap pause cleared: $${spent.toFixed(2)} ` +
|
|
4676
|
+
(d.caps.dailySpendUsd === null
|
|
4677
|
+
? "(no daily cap configured)"
|
|
4678
|
+
: `below the $${d.caps.dailySpendUsd.toFixed(2)} daily cap`) +
|
|
4679
|
+
" — dispatch resumes",
|
|
4680
|
+
);
|
|
4681
|
+
}
|
|
4682
|
+
}
|
|
4683
|
+
|
|
4033
4684
|
// A paused fleet claims nothing. Checked first so pausing takes effect on the
|
|
4034
4685
|
// next tick without signalling the process. But the pass still ran, and the
|
|
4035
4686
|
// operator has to be able to see it: record it as a held pass — the work the
|
|
@@ -4050,6 +4701,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4050
4701
|
return;
|
|
4051
4702
|
}
|
|
4052
4703
|
|
|
4704
|
+
// The project drain is the self-expiring sibling of the pause fence (#484):
|
|
4705
|
+
// the same admission boundary — settlement above it, nothing claimed below —
|
|
4706
|
+
// but the intent is durable (it survives orchestrator loss) and bounded (the
|
|
4707
|
+
// record carries an absolute deadline, so a crash can never strand
|
|
4708
|
+
// admission). Three shapes, three behaviours:
|
|
4709
|
+
// - fresh drain with live runs → a held pass, exactly like a pause;
|
|
4710
|
+
// - fresh drain with nothing left to wait for → the drain is satisfied
|
|
4711
|
+
// and clears itself, so a completed drain never needs a second operator
|
|
4712
|
+
// action and this pass proceeds normally;
|
|
4713
|
+
// - malformed record → fails closed for THIS pass (it might be a fresh
|
|
4714
|
+
// fence we cannot read), and the same consume removed it, so it can
|
|
4715
|
+
// never become an unbounded permanent drain.
|
|
4716
|
+
const drain = consumeDrain(d.project.name);
|
|
4717
|
+
if (drain.kind === "active") {
|
|
4718
|
+
// Completion is the ACTIVE set, not the live-worker set: pushed-pending
|
|
4719
|
+
// and pushed-green PRs still make the `runs-settled` release gate fail, so
|
|
4720
|
+
// a drain that cleared while one remained would admit work on top of a
|
|
4721
|
+
// batch the releases still see as unfinished (#776 review #2).
|
|
4722
|
+
if (d.store.activeRuns(d.project.name).length === 0) {
|
|
4723
|
+
cancelDrain(d.project.name);
|
|
4724
|
+
log("project drain completed: no active runs remain — drain cleared");
|
|
4725
|
+
} else {
|
|
4726
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
4727
|
+
return;
|
|
4728
|
+
}
|
|
4729
|
+
} else if (drain.kind === "error") {
|
|
4730
|
+
log(`project drain record invalid (${drain.problem}) — removed; this pass admits nothing`);
|
|
4731
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
4732
|
+
return;
|
|
4733
|
+
}
|
|
4734
|
+
|
|
4053
4735
|
const { project, caps, store } = d;
|
|
4054
4736
|
|
|
4055
4737
|
// "Nobody patches the running conductor" is a hard boundary in both briefs —
|
|
@@ -4154,12 +4836,28 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4154
4836
|
const dropped = effective.filter(
|
|
4155
4837
|
(issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
|
|
4156
4838
|
);
|
|
4839
|
+
// The operator's park label is not a stale lifecycle label: the queue query
|
|
4840
|
+
// still returns a parked-and-queued issue, route() drops it as ineligible,
|
|
4841
|
+
// and the claim gate holds it as `issue-parked` when the park lands mid-pass
|
|
4842
|
+
// (#734). Counting it as stale-lifecycle below would summon the orchestrator
|
|
4843
|
+
// to reconcile a deliberate decision. Parked is its own population, derived
|
|
4844
|
+
// from the same isEligible read the gate uses, so the number status renders
|
|
4845
|
+
// cannot disagree with what admission would hold (#507).
|
|
4846
|
+
const parkedCandidates = effective.filter(
|
|
4847
|
+
(issue) => !isEligible(issue, project) && issue.labels.includes(project.stateLabels.backlog),
|
|
4848
|
+
);
|
|
4849
|
+
const parkedNumbers = new Set(parkedCandidates.map((issue) => issue.number));
|
|
4157
4850
|
let claimed = 0;
|
|
4851
|
+
let parked = 0;
|
|
4158
4852
|
const lifecycleHolds: AdmissionHold[] = [];
|
|
4159
4853
|
for (const issue of dropped) {
|
|
4160
4854
|
const newest = store.latestRun(project.name, issue.number);
|
|
4161
4855
|
if (newest !== undefined && ACTIVE_STATES.includes(newest.state)) {
|
|
4162
4856
|
claimed += 1;
|
|
4857
|
+
} else if (parkedNumbers.has(issue.number)) {
|
|
4858
|
+
// A park never kills a live run; a parked issue with no run is inventory
|
|
4859
|
+
// the operator is deliberately holding, not reconciliation work.
|
|
4860
|
+
parked += 1;
|
|
4163
4861
|
} else {
|
|
4164
4862
|
lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
|
|
4165
4863
|
}
|
|
@@ -4173,7 +4871,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4173
4871
|
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
4174
4872
|
store.recordDispatch(
|
|
4175
4873
|
project.name,
|
|
4176
|
-
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled),
|
|
4874
|
+
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled, parked),
|
|
4177
4875
|
);
|
|
4178
4876
|
};
|
|
4179
4877
|
|
|
@@ -4218,7 +4916,8 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4218
4916
|
summary: `Daily spend cap reached on ${new Date().toISOString().slice(0, 10)} — ${project.name} is paused`,
|
|
4219
4917
|
detail: [
|
|
4220
4918
|
`Spent $${spent.toFixed(2)} of the $${caps.dailySpendUsd.toFixed(2)} daily cap.`,
|
|
4221
|
-
"
|
|
4919
|
+
"Dispatch stays paused until today's spend falls below the cap, then resumes",
|
|
4920
|
+
"automatically on the next pass — `omp-conductor resume` only speeds that up.",
|
|
4222
4921
|
].join("\n"),
|
|
4223
4922
|
});
|
|
4224
4923
|
recordDispatch(0, [
|
|
@@ -4266,7 +4965,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4266
4965
|
log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
|
|
4267
4966
|
await dispatchAdmissions(
|
|
4268
4967
|
pass.admitted,
|
|
4269
|
-
(a) => handleIssue(d, a.r, a.attempt, a.lane),
|
|
4968
|
+
(a) => handleIssue(d, a.r, a.attempt, a.lane, a.model),
|
|
4270
4969
|
workers,
|
|
4271
4970
|
);
|
|
4272
4971
|
|
|
@@ -4294,6 +4993,23 @@ export interface DaemonHealthSnapshot {
|
|
|
4294
4993
|
codeGraph?: CodeGraphHealth;
|
|
4295
4994
|
/** Live workers in a non-running pause phase; absent/empty = nothing paused. */
|
|
4296
4995
|
workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
4996
|
+
/**
|
|
4997
|
+
* The live orchestrator surface, attested by the running daemon (#832):
|
|
4998
|
+
* which mode this project's fleet is in, and — when the daemon hosts the
|
|
4999
|
+
* orchestrator session itself — the extension version that session *loaded*
|
|
5000
|
+
* and the transcript it resumed. This is the read the upgrade's
|
|
5001
|
+
* session-reload verification checks: a restarted daemon whose orchestrator
|
|
5002
|
+
* child came up on the installed release answers `loaded` equal to that
|
|
5003
|
+
* release; one that never reloaded answers an older version; one that
|
|
5004
|
+
* failed to start answers `mode: "failed"`. `external` means the pane owns
|
|
5005
|
+
* the session and the daemon cannot attest it from here.
|
|
5006
|
+
*/
|
|
5007
|
+
orchestrator?: {
|
|
5008
|
+
mode: "embedded" | "external" | "failed";
|
|
5009
|
+
loaded?: string;
|
|
5010
|
+
sessionFile?: string;
|
|
5011
|
+
alive?: boolean;
|
|
5012
|
+
};
|
|
4297
5013
|
}
|
|
4298
5014
|
|
|
4299
5015
|
export interface DaemonHealth {
|
|
@@ -4309,6 +5025,7 @@ export function daemonHealthSnapshot(
|
|
|
4309
5025
|
paused = isPaused(project),
|
|
4310
5026
|
codeGraph?: CodeGraphHealth,
|
|
4311
5027
|
workerControls?: WorkerControlRegistry,
|
|
5028
|
+
orchestrator?: DaemonHealthSnapshot["orchestrator"],
|
|
4312
5029
|
): DaemonHealthSnapshot {
|
|
4313
5030
|
const dispatch = store.latestDispatch(project);
|
|
4314
5031
|
return {
|
|
@@ -4320,6 +5037,7 @@ export function daemonHealthSnapshot(
|
|
|
4320
5037
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
4321
5038
|
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
4322
5039
|
...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
|
|
5040
|
+
...(orchestrator === undefined ? {} : { orchestrator }),
|
|
4323
5041
|
};
|
|
4324
5042
|
}
|
|
4325
5043
|
|
|
@@ -4637,6 +5355,55 @@ export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promi
|
|
|
4637
5355
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
4638
5356
|
}
|
|
4639
5357
|
|
|
5358
|
+
/**
|
|
5359
|
+
* The effective reporting surface, derived from the configured policy — never
|
|
5360
|
+
* from the legacy preset name alone: an explicit policy without the
|
|
5361
|
+
* `scopePreset` back-annotation must present the same truth from `interruptOn`
|
|
5362
|
+
* and the digest cadence (#633).
|
|
5363
|
+
*/
|
|
5364
|
+
export interface ReportingSummary {
|
|
5365
|
+
/** The legacy preset this policy came from, when the config back-annotates one. */
|
|
5366
|
+
scopePreset?: ReportScope;
|
|
5367
|
+
/** Categories allowed to interrupt the operator's phone. */
|
|
5368
|
+
interruptOn: InterruptCategory[];
|
|
5369
|
+
/** Where non-interrupting outcomes accumulate: the effective digest config. */
|
|
5370
|
+
digest: ReportingPolicy["digest"];
|
|
5371
|
+
}
|
|
5372
|
+
|
|
5373
|
+
/**
|
|
5374
|
+
* The effective reporting summary. The loader always materialises the policy,
|
|
5375
|
+
* so `undefined` means the default (`DEFAULT_REPORT_POLICY`); the preset name
|
|
5376
|
+
* is carried only when the policy actually back-annotates one, so a rendered
|
|
5377
|
+
* status names the preset without ever letting a stale or absent preset
|
|
5378
|
+
* misstate what the policy does (#633).
|
|
5379
|
+
*/
|
|
5380
|
+
export function reportingSummary(policy: ReportingPolicy | undefined): ReportingSummary {
|
|
5381
|
+
const effective = policy ?? DEFAULT_REPORT_POLICY;
|
|
5382
|
+
return {
|
|
5383
|
+
...(effective.scopePreset === undefined ? {} : { scopePreset: effective.scopePreset }),
|
|
5384
|
+
interruptOn: [...effective.interruptOn],
|
|
5385
|
+
digest: { ...effective.digest },
|
|
5386
|
+
};
|
|
5387
|
+
}
|
|
5388
|
+
|
|
5389
|
+
/**
|
|
5390
|
+
* The active drain as structured status exposes it: the creation instant, the
|
|
5391
|
+
* absolute deadline, the purpose, and the live-run count the drain is waiting
|
|
5392
|
+
* to reach zero. Only present on the snapshot while the record is fresh.
|
|
5393
|
+
*/
|
|
5394
|
+
export interface DrainStatus {
|
|
5395
|
+
/** Epoch-ms instant the drain intent was recorded. */
|
|
5396
|
+
since: number;
|
|
5397
|
+
/** Absolute epoch-ms deadline — admission resumes automatically after it. */
|
|
5398
|
+
expiresAt: number;
|
|
5399
|
+
/** Purpose recorded at creation. */
|
|
5400
|
+
reason?: string;
|
|
5401
|
+
/** Runs still in the active set (live workers plus pushed-state PRs) — the
|
|
5402
|
+
* count a drain waits to reach zero, matching the `runs-settled` release
|
|
5403
|
+
* gate (#776 review #2). */
|
|
5404
|
+
remainingRuns: number;
|
|
5405
|
+
}
|
|
5406
|
+
|
|
4640
5407
|
export interface StatusSnapshot {
|
|
4641
5408
|
project: string;
|
|
4642
5409
|
configPath: string;
|
|
@@ -4648,10 +5415,27 @@ export interface StatusSnapshot {
|
|
|
4648
5415
|
* reading like a mistake (#220).
|
|
4649
5416
|
*/
|
|
4650
5417
|
pauseReason?: string;
|
|
5418
|
+
/**
|
|
5419
|
+
* The project's active self-expiring drain (#484): present only while a
|
|
5420
|
+
* fresh, valid drain record exists. Paused and drained are deliberately two
|
|
5421
|
+
* fields — a drain is bounded and self-clearing where a pause is not, and
|
|
5422
|
+
* the structured surface has to tell them apart without reading the pass
|
|
5423
|
+
* history. Human CLI wording over it is a later #484 child.
|
|
5424
|
+
*/
|
|
5425
|
+
drain?: DrainStatus;
|
|
4651
5426
|
/** Mechanical operator availability at the moment this snapshot was read. */
|
|
4652
5427
|
availability?: AvailabilityState;
|
|
4653
5428
|
/** Next digest opportunity under the same predicate that gates submission. */
|
|
4654
5429
|
digestSchedule?: DigestScheduleState;
|
|
5430
|
+
/**
|
|
5431
|
+
* The effective reporting policy: what interrupts, where everything else
|
|
5432
|
+
* goes, and the legacy preset name when one is back-annotated. On the
|
|
5433
|
+
* snapshot so the human `status`, the dashboard API and the CLI all read
|
|
5434
|
+
* the same truth about the reporting surface without opening config — a
|
|
5435
|
+
* routine outcome that reads as "Telegram is broken" unless the policy says
|
|
5436
|
+
* it is digest-only (#633).
|
|
5437
|
+
*/
|
|
5438
|
+
reporting?: ReportingSummary;
|
|
4655
5439
|
caps: Caps;
|
|
4656
5440
|
/**
|
|
4657
5441
|
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
@@ -4773,6 +5557,19 @@ export function statusSnapshotFromStore(
|
|
|
4773
5557
|
// Read once: the provenance read touches the filesystem, and the renderer
|
|
4774
5558
|
// should never pay for it twice per status.
|
|
4775
5559
|
const reason = pauseProvenance(p.name)?.reason;
|
|
5560
|
+
// Same cost discipline as the pause read: the drain record is a file read,
|
|
5561
|
+
// and the active-drain view is only built when one is actually fresh. The
|
|
5562
|
+
// read is observational — it never mutates the record — so a status read
|
|
5563
|
+
// cannot consume a malformed marker the next dispatch pass still has to fail
|
|
5564
|
+
// closed on; the dispatch consume owns cleanup (#776 review #2).
|
|
5565
|
+
const drain = readDrain(p.name);
|
|
5566
|
+
// The live run list feeds the worker-capacity row; the drain's
|
|
5567
|
+
// remaining-runs count waits on the ACTIVE set (live workers plus
|
|
5568
|
+
// pushed-state PRs), the same population the `runs-settled` release gate
|
|
5569
|
+
// reads, so the two can never disagree about when a batch is finished
|
|
5570
|
+
// (#776 review #2).
|
|
5571
|
+
const live = store.liveRuns(p.name);
|
|
5572
|
+
const active = store.activeRuns(p.name);
|
|
4776
5573
|
// The live review-revision rounds, read from the same durable rows the
|
|
4777
5574
|
// restart recovery uses: a run whose revision is dispatched is read as
|
|
4778
5575
|
// `review-revision N` while its worker is live (#692).
|
|
@@ -4786,12 +5583,23 @@ export function statusSnapshotFromStore(
|
|
|
4786
5583
|
stateDir: stateDir(),
|
|
4787
5584
|
paused: isPaused(p.name),
|
|
4788
5585
|
...(reason === undefined ? {} : { pauseReason: reason }),
|
|
5586
|
+
...(drain.kind === "active"
|
|
5587
|
+
? {
|
|
5588
|
+
drain: {
|
|
5589
|
+
since: Date.parse(drain.drain.createdAt),
|
|
5590
|
+
expiresAt: Date.parse(drain.drain.expiresAt),
|
|
5591
|
+
...(drain.drain.reason === undefined ? {} : { reason: drain.drain.reason }),
|
|
5592
|
+
remainingRuns: active.length,
|
|
5593
|
+
} satisfies DrainStatus,
|
|
5594
|
+
}
|
|
5595
|
+
: {}),
|
|
4789
5596
|
availability: availabilityState(p.reporting, now),
|
|
4790
5597
|
digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
|
|
5598
|
+
reporting: reportingSummary(p.reporting ?? DEFAULT_REPORT_POLICY),
|
|
4791
5599
|
caps,
|
|
4792
5600
|
releaseGrants: resolveReleaseGrants(p),
|
|
4793
5601
|
review: resolveReview(p),
|
|
4794
|
-
activeRuns:
|
|
5602
|
+
activeRuns: active,
|
|
4795
5603
|
reviewRounds,
|
|
4796
5604
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
4797
5605
|
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
@@ -4799,7 +5607,7 @@ export function statusSnapshotFromStore(
|
|
|
4799
5607
|
openReports: store.openReports(p.name),
|
|
4800
5608
|
digestBacklog: store.digestBacklog(p.name),
|
|
4801
5609
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
4802
|
-
liveWorkers:
|
|
5610
|
+
liveWorkers: live.length,
|
|
4803
5611
|
runsToday: store.runsStartedSince(p.name, since),
|
|
4804
5612
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
4805
5613
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
@@ -4846,6 +5654,14 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
4846
5654
|
` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
|
|
4847
5655
|
` admitted ${summary.admitted}`,
|
|
4848
5656
|
);
|
|
5657
|
+
// The operator's park label, counted from the same eligibility read the
|
|
5658
|
+
// claim gate uses (#507): "0 claimable, 12 parked" and "0 claimable,
|
|
5659
|
+
// nothing to do" demand opposite orchestrator responses and must not
|
|
5660
|
+
// render alike. Omitted at zero so an empty queue stays the old shape.
|
|
5661
|
+
const parked = summary.parked ?? 0;
|
|
5662
|
+
if (parked > 0) {
|
|
5663
|
+
lines.push(` parked ${parked} — operator-held, never claimed`);
|
|
5664
|
+
}
|
|
4849
5665
|
if (summary.holds.length === 0) {
|
|
4850
5666
|
lines.push(" held 0");
|
|
4851
5667
|
} else {
|
|
@@ -5398,6 +6214,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5398
6214
|
log(projects.length === 1 ? message : `[${project.name}] ${message}`);
|
|
5399
6215
|
};
|
|
5400
6216
|
const caps = resolveCaps(project, cfg.defaults);
|
|
6217
|
+
const host = cfg.host;
|
|
5401
6218
|
// One transient-server-error breaker per project (#642): admission's
|
|
5402
6219
|
// GraphQL checks and the orchestrator mutation commands share it, so a 503
|
|
5403
6220
|
// observed by either side gates both instead of one provider outage being
|
|
@@ -5476,7 +6293,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5476
6293
|
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
5477
6294
|
});
|
|
5478
6295
|
const transcript = orchestrator.sessionFile();
|
|
5479
|
-
|
|
6296
|
+
const loaded = orchestrator.extensionVersion();
|
|
6297
|
+
projectLog(
|
|
6298
|
+
`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}` +
|
|
6299
|
+
`${loaded === undefined ? "" : ` · loaded omp-conductor ${loaded}`}`,
|
|
6300
|
+
);
|
|
5480
6301
|
} catch (err) {
|
|
5481
6302
|
orchestratorStartError = errText(err);
|
|
5482
6303
|
projectLog(
|
|
@@ -5529,6 +6350,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5529
6350
|
const d: Deps = {
|
|
5530
6351
|
project,
|
|
5531
6352
|
caps,
|
|
6353
|
+
host,
|
|
5532
6354
|
tracker,
|
|
5533
6355
|
store,
|
|
5534
6356
|
drain,
|
|
@@ -5718,15 +6540,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5718
6540
|
},
|
|
5719
6541
|
health: () =>
|
|
5720
6542
|
daemonHealth(
|
|
5721
|
-
runtimes.map((runtime) =>
|
|
5722
|
-
|
|
6543
|
+
runtimes.map((runtime) => {
|
|
6544
|
+
const orch = runtime.orchestrator;
|
|
6545
|
+
const mode = runtime.d.project.escalation.orchestrator;
|
|
6546
|
+
return daemonHealthSnapshot(
|
|
5723
6547
|
store,
|
|
5724
6548
|
runtime.d.project.name,
|
|
5725
6549
|
isPaused(runtime.d.project.name),
|
|
5726
6550
|
runtime.codeGraph,
|
|
5727
6551
|
workerControls,
|
|
5728
|
-
|
|
5729
|
-
|
|
6552
|
+
orch === undefined
|
|
6553
|
+
? // External mode is the pane's session: the daemon hosts no
|
|
6554
|
+
// child to attest. An embedded orchestrator that failed to
|
|
6555
|
+
// start is a live outage of the surface the upgrade must
|
|
6556
|
+
// promise reloaded (#832) — say so out loud.
|
|
6557
|
+
{ mode: mode === "external" ? "external" : "failed" }
|
|
6558
|
+
: {
|
|
6559
|
+
mode: "embedded",
|
|
6560
|
+
...(orch.extensionVersion() === undefined
|
|
6561
|
+
? {}
|
|
6562
|
+
: { loaded: orch.extensionVersion() }),
|
|
6563
|
+
...(orch.sessionFile() === undefined ? {} : { sessionFile: orch.sessionFile() }),
|
|
6564
|
+
alive: orch.alive(),
|
|
6565
|
+
},
|
|
6566
|
+
);
|
|
6567
|
+
}),
|
|
5730
6568
|
),
|
|
5731
6569
|
}),
|
|
5732
6570
|
});
|