omp-conductor 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/admission.ts +58 -14
- package/src/arm-challenge.ts +54 -3
- package/src/briefs/console.md +10 -5
- package/src/commands/arm.ts +7 -5
- package/src/daemon/groom-pass.ts +16 -6
- package/src/daemon/runtime.ts +55 -3
- package/src/daemon/settle-pass.ts +19 -2
- package/src/daemon.ts +2 -2
- package/src/diff-flags.ts +111 -6
- package/src/doctor.ts +2 -2
- package/src/failure-class.ts +182 -1
- package/src/fleet.ts +13 -20
- package/src/orchestrator-tick.ts +296 -24
- package/src/settlement.ts +35 -5
- package/src/status-render.ts +11 -7
- package/src/store.ts +14 -2
- package/src/to-spec.ts +233 -24
- package/src/types.ts +18 -3
- package/src/worker.ts +149 -35
package/src/orchestrator-tick.ts
CHANGED
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
|
|
47
47
|
import { createHash, randomUUID } from "node:crypto";
|
|
48
48
|
import { spawnSync } from "node:child_process";
|
|
49
|
-
import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
49
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
50
50
|
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
51
51
|
import { availabilityPrompt, interruptDisposition } from "./availability.ts";
|
|
52
52
|
import {
|
|
@@ -132,6 +132,20 @@ import {
|
|
|
132
132
|
type ToSpecTrackerSeam,
|
|
133
133
|
} from "./groom.ts";
|
|
134
134
|
import { heldNoticeId } from "./notices.ts";
|
|
135
|
+
import {
|
|
136
|
+
clearArmTransaction,
|
|
137
|
+
expiredArmChallenges,
|
|
138
|
+
FLEET_ARM_KEY,
|
|
139
|
+
resolveArmReply,
|
|
140
|
+
type ArmChallengeSighting,
|
|
141
|
+
type ArmReplyMatch,
|
|
142
|
+
} from "./arm-challenge.ts";
|
|
143
|
+
import {
|
|
144
|
+
readTelegramChannel,
|
|
145
|
+
readTelegramToken,
|
|
146
|
+
resolveProjectTopicId,
|
|
147
|
+
sendTelegram,
|
|
148
|
+
} from "./escalate.ts";
|
|
135
149
|
|
|
136
150
|
/** The activation file. Absent means "this is not an orchestrator session". */
|
|
137
151
|
export const TICK_CONFIG_FILE = ".conductor-tick.json";
|
|
@@ -489,6 +503,10 @@ const AUTONOMOUS_RECOVERY_ACTIONS: Record<RecoveryAction, boolean> = {
|
|
|
489
503
|
"rerun-checks": true,
|
|
490
504
|
settle: true,
|
|
491
505
|
escalate: false,
|
|
506
|
+
// An observation row never stamps `recoveredAt` (the recovery IS the later
|
|
507
|
+
// sweep pass), so it never lands in this digest; `false` is the honest value
|
|
508
|
+
// if one ever did — it is not "already handled", it is waiting (#1068).
|
|
509
|
+
observe: false,
|
|
492
510
|
hold: false,
|
|
493
511
|
none: false,
|
|
494
512
|
};
|
|
@@ -1638,6 +1656,20 @@ export function resolveArmState(armedFile: string, projectName?: string): ArmSta
|
|
|
1638
1656
|
return shared ? { armed: true } : { armed: true, legacy: "honoured" };
|
|
1639
1657
|
}
|
|
1640
1658
|
|
|
1659
|
+
/**
|
|
1660
|
+
* The one armed-marker write both settlement paths share — the CLI's
|
|
1661
|
+
* `arm --reply` half and the orchestrator session's mechanical reply consumer
|
|
1662
|
+
* (#1061): same content, same mode, and the same restamp of the pre-per-project
|
|
1663
|
+
* shared marker the heartbeat still honours (#316). It lives here, beside
|
|
1664
|
+
* {@link resolveArmState}, so the session consumer can write the gate it reads
|
|
1665
|
+
* — the reverse import (into `fleet.ts`) would cycle.
|
|
1666
|
+
*/
|
|
1667
|
+
export function writeArmedMarker(path: string, owner: string, arm: ArmState): void {
|
|
1668
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1669
|
+
writeFileSync(path, `armed ${new Date().toISOString()} owner=${owner}\n`, { mode: 0o600 });
|
|
1670
|
+
if (arm.legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1641
1673
|
/**
|
|
1642
1674
|
* Best-effort recompose of `ORCHESTRATOR.md` from the installed package floor +
|
|
1643
1675
|
* live `POLICY.md`.
|
|
@@ -2854,6 +2886,7 @@ async function tick(
|
|
|
2854
2886
|
config: TickConfig,
|
|
2855
2887
|
session: TickSession,
|
|
2856
2888
|
toSpecTrackerSeam: ToSpecTrackerSeam | undefined,
|
|
2889
|
+
armSeam: ArmConsumerSeam | undefined,
|
|
2857
2890
|
): Promise<void> {
|
|
2858
2891
|
const live = currentConfig(ctx.cwd, config);
|
|
2859
2892
|
const arm = live.armedFile === undefined ? undefined : resolveArmState(live.armedFile, live.project);
|
|
@@ -2863,6 +2896,29 @@ async function tick(
|
|
|
2863
2896
|
`[omp-conductor] ${legacyArmedMarkerPath()}: ${LEGACY_ARM_MARKER_DETAIL} — this heartbeat stays disarmed`,
|
|
2864
2897
|
);
|
|
2865
2898
|
}
|
|
2899
|
+
// An unanswered ceremony that passed its window is a dead ceremony, not a
|
|
2900
|
+
// silent one (#1061): the tick that notices tells the operator why and how
|
|
2901
|
+
// to re-run it, on the surface the challenge was sent to. Deliberately
|
|
2902
|
+
// before the tick gate — a disarmed fleet is exactly the state an
|
|
2903
|
+
// unanswered challenge leaves, and its death notice must not wait for an
|
|
2904
|
+
// arm to be heard. The scan is synchronous so an idle host (no expired
|
|
2905
|
+
// records) costs the tick nothing; only a send is awaited.
|
|
2906
|
+
const armNow = (armSeam?.now ?? Date.now)();
|
|
2907
|
+
for (const expired of expiredArmChallenges(live.project, armNow)) {
|
|
2908
|
+
const surface: ArmNoticeSurface = { accessFile: live.accessFile, project: live.project };
|
|
2909
|
+
const send = armSeam?.notify ?? sendArmNotice;
|
|
2910
|
+
try {
|
|
2911
|
+
await send(armExpiryNotice(expired, live.project), surface);
|
|
2912
|
+
} catch (err) {
|
|
2913
|
+
// Keep the record: the next heartbeat retries, and `doctor` still
|
|
2914
|
+
// reports it.
|
|
2915
|
+
pi.logger.error(
|
|
2916
|
+
`[omp-conductor] arm expiry notice not delivered: ${err instanceof Error ? err.message : String(err)}`,
|
|
2917
|
+
);
|
|
2918
|
+
continue;
|
|
2919
|
+
}
|
|
2920
|
+
clearArmTransaction(expired.key, expired.id);
|
|
2921
|
+
}
|
|
2866
2922
|
const decision = tickDecision({
|
|
2867
2923
|
armed: arm === undefined || arm.armed,
|
|
2868
2924
|
channelOk: live.accessFile === undefined || channelIsUp(live.accessFile),
|
|
@@ -3371,11 +3427,12 @@ function armTickHeartbeat(
|
|
|
3371
3427
|
config: TickConfig,
|
|
3372
3428
|
session: TickSession,
|
|
3373
3429
|
toSpecTrackerSeam: ToSpecTrackerSeam | undefined,
|
|
3430
|
+
armSeam: ArmConsumerSeam | undefined,
|
|
3374
3431
|
): void {
|
|
3375
3432
|
writeTickRuntimeStatus(pi, ctx.cwd, config.intervalSeconds);
|
|
3376
3433
|
const runScheduledTick = async (): Promise<void> => {
|
|
3377
3434
|
try {
|
|
3378
|
-
await tick(pi, ctx, config, session, toSpecTrackerSeam);
|
|
3435
|
+
await tick(pi, ctx, config, session, toSpecTrackerSeam, armSeam);
|
|
3379
3436
|
} finally {
|
|
3380
3437
|
writeTickRuntimeStatus(pi, ctx.cwd, config.intervalSeconds);
|
|
3381
3438
|
}
|
|
@@ -3431,7 +3488,7 @@ function armTickHeartbeat(
|
|
|
3431
3488
|
// call runs inside the session_start handler dispatch, which cannot see an
|
|
3432
3489
|
// async rejection — an escaped one reaches the process-level
|
|
3433
3490
|
// unhandledRejection handler and takes the session down.
|
|
3434
|
-
void tick(pi, ctx, config, session, toSpecTrackerSeam).catch((err) => {
|
|
3491
|
+
void tick(pi, ctx, config, session, toSpecTrackerSeam, armSeam).catch((err) => {
|
|
3435
3492
|
pi.logger.error(
|
|
3436
3493
|
`[omp-conductor] arm-time tick failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
3437
3494
|
);
|
|
@@ -3496,6 +3553,209 @@ export interface OrchestratorTickExtensionOptions {
|
|
|
3496
3553
|
* present-tense empty queue.
|
|
3497
3554
|
*/
|
|
3498
3555
|
toSpec?: { tracker?: ToSpecTrackerSeam };
|
|
3556
|
+
/**
|
|
3557
|
+
* The arm ceremony's mechanical consumer seams (#1061). Production omits
|
|
3558
|
+
* them and the extension classifies inbound turns against the pending
|
|
3559
|
+
* challenge records itself, sending confirmations and expiry notices over
|
|
3560
|
+
* the same bridge transport `armTicks` sends the challenge on. Tests inject
|
|
3561
|
+
* a recorder (`notify`) and a fixed clock (`now`) so the settle and the
|
|
3562
|
+
* notices are provable without Telegram or timers.
|
|
3563
|
+
*/
|
|
3564
|
+
arm?: {
|
|
3565
|
+
now?: () => number;
|
|
3566
|
+
notify?: ArmNoticeFn;
|
|
3567
|
+
};
|
|
3568
|
+
}
|
|
3569
|
+
|
|
3570
|
+
/** The send surface an arm notice rides: the live tick config's channel and project. */
|
|
3571
|
+
export interface ArmNoticeSurface {
|
|
3572
|
+
accessFile?: string;
|
|
3573
|
+
project?: string;
|
|
3574
|
+
}
|
|
3575
|
+
|
|
3576
|
+
/**
|
|
3577
|
+
* One mechanical Telegram send the arm consumer raises: the confirmation when
|
|
3578
|
+
* a reply settles the ceremony, or the expiry notice when a challenge dies
|
|
3579
|
+
* unanswered. The default is {@link sendArmNotice}; tests record instead.
|
|
3580
|
+
*/
|
|
3581
|
+
export type ArmNoticeFn = (text: string, surface: ArmNoticeSurface) => Promise<void> | void;
|
|
3582
|
+
|
|
3583
|
+
/** The consumer's injectable clock and transport, carried into the tick's helpers. */
|
|
3584
|
+
export interface ArmConsumerSeam {
|
|
3585
|
+
now?: () => number;
|
|
3586
|
+
notify?: ArmNoticeFn;
|
|
3587
|
+
}
|
|
3588
|
+
|
|
3589
|
+
/**
|
|
3590
|
+
* The plain text of an inbound message, for the arm classifier. Content blocks
|
|
3591
|
+
* join with newlines so a code split across block boundaries still tokenises
|
|
3592
|
+
* whole; non-text blocks contribute nothing. `undefined` for an absent or
|
|
3593
|
+
* empty message, which is ordinary chat either way.
|
|
3594
|
+
*/
|
|
3595
|
+
function inboundMessageText(content: readonly { type?: string; text?: string }[] | undefined): string | undefined {
|
|
3596
|
+
if (content === undefined || content.length === 0) return undefined;
|
|
3597
|
+
const text = content.map((block) => block.text ?? "").join("\n");
|
|
3598
|
+
return text.length === 0 ? undefined : text;
|
|
3599
|
+
}
|
|
3600
|
+
|
|
3601
|
+
/**
|
|
3602
|
+
* The ready-to-send expiry notice: the reason — never answered, or answered
|
|
3603
|
+
* but never settled — and the exact command that re-runs the ceremony. The
|
|
3604
|
+
* code is the proof and is never named in a notice; only the window is.
|
|
3605
|
+
*/
|
|
3606
|
+
function armExpiryNotice(expired: { key: string; sighting: ArmChallengeSighting }, project: string | undefined): string {
|
|
3607
|
+
const reissue =
|
|
3608
|
+
expired.key === FLEET_ARM_KEY || project === undefined || project === ""
|
|
3609
|
+
? "omp-conductor arm"
|
|
3610
|
+
: `omp-conductor arm --project ${project}`;
|
|
3611
|
+
const sentAt =
|
|
3612
|
+
expired.sighting.sentAt === undefined
|
|
3613
|
+
? ""
|
|
3614
|
+
: ` sent ${new Date(expired.sighting.sentAt).toISOString()}`;
|
|
3615
|
+
const reason =
|
|
3616
|
+
expired.sighting.acknowledgedAt === undefined
|
|
3617
|
+
? `The arming challenge${sentAt} expired without being answered — nothing is armed.`
|
|
3618
|
+
: `The arming challenge${sentAt} was answered but never settled before its window closed — nothing is armed.`;
|
|
3619
|
+
return `${reason} Send a fresh one: ${reissue}`;
|
|
3620
|
+
}
|
|
3621
|
+
|
|
3622
|
+
/**
|
|
3623
|
+
* The default transport for the arm consumer's confirmation and expiry
|
|
3624
|
+
* notices: the same bot token, paired channel and project topic `armTicks`
|
|
3625
|
+
* sends the challenge on, so the notice lands on the surface the operator was
|
|
3626
|
+
* asked in. Deliberately not availability-gated — the ceremony is the
|
|
3627
|
+
* operator's own interaction, exactly like the challenge send itself, and a
|
|
3628
|
+
* notice held for quiet hours would be the silent failure #1061 removes. Any
|
|
3629
|
+
* missing or down fact throws; the caller logs and keeps the challenge record
|
|
3630
|
+
* so the next heartbeat retries and `doctor` still reports it.
|
|
3631
|
+
*/
|
|
3632
|
+
export async function sendArmNotice(text: string, surface: ArmNoticeSurface): Promise<void> {
|
|
3633
|
+
if (surface.accessFile === undefined) {
|
|
3634
|
+
throw new Error("no accessFile — the arm notice has no channel");
|
|
3635
|
+
}
|
|
3636
|
+
const channel = readTelegramChannel(surface.accessFile);
|
|
3637
|
+
if (channel.kind === "down") {
|
|
3638
|
+
throw new Error(`escalation channel is not up (${surface.accessFile}): ${channel.reason}`);
|
|
3639
|
+
}
|
|
3640
|
+
const token = readTelegramToken();
|
|
3641
|
+
if (token === undefined) {
|
|
3642
|
+
throw new Error("no Telegram bot token readable — the arm notice cannot send");
|
|
3643
|
+
}
|
|
3644
|
+
let topicId: number | undefined;
|
|
3645
|
+
if (surface.project !== undefined) {
|
|
3646
|
+
try {
|
|
3647
|
+
topicId = resolveProjectTopicId(findProject(loadConfig(), surface.project));
|
|
3648
|
+
} catch {
|
|
3649
|
+
/* no project config — flat chat, exactly like armTicks */
|
|
3650
|
+
}
|
|
3651
|
+
}
|
|
3652
|
+
await sendTelegram(token, channel.owner, text, { topicId });
|
|
3653
|
+
}
|
|
3654
|
+
|
|
3655
|
+
/**
|
|
3656
|
+
* The settle half of the mechanical consumer: write the markers the matched
|
|
3657
|
+
* challenge recorded and clear its transaction, then confirm on the
|
|
3658
|
+
* challenge's surface. The recorded targets are the contract — the exact rule
|
|
3659
|
+
* `armReply` applies — so nothing is ever armed that the challenge did not
|
|
3660
|
+
* name. A pre-targets record (one release of overlap) is not settled here:
|
|
3661
|
+
* nothing in-session may guess a marker the challenge never recorded; the CLI
|
|
3662
|
+
* half still settles it, and the operator is told so.
|
|
3663
|
+
*/
|
|
3664
|
+
function settleInboundArmMatch(
|
|
3665
|
+
pi: TickApi,
|
|
3666
|
+
match: ArmReplyMatch,
|
|
3667
|
+
notify: (text: string) => void,
|
|
3668
|
+
): void {
|
|
3669
|
+
if (match.targets === undefined || match.owner === undefined) {
|
|
3670
|
+
pi.logger.error(
|
|
3671
|
+
"[omp-conductor] an inbound arming code matched a pre-targets challenge — nothing armed in-session; " +
|
|
3672
|
+
'run `omp-conductor arm --reply "<the reply>"` to settle it',
|
|
3673
|
+
);
|
|
3674
|
+
return;
|
|
3675
|
+
}
|
|
3676
|
+
for (const target of match.targets) {
|
|
3677
|
+
const state = resolveArmState(target.armedFile, target.project);
|
|
3678
|
+
writeArmedMarker(target.armedFile, match.owner, state);
|
|
3679
|
+
}
|
|
3680
|
+
clearArmTransaction(match.key, match.id);
|
|
3681
|
+
const label =
|
|
3682
|
+
match.targets.length === 1 && match.targets[0]!.project !== undefined
|
|
3683
|
+
? `project ${match.targets[0]!.project}`
|
|
3684
|
+
: `${match.targets.length} project(s)`;
|
|
3685
|
+
notify(`Arming verified — ${label} armed. The reply was consumed; nothing else to do.`);
|
|
3686
|
+
}
|
|
3687
|
+
|
|
3688
|
+
/**
|
|
3689
|
+
* The arm ceremony's mechanical consumer (#1061): classify one inbound user
|
|
3690
|
+
* turn against this host's pending challenges and, on a match, settle the
|
|
3691
|
+
* ceremony exactly as `omp-conductor arm --reply` would — markers for the
|
|
3692
|
+
* recorded targets, transaction cleared, a confirmation on the surface the
|
|
3693
|
+
* challenge was sent to. This is the real owner the ceremony needed on a host
|
|
3694
|
+
* with no console session: the challenge goes to the project topic, which is
|
|
3695
|
+
* this pane's own claimed surface, so the reply lands here and the ceremony
|
|
3696
|
+
* completes with no second human command. A code past its window is answered
|
|
3697
|
+
* with the re-run command and the dead record is cleared; a lookalike matching
|
|
3698
|
+
* nothing stays inert (#415's non-disclosure bound is unchanged).
|
|
3699
|
+
*
|
|
3700
|
+
* Never throws: an inbound message must not take the session down. Failures
|
|
3701
|
+
* are logged and the CLI half of the ceremony stays available.
|
|
3702
|
+
*/
|
|
3703
|
+
function consumeInboundArmReply(
|
|
3704
|
+
pi: TickApi,
|
|
3705
|
+
cwd: string,
|
|
3706
|
+
startupProject: string | undefined,
|
|
3707
|
+
text: string,
|
|
3708
|
+
seam: ArmConsumerSeam | undefined,
|
|
3709
|
+
): void {
|
|
3710
|
+
// The state key the send half derived from the same file, read fresh so a
|
|
3711
|
+
// re-stamp between the send and the reply cannot split the ceremony. An
|
|
3712
|
+
// invalid config cannot be trusted to name either the key or a marker —
|
|
3713
|
+
// fail closed exactly like `armReply`, but without throwing at a message.
|
|
3714
|
+
const reread = readTickConfig(cwd);
|
|
3715
|
+
if (reread.kind === "invalid") {
|
|
3716
|
+
pi.logger.error(
|
|
3717
|
+
`[omp-conductor] inbound arm reply not classified — tick config invalid at ${reread.path}: ` +
|
|
3718
|
+
`${reread.problem}; use \`omp-conductor arm --reply\` on the host`,
|
|
3719
|
+
);
|
|
3720
|
+
return;
|
|
3721
|
+
}
|
|
3722
|
+
const project = reread.kind === "ok" ? reread.config.project : startupProject;
|
|
3723
|
+
const surface: ArmNoticeSurface = {
|
|
3724
|
+
...(reread.kind === "ok" ? { accessFile: reread.config.accessFile } : {}),
|
|
3725
|
+
project,
|
|
3726
|
+
};
|
|
3727
|
+
const now = (seam?.now ?? Date.now)();
|
|
3728
|
+
const resolved = resolveArmReply(project, text, now);
|
|
3729
|
+
const notify = (notice: string): void => {
|
|
3730
|
+
const send = seam?.notify ?? sendArmNotice;
|
|
3731
|
+
try {
|
|
3732
|
+
// The send is async on the production path; message_start handlers are
|
|
3733
|
+
// not, so delivery rides the microtask queue and its failure lands in
|
|
3734
|
+
// the log, never on the session.
|
|
3735
|
+
void Promise.resolve(send(notice, surface)).catch((err: unknown) => {
|
|
3736
|
+
pi.logger.error(
|
|
3737
|
+
`[omp-conductor] arm notice not delivered: ${err instanceof Error ? err.message : String(err)}`,
|
|
3738
|
+
);
|
|
3739
|
+
});
|
|
3740
|
+
} catch (err) {
|
|
3741
|
+
pi.logger.error(
|
|
3742
|
+
`[omp-conductor] arm notice not delivered: ${err instanceof Error ? err.message : String(err)}`,
|
|
3743
|
+
);
|
|
3744
|
+
}
|
|
3745
|
+
};
|
|
3746
|
+
if (resolved.verdict === "matched") {
|
|
3747
|
+
settleInboundArmMatch(pi, resolved.match, notify);
|
|
3748
|
+
return;
|
|
3749
|
+
}
|
|
3750
|
+
// A code past its window: the ceremony is dead, so answer the person who is
|
|
3751
|
+
// clearly trying to complete it — and clear the expired record so neither
|
|
3752
|
+
// this path nor the heartbeat ever notifies twice.
|
|
3753
|
+
if (resolved.verdict === "expired") {
|
|
3754
|
+
for (const expired of expiredArmChallenges(project, now)) {
|
|
3755
|
+
notify(armExpiryNotice(expired, project));
|
|
3756
|
+
clearArmTransaction(expired.key, expired.id);
|
|
3757
|
+
}
|
|
3758
|
+
}
|
|
3499
3759
|
}
|
|
3500
3760
|
|
|
3501
3761
|
export default function orchestratorTickExtension(
|
|
@@ -3634,8 +3894,10 @@ export default function orchestratorTickExtension(
|
|
|
3634
3894
|
|
|
3635
3895
|
/** `configuredProject` carries {@link TickConfig.project} for the same reason
|
|
3636
3896
|
* {@link armReleaseGate} takes it: a recovered tick must reconstruct *this*
|
|
3637
|
-
* fleet's availability policy, not refuse to guess between two projects.
|
|
3638
|
-
|
|
3897
|
+
* fleet's availability policy, not refuse to guess between two projects.
|
|
3898
|
+
* `cwd` is the fleet directory itself, which the arm consumer re-reads the
|
|
3899
|
+
* tick config from at reply time (#1061). */
|
|
3900
|
+
const armAvailabilityGate = (configuredProject?: string, cwd?: string): void => {
|
|
3639
3901
|
if (availabilityGateArmed) return;
|
|
3640
3902
|
availabilityGateArmed = true;
|
|
3641
3903
|
|
|
@@ -3667,23 +3929,33 @@ export default function orchestratorTickExtension(
|
|
|
3667
3929
|
}
|
|
3668
3930
|
return;
|
|
3669
3931
|
}
|
|
3670
|
-
// An inbound user turn on THIS session is
|
|
3932
|
+
// An inbound user turn on THIS session is an anomaly, not a
|
|
3671
3933
|
// conversation: operator DMs belong to the 24/7 console session, so a
|
|
3672
|
-
// turn arriving here is an accidental topic post.
|
|
3673
|
-
//
|
|
3674
|
-
//
|
|
3675
|
-
//
|
|
3676
|
-
// (
|
|
3677
|
-
//
|
|
3678
|
-
//
|
|
3679
|
-
//
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
|
|
3934
|
+
// turn arriving here is an accidental topic post. Two things happen,
|
|
3935
|
+
// and both are mechanisms — nothing here is left to model behaviour:
|
|
3936
|
+
//
|
|
3937
|
+
// 1. The arm ceremony's reply is classified against the pending
|
|
3938
|
+
// challenge records (#1061). The pre-#415 tombstone is retired: it
|
|
3939
|
+
// reasoned the reply lands "where no tick extension runs", but the
|
|
3940
|
+
// send half delivers the challenge to the project topic — this
|
|
3941
|
+
// pane's own claimed surface — so the reply lands exactly here, and
|
|
3942
|
+
// on a host with no console session the CLI's `arm --reply` step had
|
|
3943
|
+
// no owner at all. A matching code settles the ceremony
|
|
3944
|
+
// (markers written, transaction cleared) and answers with a
|
|
3945
|
+
// confirmation; a code past its window is answered with the re-run
|
|
3946
|
+
// command; a lookalike matching nothing stays inert, exactly as
|
|
3947
|
+
// #415 required.
|
|
3948
|
+
// 2. {@link tickGuardDecision}'s preemption, unchanged: a person who
|
|
3949
|
+
// posts into the fleet topic by accident is still a person waiting,
|
|
3950
|
+
// and the autonomous-interrupt refusal must not fire at them.
|
|
3951
|
+
if (message.role === "user" && message.synthetic !== true && message.attribution !== "agent") {
|
|
3952
|
+
if (cwd !== undefined) {
|
|
3953
|
+
const text = inboundMessageText(message.content);
|
|
3954
|
+
if (text !== undefined) consumeInboundArmReply(pi, cwd, configuredProject, text, options.arm);
|
|
3955
|
+
}
|
|
3956
|
+
if (session.activeLocalTick !== undefined) {
|
|
3957
|
+
session.activeLocalTick.humanWaiting = true;
|
|
3958
|
+
}
|
|
3687
3959
|
}
|
|
3688
3960
|
});
|
|
3689
3961
|
pi.on("agent_end", () => {
|
|
@@ -3921,7 +4193,7 @@ export default function orchestratorTickExtension(
|
|
|
3921
4193
|
// invalid config names none — it cannot be trusted to.
|
|
3922
4194
|
const configuredProject = result.kind === "ok" ? result.config.project : undefined;
|
|
3923
4195
|
armReleaseGate(configuredProject);
|
|
3924
|
-
armAvailabilityGate(configuredProject);
|
|
4196
|
+
armAvailabilityGate(configuredProject, ctx.cwd);
|
|
3925
4197
|
// The tool is already registered (extension-factory time); only the session
|
|
3926
4198
|
// state it acts on is filled, and only once ownership is accepted below.
|
|
3927
4199
|
// `configuredProject` isn't closed over at all — routing is re-read from
|
|
@@ -4011,7 +4283,7 @@ export default function orchestratorTickExtension(
|
|
|
4011
4283
|
// not create or deliver decisions). `cwd` is for re-reading the live
|
|
4012
4284
|
// tick config at execution, `config` for the startup-only ceiling.
|
|
4013
4285
|
askSession = { cwd: ctx.cwd, config };
|
|
4014
|
-
armTickHeartbeat(pi, ctx, config, session, toSpecTrackerSeam);
|
|
4286
|
+
armTickHeartbeat(pi, ctx, config, session, toSpecTrackerSeam, options.arm);
|
|
4015
4287
|
if (!guardArmed) {
|
|
4016
4288
|
guardArmed = true;
|
|
4017
4289
|
armTickGuard(pi, ctx, (config.budgetSeconds ?? DEFAULT_TICK_BUDGET_SECONDS) * 1000);
|
|
@@ -4033,7 +4305,7 @@ export default function orchestratorTickExtension(
|
|
|
4033
4305
|
// why): a declined or unresolved session keeps `askSession` undefined and
|
|
4034
4306
|
// the tool fails closed.
|
|
4035
4307
|
askSession = { cwd: ctx.cwd, config };
|
|
4036
|
-
armTickHeartbeat(pi, ctx, config, session, toSpecTrackerSeam);
|
|
4308
|
+
armTickHeartbeat(pi, ctx, config, session, toSpecTrackerSeam, options.arm);
|
|
4037
4309
|
if (!guardArmed) {
|
|
4038
4310
|
guardArmed = true;
|
|
4039
4311
|
armTickGuard(pi, ctx, (config.budgetSeconds ?? DEFAULT_TICK_BUDGET_SECONDS) * 1000);
|
package/src/settlement.ts
CHANGED
|
@@ -25,7 +25,13 @@ import {
|
|
|
25
25
|
analyseSettlement,
|
|
26
26
|
deriveChangedLine,
|
|
27
27
|
} from "./diff-flags.ts";
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
SPINNING_CAP_CLASSES,
|
|
30
|
+
COMPOSE_DEPENDENCY_STARTUP_SIGNATURE,
|
|
31
|
+
classifyRun,
|
|
32
|
+
normalise,
|
|
33
|
+
type ClassifyFacts,
|
|
34
|
+
} from "./failure-class.ts";
|
|
29
35
|
import { GhPrMissingError } from "./tracker/github.ts";
|
|
30
36
|
import { formatModelsTried, modelsTried } from "./model-fallback.ts";
|
|
31
37
|
import { PR_LOOKUP_WINDOW_MS } from "./decisions.ts";
|
|
@@ -1558,6 +1564,26 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
|
|
|
1558
1564
|
if (firstFailure?.link !== undefined) {
|
|
1559
1565
|
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
1560
1566
|
}
|
|
1567
|
+
// A Compose dependency-startup failure ("docker compose up" aborted
|
|
1568
|
+
// because a dependency container failed its healthcheck) is only
|
|
1569
|
+
// outside the diff when the PR touched no container configuration —
|
|
1570
|
+
// a PR that breaks its own `docker-compose.yml` produces this exact
|
|
1571
|
+
// sentence and must still charge its attempt (#1059). So the
|
|
1572
|
+
// classifier needs the PR's changed-file list, derived from the
|
|
1573
|
+
// diff this settlement already knows how to fetch
|
|
1574
|
+
// (`Tracker.prDiff`, the same call the settlement audit uses) —
|
|
1575
|
+
// one fetch, and only for a row whose log actually carries
|
|
1576
|
+
// Compose's sentence; every other row pays nothing. A diff that
|
|
1577
|
+
// cannot be read or was cut short (`truncated`) stays undefined: an
|
|
1578
|
+
// unknown list must never waive an attempt.
|
|
1579
|
+
if (
|
|
1580
|
+
facts.failingLog !== undefined &&
|
|
1581
|
+
facts.failingLog.toLowerCase().includes(COMPOSE_DEPENDENCY_STARTUP_SIGNATURE)
|
|
1582
|
+
) {
|
|
1583
|
+
const diff = await tracker.prDiff(run.prUrl);
|
|
1584
|
+
facts.changedFiles =
|
|
1585
|
+
diff === undefined || diff.truncated ? undefined : diff.files.map((f) => f.path);
|
|
1586
|
+
}
|
|
1561
1587
|
}
|
|
1562
1588
|
}
|
|
1563
1589
|
} catch (err) {
|
|
@@ -1990,10 +2016,14 @@ async function recoverRun(
|
|
|
1990
2016
|
return;
|
|
1991
2017
|
}
|
|
1992
2018
|
|
|
1993
|
-
// `hold` (orphan-dirty) and `none`:
|
|
1994
|
-
//
|
|
1995
|
-
//
|
|
1996
|
-
//
|
|
2019
|
+
// `observe` (awaiting-observation), `hold` (orphan-dirty) and `none`:
|
|
2020
|
+
// recorded, nothing performed here. `observe`'s recovery IS the later observation —
|
|
2021
|
+
// the settle sweep re-offers the blocked row until its PR resolves and settles it
|
|
2022
|
+
// the way it settles `settlement-stuck`, so no action exists to perform today and
|
|
2023
|
+
// `recoveredAt` stays NULL by design (#1068). The existing unsalvaged-WIP
|
|
2024
|
+
// admission hold already fails dispatch closed until an operator acknowledges
|
|
2025
|
+
// the tree, which is the only safe move when the worktree holds the only copy
|
|
2026
|
+
// of real work.
|
|
1997
2027
|
}
|
|
1998
2028
|
|
|
1999
2029
|
/**
|
package/src/status-render.ts
CHANGED
|
@@ -254,12 +254,14 @@ const MECHANICAL_GROOMING_REASONS: Record<string, true> = {
|
|
|
254
254
|
};
|
|
255
255
|
|
|
256
256
|
/**
|
|
257
|
-
* The to-spec refusal classes persisted as blocked rows (#772) — a
|
|
258
|
-
* that failed validation
|
|
259
|
-
*
|
|
257
|
+
* The to-spec refusal classes persisted as blocked rows (#772, #1064) — a
|
|
258
|
+
* result that failed validation, or a pass that produced no answer at all, is
|
|
259
|
+
* a mechanical block, never a verdict. Mirrors the failure kinds of
|
|
260
|
+
* `ToSpecFailure` in `to-spec.ts`.
|
|
260
261
|
*/
|
|
261
262
|
const REFUSED_GROOMING_REASONS: Record<string, true> = {
|
|
262
263
|
malformed: true,
|
|
264
|
+
"no-answer": true,
|
|
263
265
|
"missing-source": true,
|
|
264
266
|
"stale-source": true,
|
|
265
267
|
};
|
|
@@ -289,10 +291,12 @@ const REFUSED_GROOMING_REASONS: Record<string, true> = {
|
|
|
289
291
|
* (`blocked` rows whose reason is a groomer verdict or a product-judgement
|
|
290
292
|
* label, e.g. `needs-product-decision`).
|
|
291
293
|
* - `mechanically blocked` — admission's lane/dependency holds.
|
|
292
|
-
* - `refused` — to-spec
|
|
293
|
-
* `missing-source`, `stale-source`), told apart from the
|
|
294
|
-
* operator sees whether the runway cannot move or a result
|
|
295
|
-
* trusted.
|
|
294
|
+
* - `refused` — to-spec passes that produced no usable verdict (`malformed`,
|
|
295
|
+
* `no-answer`, `missing-source`, `stale-source`), told apart from the
|
|
296
|
+
* holds so an operator sees whether the runway cannot move or a result
|
|
297
|
+
* cannot be trusted. Each row's line names its own class (`#19 malformed`,
|
|
298
|
+
* `#22 no-answer`), so a run of identical refusals is visible as a pattern
|
|
299
|
+
* rather than a wall of one word.
|
|
296
300
|
* - `in-flight` — a launched batch is running (#777).
|
|
297
301
|
* - `operator-parked` — the dispatch snapshot's parked count (#507).
|
|
298
302
|
*
|
package/src/store.ts
CHANGED
|
@@ -2973,6 +2973,17 @@ export function openStore(dbPath: string): Store {
|
|
|
2973
2973
|
// "retrying next tick" is a lie and the row is stranded with a class and no
|
|
2974
2974
|
// action. `hold` is excluded because it is *recorded only* by design — its
|
|
2975
2975
|
// `recoveredAt` stays NULL forever, and re-offering it would spin the sweep.
|
|
2976
|
+
//
|
|
2977
|
+
// One deliberate exception re-enters the sweep after classification: a
|
|
2978
|
+
// blocked row with a PR. It is the observation half of the
|
|
2979
|
+
// `awaiting-observation` recovery (#1068) — the sweep re-reads the PR every
|
|
2980
|
+
// pass, and the classifier's merged-PR branch names `settlement-stuck` the
|
|
2981
|
+
// way it does for any other stuck row, so a blocked run whose PR later merges
|
|
2982
|
+
// (the #1062 shape: merged at 15:02, row still `blocked`/`question`) settles
|
|
2983
|
+
// instead of listing for Duty 1 triage forever. A genuine question row is
|
|
2984
|
+
// re-offered too and its escalation is not repeated: the notifications ledger
|
|
2985
|
+
// dedupes on the class+run summary of the stable evidence. Rows without a PR
|
|
2986
|
+
// have nothing to observe and are only re-offered by the ordinary clauses.
|
|
2976
2987
|
const selectUnclassified = db.query<RunRow, [string, number]>(
|
|
2977
2988
|
`SELECT * FROM runs
|
|
2978
2989
|
WHERE project = ?
|
|
@@ -2980,6 +2991,7 @@ export function openStore(dbPath: string): Store {
|
|
|
2980
2991
|
AND (
|
|
2981
2992
|
failureClass IS NULL
|
|
2982
2993
|
OR (recoveredAt IS NULL AND recoveryAction IN ('settle', 'continue', 'requeue', 'rerun-checks'))
|
|
2994
|
+
OR (state = 'blocked' AND prUrl IS NOT NULL)
|
|
2983
2995
|
)
|
|
2984
2996
|
ORDER BY startedAt DESC, rowid DESC
|
|
2985
2997
|
LIMIT ?`,
|
|
@@ -3062,7 +3074,7 @@ export function openStore(dbPath: string): Store {
|
|
|
3062
3074
|
`SELECT failureClass AS cls, COUNT(*) AS n FROM runs
|
|
3063
3075
|
WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
|
|
3064
3076
|
AND state IN ('blocked', 'killed', 'failed', 'orphaned', 'pushed-green')
|
|
3065
|
-
AND recoveryAction NOT IN ('none', 'hold')
|
|
3077
|
+
AND recoveryAction NOT IN ('none', 'hold', 'observe')
|
|
3066
3078
|
GROUP BY failureClass
|
|
3067
3079
|
ORDER BY n DESC, failureClass ASC`,
|
|
3068
3080
|
);
|
|
@@ -3070,7 +3082,7 @@ export function openStore(dbPath: string): Store {
|
|
|
3070
3082
|
`SELECT failureClass AS cls, COUNT(*) AS n FROM runs
|
|
3071
3083
|
WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
|
|
3072
3084
|
AND state IN ('blocked', 'killed', 'failed', 'orphaned', 'pushed-green')
|
|
3073
|
-
AND recoveryAction IN ('none', 'hold')
|
|
3085
|
+
AND recoveryAction IN ('none', 'hold', 'observe')
|
|
3074
3086
|
GROUP BY failureClass
|
|
3075
3087
|
ORDER BY n DESC, failureClass ASC`,
|
|
3076
3088
|
);
|