omp-conductor 0.19.1 → 0.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +1 -1
- package/package.json +2 -2
- package/schema/config.schema.json +13 -1
- package/src/arm-challenge.ts +92 -16
- package/src/board.ts +50 -9
- package/src/check-browser-js.ts +72 -0
- package/src/cli.ts +2 -0
- package/src/command-manifest.ts +10 -0
- package/src/commands/arm.ts +30 -2
- package/src/commands/companion.ts +103 -0
- package/src/commands/context.ts +1 -0
- package/src/commands/worker.ts +1 -0
- package/src/companion-view.ts +81 -0
- package/src/config-schema.ts +7 -1
- package/src/config.ts +6 -3
- package/src/daemon.ts +197 -14
- package/src/dashboard/app.js +4 -4
- package/src/dashboard/controls.ts +1 -1
- package/src/doctor.ts +126 -7
- package/src/escalate.ts +47 -1
- package/src/failure-class.fixture.json +546 -0
- package/src/failure-class.ts +192 -0
- package/src/fleet.ts +275 -27
- package/src/orchestrator-tick.ts +115 -34
- package/src/settlement.ts +167 -0
- package/src/setup.ts +6 -4
- package/src/spend-telemetry.ts +74 -2
- package/src/status-render.ts +25 -6
- package/src/store.ts +38 -2
- package/src/types.ts +39 -2
- package/src/usage.ts +25 -0
package/src/doctor.ts
CHANGED
|
@@ -64,6 +64,7 @@ import {
|
|
|
64
64
|
import { pauseInstance } from "./pause.ts";
|
|
65
65
|
import type { TelegramHealth } from "./status-render.ts";
|
|
66
66
|
import {
|
|
67
|
+
claimIsLive,
|
|
67
68
|
claimedTelegramTopics,
|
|
68
69
|
lockPidAlive,
|
|
69
70
|
pidAlive,
|
|
@@ -100,7 +101,7 @@ import {
|
|
|
100
101
|
readTickConfig,
|
|
101
102
|
type HerdrAgentList,
|
|
102
103
|
} from "./orchestrator-tick.ts";
|
|
103
|
-
import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
|
|
104
|
+
import type { Caps, ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
|
|
104
105
|
import { DEFAULT_ARM_PROOF, type ArmProof } from "./types.ts";
|
|
105
106
|
import {
|
|
106
107
|
DEFAULT_DEPS as UPGRADE_DEPS,
|
|
@@ -114,7 +115,8 @@ import {
|
|
|
114
115
|
checkTelegramFreshness,
|
|
115
116
|
type TelegramFreshness,
|
|
116
117
|
} from "./telegram-freshness.ts";
|
|
117
|
-
import {
|
|
118
|
+
import { bindingAllowanceWindow, sharedUsageSource } from "./usage.ts";
|
|
119
|
+
import { judgeSpendTelemetry, spendTelemetryDetail, type SpendBilling } from "./spend-telemetry.ts";
|
|
118
120
|
|
|
119
121
|
/** One read of this host's installed surfaces: the three identities, or why
|
|
120
122
|
* they could not be read. A failed read is a finding (`warn`, "unverified"),
|
|
@@ -308,6 +310,14 @@ export interface DoctorDeps {
|
|
|
308
310
|
/** Whether a recorded claim or dm-owner pid is live, with omp-telegram's
|
|
309
311
|
* topics.ts semantics (EPERM is dead). */
|
|
310
312
|
pidAlive?: (pid: number) => boolean;
|
|
313
|
+
/** Whether one claimed topic is live — the single fact `topic-pin` and
|
|
314
|
+
* `telegram-plumbing` both read, so they cannot disagree about a claim
|
|
315
|
+
* (#987). Defaults to {@link claimIsLive}. */
|
|
316
|
+
claimIsLive?: (claim: { pid?: number }) => boolean;
|
|
317
|
+
/** The provider allowance window nearest its ceiling, from `omp usage --json`
|
|
318
|
+
* — what binds a subscription-billed fleet instead of a dollar cap (#984).
|
|
319
|
+
* `undefined` when the provider reports no comparable window. */
|
|
320
|
+
allowanceWindow?: () => Promise<string | undefined>;
|
|
311
321
|
/** Whether a bot.lock owner pid is live, with omp-telegram's api.ts
|
|
312
322
|
* semantics (EPERM is live). Distinct from {@link pidAlive} because the
|
|
313
323
|
* bridge itself uses two rules. */
|
|
@@ -1096,6 +1106,62 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
|
|
|
1096
1106
|
);
|
|
1097
1107
|
}
|
|
1098
1108
|
|
|
1109
|
+
/**
|
|
1110
|
+
* Is any economic control actually governing this project? (#985)
|
|
1111
|
+
*
|
|
1112
|
+
* Two exist, and on this fleet neither was active. `caps.planUsage` was `null`,
|
|
1113
|
+
* so the allowance gate that admission already applies — it holds every routed
|
|
1114
|
+
* candidate and escalates once (`admission.ts`) — was configured off. And
|
|
1115
|
+
* `caps.dailySpendUsd` was a ceiling that #984 established cannot fire under
|
|
1116
|
+
* subscription billing, because a subscription request carries no per-request
|
|
1117
|
+
* price. So the fleet ran with no ceiling of any kind, and no surface said so:
|
|
1118
|
+
* `status` reported `unmetered — no plan allowance cap configured`, which reads
|
|
1119
|
+
* as a fact about the plan rather than as a missing control.
|
|
1120
|
+
*
|
|
1121
|
+
* This row states the gap once, per project, with the remedy that closes it.
|
|
1122
|
+
* It is deliberately NOT a new gate: the gate exists and works. What was
|
|
1123
|
+
* missing was anyone being told it is switched off.
|
|
1124
|
+
*/
|
|
1125
|
+
function economicGateProbe(
|
|
1126
|
+
project: ProjectConfig,
|
|
1127
|
+
caps: Caps,
|
|
1128
|
+
subscriptionBilled: boolean,
|
|
1129
|
+
allowanceWindow: string | undefined,
|
|
1130
|
+
): Finding {
|
|
1131
|
+
const allowanceGate = caps.planUsage !== null;
|
|
1132
|
+
// A dollar cap governs only if it can fire, and under subscription billing it
|
|
1133
|
+
// cannot — that is #984's finding, reused here rather than re-derived.
|
|
1134
|
+
const dollarGate = caps.dailySpendUsd !== null && !subscriptionBilled;
|
|
1135
|
+
|
|
1136
|
+
if (allowanceGate) {
|
|
1137
|
+
return passFinding(
|
|
1138
|
+
"economic-gate",
|
|
1139
|
+
`[${project.name}] the provider allowance gate is configured (window ${caps.planUsage?.windowId ?? "?"}, ` +
|
|
1140
|
+
`hold at ${String(Math.round((caps.planUsage?.maxUsedFraction ?? 0) * 100))}% used)` +
|
|
1141
|
+
(dollarGate ? ` beside a $${caps.dailySpendUsd?.toFixed(2) ?? "?"} daily cap` : ""),
|
|
1142
|
+
);
|
|
1143
|
+
}
|
|
1144
|
+
if (dollarGate) {
|
|
1145
|
+
return passFinding(
|
|
1146
|
+
"economic-gate",
|
|
1147
|
+
`[${project.name}] the $${caps.dailySpendUsd?.toFixed(2) ?? "?"} daily spend cap governs; ` +
|
|
1148
|
+
`no provider allowance cap is configured`,
|
|
1149
|
+
);
|
|
1150
|
+
}
|
|
1151
|
+
return warnFinding(
|
|
1152
|
+
"economic-gate",
|
|
1153
|
+
`[${project.name}] no economic control is active: caps.planUsage is null, and ` +
|
|
1154
|
+
(caps.dailySpendUsd === null
|
|
1155
|
+
? "no daily spend cap is set"
|
|
1156
|
+
: `the $${caps.dailySpendUsd.toFixed(2)} daily cap cannot fire on subscription billing`) +
|
|
1157
|
+
` — admission has nothing to weigh and will keep claiming until the provider itself refuses` +
|
|
1158
|
+
(allowanceWindow === undefined ? "" : `; the window that would bind is ${allowanceWindow}`),
|
|
1159
|
+
allowanceWindow === undefined
|
|
1160
|
+
? "run `omp usage --json` on the fleet host and copy an allowance `id` into `caps.planUsage.windowId` (with `maxUsedFraction`), or set a `caps.dailySpendUsd` the billing model can actually spend"
|
|
1161
|
+
: `set caps.planUsage to { windowId: "${allowanceWindow.split(" ")[0] ?? ""}", maxUsedFraction: 0.9 } — the gate is already wired into admission, it is simply switched off`,
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1099
1165
|
/**
|
|
1100
1166
|
* Spend telemetry, through the shared judgement (#970).
|
|
1101
1167
|
*
|
|
@@ -1105,9 +1171,17 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
|
|
|
1105
1171
|
* that had lost a majority of their telemetry — so partial loss, the common
|
|
1106
1172
|
* case, was the invisible one. See `spend-telemetry.ts` for the measurements.
|
|
1107
1173
|
*/
|
|
1108
|
-
function spendProbe(rows: RunSpendRow[], limit: number): Finding {
|
|
1109
|
-
const verdict = judgeSpendTelemetry(rows, limit);
|
|
1174
|
+
function spendProbe(rows: RunSpendRow[], limit: number, billing: SpendBilling = {}): Finding {
|
|
1175
|
+
const verdict = judgeSpendTelemetry(rows, limit, billing);
|
|
1110
1176
|
const detail = spendTelemetryDetail(verdict);
|
|
1177
|
+
// A subscription-billed fleet's zeros are the truth, so this passes and names
|
|
1178
|
+
// the constraint that does bind (#984). It still says the sentence: a reader
|
|
1179
|
+
// who expected a dollar figure needs to know why there isn't one. What it
|
|
1180
|
+
// must never do is warn forever with advice that cannot be acted on — there
|
|
1181
|
+
// is nothing to repair, and a permanent warning is one nobody reads.
|
|
1182
|
+
if (verdict.kind === "subscription") {
|
|
1183
|
+
return passFinding("spend-telemetry", detail ?? "subscription-billed; no USD cap applies");
|
|
1184
|
+
}
|
|
1111
1185
|
if (detail !== undefined) {
|
|
1112
1186
|
return warnFinding(
|
|
1113
1187
|
"spend-telemetry",
|
|
@@ -1296,15 +1370,35 @@ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
|
1296
1370
|
`check omp-telegram's claim registry (threads.json in its state dir) is readable and the bridge is running, then re-run doctor — until then sends keep the pinned topic and degrade to the flat chat on a missing thread (#318)`,
|
|
1297
1371
|
);
|
|
1298
1372
|
}
|
|
1299
|
-
|
|
1373
|
+
// One liveness fact, shared with `telegram-plumbing` (#987). Before this the
|
|
1374
|
+
// registry's rows were all treated as live here, so a pin aimed at a corpse
|
|
1375
|
+
// read PASS while the plumbing row called the same claim dead.
|
|
1376
|
+
const all = result.claims;
|
|
1377
|
+
const claims = all.filter((claim) => probes.claimIsLive(claim));
|
|
1378
|
+
const dead = all.filter((claim) => !probes.claimIsLive(claim));
|
|
1379
|
+
// Named, never dropped: the row is the only index to the remote topic, so
|
|
1380
|
+
// deleting it strands the topic instead of cleaning it up.
|
|
1381
|
+
const deadNote =
|
|
1382
|
+
dead.length === 0
|
|
1383
|
+
? ""
|
|
1384
|
+
: ` (dead claims still recorded, and left alone: ${dead
|
|
1385
|
+
.map((c) => `${c.threadId}${c.pid === undefined ? " no pid" : ` pid ${c.pid}`}`)
|
|
1386
|
+
.join(", ")} — run the bridge's /cleanup to remove those topics)`;
|
|
1300
1387
|
if (claims.length === 0) {
|
|
1388
|
+
if (dead.length > 0) {
|
|
1389
|
+
return warnFinding(
|
|
1390
|
+
"topic-pin",
|
|
1391
|
+
`[${p.name}] pinned topic ${pinned} — every claim in omp-telegram's registry is dead${deadNote}`,
|
|
1392
|
+
`run /cleanup in Telegram so the bridge closes those topics and drops their rows itself, then re-run setup to re-pin escalation.telegramTopicId to a live claim — never delete rows from threads.json by hand, because the row is the only index to the remote topic (#987)`,
|
|
1393
|
+
);
|
|
1394
|
+
}
|
|
1301
1395
|
return passFinding(
|
|
1302
1396
|
"topic-pin",
|
|
1303
1397
|
`[${p.name}] pinned topic ${pinned} — no live claims to compare (the bridge has claimed no topics yet); the pin stands`,
|
|
1304
1398
|
);
|
|
1305
1399
|
}
|
|
1306
1400
|
if (claims.some((claim) => claim.threadId === pinned)) {
|
|
1307
|
-
return passFinding("topic-pin", `[${p.name}] pinned topic ${pinned} is a live claim`);
|
|
1401
|
+
return passFinding("topic-pin", `[${p.name}] pinned topic ${pinned} is a live claim${deadNote}`);
|
|
1308
1402
|
}
|
|
1309
1403
|
const match = resolveProjectClaim(claims, p.name);
|
|
1310
1404
|
if (match.kind === "match") {
|
|
@@ -1924,7 +2018,30 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
1924
2018
|
// reported against another's name.
|
|
1925
2019
|
for (const p of projects) findings.push(fenceProbe(probes, p.name));
|
|
1926
2020
|
}
|
|
1927
|
-
|
|
2021
|
+
// The billing class the spend row needs (#984). Declared only when EVERY
|
|
2022
|
+
// project requires a subscription credential: on a mixed fleet the zeros of
|
|
2023
|
+
// an API-key project are still a fault, and one project's declaration must
|
|
2024
|
+
// not excuse another's telemetry loss.
|
|
2025
|
+
const declaredSubscription =
|
|
2026
|
+
projects.length > 0 && projects.every((p) => (p.requireOauthProviders ?? []).length > 0);
|
|
2027
|
+
const allowanceWindow = probes.allowanceWindow === undefined ? undefined : await probes.allowanceWindow();
|
|
2028
|
+
const billing = {
|
|
2029
|
+
declaredSubscription,
|
|
2030
|
+
...(allowanceWindow === undefined ? {} : { allowanceWindow }),
|
|
2031
|
+
};
|
|
2032
|
+
findings.push(spendProbe(spendRows, SPEND_SAMPLE_RUNS, billing));
|
|
2033
|
+
// Per project, because the caps are: one project's configured ceiling says
|
|
2034
|
+
// nothing about another's (#985).
|
|
2035
|
+
const subscriptionBilled =
|
|
2036
|
+
judgeSpendTelemetry(spendRows, SPEND_SAMPLE_RUNS, billing).kind === "subscription";
|
|
2037
|
+
if (cfg !== undefined) {
|
|
2038
|
+
const defaults = cfg.defaults;
|
|
2039
|
+
for (const p of projects) {
|
|
2040
|
+
findings.push(
|
|
2041
|
+
economicGateProbe(p, resolveCaps(p, defaults), subscriptionBilled, allowanceWindow),
|
|
2042
|
+
);
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
1928
2045
|
|
|
1929
2046
|
const status: ReportStatus = findings.some((f) => f.status === "fail")
|
|
1930
2047
|
? "fail"
|
|
@@ -2042,6 +2159,8 @@ export function defaultProbes(): Probes {
|
|
|
2042
2159
|
return claimDir === cwdDir ? [cwdDir] : [cwdDir, claimDir];
|
|
2043
2160
|
},
|
|
2044
2161
|
pidAlive,
|
|
2162
|
+
claimIsLive: (claim) => claimIsLive(claim),
|
|
2163
|
+
allowanceWindow: async () => bindingAllowanceWindow(await sharedUsageSource().read()),
|
|
2045
2164
|
lockPidAlive,
|
|
2046
2165
|
lockFresh: (mtimeMs) => Date.now() - mtimeMs < TELEGRAM_LOCK_FRESH_MS,
|
|
2047
2166
|
tickAgentName: (p) => {
|
package/src/escalate.ts
CHANGED
|
@@ -750,6 +750,37 @@ export function pidAlive(
|
|
|
750
750
|
}
|
|
751
751
|
}
|
|
752
752
|
|
|
753
|
+
/**
|
|
754
|
+
* Whether one claimed topic is live — **the** liveness fact about a claim
|
|
755
|
+
* (#987).
|
|
756
|
+
*
|
|
757
|
+
* Two doctor rows used to answer this question separately: `topic-pin` treated
|
|
758
|
+
* every row in the registry as live (the reader returns rows, it never filtered
|
|
759
|
+
* them), while `telegram-plumbing` applied `pidAlive` and called the same claim
|
|
760
|
+
* `claim-dead`. One fact, two rules, opposite answers — and whichever row the
|
|
761
|
+
* reader happened to trust decided whether the fleet looked healthy. A doctor
|
|
762
|
+
* that contradicts itself is worse than a silent one, because it teaches the
|
|
763
|
+
* reader to discount its rows.
|
|
764
|
+
*
|
|
765
|
+
* A claim with no pid is not live: the bridge records a number or nothing, and
|
|
766
|
+
* `telegramPlumbingVerdict` already refuses a pidless claim as dead. `kill` is
|
|
767
|
+
* injectable for the same reason it is on {@link pidAlive}.
|
|
768
|
+
*
|
|
769
|
+
* Being dead is **not** a licence to delete the row. No Bot API call lists
|
|
770
|
+
* forum topics, so a claim row is the only index to a remote topic
|
|
771
|
+
* (`omp-telegram/src/topics.ts:149-151`): dropping it strands the topic with no
|
|
772
|
+
* supported way to remove it — 70 were stranded that way in one day, and every
|
|
773
|
+
* recovery depended on an artifact nothing guarantees. Callers report dead
|
|
774
|
+
* claims; the bridge's own `/cleanup` removes them, remote side first.
|
|
775
|
+
*/
|
|
776
|
+
export function claimIsLive(
|
|
777
|
+
claim: Pick<ClaimedTopic, "pid">,
|
|
778
|
+
kill?: (target: number, signal: number) => void,
|
|
779
|
+
): boolean {
|
|
780
|
+
if (claim.pid === undefined) return false;
|
|
781
|
+
return kill === undefined ? pidAlive(claim.pid) : pidAlive(claim.pid, kill);
|
|
782
|
+
}
|
|
783
|
+
|
|
753
784
|
/**
|
|
754
785
|
* Whether a bot.lock owner pid is a live process as the bridge's `api.ts`
|
|
755
786
|
* judges it (`src/api.ts:pidAlive`): `EPERM` means the process exists but is
|
|
@@ -1073,6 +1104,19 @@ export type TelegramPlumbingScan = { dirs: readonly string[] };
|
|
|
1073
1104
|
* stops its heartbeat while its own pid stays live (#612 ecosystem
|
|
1074
1105
|
* correction).
|
|
1075
1106
|
*/
|
|
1107
|
+
/**
|
|
1108
|
+
* Adapts an injected pid-liveness predicate to the `kill` seam
|
|
1109
|
+
* {@link claimIsLive} owns, so the verdict can read the shared claim fact
|
|
1110
|
+
* without a second pid rule and without changing its own probe shape (#987).
|
|
1111
|
+
* Throwing is how `pidAlive` spells "dead", which is the contract it applies to
|
|
1112
|
+
* whatever this returns.
|
|
1113
|
+
*/
|
|
1114
|
+
function killOf(alive: (pid: number) => boolean): (target: number, signal: number) => void {
|
|
1115
|
+
return (target) => {
|
|
1116
|
+
if (!alive(target)) throw new Error("not alive");
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1076
1120
|
export function telegramPlumbingVerdict(
|
|
1077
1121
|
sendTopic: number | undefined,
|
|
1078
1122
|
scan: TelegramPlumbingScan | undefined,
|
|
@@ -1085,7 +1129,9 @@ export function telegramPlumbingVerdict(
|
|
|
1085
1129
|
if (registry.kind !== "ok") return { ok: false, reason: "registry-unreadable" };
|
|
1086
1130
|
const claim = registry.claims.find((c) => c.threadId === sendTopic);
|
|
1087
1131
|
if (claim === undefined) return { ok: false, reason: "no-topic-claim" };
|
|
1088
|
-
|
|
1132
|
+
// The shared fact, not a second copy of the pid rule (#987): `topic-pin`
|
|
1133
|
+
// reads the same predicate, so the two rows cannot disagree about a claim.
|
|
1134
|
+
if (!claimIsLive(claim, killOf(alive))) return { ok: false, reason: "claim-dead" };
|
|
1089
1135
|
if (
|
|
1090
1136
|
scan !== undefined &&
|
|
1091
1137
|
claim.sessionFile !== undefined &&
|