@cohortapp/agent-sdk 2.19.0 → 2.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +7 -1
- package/lib/budget-guard.mjs +69 -2
- package/lib/comms/receipts.mjs +15 -3
- package/lib/comms/send-gate.mjs +20 -0
- package/lib/org/registry.mjs +7 -1
- package/lib/telemetry/alerts.mjs +74 -1
- package/package.json +1 -1
- package/scripts/healthcheck.sh +35 -0
- package/scripts/local-triggers/autoupdate.sh +61 -5
- package/scripts/maintenance/check-stamp-freshness.mjs +314 -0
- package/scripts/org/send-orgmail.mjs +112 -25
- package/scripts/session/supervisor.mjs +14 -0
package/bin/maestro.mjs
CHANGED
|
@@ -266,7 +266,13 @@ function create(targetName) {
|
|
|
266
266
|
|
|
267
267
|
log("Generating operational templates...");
|
|
268
268
|
|
|
269
|
-
// Queue templates — 16 operational queues
|
|
269
|
+
// Queue templates — 16 operational queues.
|
|
270
|
+
//
|
|
271
|
+
// A queue added here is a new machine-internal name, so it must also be named
|
|
272
|
+
// by the `queue-file` rule in lib/comms/send-gate.mjs (MACHINE_LEAK_RULES) —
|
|
273
|
+
// lib/comms/fallback-leak-corpus.test.mjs fails if it is not. A new queue-ID
|
|
274
|
+
// prefix for it goes into `queue-item-id` and that corpus's DEDICATED cases in
|
|
275
|
+
// the same change (docs/engineering/functional-architecture.md §9).
|
|
270
276
|
const queueTemplates = {
|
|
271
277
|
"action-stack": "Active action items requiring attention or follow-through",
|
|
272
278
|
"decision-queue": "Key decisions awaiting principal input or closure",
|
package/lib/budget-guard.mjs
CHANGED
|
@@ -56,7 +56,7 @@ import {
|
|
|
56
56
|
import { join, resolve, dirname } from "node:path";
|
|
57
57
|
import { randomBytes } from "node:crypto";
|
|
58
58
|
|
|
59
|
-
import { summariseRows, imputeUnmeasured } from "./cost/ledger-row.mjs";
|
|
59
|
+
import { summariseRows, imputeUnmeasured, billableUsd } from "./cost/ledger-row.mjs";
|
|
60
60
|
import { quotaBand, quotaStatePath, readQuotaSnapshot } from "./org/quota.mjs";
|
|
61
61
|
import { charterMonthlyCapCents } from "./org/governance-store.mjs";
|
|
62
62
|
import { load as yamlLoad } from "js-yaml";
|
|
@@ -204,6 +204,22 @@ const microsToUsd = (m) => {
|
|
|
204
204
|
export const DEFAULT_DAILY_SPEND_CAP_USD = 25;
|
|
205
205
|
export const DEFAULT_DAILY_ITERATION_CAP = 0; // 0 = no session-count cap
|
|
206
206
|
|
|
207
|
+
/**
|
|
208
|
+
* Flat-fee (subscription) seats have no dollar meter, but they do have a weekly
|
|
209
|
+
* quota that real work drains. FU-1046 measured 10-13..10-20: $253 API-equivalent
|
|
210
|
+
* in a week, 58% of it inbox replies, while the dollar band read 0 because the
|
|
211
|
+
* guard ignores dollars on a subscription login. These two bounds put a ceiling
|
|
212
|
+
* back on the two things that actually ate the quota:
|
|
213
|
+
* - WEEKLY: rolling-7d API-equivalent spend vs a weekly budget ($90).
|
|
214
|
+
* - REPLY RUNS: inbox-reply sessions per UTC day (~30; a normal day is 30-37,
|
|
215
|
+
* the 10-20 storm was 187).
|
|
216
|
+
* Both band through the same ladder as every other cap (100 = degrade, which
|
|
217
|
+
* keeps inbox replies; 125 = defer non-inbox work). Override via deps, or
|
|
218
|
+
* WEEKLY_SPEND_CAP_USD / DAILY_REPLY_RUN_CEILING; 0 disables that bound.
|
|
219
|
+
*/
|
|
220
|
+
export const DEFAULT_SUBSCRIPTION_WEEKLY_CAP_USD = 90;
|
|
221
|
+
export const DEFAULT_SUBSCRIPTION_REPLY_RUN_CEILING = 30;
|
|
222
|
+
|
|
207
223
|
/**
|
|
208
224
|
* Envelope freshness, mirroring `lib/mandate/cache.mjs` TIERS (duplicated as two
|
|
209
225
|
* numbers rather than imported: this module's contract is Node built-ins only
|
|
@@ -276,6 +292,12 @@ function cfgSpendCap(deps) {
|
|
|
276
292
|
const env = Number(process.env.DAILY_SPEND_CAP_USD);
|
|
277
293
|
return Number.isFinite(env) && env > 0 ? env : DEFAULT_DAILY_SPEND_CAP_USD;
|
|
278
294
|
}
|
|
295
|
+
function cfgBound(depKey, envKey, dflt, deps) {
|
|
296
|
+
if (deps && Number.isFinite(deps[depKey]) && deps[depKey] >= 0) return deps[depKey];
|
|
297
|
+
const env = process.env[envKey];
|
|
298
|
+
if (env !== undefined && env !== "" && Number.isFinite(Number(env)) && Number(env) >= 0) return Number(env);
|
|
299
|
+
return dflt;
|
|
300
|
+
}
|
|
279
301
|
function cfgIterationCap(deps) {
|
|
280
302
|
if (deps && Number.isFinite(deps.iterationCap)) return deps.iterationCap;
|
|
281
303
|
const env = Number(process.env.DAILY_ITERATION_CAP);
|
|
@@ -873,6 +895,39 @@ function worseOfLedgerAndCohort(local, cohort) {
|
|
|
873
895
|
return out;
|
|
874
896
|
}
|
|
875
897
|
|
|
898
|
+
/**
|
|
899
|
+
* What a flat-fee seat's quota has absorbed: rolling-7d API-equivalent dollars
|
|
900
|
+
* (known prices only; no imputation, so a telemetry gap under-reads rather than
|
|
901
|
+
* invents) and today's inbox-reply session count. Never throws.
|
|
902
|
+
*/
|
|
903
|
+
function subscriptionLoad(deps) {
|
|
904
|
+
const now = clock(deps)();
|
|
905
|
+
const today = todayUtc(deps);
|
|
906
|
+
let weekUSD = 0, replyRuns = 0;
|
|
907
|
+
let files = [];
|
|
908
|
+
try {
|
|
909
|
+
files = readdirSync(ledgerDir(deps)).filter((f) => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)).sort().slice(-8);
|
|
910
|
+
} catch { return { weekUSD, replyRuns }; }
|
|
911
|
+
for (const f of files) {
|
|
912
|
+
let body;
|
|
913
|
+
try { body = readFileSync(join(ledgerDir(deps), f), "utf-8"); } catch { continue; }
|
|
914
|
+
for (const line of body.split("\n")) {
|
|
915
|
+
if (!line.trim()) continue;
|
|
916
|
+
let r;
|
|
917
|
+
try { r = JSON.parse(line); } catch { continue; }
|
|
918
|
+
if (isCohortBilledRow(r)) continue;
|
|
919
|
+
const at = r && r.ts ? Date.parse(r.ts) : NaN;
|
|
920
|
+
if (!Number.isFinite(at)) continue;
|
|
921
|
+
const dt = now - at;
|
|
922
|
+
if (dt < 0 || dt > WINDOW_WEEK_MS) continue;
|
|
923
|
+
const usd = billableUsd(r).usd;
|
|
924
|
+
if (Number.isFinite(usd)) weekUSD += usd;
|
|
925
|
+
if (r.cadence === "inbox" && f === `${today}.jsonl`) replyRuns += 1;
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
return { weekUSD: +weekUSD.toFixed(6), replyRuns };
|
|
929
|
+
}
|
|
930
|
+
|
|
876
931
|
function ledgerDailyStatus(deps) {
|
|
877
932
|
const subscriptionAuth = isSubscriptionAuth(deps);
|
|
878
933
|
const iterationCap = cfgIterationCap(deps);
|
|
@@ -924,8 +979,16 @@ function ledgerDailyStatus(deps) {
|
|
|
924
979
|
// flat-fee seat and halts its autonomous work. Band it on SESSION VOLUME only
|
|
925
980
|
// (a real limit); concurrency (resource-governor) + the 429 rate-breaker are
|
|
926
981
|
// its other guards.
|
|
982
|
+
// The two quota-draining bounds (see DEFAULT_SUBSCRIPTION_WEEKLY_CAP_USD) also
|
|
983
|
+
// apply, since neither is a dollar meter: one is a weekly budget, the other a
|
|
984
|
+
// run count.
|
|
985
|
+
const weekCapUSD = cfgBound("weeklyCapUSD", "WEEKLY_SPEND_CAP_USD", DEFAULT_SUBSCRIPTION_WEEKLY_CAP_USD, deps);
|
|
986
|
+
const replyCeiling = cfgBound("replyRunCeiling", "DAILY_REPLY_RUN_CEILING", DEFAULT_SUBSCRIPTION_REPLY_RUN_CEILING, deps);
|
|
987
|
+
const load = subscriptionAuth ? subscriptionLoad(deps) : { weekUSD: 0, replyRuns: 0 };
|
|
988
|
+
const weekPct = subscriptionAuth && weekCapUSD > 0 ? (load.weekUSD / weekCapUSD) * 100 : 0;
|
|
989
|
+
const replyPct = subscriptionAuth && replyCeiling > 0 ? (load.replyRuns / replyCeiling) * 100 : 0;
|
|
927
990
|
const pct = subscriptionAuth
|
|
928
|
-
? iterPct
|
|
991
|
+
? Math.max(iterPct, weekPct, replyPct)
|
|
929
992
|
: Math.max(spendPct ?? 0, monthPct, iterPct);
|
|
930
993
|
let band = bandForPct(pct);
|
|
931
994
|
|
|
@@ -981,6 +1044,10 @@ function ledgerDailyStatus(deps) {
|
|
|
981
1044
|
essentialOnly: band >= 125,
|
|
982
1045
|
sessions,
|
|
983
1046
|
iterationCap,
|
|
1047
|
+
...(subscriptionAuth ? {
|
|
1048
|
+
weekUSD: load.weekUSD, weekCapUSD, weekPct: +weekPct.toFixed(2),
|
|
1049
|
+
replyRuns: load.replyRuns, replyRunCeiling: replyCeiling, replyPct: +replyPct.toFixed(2),
|
|
1050
|
+
} : {}),
|
|
984
1051
|
date: todayUtc(deps),
|
|
985
1052
|
// Where the ceiling came from. `funded:false` means hq published no envelope
|
|
986
1053
|
// and the number below is the seat's own config, not the org's decision.
|
package/lib/comms/receipts.mjs
CHANGED
|
@@ -392,13 +392,25 @@ export function noSendFor(q = {}) {
|
|
|
392
392
|
const rec = q.obligation || {};
|
|
393
393
|
const ch = channelKey(rec.channel);
|
|
394
394
|
const since = Number.isFinite(rec.openedAt) ? rec.openedAt : q.now;
|
|
395
|
-
if (!
|
|
395
|
+
if (!Number.isFinite(since)) return null;
|
|
396
|
+
// The obligation key alone already names this exact debt — a row stamped
|
|
397
|
+
// with it IS this debt's row, whatever channel string rode in with it. The
|
|
398
|
+
// channel-equality check used to be ANDed on top of the key match too, on
|
|
399
|
+
// the theory that it added a second disambiguator; in practice the two
|
|
400
|
+
// sides are populated from different sources (the obligation's channel is
|
|
401
|
+
// always an id — `replyTargetOf` never returns a label — while a session
|
|
402
|
+
// only ever sees the channel's display name in its own prompt context), so
|
|
403
|
+
// requiring both never found anything and every key-matched decline was
|
|
404
|
+
// read as "no row — genuinely unexplained", firing the fallback notice over
|
|
405
|
+
// a decline that had, in fact, already been recorded. The channel check is
|
|
406
|
+
// kept for the sessionId-only fallback below, where it is the sole
|
|
407
|
+
// disambiguator between this session's obligations and still earns its
|
|
408
|
+
// keep.
|
|
396
409
|
const rows = readReceiptsSince(since, q).filter(
|
|
397
410
|
(r) =>
|
|
398
411
|
r.kind === NO_SEND_KIND &&
|
|
399
|
-
r.channel === ch &&
|
|
400
412
|
((rec.key && r.obligationKey === rec.key) ||
|
|
401
|
-
(rec.sessionId && r.sessionId && r.sessionId === rec.sessionId)),
|
|
413
|
+
(ch && rec.sessionId && r.sessionId && r.sessionId === rec.sessionId && r.channel === ch)),
|
|
402
414
|
);
|
|
403
415
|
if (!rows.length) return null;
|
|
404
416
|
const refused = rows.filter((r) => r.decision === "refused");
|
package/lib/comms/send-gate.mjs
CHANGED
|
@@ -229,6 +229,26 @@ function agentDirName(agentRoot) {
|
|
|
229
229
|
* appear in those queues (`ACT-…`, `RSK-…`, `HR-…`) belong to the org, are used
|
|
230
230
|
* by people, and are deliberately NOT matched.
|
|
231
231
|
*
|
|
232
|
+
* ── A NEW SHAPE LANDS IN THE RULE AND THE CORPUS TOGETHER ───────────────────
|
|
233
|
+
*
|
|
234
|
+
* These rules are a vocabulary, and a vocabulary only covers the shapes it was
|
|
235
|
+
* told about. A newly minted queue file, queue-id prefix or inbox/obligation
|
|
236
|
+
* handle that is not added here is a shape that leaks with nothing going red.
|
|
237
|
+
* So, in the SAME change that introduces it:
|
|
238
|
+
*
|
|
239
|
+
* 1. add it to the rule (`queue-file`, `queue-item-id` or `inbox-item-ref`);
|
|
240
|
+
* 2. add a must-block case for it under that rule's id in `DEDICATED` in
|
|
241
|
+
* lib/comms/fallback-leak-corpus.test.mjs — the in-file ablation then
|
|
242
|
+
* proves the rule is what catches it;
|
|
243
|
+
* 3. add a must-pass case for any legitimate look-alike it could swallow
|
|
244
|
+
* (an org register id, a product path) to the corpus's PASS lists.
|
|
245
|
+
*
|
|
246
|
+
* Queue FILE names are minted in one place (`maestro init`'s queue scaffold,
|
|
247
|
+
* bin/maestro.mjs), so that half is also enforced: the corpus fails if a
|
|
248
|
+
* scaffolded queue is not named by `queue-file`. Queue ID prefixes are minted by
|
|
249
|
+
* seats, with no central declaration to check against, so that half is this
|
|
250
|
+
* rule — recorded again in docs/engineering/functional-architecture.md §9.
|
|
251
|
+
*
|
|
232
252
|
* @type {ReadonlyArray<{id:string, kind:string, hint:string, find:(text:string, ctx?:object)=>{match:string,index:number}[]}>}
|
|
233
253
|
*/
|
|
234
254
|
export const MACHINE_LEAK_RULES = Object.freeze([
|
package/lib/org/registry.mjs
CHANGED
|
@@ -99,6 +99,12 @@ export function buildSelfEntry(o = {}) {
|
|
|
99
99
|
* @param {object} [opts] - { limit=5, queues=["action-stack","backlog"], items? }
|
|
100
100
|
* @returns {Promise<object[]>}
|
|
101
101
|
*/
|
|
102
|
+
// An allow list, not a deny list: a deny list publishes every terminal status
|
|
103
|
+
// it forgot to name (resolved, superseded …) as fleet focus — one resolved row
|
|
104
|
+
// was a seat's top focus eight days after it closed. Unknown statuses are
|
|
105
|
+
// omitted; a missing status still reads as open.
|
|
106
|
+
const LIVE_FOCUS_STATUSES = new Set(["open", "in_progress", "blocked"]);
|
|
107
|
+
|
|
102
108
|
export async function currentFocus(agentRoot, opts = {}) {
|
|
103
109
|
const limit = Number.isFinite(opts.limit) ? opts.limit : 5;
|
|
104
110
|
if (Array.isArray(opts.items)) return opts.items.slice(0, limit);
|
|
@@ -115,7 +121,7 @@ export async function currentFocus(agentRoot, opts = {}) {
|
|
|
115
121
|
for (const it of items) {
|
|
116
122
|
if (!it || typeof it !== "object") continue;
|
|
117
123
|
const status = String(it.status || "open").toLowerCase();
|
|
118
|
-
if (status
|
|
124
|
+
if (!LIVE_FOCUS_STATUSES.has(status)) continue;
|
|
119
125
|
out.push({ title: String(it.title || it.next_action || "").slice(0, 140), queue: q, priority: it.priority || "normal" });
|
|
120
126
|
if (out.length >= limit) return out;
|
|
121
127
|
}
|
package/lib/telemetry/alerts.mjs
CHANGED
|
@@ -19,6 +19,10 @@
|
|
|
19
19
|
* - subAgentsRunning over the cap → INFO (more children than expected).
|
|
20
20
|
* - machine.sessionNote present → WARNING/CRITICAL (frontdoor) — the
|
|
21
21
|
* seat's front door is not answering.
|
|
22
|
+
* - machine.wedge present → WARNING/CRITICAL (wedge) — the
|
|
23
|
+
* seat's front door is BEATING but not
|
|
24
|
+
* READING (risk-2026-09-25-03):
|
|
25
|
+
* presence alone reads idle/healthy.
|
|
22
26
|
* - machine.upgrade.failStreak >= N → WARNING/CRITICAL (rollout) — N
|
|
23
27
|
* consecutive attempts to reach
|
|
24
28
|
* @latest have failed; the seat is
|
|
@@ -99,6 +103,34 @@ export const DEFAULT_THRESHOLDS = Object.freeze({
|
|
|
99
103
|
warnMs: 15 * 60 * 1000,
|
|
100
104
|
critMs: 2 * 60 * 60 * 1000,
|
|
101
105
|
}),
|
|
106
|
+
/**
|
|
107
|
+
* How long the seat's front door may be WEDGED — beating, but not reading
|
|
108
|
+
* its inbound (`machine.wedge`, `lib/telemetry/collect.sessionWedge`) —
|
|
109
|
+
* before the fleet is told. `frontDoor` above catches a door that stopped
|
|
110
|
+
* beating; this catches the inverse the same episode exposed
|
|
111
|
+
* (risk-2026-09-25-03, A029 2026-09-25): claudeAuth ok, state=idle,
|
|
112
|
+
* lastSeenMs fresh, and the seat could not read a message sent 23 minutes
|
|
113
|
+
* earlier. Presence alone cannot tell the two apart — this rule is the
|
|
114
|
+
* difference.
|
|
115
|
+
*
|
|
116
|
+
* `warnMs` 0: unlike `frontDoor`, `sessionWedge` does not fire on a single
|
|
117
|
+
* bad read — it already requires a handoff to be overdue past its OWN
|
|
118
|
+
* deadline (`DEFAULT_HANDOFF_DEADLINE_MS`, 30 min) before returning
|
|
119
|
+
* `wedged:true`. The grace this rule's `frontDoor` sibling gives a resume
|
|
120
|
+
* gap is already spent by the time `machine.wedge` appears, so nothing here
|
|
121
|
+
* should re-spend it — the first beat that carries the field is worth
|
|
122
|
+
* surfacing.
|
|
123
|
+
*
|
|
124
|
+
* `critMs` 30 min: a wedge that outlives another half hour past first
|
|
125
|
+
* detection is not resolving on its own — the same order of magnitude as
|
|
126
|
+
* the handoff deadline that opened it, not the frontDoor rule's 2h (that
|
|
127
|
+
* window exists to tolerate a LEGITIMATE long silence; a wedge is never
|
|
128
|
+
* legitimate once the deadline it is keyed to has already passed).
|
|
129
|
+
*/
|
|
130
|
+
wedge: Object.freeze({
|
|
131
|
+
warnMs: 0,
|
|
132
|
+
critMs: 30 * 60 * 1000,
|
|
133
|
+
}),
|
|
102
134
|
/**
|
|
103
135
|
* How many CONSECUTIVE failed attempts to reach @latest make a seat STUCK.
|
|
104
136
|
*
|
|
@@ -206,6 +238,7 @@ export function deriveAlerts(status, thresholds, now = Date.now()) {
|
|
|
206
238
|
push(alerts, diskRule(machine, t.disk));
|
|
207
239
|
push(alerts, subAgentsRule(s, t.subAgents));
|
|
208
240
|
push(alerts, frontDoorRule(machine, t.frontDoor, now));
|
|
241
|
+
push(alerts, wedgeRule(machine, t.wedge, now));
|
|
209
242
|
push(alerts, scopeFaultRule(machine));
|
|
210
243
|
push(alerts, repliesWithheldRule(machine, t.replyDebt));
|
|
211
244
|
// NO THRESHOLD ARGUMENT, DELIBERATELY. `planeAlert` derives its severity from
|
|
@@ -462,6 +495,46 @@ function frontDoorRule(machine, cfg = {}, now = Date.now()) {
|
|
|
462
495
|
};
|
|
463
496
|
}
|
|
464
497
|
|
|
498
|
+
/**
|
|
499
|
+
* The front door is BEATING but not READING — `machine.wedge`
|
|
500
|
+
* (`collect.sessionWedge`), present only while a handoff sits unclaimed past
|
|
501
|
+
* its own deadline with a fresh heartbeat underneath it. This is the inverse
|
|
502
|
+
* of `frontDoorRule` above: that rule fires when the beat itself goes stale;
|
|
503
|
+
* this one fires when the beat stays fresh and everything else is still
|
|
504
|
+
* wrong. Neither rule can see the other's case — `sessionLiveness` reports
|
|
505
|
+
* exactly one `sessionLive` value, so a snapshot is in at most one of the two
|
|
506
|
+
* states, never both.
|
|
507
|
+
*
|
|
508
|
+
* Anchored on `wedge.since`, same as `frontDoorRule` and for the same reason:
|
|
509
|
+
* a detail carrying a ticking duration opens a new hq alert row every beat
|
|
510
|
+
* (`alertClientKey` keys on severity\0kind\0detail). At most two rows per
|
|
511
|
+
* episode this way — the warn one, then the critical one — not one per beat.
|
|
512
|
+
*
|
|
513
|
+
* A `since` that fails to parse raises nothing, same as `frontDoorRule`: an
|
|
514
|
+
* alert is a claim about duration, and an undatable wedge cannot support one.
|
|
515
|
+
*/
|
|
516
|
+
function wedgeRule(machine, cfg = {}, now = Date.now()) {
|
|
517
|
+
const wedge = machine && typeof machine.wedge === "object" && machine.wedge !== null && !Array.isArray(machine.wedge)
|
|
518
|
+
? machine.wedge
|
|
519
|
+
: null;
|
|
520
|
+
if (!wedge) return null;
|
|
521
|
+
const reason = typeof wedge.reason === "string" ? wedge.reason.trim() : "";
|
|
522
|
+
if (!reason) return null;
|
|
523
|
+
const since = Date.parse(typeof wedge.since === "string" ? wedge.since : "");
|
|
524
|
+
if (!Number.isFinite(since)) return null;
|
|
525
|
+
const ageMs = num(now) - since;
|
|
526
|
+
if (!(ageMs >= 0)) return null; // a future stamp is skew, not a duration
|
|
527
|
+
const sev = severityFor(ageMs, num(cfg.warnMs, Infinity), num(cfg.critMs, Infinity));
|
|
528
|
+
if (!sev) return null;
|
|
529
|
+
return {
|
|
530
|
+
id: "front_door_wedged",
|
|
531
|
+
severity: sev,
|
|
532
|
+
kind: "wedge",
|
|
533
|
+
// An INSTANT, never a duration — see `alertClientKey` above.
|
|
534
|
+
detail: `the seat's front door is beating but not reading since ${new Date(since).toISOString()} (${reason}) — presence alone reads this seat as idle and healthy.`,
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
|
|
465
538
|
/**
|
|
466
539
|
* Does this alert list contain a critical alert? Convenience for the mesh's
|
|
467
540
|
* local-escalation decision. Never throws.
|
|
@@ -474,6 +547,6 @@ export function hasCritical(alerts) {
|
|
|
474
547
|
}
|
|
475
548
|
}
|
|
476
549
|
|
|
477
|
-
export const _internals = { severityFor, reloginRule, tempRule, memRule, diskRule, subAgentsRule, frontDoorRule };
|
|
550
|
+
export const _internals = { severityFor, reloginRule, tempRule, memRule, diskRule, subAgentsRule, frontDoorRule, wedgeRule };
|
|
478
551
|
|
|
479
552
|
export default { deriveAlerts, hasCritical, mergeThresholds, DEFAULT_THRESHOLDS };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cohortapp/agent-sdk",
|
|
3
|
-
"version": "2.19.
|
|
3
|
+
"version": "2.19.1",
|
|
4
4
|
"description": "Cohort Agent SDK — autonomous AI colleague runtime. Deploy senior AI colleagues on dedicated Mac minis, wired to the Cohort operating surface.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/scripts/healthcheck.sh
CHANGED
|
@@ -187,6 +187,41 @@ for config in priorities.yaml environment.yaml contacts.yaml; do
|
|
|
187
187
|
fi
|
|
188
188
|
done
|
|
189
189
|
|
|
190
|
+
# Check 9: Artefact stamp freshness — a typed `generated:` must never lead the
|
|
191
|
+
# file's own mtime, and under --strict a stamp that cannot witness same-day
|
|
192
|
+
# freshness (date-only, null, unparseable) must not pass silently either.
|
|
193
|
+
# Typed-shape queue-row stamps are reported by the checker and never gated.
|
|
194
|
+
# `|| STAMP_RC=$?` is load-bearing: `set -e` is on, so an unguarded `$(...)`
|
|
195
|
+
# would abort the whole healthcheck the moment the checker exits non-zero —
|
|
196
|
+
# before this [FAIL] branch and before the summary, so the run would read as
|
|
197
|
+
# DEGRADED with the error count and the CRITICAL verdict never computed.
|
|
198
|
+
if [ -f "$AGENT_DIR/scripts/maintenance/check-stamp-freshness.mjs" ]; then
|
|
199
|
+
STAMP_RC=0
|
|
200
|
+
STAMP_OUT=$(cd "$AGENT_DIR" && node scripts/maintenance/check-stamp-freshness.mjs --strict 2>&1) || STAMP_RC=$?
|
|
201
|
+
if [ "$STAMP_RC" -eq 0 ]; then
|
|
202
|
+
# Summary is the last line on stdout; WEAK/INFO detail precedes it on stderr.
|
|
203
|
+
echo "[OK] Artefact stamps: $(printf '%s' "$STAMP_OUT" | tail -1)"
|
|
204
|
+
elif [ "$STAMP_RC" -eq 2 ] || ! printf '%s\n' "$STAMP_OUT" | grep -Eq '^(OK|OK-WEAK|FAIL): '; then
|
|
205
|
+
# The checker could not run (rc 2), or it died before printing its verdict
|
|
206
|
+
# line (a crash exits 1 like a finding does). Never read "did not measure"
|
|
207
|
+
# as "measured green", and never read a crash as a stamp finding either.
|
|
208
|
+
echo "[FAIL] Artefact stamps NOT CHECKED — checker could not run:"
|
|
209
|
+
printf '%s\n' "$STAMP_OUT" | sed 's/^/ /'
|
|
210
|
+
ERRORS=$((ERRORS + 1))
|
|
211
|
+
elif printf '%s\n' "$STAMP_OUT" | grep -Eq '^(FAIL|LEAD) '; then
|
|
212
|
+
echo "[FAIL] Artefact stamp(s) dated ahead of their own write:"
|
|
213
|
+
printf '%s\n' "$STAMP_OUT" | sed 's/^/ /'
|
|
214
|
+
ERRORS=$((ERRORS + 1))
|
|
215
|
+
else
|
|
216
|
+
echo "[FAIL] Artefact stamp(s) cannot witness same-day freshness (date-only, null or unparseable under --strict):"
|
|
217
|
+
printf '%s\n' "$STAMP_OUT" | sed 's/^/ /'
|
|
218
|
+
ERRORS=$((ERRORS + 1))
|
|
219
|
+
fi
|
|
220
|
+
else
|
|
221
|
+
echo "[WARN] scripts/maintenance/check-stamp-freshness.mjs absent — stamp freshness unchecked"
|
|
222
|
+
WARNINGS=$((WARNINGS + 1))
|
|
223
|
+
fi
|
|
224
|
+
|
|
190
225
|
# Summary. `Blocking:` / `Warnings:` are the machine-readable contract:
|
|
191
226
|
# resume-operations.sh parses `Blocking:` to name the condition it refused on.
|
|
192
227
|
echo ""
|
|
@@ -612,6 +612,37 @@ session_beating(){ # is the front door actually ANSWERING, not merely loaded?
|
|
|
612
612
|
[ "$age" -le "$SESSION_STALE_S" ]
|
|
613
613
|
}
|
|
614
614
|
|
|
615
|
+
front_door_wedged(){ # is a FRESH critical wedge alert on record — beating, but not reading (risk-2026-09-25-03)?
|
|
616
|
+
# `session_beating` proves the heartbeat is fresh; it proves nothing about
|
|
617
|
+
# inbox consumption. state/org/alert-critical.json is written by the SAME
|
|
618
|
+
# process's own beat loop (lib/org/mesh.mjs#escalateCriticalAlerts) every
|
|
619
|
+
# ~30s while a critical alert is open, INCLUDING a "wedge" kind
|
|
620
|
+
# (lib/telemetry/alerts.mjs#wedgeRule) — the local, path-independent read of
|
|
621
|
+
# exactly the signal that made A029's seat look healthy from presence alone.
|
|
622
|
+
# The file is never deleted, only re-written on the next critical beat, so a
|
|
623
|
+
# RESOLVED wedge would read as still-live forever without the freshness
|
|
624
|
+
# check below.
|
|
625
|
+
local f="$AGENT_DIR/state/org/alert-critical.json"
|
|
626
|
+
[ -f "$f" ] || return 1
|
|
627
|
+
node -e '
|
|
628
|
+
try {
|
|
629
|
+
const j = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
|
630
|
+
const at = Date.parse(j && j.at);
|
|
631
|
+
const freshMs = Number(process.argv[2]) * 1000;
|
|
632
|
+
if (!Number.isFinite(at) || (Date.now() - at) > freshMs) process.exit(1);
|
|
633
|
+
const hit = Array.isArray(j.alerts) && j.alerts.some((a) => a && a.kind === "wedge");
|
|
634
|
+
process.exit(hit ? 0 : 1);
|
|
635
|
+
} catch { process.exit(1); }
|
|
636
|
+
' "$f" "$WEDGE_ALERT_FRESH_S"
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
kickstart_session(){ # the shared "actually restart it" tail for both revive_session branches below
|
|
640
|
+
launchctl kickstart -k "gui/$(id -u)/$SESSION_LABEL" >> "$LOG" 2>&1 \
|
|
641
|
+
&& { log "front door $SESSION_LABEL restarted"; return 0; }
|
|
642
|
+
log "WARN: could not kickstart $SESSION_LABEL (maestro session restart --force)"
|
|
643
|
+
return 1
|
|
644
|
+
}
|
|
645
|
+
|
|
615
646
|
revive_session(){ # a wedged front door cannot restart itself; restart it FOR it
|
|
616
647
|
# The upgrade notice asks the session to restart "at an idle moment". A
|
|
617
648
|
# session sitting on a modal never reaches an idle moment, so the polite ask
|
|
@@ -624,7 +655,23 @@ revive_session(){ # a wedged front door cannot restart itself; restart it FOR it
|
|
|
624
655
|
# session start`), and kickstarting a label launchd does not know just fails.
|
|
625
656
|
# Leave that one to session_reconciled, which already names it.
|
|
626
657
|
label_loaded "$SESSION_LABEL" || return 0
|
|
627
|
-
session_beating
|
|
658
|
+
if session_beating; then
|
|
659
|
+
# risk-2026-09-25-03: a fresh beat is not proof of health — it proves the
|
|
660
|
+
# heartbeat process is alive, and nothing about whether it is reading its
|
|
661
|
+
# inbox. Only kickstart on TOP of a fresh beat when a fresh critical wedge
|
|
662
|
+
# alert is on record; otherwise this is the ordinary healthy case.
|
|
663
|
+
front_door_wedged || return 0
|
|
664
|
+
log "front door $SESSION_LABEL is beating but a fresh critical wedge alert is on record (beating but not reading) — confirming over a second read ${SESSION_REVIVE_RECHECK_S}s apart before kickstarting"
|
|
665
|
+
sleep "$SESSION_REVIVE_RECHECK_S"
|
|
666
|
+
# Re-check the WEDGE, not the beat: the beat was never the thing in
|
|
667
|
+
# question here, and re-reading it would tell us nothing new. If the
|
|
668
|
+
# handoff got claimed during the sleep the alert stops being fresh (the
|
|
669
|
+
# daemon's beat loop stops re-writing it) and this reads as cleared.
|
|
670
|
+
front_door_wedged || { log "front door $SESSION_LABEL wedge alert cleared on the second read — leaving it"; return 0; }
|
|
671
|
+
log "front door $SESSION_LABEL still shows a fresh critical wedge alert on the second read — kickstarting it; a beating front door proves nothing about inbox consumption"
|
|
672
|
+
kickstart_session
|
|
673
|
+
return $?
|
|
674
|
+
fi
|
|
628
675
|
# Two-read hysteresis (see SESSION_REVIVE_RECHECK_S): the first stale read is
|
|
629
676
|
# not enough to drop a session, because a supervisor resume gaps the beat for
|
|
630
677
|
# a moment. Confirm the door is STILL silent on a second read 60s later
|
|
@@ -647,10 +694,8 @@ revive_session(){ # a wedged front door cannot restart itself; restart it FOR it
|
|
|
647
694
|
log "front door $SESSION_LABEL dark past grace while a supervisor (pid $spid) still holds the session lock — parked in its launch probe; hard-kickstarting over it"
|
|
648
695
|
fi
|
|
649
696
|
log "front door $SESSION_LABEL not beating on two reads ${SESSION_REVIVE_RECHECK_S}s apart — kickstarting it rather than waiting for a session that cannot answer"
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
log "WARN: could not kickstart $SESSION_LABEL (maestro session restart --force)"
|
|
653
|
-
return 1
|
|
697
|
+
kickstart_session
|
|
698
|
+
return $?
|
|
654
699
|
}
|
|
655
700
|
|
|
656
701
|
daemon_launchctl_state(){ # "pid" | "dash" | "absent" for DAEMON_LABEL, read from column 1 (PID) of `launchctl list` — a number is a live pid, "-" is loaded-with-no-pid, missing is absent
|
|
@@ -1088,6 +1133,17 @@ SESSION_STALE_S="${MAESTRO_SESSION_STALE_S:-90}"
|
|
|
1088
1133
|
# gap must not kickstart a session that is mid-resume. The two reads are 60s
|
|
1089
1134
|
# apart; the hourly backstop can afford the one wait.
|
|
1090
1135
|
SESSION_REVIVE_RECHECK_S="${MAESTRO_SESSION_REVIVE_RECHECK_S:-60}"
|
|
1136
|
+
# How fresh a critical "wedge" alert (state/org/alert-critical.json, kind
|
|
1137
|
+
# "wedge" — lib/org/mesh.mjs#escalateCriticalAlerts, fed by
|
|
1138
|
+
# lib/telemetry/alerts.mjs#wedgeRule reading machine.wedge from
|
|
1139
|
+
# lib/telemetry/collect.sessionWedge) must be for `front_door_wedged` to trust
|
|
1140
|
+
# it. risk-2026-09-25-03: a front door can be BEATING (session_beating true)
|
|
1141
|
+
# while unable to read its inbox — claudeAuth ok, state=idle, indistinguishable
|
|
1142
|
+
# from a seat between tasks — so a fresh beat alone is not proof of health.
|
|
1143
|
+
# 90s — the same 3x-the-30s-beat-cadence margin as SESSION_STALE_S above, so a
|
|
1144
|
+
# wedge that cleared is not read as still-live off a stale flag file (the file
|
|
1145
|
+
# is never deleted, only re-written on the next critical beat).
|
|
1146
|
+
WEDGE_ALERT_FRESH_S="${MAESTRO_WEDGE_ALERT_FRESH_S:-90}"
|
|
1091
1147
|
failed_hold_reason(){ # prints "<reason> at <at>" when LATEST failed health within FAILED_HOLD_S
|
|
1092
1148
|
node -e '
|
|
1093
1149
|
const [file, latest, holdS] = process.argv.slice(1);
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* check-stamp-freshness.mjs
|
|
4
|
+
*
|
|
5
|
+
* Fails when a maintenance artefact's typed `generated:` stamp LEADS the file's
|
|
6
|
+
* own mtime — i.e. the record claims it was written at a time that had not yet
|
|
7
|
+
* happened — and, under --strict, when a stamp cannot witness same-day
|
|
8
|
+
* freshness at all (date-only, null or unparseable).
|
|
9
|
+
*
|
|
10
|
+
* Why this exists: an evening wrap once stamped six artefacts with a generation
|
|
11
|
+
* time it estimated from the volume of work done rather than read from the
|
|
12
|
+
* clock. Every stamp was in the future relative to its own write and the drift
|
|
13
|
+
* grew monotonically through the run (+8 min on the first index, +66 min on the
|
|
14
|
+
* last brief). The timezone conversion was correct in every row, so this was not
|
|
15
|
+
* a timezone bug. Because the error was self-consistent across all six files, no
|
|
16
|
+
* cross-artefact check could catch it. The only independent witness is the
|
|
17
|
+
* filesystem mtime — which nothing was comparing against.
|
|
18
|
+
*
|
|
19
|
+
* The asymmetry is deliberate: leading the mtime is always an error (a file
|
|
20
|
+
* cannot be authored after it was written), while lagging it is normal and
|
|
21
|
+
* expected — later edits touch mtime without regenerating content. So this
|
|
22
|
+
* checker is strict above the line and silent below it.
|
|
23
|
+
*
|
|
24
|
+
* Every target is named in exactly one bucket, and the classified total is
|
|
25
|
+
* asserted against the file total, so a bucket that fails to add up is a hard
|
|
26
|
+
* error rather than a quiet gap. The buckets:
|
|
27
|
+
*
|
|
28
|
+
* - OK precise stamp, at or behind its mtime.
|
|
29
|
+
* - LEAD stamp ahead of its mtime by more than the tolerance. Always FAIL.
|
|
30
|
+
* - COARSE date-only stamp (`generated: '2026-09-02'`). Parses to midnight
|
|
31
|
+
* UTC and therefore sits hours BEHIND its own mtime by construction,
|
|
32
|
+
* so it cannot witness the intra-day forward-dating this checker
|
|
33
|
+
* exists to catch. Not literally unfailable (a future DAY still
|
|
34
|
+
* trips) but blind to same-day errors. WEAK.
|
|
35
|
+
* - NULL the literal `generated: null` / `~` / empty. Scaffold writers seed
|
|
36
|
+
* this when they first create a file, so it means THE GENERATOR HAS
|
|
37
|
+
* NEVER RUN — a real finding, not "nothing to check". WEAK.
|
|
38
|
+
* - UNPARSEABLE present but not a date. WEAK.
|
|
39
|
+
* - ABSENT no `generated:` key at all (e.g. a file that uses `updated:`,
|
|
40
|
+
* machine-written every cycle). INFO only.
|
|
41
|
+
* - ALLOWED a NULL/ABSENT target the operator has declared a design state
|
|
42
|
+
* via --allow-null (a capability deliberately not run on this seat).
|
|
43
|
+
* Printed, never counted weak. The allowance is SELF-INVALIDATING:
|
|
44
|
+
* if a stamp ever appears, something is generating the file after
|
|
45
|
+
* all and it falls through to ordinary checking.
|
|
46
|
+
*
|
|
47
|
+
* Default mode gates LEAD only (exit 1) and reports WEAK as OK-WEAK (exit 0).
|
|
48
|
+
* --strict also gates the WEAK buckets: a stamp that cannot prove same-day
|
|
49
|
+
* freshness must not pass silently.
|
|
50
|
+
*
|
|
51
|
+
* A second class of typed stamp is detectable by SHAPE rather than by lead: every
|
|
52
|
+
* deterministic writer emits sub-second precision (12:00:28.851Z); a hand-typed
|
|
53
|
+
* time lands on a round minute with zero milliseconds. That shape survives any
|
|
54
|
+
* later write, whereas the mtime witness does not — once any tick rewrites the
|
|
55
|
+
* file its mtime moves past the stamp and a forward-dated value drops silently out
|
|
56
|
+
* of LEAD. Typed-shape stamps are REPORTED, NEVER GATED, in either mode: the shape
|
|
57
|
+
* proves the stamp was typed, not that it is stale, and failing a build on a proof
|
|
58
|
+
* gap rather than a staleness signal is the wrong trade.
|
|
59
|
+
*
|
|
60
|
+
* A queue-row pass applies the same mtime witness to `last_updated:` / `opened:` /
|
|
61
|
+
* `created:` / `resolved:` / `closed:` stamps inside state/queues/*.yaml, by regex
|
|
62
|
+
* over the raw text (no YAML dependency, stays fast over multi-MB files). Leads
|
|
63
|
+
* there FAIL in both modes; typed-shape rows are reported, never gated.
|
|
64
|
+
*
|
|
65
|
+
* OUT OF SCOPE, deliberately: stamps written inside JSON note fields. Those need a
|
|
66
|
+
* different reader and that tree may be under concurrent edit.
|
|
67
|
+
*
|
|
68
|
+
* Usage: node scripts/maintenance/check-stamp-freshness.mjs
|
|
69
|
+
* [--strict] [--tolerance-sec N] [--allow-null PATH]... [paths...]
|
|
70
|
+
* Exit: 0 all clear (or OK-WEAK in default mode),
|
|
71
|
+
* 1 a stamp leads its mtime, or --strict and a weak stamp,
|
|
72
|
+
* 2 usage/read error, or the classification did not account for every file.
|
|
73
|
+
*/
|
|
74
|
+
|
|
75
|
+
import { readFileSync, statSync, existsSync } from 'node:fs';
|
|
76
|
+
import { readdirSync } from 'node:fs';
|
|
77
|
+
|
|
78
|
+
// Node 20 has no fs.globSync (it arrived in 22), and package.json engines says
|
|
79
|
+
// >=20, so expand the one pattern shape this file uses by hand: a literal
|
|
80
|
+
// directory, then a single '*' segment with an optional literal suffix.
|
|
81
|
+
function globSync(pattern) {
|
|
82
|
+
const slash = pattern.lastIndexOf('/');
|
|
83
|
+
const dir = slash === -1 ? '.' : pattern.slice(0, slash);
|
|
84
|
+
const leaf = pattern.slice(slash + 1);
|
|
85
|
+
const star = leaf.indexOf('*');
|
|
86
|
+
if (star === -1 || dir.includes('*')) throw new Error(`unsupported pattern ${pattern}`);
|
|
87
|
+
const prefix = leaf.slice(0, star), suffix = leaf.slice(star + 1);
|
|
88
|
+
let names;
|
|
89
|
+
try { names = readdirSync(dir); } catch { return []; }
|
|
90
|
+
return names
|
|
91
|
+
.filter((n) => n.startsWith(prefix) && n.endsWith(suffix) && n.length >= prefix.length + suffix.length)
|
|
92
|
+
.map((n) => (dir === '.' ? n : `${dir}/${n}`));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// tmp/ is excluded on purpose: those are pre-edit backups whose mtime is the COPY
|
|
96
|
+
// time, so their stamps legitimately lag by hours and would be pure noise.
|
|
97
|
+
const DEFAULT_TARGETS = [
|
|
98
|
+
'state/dashboards/*.yaml',
|
|
99
|
+
'state/indexes/*.yaml',
|
|
100
|
+
'state/registers/*.yaml',
|
|
101
|
+
'state/backlog/*.yaml',
|
|
102
|
+
'memory/indexes/*.yaml',
|
|
103
|
+
'knowledge/decisions/index.yaml',
|
|
104
|
+
];
|
|
105
|
+
|
|
106
|
+
// A write takes a few seconds; allow for it. Anything beyond this is a typed guess.
|
|
107
|
+
const DEFAULT_TOLERANCE_SEC = 120;
|
|
108
|
+
|
|
109
|
+
const USAGE =
|
|
110
|
+
'usage: check-stamp-freshness.mjs [--strict] [--tolerance-sec N] [--allow-null PATH]... [paths...]';
|
|
111
|
+
|
|
112
|
+
function parseArgs(argv) {
|
|
113
|
+
const out = { tolerance: DEFAULT_TOLERANCE_SEC, strict: false, targets: [], allowNull: new Set() };
|
|
114
|
+
for (let i = 0; i < argv.length; i++) {
|
|
115
|
+
const a = argv[i];
|
|
116
|
+
if (a === '--help' || a === '-h') { console.log(USAGE); process.exit(0); }
|
|
117
|
+
if (a === '--tolerance-sec') {
|
|
118
|
+
out.tolerance = Number(argv[++i]);
|
|
119
|
+
if (!Number.isFinite(out.tolerance) || out.tolerance < 0) { console.error(USAGE); process.exit(2); }
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
if (a === '--strict') { out.strict = true; continue; }
|
|
123
|
+
if (a === '--allow-null') {
|
|
124
|
+
const p = argv[++i];
|
|
125
|
+
if (!p) { console.error(USAGE); process.exit(2); }
|
|
126
|
+
out.allowNull.add(p);
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
if (a.startsWith('--')) { console.error(`unknown option ${a}\n${USAGE}`); process.exit(2); }
|
|
130
|
+
out.targets.push(a);
|
|
131
|
+
}
|
|
132
|
+
if (!out.targets.length) out.targets = DEFAULT_TARGETS;
|
|
133
|
+
return out;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Classify the `generated:` scalar without a YAML parse — these files must stay
|
|
138
|
+
* checkable even when malformed. Returns a KIND rather than a bare value, so the
|
|
139
|
+
* caller can tell "absent", "null", "garbage" and "imprecise" apart.
|
|
140
|
+
*/
|
|
141
|
+
function readStamp(text) {
|
|
142
|
+
const m = text.match(/^generated: *(.*?) *$/m);
|
|
143
|
+
if (!m) return { kind: 'ABSENT' };
|
|
144
|
+
|
|
145
|
+
const raw = m[1].replace(/^['"]|['"]$/g, '').trim();
|
|
146
|
+
if (raw === '' || raw === 'null' || raw === '~') return { kind: 'NULL', raw: m[1].trim() };
|
|
147
|
+
|
|
148
|
+
const d = new Date(raw);
|
|
149
|
+
if (Number.isNaN(d.getTime())) return { kind: 'UNPARSEABLE', raw };
|
|
150
|
+
|
|
151
|
+
// No time-of-day component: parses to midnight UTC, so it cannot witness
|
|
152
|
+
// same-day forward-dating even though it is a valid date.
|
|
153
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(raw)) return { kind: 'COARSE', raw, date: d };
|
|
154
|
+
|
|
155
|
+
return { kind: 'PRECISE', raw, date: d };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const { tolerance, strict, targets, allowNull } = parseArgs(process.argv.slice(2));
|
|
159
|
+
|
|
160
|
+
let files = [];
|
|
161
|
+
for (const t of targets) {
|
|
162
|
+
try {
|
|
163
|
+
files.push(...(t.includes('*') ? globSync(t) : existsSync(t) ? [t] : []));
|
|
164
|
+
} catch (e) {
|
|
165
|
+
console.error(`cannot expand ${t}: ${e.message}`);
|
|
166
|
+
process.exit(2);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
files = [...new Set(files)].sort();
|
|
170
|
+
|
|
171
|
+
if (!files.length) {
|
|
172
|
+
console.error('no files matched');
|
|
173
|
+
process.exit(2);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const buckets = { OK: [], LEAD: [], NULL: [], UNPARSEABLE: [], COARSE: [], ABSENT: [], ALLOWED: [] };
|
|
177
|
+
|
|
178
|
+
for (const f of files) {
|
|
179
|
+
let text, st;
|
|
180
|
+
try { text = readFileSync(f, 'utf8'); st = statSync(f); }
|
|
181
|
+
catch (e) { console.error(`cannot read ${f}: ${e.message}`); process.exit(2); }
|
|
182
|
+
|
|
183
|
+
const stamp = readStamp(text);
|
|
184
|
+
|
|
185
|
+
if (allowNull.has(f) && (stamp.kind === 'NULL' || stamp.kind === 'ABSENT')) {
|
|
186
|
+
buckets.ALLOWED.push({ file: f, raw: stamp.raw ?? '(no key)' });
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if (stamp.kind === 'ABSENT' || stamp.kind === 'NULL' || stamp.kind === 'UNPARSEABLE') {
|
|
191
|
+
buckets[stamp.kind].push({ file: f, raw: stamp.raw });
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const leadSec = (stamp.date.getTime() - st.mtime.getTime()) / 1000;
|
|
196
|
+
const row = {
|
|
197
|
+
file: f, stamp: stamp.raw, mtime: st.mtime.toISOString(),
|
|
198
|
+
leadMin: (leadSec / 60).toFixed(1),
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
// A lead is an error regardless of precision — a date-only stamp naming a future
|
|
202
|
+
// day is still a stamp ahead of its own write.
|
|
203
|
+
if (leadSec > tolerance) buckets.LEAD.push(row);
|
|
204
|
+
else if (stamp.kind === 'COARSE') buckets.COARSE.push(row);
|
|
205
|
+
else buckets.OK.push(row);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Assert the classification adds up, so a bucket that swallows a file is a hard
|
|
209
|
+
// error rather than a quiet gap.
|
|
210
|
+
const classified = Object.values(buckets).reduce((n, b) => n + b.length, 0);
|
|
211
|
+
if (classified !== files.length) {
|
|
212
|
+
console.error(`INTERNAL: classified ${classified} of ${files.length} files — a bucket is dropping rows`);
|
|
213
|
+
process.exit(2);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
for (const x of buckets.LEAD) {
|
|
217
|
+
console.error(`FAIL ${x.file}`);
|
|
218
|
+
console.error(` generated: ${x.stamp} leads mtime ${x.mtime} by ${x.leadMin} min`);
|
|
219
|
+
}
|
|
220
|
+
for (const x of buckets.NULL) {
|
|
221
|
+
console.error(`WEAK ${x.file}`);
|
|
222
|
+
console.error(` generated: ${x.raw} — generator has never run; stamp is an unfilled scaffold seed`);
|
|
223
|
+
}
|
|
224
|
+
for (const x of buckets.UNPARSEABLE) {
|
|
225
|
+
console.error(`WEAK ${x.file}`);
|
|
226
|
+
console.error(` generated: ${x.raw} — present but not a date; cannot be checked`);
|
|
227
|
+
}
|
|
228
|
+
for (const x of buckets.COARSE) {
|
|
229
|
+
console.error(`WEAK ${x.file}`);
|
|
230
|
+
console.error(` generated: ${x.stamp} — date-only, parses to midnight UTC and sits ${Math.abs(x.leadMin)} min behind mtime; cannot witness same-day freshness`);
|
|
231
|
+
}
|
|
232
|
+
for (const x of buckets.ALLOWED) {
|
|
233
|
+
console.error(`INFO ${x.file} — generated: ${x.raw}, NOT COUNTED WEAK: allowed by --allow-null (generator deliberately not run here)`);
|
|
234
|
+
}
|
|
235
|
+
for (const x of buckets.ABSENT) {
|
|
236
|
+
console.error(`INFO ${x.file} — no generated: key`);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Typed-shape stamps: round minute, zero milliseconds. Reported, never gated.
|
|
240
|
+
const TYPED_SHAPE = /T\d{2}:\d{2}:00\.000Z$/;
|
|
241
|
+
const typed = [...buckets.LEAD, ...buckets.COARSE, ...buckets.OK].filter((x) => TYPED_SHAPE.test(x.stamp));
|
|
242
|
+
for (const x of typed) {
|
|
243
|
+
console.error(
|
|
244
|
+
`INFO ${x.file} — generated: ${x.stamp} is a round minute with zero ms (typed, not clock-read)` +
|
|
245
|
+
(buckets.LEAD.includes(x)
|
|
246
|
+
? '; witnessed above as a lead'
|
|
247
|
+
: '; mtime has moved since, so no lead can be proven either way')
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ---------------------------------------------------------------------------
|
|
252
|
+
// QUEUE-ROW PASS. A queue row cannot legitimately claim to have been updated after
|
|
253
|
+
// the file that contains it was last written, so file mtime is the same witness
|
|
254
|
+
// used above. Note the typed shape here covers both `T08:45:00Z` (no millisecond
|
|
255
|
+
// field) and `T08:45:00.000Z`.
|
|
256
|
+
const QUEUE_GLOB = 'state/queues/*.yaml';
|
|
257
|
+
const QUEUE_STAMP_KEY = /^\s+(last_updated|opened|created|resolved|closed): *(.+?) *$/;
|
|
258
|
+
const QUEUE_ROW_ID = /^\s*- id: *(.+?) *$/;
|
|
259
|
+
const QUEUE_TYPED_SHAPE = /T\d{2}:\d{2}:00(\.000)?Z$/;
|
|
260
|
+
const queueLeads = [];
|
|
261
|
+
const queueTyped = [];
|
|
262
|
+
let queueRowsChecked = 0;
|
|
263
|
+
let queueFilesChecked = 0;
|
|
264
|
+
for (const qp of globSync(QUEUE_GLOB).sort()) {
|
|
265
|
+
let text;
|
|
266
|
+
try { text = readFileSync(qp, 'utf8'); } catch { continue; }
|
|
267
|
+
const mtime = statSync(qp).mtime.getTime();
|
|
268
|
+
queueFilesChecked++;
|
|
269
|
+
let currentId = '(before first row)';
|
|
270
|
+
for (const line of text.split('\n')) {
|
|
271
|
+
const idm = line.match(QUEUE_ROW_ID);
|
|
272
|
+
if (idm) { currentId = idm[1]; continue; }
|
|
273
|
+
const m = line.match(QUEUE_STAMP_KEY);
|
|
274
|
+
if (!m) continue;
|
|
275
|
+
const raw = m[2].replace(/^['"]|['"]$/g, '').trim();
|
|
276
|
+
// Date-only values carry no intra-day information; skip rather than guess.
|
|
277
|
+
if (!/T\d{2}:\d{2}/.test(raw)) continue;
|
|
278
|
+
const t = Date.parse(raw);
|
|
279
|
+
if (Number.isNaN(t)) continue;
|
|
280
|
+
queueRowsChecked++;
|
|
281
|
+
const leadSec = (t - mtime) / 1000;
|
|
282
|
+
if (leadSec > tolerance) queueLeads.push({ file: qp, id: currentId, key: m[1], stamp: raw, leadSec });
|
|
283
|
+
if (QUEUE_TYPED_SHAPE.test(raw)) queueTyped.push({ file: qp, id: currentId, key: m[1], stamp: raw });
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
for (const x of queueLeads) {
|
|
287
|
+
console.error(
|
|
288
|
+
`LEAD ${x.file} :: ${x.id} — ${x.key}: ${x.stamp} is ` +
|
|
289
|
+
`${(x.leadSec / 60).toFixed(1)} min AHEAD of that file's own mtime`
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
if (queueTyped.length) {
|
|
293
|
+
console.error(
|
|
294
|
+
`INFO ${queueTyped.length} queue row stamp(s) carry the typed shape (round minute, zero or absent ms) — ` +
|
|
295
|
+
'reported, not gated in either mode'
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
const weak = buckets.NULL.length + buckets.UNPARSEABLE.length + buckets.COARSE.length;
|
|
300
|
+
const leads = buckets.LEAD.length + queueLeads.length;
|
|
301
|
+
const failed = leads + (strict ? weak : 0);
|
|
302
|
+
const verdict = leads ? 'FAIL' : weak && strict ? 'FAIL' : weak ? 'OK-WEAK' : 'OK';
|
|
303
|
+
|
|
304
|
+
console.log(
|
|
305
|
+
`${verdict}: ${files.length} target(s) — ${buckets.OK.length} checkable and clean, ` +
|
|
306
|
+
`${buckets.LEAD.length} ahead of their own mtime, ${buckets.COARSE.length} date-only (cannot witness same-day freshness), ` +
|
|
307
|
+
`${buckets.NULL.length} never generated, ${buckets.UNPARSEABLE.length} unparseable, ${buckets.ABSENT.length} unstamped, ` +
|
|
308
|
+
`${buckets.ALLOWED.length} allowed null(s) excluded from weak, ` +
|
|
309
|
+
`${typed.length} typed-shape (round-minute .000Z, not gated) ` +
|
|
310
|
+
`| queue rows: ${queueRowsChecked} stamp(s) over ${queueFilesChecked} file(s), ` +
|
|
311
|
+
`${queueLeads.length} ahead of their own mtime, ${queueTyped.length} typed-shape ` +
|
|
312
|
+
`(tolerance ${tolerance}s${strict ? ', strict' : ''})`
|
|
313
|
+
);
|
|
314
|
+
process.exit(failed ? 1 : 0);
|
|
@@ -50,10 +50,6 @@ import { call, approvalWait } from "../../lib/org/client.mjs";
|
|
|
50
50
|
import { resolveOrgToolConfig } from "../../lib/org/tool-surface.mjs";
|
|
51
51
|
import { applyBrandEnvCompat } from "../../lib/env-compat.mjs";
|
|
52
52
|
|
|
53
|
-
// Fleet back-compat FIRST: bridge NEOLITH_* ⇄ COHORT_* env names before the
|
|
54
|
-
// env-first credential resolution (COHORT_API_TOKEN et al.) kicks in.
|
|
55
|
-
applyBrandEnvCompat();
|
|
56
|
-
|
|
57
53
|
function parseArgs(argv) {
|
|
58
54
|
const flags = { to: [], cc: [], bcc: [] };
|
|
59
55
|
for (let i = 0; i < argv.length; i += 1) {
|
|
@@ -88,7 +84,30 @@ function usage() {
|
|
|
88
84
|
}
|
|
89
85
|
|
|
90
86
|
/**
|
|
91
|
-
*
|
|
87
|
+
* The local screen — the same send-gate every other outbound lane runs, and
|
|
88
|
+
* FAIL-CLOSED. The server guardrails stay authoritative for what they cover;
|
|
89
|
+
* they do not cover what this gate covers (persona leaks, machine internals,
|
|
90
|
+
* scaffold markers, the retired register), so this is not belt-and-braces.
|
|
91
|
+
*
|
|
92
|
+
* ── WHY A GATE THAT CANNOT LOAD IS A REFUSAL ────────────────────────────────
|
|
93
|
+
*
|
|
94
|
+
* This used to `catch { return text; }` around the whole screen: a send-gate
|
|
95
|
+
* that failed to import, or threw, meant the body went out unscreened on "the
|
|
96
|
+
* server guardrails still apply". That made this CLI the one outbound lane a
|
|
97
|
+
* direct invocation could take past the gate — `lib/org/messaging.mjs` fails
|
|
98
|
+
* closed on a gate fault ("never send what we could not screen"), and so does
|
|
99
|
+
* `deliver()`. A send this lane cannot screen is now a send it does not make,
|
|
100
|
+
* with the reason on stderr and a no-send row on this seat's ledger.
|
|
101
|
+
*
|
|
102
|
+
* RESULTS, NOT THROWS. Returns `{ok:true, text}` or `{ok:false, code, reason}`
|
|
103
|
+
* with `code` one of:
|
|
104
|
+
* BLOCKED the gate refused the body (its reason names the rule and
|
|
105
|
+
* the span — rewrite and send again)
|
|
106
|
+
* GATE_UNAVAILABLE the gate module would not load, or exports no screen
|
|
107
|
+
* GATE_ERROR the screen itself threw
|
|
108
|
+
* so the caller decides the exit and the test drives every branch without a
|
|
109
|
+
* process exit. `deps.loadGate` is the test seam for the import; production
|
|
110
|
+
* never passes it.
|
|
92
111
|
*
|
|
93
112
|
* `firstContact` MUST be passed. `screenOutbound` defaults it to TRUE, and under
|
|
94
113
|
* the `disclose-always` posture that limb is an outright block for any body that
|
|
@@ -104,27 +123,82 @@ function usage() {
|
|
|
104
123
|
* (`--reply-to-message`) is by construction NOT first contact. A bare `--to` is,
|
|
105
124
|
* and still blocks — that limb is left exactly as it was.
|
|
106
125
|
*/
|
|
107
|
-
async function screenLocally(text, recipients, agentRoot, firstContact) {
|
|
126
|
+
export async function screenLocally(text, recipients, agentRoot, firstContact, deps = {}) {
|
|
127
|
+
const errText = (err) => (err && err.message ? err.message : String(err));
|
|
128
|
+
let screenOutbound;
|
|
108
129
|
try {
|
|
109
|
-
const
|
|
110
|
-
|
|
130
|
+
const load = typeof deps.loadGate === "function" ? deps.loadGate : () => import("../../lib/comms/send-gate.mjs");
|
|
131
|
+
({ screenOutbound } = await load());
|
|
132
|
+
} catch (err) {
|
|
133
|
+
return { ok: false, code: "GATE_UNAVAILABLE", reason: `the outbound send-gate could not be loaded (${errText(err)}) — nothing was sent: this lane does not send what it cannot screen` };
|
|
134
|
+
}
|
|
135
|
+
if (typeof screenOutbound !== "function") {
|
|
136
|
+
return { ok: false, code: "GATE_UNAVAILABLE", reason: "the outbound send-gate module exports no screenOutbound — nothing was sent: this lane does not send what it cannot screen" };
|
|
137
|
+
}
|
|
138
|
+
let verdict;
|
|
139
|
+
try {
|
|
140
|
+
verdict = await screenOutbound({
|
|
111
141
|
channel: "orgmail",
|
|
112
142
|
recipient: recipients.join(",") || "(thread)",
|
|
113
143
|
text,
|
|
114
144
|
agentRoot,
|
|
115
145
|
firstContact,
|
|
116
146
|
});
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
process.exit(1);
|
|
120
|
-
}
|
|
121
|
-
return typeof verdict?.redactedText === "string" ? verdict.redactedText : text;
|
|
122
|
-
} catch {
|
|
123
|
-
return text; // gate unavailable → server guardrails still apply
|
|
147
|
+
} catch (err) {
|
|
148
|
+
return { ok: false, code: "GATE_ERROR", reason: `the outbound send-gate failed while screening (${errText(err)}) — nothing was sent` };
|
|
124
149
|
}
|
|
150
|
+
// `allow !== true`, not `allow === false`: a verdict that does not say "allow"
|
|
151
|
+
// has not allowed anything.
|
|
152
|
+
if (!verdict || verdict.allow !== true) {
|
|
153
|
+
return {
|
|
154
|
+
ok: false,
|
|
155
|
+
code: "BLOCKED",
|
|
156
|
+
reason: (verdict && verdict.reason) || "policy",
|
|
157
|
+
...(verdict && verdict.rule ? { rule: verdict.rule, match: verdict.match } : {}),
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
return { ok: true, text: typeof verdict.redactedText === "string" ? verdict.redactedText : text };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* A refusal from `screenLocally`, made loud and durable: the reason on stderr,
|
|
165
|
+
* and a no-send row on THIS seat's ledger (the seat whose body was refused),
|
|
166
|
+
* keyed the way `tool-surface.executeOrgTool` keys its own refusals, so the
|
|
167
|
+
* daemon that spawned the session records "the send was refused" rather than
|
|
168
|
+
* an unexplained silence. Never throws; the bookkeeping cannot change the exit.
|
|
169
|
+
*
|
|
170
|
+
* @param {{code:string, reason:string}} refusal
|
|
171
|
+
* @param {{channel:string, agentRoot:string}} where
|
|
172
|
+
*/
|
|
173
|
+
async function reportRefusal(refusal, where) {
|
|
174
|
+
console.error(`send-orgmail: BLOCKED (${refusal.code}) — ${refusal.reason}`);
|
|
175
|
+
if (refusal.code === "BLOCKED") {
|
|
176
|
+
console.error("send-orgmail: nothing was sent and nothing was edited. Rewrite the body to clear the stated rule and run this again.");
|
|
177
|
+
}
|
|
178
|
+
try {
|
|
179
|
+
const { recordNoSend } = await import("../../lib/comms/receipts.mjs");
|
|
180
|
+
recordNoSend({
|
|
181
|
+
decision: "refused",
|
|
182
|
+
service: "orgmail",
|
|
183
|
+
channel: where.channel,
|
|
184
|
+
code: refusal.code === "BLOCKED" ? "CONTENT_POLICY" : refusal.code,
|
|
185
|
+
reason: refusal.reason,
|
|
186
|
+
surface: "session",
|
|
187
|
+
// A content verdict clears on a rewrite and an unloadable gate on a fix,
|
|
188
|
+
// so neither is final — calling it permanent is how a fixable refusal
|
|
189
|
+
// becomes a final apology.
|
|
190
|
+
permanent: false,
|
|
191
|
+
agentRoot: where.agentRoot,
|
|
192
|
+
});
|
|
193
|
+
} catch { /* bookkeeping must never change a refusal into anything else */ }
|
|
125
194
|
}
|
|
126
195
|
|
|
127
196
|
async function main() {
|
|
197
|
+
// Fleet back-compat FIRST: bridge NEOLITH_* ⇄ COHORT_* env names before the
|
|
198
|
+
// env-first credential resolution (COHORT_API_TOKEN et al.) kicks in. Inside
|
|
199
|
+
// main, not at module scope, so a test can import `screenLocally` without
|
|
200
|
+
// this mutating its environment.
|
|
201
|
+
applyBrandEnvCompat();
|
|
128
202
|
const flags = parseArgs(process.argv.slice(2));
|
|
129
203
|
if (flags.help) { usage(); process.exit(0); }
|
|
130
204
|
|
|
@@ -141,20 +215,29 @@ async function main() {
|
|
|
141
215
|
}
|
|
142
216
|
|
|
143
217
|
const cfg = resolveOrgToolConfig({});
|
|
144
|
-
if (!cfg.token || !cfg.base) {
|
|
145
|
-
console.error("send-orgmail: no org credential — set COHORT_API_TOKEN or run `cohort setup --only org`");
|
|
146
|
-
process.exit(1);
|
|
147
|
-
}
|
|
148
|
-
const o = { base: cfg.base, token: cfg.token, orgId: cfg.orgId };
|
|
149
218
|
|
|
219
|
+
// THE SCREEN RUNS BEFORE THE CREDENTIAL CHECK, as in `lib/org/messaging.mjs`:
|
|
220
|
+
// a body the gate refuses is refused on every seat, enrolled or not, and the
|
|
221
|
+
// sender hears the rule rather than an unrelated configuration error.
|
|
150
222
|
// Not first contact when we are replying into an existing thread or message.
|
|
151
223
|
const isFirstContact = !flags.thread && !flags.replyToMessageId;
|
|
152
|
-
const
|
|
224
|
+
const screened = await screenLocally(
|
|
153
225
|
String(body),
|
|
154
226
|
[...flags.to, ...flags.cc, ...flags.bcc],
|
|
155
227
|
cfg.agentRoot,
|
|
156
228
|
isFirstContact,
|
|
157
229
|
);
|
|
230
|
+
if (!screened.ok) {
|
|
231
|
+
await reportRefusal(screened, { channel: flags.thread || flags.to[0] || flags.replyToMessageId, agentRoot: cfg.agentRoot });
|
|
232
|
+
process.exit(1);
|
|
233
|
+
}
|
|
234
|
+
const text = screened.text;
|
|
235
|
+
|
|
236
|
+
if (!cfg.token || !cfg.base) {
|
|
237
|
+
console.error("send-orgmail: no org credential — set COHORT_API_TOKEN or run `cohort setup --only org`");
|
|
238
|
+
process.exit(1);
|
|
239
|
+
}
|
|
240
|
+
const o = { base: cfg.base, token: cfg.token, orgId: cfg.orgId };
|
|
158
241
|
|
|
159
242
|
// ONE business id per invocation; hq dedups on (mailbox, idempotencyId).
|
|
160
243
|
const idempotencyId = randomUUID();
|
|
@@ -261,7 +344,11 @@ async function main() {
|
|
|
261
344
|
process.exit(1);
|
|
262
345
|
}
|
|
263
346
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
347
|
+
// Run as a CLI only when invoked directly, so a test can import `screenLocally`
|
|
348
|
+
// (the same guard `send-cohort-message.mjs` uses).
|
|
349
|
+
if (process.argv[1] && process.argv[1].endsWith("send-orgmail.mjs")) {
|
|
350
|
+
main().catch((err) => {
|
|
351
|
+
console.error(`send-orgmail: ${err && err.message ? err.message : err}`);
|
|
352
|
+
process.exit(1);
|
|
353
|
+
});
|
|
354
|
+
}
|
|
@@ -453,6 +453,20 @@ export async function runSupervisor(deps = {}) {
|
|
|
453
453
|
}
|
|
454
454
|
}
|
|
455
455
|
}
|
|
456
|
+
// 4d. A restart requested via `maestro session restart` is honoured by
|
|
457
|
+
// THIS launch: whatever process the directive reached has already
|
|
458
|
+
// exited (or the job was bounced outright), and this launch is the
|
|
459
|
+
// relaunch it asked for. Retire the marker, so `maestro session
|
|
460
|
+
// status` stops reporting "restart requested" for a restart that has
|
|
461
|
+
// already happened — left in place, it never clears on its own.
|
|
462
|
+
{
|
|
463
|
+
let restartRequested = false;
|
|
464
|
+
try { d.readFileSync(paths.restartRequestedFile, "utf8"); restartRequested = true; } catch { restartRequested = false; }
|
|
465
|
+
if (restartRequested) {
|
|
466
|
+
try { d.unlinkSync(paths.restartRequestedFile); d.log("restart requested honoured by this launch; marker retired"); }
|
|
467
|
+
catch (e) { d.log(`restart requested honoured but not retired: ${e && e.message ? e.message : e}`); }
|
|
468
|
+
}
|
|
469
|
+
}
|
|
456
470
|
const launched = recordLaunch(record, { now: d.now });
|
|
457
471
|
const saved = saveMainSession(agentRoot, launched, fsDeps);
|
|
458
472
|
if (!saved.ok) d.log(`main-session.json not saved: ${saved.error}`);
|