@cohortapp/agent-sdk 2.18.16 → 2.18.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -174,11 +174,29 @@ export function frontDoorServices(config) {
|
|
|
174
174
|
* `sessionConsumed` (an older reader, a direct caller) never steals the item,
|
|
175
175
|
* so behaviour without the field is exactly the pre-DM-latency gate.
|
|
176
176
|
*
|
|
177
|
-
*
|
|
177
|
+
* PER-ITEM OVERRIDE — `itemHandled` (DM-LATENCY DOUBLE-ANSWER, 2026-09-27). The
|
|
178
|
+
* `sessionConsumed` clause is a SEAT-WIDE fact: a live door that has not stamped
|
|
179
|
+
* `last-consumed.json` looks not-consuming for EVERY item, so the daemon takes
|
|
180
|
+
* over the whole service. But a door that has already answered a SPECIFIC item
|
|
181
|
+
* left a marker on that item on disk — a terminal `.yaml.processed`, or a live
|
|
182
|
+
* `claimed_by: session` `.yaml.dispatched` — through `maestro inbox
|
|
183
|
+
* claim/reply/done`, which is OLD code every door already runs REGARDLESS of
|
|
184
|
+
* whether it restarted onto the stamp path. So when the caller has read that
|
|
185
|
+
* marker and passes `itemHandled:true`, the daemon must NOT re-take that item,
|
|
186
|
+
* even mid-takeover — this is what closes the double-answer without requiring a
|
|
187
|
+
* fleet restart. It short-circuits FIRST and unconditionally: an already-handled
|
|
188
|
+
* item is the door's whether the door is live, wedged or dead. Opt-in on an
|
|
189
|
+
* explicit `true`: an absent `itemHandled` is exactly the pre-fix gate.
|
|
190
|
+
*
|
|
191
|
+
* @param {{frontDoor?:string, sessionLive?:boolean, sessionConsumed?:boolean, itemHandled?:boolean, service?:string, services?:string[]}} a
|
|
178
192
|
* @returns {{dispatch:boolean, reason:string}}
|
|
179
193
|
*/
|
|
180
194
|
export function shouldDaemonDispatch(a) {
|
|
181
195
|
const x = a && typeof a === "object" ? a : {};
|
|
196
|
+
// A specific item the door already holds on disk is never re-taken — not even
|
|
197
|
+
// when the seat-wide gate would open because a live door has not stamped
|
|
198
|
+
// last-consumed. See PER-ITEM OVERRIDE above.
|
|
199
|
+
if (x.itemHandled === true) return { dispatch: false, reason: "item-already-handled" };
|
|
182
200
|
if (x.frontDoor !== "session") return { dispatch: true, reason: "front-door-daemon" };
|
|
183
201
|
if (x.sessionLive !== true) return { dispatch: true, reason: "session-not-live" };
|
|
184
202
|
const services = Array.isArray(x.services) && x.services.length ? x.services : DEFAULT_FRONT_DOOR_SERVICES;
|
|
@@ -210,6 +210,64 @@ export function findInboxFile(agentRoot, id, o = {}) {
|
|
|
210
210
|
return looseIds.size === 1 ? loose : null;
|
|
211
211
|
}
|
|
212
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Does the front door ALREADY hold this specific inbound item on disk? A
|
|
215
|
+
* terminal `.yaml.processed` / `.yaml.processed-bundled` marker (the door read
|
|
216
|
+
* it and ran `inbox done`), or a live `claimed_by: session` `.yaml.dispatched`
|
|
217
|
+
* claim, both mean the door has this item — so a daemon takeover must NOT answer
|
|
218
|
+
* it a second time in the seat's name (DM-LATENCY DOUBLE-ANSWER, 2026-09-27),
|
|
219
|
+
* even when `last-consumed.json` was never stamped because the door has not
|
|
220
|
+
* restarted onto the stamp path.
|
|
221
|
+
*
|
|
222
|
+
* Keyed on the inbound id / raw_ref across the WHOLE service dir, NOT on an
|
|
223
|
+
* exact filename — and that is the crux of the fix. `writeInboxItem`
|
|
224
|
+
* (scripts/poller/utils.mjs) dedups on the exact `<ts>-<id>` base, so a re-fetch
|
|
225
|
+
* that lands the SAME message under a different timestamp base slips its dedup
|
|
226
|
+
* and writes a fresh live `.yaml` beside the door's terminal file. The seat then
|
|
227
|
+
* carries two files for one message — one answered, one live — and the live one
|
|
228
|
+
* is what the daemon scans. An id-keyed sweep sees the answered sibling; the
|
|
229
|
+
* exact-path dedup structurally cannot.
|
|
230
|
+
*
|
|
231
|
+
* A daemon-owned `.dispatched` (no session marker) is NOT the door holding it —
|
|
232
|
+
* it is the daemon's own in-flight admission, and must never block the daemon's
|
|
233
|
+
* own reclaim — so only a `claimed_by` claim counts. Fail-open in the safe
|
|
234
|
+
* direction: an unreadable dir or claim body reads as NOT handled (the item is
|
|
235
|
+
* answered, never dropped), the same bias as the rest of the DM-latency
|
|
236
|
+
* contract; the assurance sweep releases a genuinely stale claim (>20 min) back
|
|
237
|
+
* to `.yaml`, so a dead door can never permanently hold an item this way.
|
|
238
|
+
*
|
|
239
|
+
* @param {string} agentRoot
|
|
240
|
+
* @param {string} service
|
|
241
|
+
* @param {{id?:string, raw_ref?:string}} item
|
|
242
|
+
* @returns {{handled:boolean, reason:string}}
|
|
243
|
+
*/
|
|
244
|
+
export function itemHandledByDoor(agentRoot, service, item) {
|
|
245
|
+
const needles = [item && item.id, item && item.raw_ref]
|
|
246
|
+
.filter(Boolean)
|
|
247
|
+
.map((n) => String(n).trim())
|
|
248
|
+
.filter((n) => n.length > 0);
|
|
249
|
+
if (!needles.length) return { handled: false, reason: "no-id" };
|
|
250
|
+
const dir = join(agentRoot, "state", "inbox", service);
|
|
251
|
+
let names;
|
|
252
|
+
try { names = readdirSync(dir); } catch { return { handled: false, reason: "no-dir" }; }
|
|
253
|
+
for (const file of names) {
|
|
254
|
+
if (!needles.some((n) => file.includes(n))) continue;
|
|
255
|
+
if (file.endsWith(".yaml.processed") || file.endsWith(".yaml.processed-bundled")) {
|
|
256
|
+
return { handled: true, reason: "processed" };
|
|
257
|
+
}
|
|
258
|
+
if (file.endsWith(".yaml.dispatched")) {
|
|
259
|
+
// Only a SESSION claim (`claimed_by`) is the door holding it; a
|
|
260
|
+
// daemon-owned `.dispatched` carries no marker and is the daemon's own.
|
|
261
|
+
try {
|
|
262
|
+
if (parseSessionMarkers(readFileSync(join(dir, file), "utf-8")).claimedBy) {
|
|
263
|
+
return { handled: true, reason: "claimed-by-session" };
|
|
264
|
+
}
|
|
265
|
+
} catch { /* unreadable claim body → not proven handled (fail-open to answering) */ }
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return { handled: false, reason: "not-handled" };
|
|
269
|
+
}
|
|
270
|
+
|
|
213
271
|
/**
|
|
214
272
|
* Claim an item for the session: `.yaml` → `.yaml.dispatched` (the shared
|
|
215
273
|
* durable-admission marker) + `claimed_by: "session"`. Only a NEW item is
|
|
@@ -470,6 +528,7 @@ export default {
|
|
|
470
528
|
parseSessionMarkers,
|
|
471
529
|
stripSessionMarkers,
|
|
472
530
|
stripMarkersInPlace,
|
|
531
|
+
itemHandledByDoor,
|
|
473
532
|
findInboxFile,
|
|
474
533
|
markSessionClaim,
|
|
475
534
|
markSessionDeferral,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cohortapp/agent-sdk",
|
|
3
|
-
"version": "2.18.
|
|
3
|
+
"version": "2.18.17",
|
|
4
4
|
"description": "Cohort Agent SDK — autonomous AI colleague runtime. Deploy senior AI colleagues on dedicated Mac minis, wired to the Cohort operating surface.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -163,9 +163,9 @@ import { processOne } from "../../lib/execution/pipeline.mjs";
|
|
|
163
163
|
import { shouldReviveFrontDoor, reviveCommand, sessionJobLabelOnDisk, confirmRevived, isTerminalReviveFailure, detectUnrequestedRestart } from "../../lib/session/revive.mjs";
|
|
164
164
|
import { budgetSpentNote } from "../../lib/telemetry/collect.mjs";
|
|
165
165
|
import { checkLock as checkProcessLock } from "../../lib/singleton.js";
|
|
166
|
-
import { makeFrontDoorGate, sessionLiveFromHeartbeat, readFrontDoorState, DEFAULT_STALE_MS } from "../../lib/session/frontdoor.mjs";
|
|
166
|
+
import { makeFrontDoorGate, shouldDaemonDispatch, sessionLiveFromHeartbeat, readFrontDoorState, DEFAULT_STALE_MS } from "../../lib/session/frontdoor.mjs";
|
|
167
167
|
import { CADENCE_REGISTRY } from "./cadence-handlers.mjs";
|
|
168
|
-
import { sweepSessionInboxForDaemon } from "../../lib/session/inbox-claims.mjs";
|
|
168
|
+
import { sweepSessionInboxForDaemon, itemHandledByDoor } from "../../lib/session/inbox-claims.mjs";
|
|
169
169
|
import { defaultEffects, scheduleToQueue } from "../../lib/execution/effects.mjs";
|
|
170
170
|
import { loadReactObligations } from "../../lib/execution/match.mjs";
|
|
171
171
|
import { policyFor } from "../../lib/execution/surface-policy.mjs";
|
|
@@ -595,6 +595,25 @@ export async function pollService(svc, deps = {}) {
|
|
|
595
595
|
return false;
|
|
596
596
|
}
|
|
597
597
|
|
|
598
|
+
// DM-LATENCY DOUBLE-ANSWER GUARD (2026-09-27). Past the seat-wide front-door
|
|
599
|
+
// gate, this is the per-item check the gate cannot make: has the door
|
|
600
|
+
// ALREADY handled THIS item on disk? The gate opened for the service because
|
|
601
|
+
// a live door had not stamped last-consumed (reason `session-not-consuming`),
|
|
602
|
+
// but the door that read+closed a specific message left its marker
|
|
603
|
+
// (`.yaml.processed`, or a live `claimed_by: session` `.yaml.dispatched`) on
|
|
604
|
+
// that item regardless of whether it restarted onto the stamp path. Taking
|
|
605
|
+
// it here posts a SECOND reply in the seat owner's name and races the door's
|
|
606
|
+
// own queue writer — the exact regression (Hannah Brooks, James's 18:42Z DM).
|
|
607
|
+
// `shouldDaemonDispatch` makes the call so the one predicate owns the whole
|
|
608
|
+
// gate; `itemHandledByDoor` is the id-keyed disk read it needs. Fail-open:
|
|
609
|
+
// an unreadable marker reads as not-handled, so a real ask is never dropped.
|
|
610
|
+
const handled = itemHandledByDoor(AGENT_REPO_DIR, svc.name, item);
|
|
611
|
+
const perItem = shouldDaemonDispatch({ ...(fd.state || {}), service: svc.name, itemHandled: handled.handled });
|
|
612
|
+
if (!perItem.dispatch) {
|
|
613
|
+
console.log(`[daemon] leaving ${item.id || item.raw_ref} to the front door on ${svc.name} (${perItem.reason}${handled.reason ? `: ${handled.reason}` : ""})`);
|
|
614
|
+
return false;
|
|
615
|
+
}
|
|
616
|
+
|
|
598
617
|
const lockKey = item.raw_ref || item.id || `${svc.name}-${Date.now()}`;
|
|
599
618
|
if (seenRefs.has(lockKey)) return false;
|
|
600
619
|
seenRefs.add(lockKey);
|
|
@@ -714,6 +714,72 @@ export function scaffoldMarkerLeak(text) {
|
|
|
714
714
|
return m ? m[0] : null;
|
|
715
715
|
}
|
|
716
716
|
|
|
717
|
+
// ── A REPLY IN THE SEAT'S VOICE NEVER CARRIES A MACHINE-LOCAL PATH ────────────
|
|
718
|
+
// (DM-LATENCY, 2026-09-27.) When the daemon takeover composed a reply in the
|
|
719
|
+
// seat owner's name, it leaked an absolute local filesystem path — a string no
|
|
720
|
+
// one off this Mac mini can open, and one CLAUDE.md forbids in any outbound.
|
|
721
|
+
// The absolute forms below (`/Users/…`, `/Volumes/…`, `/private/…`,
|
|
722
|
+
// `/var/folders/…`, `/tmp/…`, `/home/…`) are unambiguously machine-local: unlike
|
|
723
|
+
// a bare `lib/foo.mjs` a person might legitimately name in an engineering
|
|
724
|
+
// channel, none of these is ever a shareable reference, so refusing them is
|
|
725
|
+
// high-precision. Permanent refusal (no retry), the same shape as the scaffold
|
|
726
|
+
// gate: the caller escalates rather than re-send, and the double-answer this
|
|
727
|
+
// backstops should not have been composed at all (FIX 1 stops it upstream).
|
|
728
|
+
const LOCAL_PATH_RE = /(?:\/(?:Users|Volumes|private|home)\/|\/var\/folders\/|\/tmp\/)[^\s`)"'<>|]+/;
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* The first machine-local absolute path in `text`, or null when it carries none.
|
|
732
|
+
* @param {string} text
|
|
733
|
+
* @returns {string|null}
|
|
734
|
+
*/
|
|
735
|
+
export function localPathLeak(text) {
|
|
736
|
+
const m = typeof text === "string" ? text.match(LOCAL_PATH_RE) : null;
|
|
737
|
+
return m ? m[0] : null;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// ── STRIP A LEADING SEAT-FLUSH / RECOVERY FILLER OPENER ───────────────────────
|
|
741
|
+
// The takeover posts the owner would never write opened with a flush apology —
|
|
742
|
+
// "finished this but didn't get a reply out", "catching up on my last" and the
|
|
743
|
+
// like (the Layla/Eli/Candace symptom, 2026-09-27). It is the daemon narrating
|
|
744
|
+
// its OWN plumbing to a human who asked about something else. Unlike a leaked
|
|
745
|
+
// path this is safely removable: only a LEADING filler sentence is stripped, and
|
|
746
|
+
// the substantive answer beneath it is left byte-for-byte. Conservative by
|
|
747
|
+
// design — it matches the flush family and a couple of generic recovery openers,
|
|
748
|
+
// never an arbitrary first sentence, so a real answer that merely starts with
|
|
749
|
+
// "I finished the audit…" is untouched (no reply/flush verb follows).
|
|
750
|
+
const REPLY_FILLER_OPENERS = [
|
|
751
|
+
// "(I've) finished/completed/did this, but the reply didn't get out / never came back / catching up …"
|
|
752
|
+
/^\s*(?:i(?:['’]ve|['’]m| have| am)?\s+)?(?:finished|completed|did|wrapped\s+up|ran)\s+this\b[^.!?\n]*?\b(?:repl(?:y|ies)|did(?:n['’]?t| not)|never|go(?:t)?\s+out|come\s+back|came\s+back|catch(?:ing)?\s+up|flush)[^.!?\n]*[.!?]+\s*/i,
|
|
753
|
+
// Generic flush / recovery openers.
|
|
754
|
+
/^\s*(?:heads[- ]up[—,\-: ]+)?(?:just\s+)?(?:flushing|catching\s+up|picking\s+this\s+back\s+up|circling\s+back|following\s+up\s+on\s+my\s+last|sorry[—,\-: ]+(?:for\s+)?the\s+(?:double|delay|resend))[^.!?\n]*[.!?]+\s*/i,
|
|
755
|
+
];
|
|
756
|
+
|
|
757
|
+
/**
|
|
758
|
+
* Remove a single leading seat-flush / recovery filler sentence, but ONLY when a
|
|
759
|
+
* real answer remains beneath it. Returns the input unchanged when there is no
|
|
760
|
+
* leading filler, when the input is not a string, OR when stripping the opener
|
|
761
|
+
* would leave nothing — because that last case cannot be told apart from a
|
|
762
|
+
* filler CLAUSE that runs into the answer on one line ("finished this but the
|
|
763
|
+
* reply didn't go out — the deploy is green"), and destroying a real answer is
|
|
764
|
+
* far worse than letting a bare filler line through. That bare-filler case is
|
|
765
|
+
* the double-post itself, which FIX 1 stops upstream; here the rule is simply
|
|
766
|
+
* "never eat content". Only the first matching opener is considered.
|
|
767
|
+
*
|
|
768
|
+
* @param {unknown} text
|
|
769
|
+
* @returns {string}
|
|
770
|
+
*/
|
|
771
|
+
export function stripReplyFiller(text) {
|
|
772
|
+
if (typeof text !== "string") return text;
|
|
773
|
+
for (const re of REPLY_FILLER_OPENERS) {
|
|
774
|
+
const m = text.match(re);
|
|
775
|
+
if (m) {
|
|
776
|
+
const rest = text.slice(m[0].length).trimStart();
|
|
777
|
+
return rest ? rest : text; // strip only when an answer survives; never eat content
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
return text;
|
|
781
|
+
}
|
|
782
|
+
|
|
717
783
|
/**
|
|
718
784
|
* Deliver `text` to the human behind `item`. No generation, no model, no
|
|
719
785
|
* budget — only transport.
|
|
@@ -805,6 +871,10 @@ export async function deliverReaction(item, emoji, o = {}) {
|
|
|
805
871
|
|
|
806
872
|
export async function deliver(item, text, o = {}) {
|
|
807
873
|
const kind = o.kind || "reply";
|
|
874
|
+
// FIX 2 (DM-LATENCY, 2026-09-27): strip a leading seat-flush / recovery filler
|
|
875
|
+
// opener the door would never write, BEFORE the empty check — a body that was
|
|
876
|
+
// nothing but filler then correctly falls through to "nothing to deliver".
|
|
877
|
+
text = stripReplyFiller(text);
|
|
808
878
|
if (!item || !text || !String(text).trim()) {
|
|
809
879
|
return { sent: false, via: null, error: "nothing to deliver" };
|
|
810
880
|
}
|
|
@@ -859,6 +929,16 @@ export async function deliver(item, text, o = {}) {
|
|
|
859
929
|
try { console.error(`[deliver] refused: raw scaffold marker ${JSON.stringify(leak)} in outbound body (permanent, not retrying)`); } catch { /* */ }
|
|
860
930
|
return { sent: false, via: null, permanent: true, channel: item && item.channel_id, surface, code: "FORBIDDEN_SCOPE", error: `FORBIDDEN_SCOPE: raw scaffold marker ${leak} in outbound body — strip the marker; never send reasoning or transcript wrappers` };
|
|
861
931
|
}
|
|
932
|
+
// FIX 2 (DM-LATENCY, 2026-09-27): a reply in the seat's voice must never carry a
|
|
933
|
+
// machine-local path — no one off this box can open it, and CLAUDE.md forbids
|
|
934
|
+
// it in any outbound. Permanent refusal, same shape as the scaffold gate: the
|
|
935
|
+
// caller escalates rather than leak the path in the seat owner's name.
|
|
936
|
+
const pathLeak = localPathLeak(text);
|
|
937
|
+
if (pathLeak) {
|
|
938
|
+
const surface = (item && item.channel_id) ? "room" : "unknown";
|
|
939
|
+
try { console.error(`[deliver] refused: machine-local path ${JSON.stringify(pathLeak)} in outbound body (permanent, not retrying)`); } catch { /* */ }
|
|
940
|
+
return { sent: false, via: null, permanent: true, channel: item && item.channel_id, surface, code: "FORBIDDEN_SCOPE", error: `FORBIDDEN_SCOPE: machine-local path ${pathLeak} in outbound body — no one off this machine can open it; a reply must not carry a local path` };
|
|
941
|
+
}
|
|
862
942
|
// `permanent` says retrying changes nothing. A caller that cannot tell the
|
|
863
943
|
// difference between "hq blipped" and "there is no transport for telegram"
|
|
864
944
|
// retries both at the same cadence, which is how an impossible send becomes an
|
|
@@ -65,9 +65,20 @@
|
|
|
65
65
|
# and the running daemon's own boot marker — "[DAEMON] boot pid=<pid>"
|
|
66
66
|
# (maestro-daemon.mjs, from the release that added this gate), else the last "[daemon] Running" /
|
|
67
67
|
# "org-mesh connected" line. A crash earlier the same day that launchd
|
|
68
|
-
# (or an operator) already recovered from is history, not health.
|
|
69
|
-
#
|
|
70
|
-
#
|
|
68
|
+
# (or an operator) already recovered from is history, not health.
|
|
69
|
+
# The log is NOT always under $AGENT_DIR: launchd-wrapper.sh redirects it
|
|
70
|
+
# to an external SSD ($SSD_VOLUME/maestro/<agent>/logs/daemon) when one is
|
|
71
|
+
# mounted+writable. This scan resolves that same path (daemon_log_base,
|
|
72
|
+
# mirroring the wrapper's own SSD derivation) so a redirected seat is gated
|
|
73
|
+
# on its REAL log — not a fixed $AGENT_DIR path that never exists there,
|
|
74
|
+
# which silently disabled this whole check (and, on gates before the
|
|
75
|
+
# missing-log fail-open, rolled EVERY upgrade back on SSD seats). When even
|
|
76
|
+
# the resolved log cannot be read, the gate does NOT blindly fail-open:
|
|
77
|
+
# it falls back to the path-independent cadence heartbeat
|
|
78
|
+
# state/cadence-bus/health.json (lib/cadence-bus.mjs writeHealth, every
|
|
79
|
+
# ~15s, under the symlinked state/ tree) — fresh (< HEALTH_FRESH_S, 120s)
|
|
80
|
+
# proves the daemon loop is cycling → pass; stale/absent → no log AND no
|
|
81
|
+
# heartbeat is genuine unhealth → fail (checks (a)+(c) still corroborate).
|
|
71
82
|
# (c) A SERVER-ACKNOWLEDGED BEAT — state/org/last-beat.json {at, ok, code?,
|
|
72
83
|
# sdkVersion?}, which lib/org/mesh.mjs overwrites on EVERY presence beat
|
|
73
84
|
# (ok:true only when the org answered 2xx; a rejected or unreachable beat
|
|
@@ -147,7 +158,10 @@
|
|
|
147
158
|
# (default 600) and MAESTRO_AUTOUPDATE_HEALTH_WAIT (default 30), both seconds;
|
|
148
159
|
# MAESTRO_AUTOUPDATE_STABLE_S (default 90; MAESTRO_AUTOUPDATE_STABLE_GAP_S is the
|
|
149
160
|
# sleep between the two pid samples, default = STABLE_S, zeroed by the test) and
|
|
150
|
-
# MAESTRO_AUTOUPDATE_BEAT_FRESH_S (default 300);
|
|
161
|
+
# MAESTRO_AUTOUPDATE_BEAT_FRESH_S (default 300); MAESTRO_AUTOUPDATE_HEALTH_FRESH_S
|
|
162
|
+
# (default 120 — the cadence-heartbeat staleness bar for the log-absent fallback);
|
|
163
|
+
# MAESTRO_SSD_VOLUME (the daemon-log SSD volume, mirroring launchd-wrapper.sh; the
|
|
164
|
+
# test points it at a fixture dir); MAESTRO_AUTOUPDATE_RETRY_SLEEP
|
|
151
165
|
# (default 20 s, ×attempt);
|
|
152
166
|
# MAESTRO_AUTOUPDATE_LOCK_STALE_S (default 7200); MAESTRO_AUTOUPDATE_FAILED_HOLD_S
|
|
153
167
|
# (default 86400); MAESTRO_AUTOUPDATE_PATH_PREFIX (a stub-binary dir that wins
|
|
@@ -264,6 +278,42 @@ DAEMON_PATTERN="$AGENT_DIR/scripts/daemon/maestro-daemon.mjs"
|
|
|
264
278
|
# a pid-set change, i.e. "crash loop".
|
|
265
279
|
DAEMON_ARGV_RE="^[^ ]*/node $(printf '%s' "$DAEMON_PATTERN" | sed 's/[][\.*^$]/\\&/g')\$"
|
|
266
280
|
DLOG_DIR="$AGENT_DIR/logs/daemon"
|
|
281
|
+
# The daemon log does NOT always live under $AGENT_DIR. launchd-wrapper.sh
|
|
282
|
+
# redirects the daemon's stdout/stderr to an external SSD when one is mounted
|
|
283
|
+
# and writable — $SSD_VOLUME/maestro/<agent>/logs/daemon/ — falling back to
|
|
284
|
+
# $AGENT_DIR/logs/daemon only when that write is denied. is_healthy reads the
|
|
285
|
+
# daemon log for its fatal scan (b); keyed to a fixed $AGENT_DIR path it reads a
|
|
286
|
+
# file that never exists on a redirected seat, which SILENTLY DISABLES the fatal
|
|
287
|
+
# scan there (and, on gates older than the missing-log fail-open, rolled EVERY
|
|
288
|
+
# upgrade back — 2026-09 SSD seats). Resolve the SSD dir the SAME way the wrapper
|
|
289
|
+
# does (its own MAESTRO_SSD_VOLUME / /Volumes/*-SSD derivation) — this is the
|
|
290
|
+
# real path, not a widened guess — and, as a path-independent backstop robust to
|
|
291
|
+
# ANY redirect, cross-check the cadence heartbeat state/cadence-bus/health.json
|
|
292
|
+
# (written under the symlinked state/ tree every ~15s by lib/cadence-bus.mjs
|
|
293
|
+
# writeHealth) whenever the resolved log still cannot be read.
|
|
294
|
+
ssd_daemon_log_dir(){ # echo the SSD daemon-log dir the wrapper would have opened, or nothing when no SSD is mounted (mirrors launchd-wrapper.sh)
|
|
295
|
+
local vol name v
|
|
296
|
+
vol="${MAESTRO_SSD_VOLUME:-}"
|
|
297
|
+
if [ -z "$vol" ]; then
|
|
298
|
+
for v in /Volumes/*-SSD /Volumes/*SSD* /Volumes/maestro-data; do
|
|
299
|
+
if [ -d "$v" ] && [ "$v" != "/Volumes/Macintosh HD" ]; then vol="$v"; break; fi
|
|
300
|
+
done
|
|
301
|
+
fi
|
|
302
|
+
[ -n "$vol" ] && [ -d "$vol" ] || return 0
|
|
303
|
+
name="$(basename "$AGENT_DIR" | sed 's/-ai$//')"
|
|
304
|
+
echo "$vol/maestro/$name/logs/daemon"
|
|
305
|
+
}
|
|
306
|
+
DLOG_DIR_SSD="$(ssd_daemon_log_dir)"
|
|
307
|
+
daemon_log_base(){ # $1 = YYYY-MM-DD → the dir holding that day's daemon log: the SSD copy when the wrapper wrote it there, else $AGENT_DIR. Resolves by where the file IS (the wrapper picks per-boot and can fall back), so a redirected AND a fallback seat both read the right file.
|
|
308
|
+
if [ -n "$DLOG_DIR_SSD" ] && [ -f "$DLOG_DIR_SSD/daemon-$1.log" ]; then echo "$DLOG_DIR_SSD"; else echo "$DLOG_DIR"; fi
|
|
309
|
+
}
|
|
310
|
+
# The cadence heartbeat: state/cadence-bus/health.json {version,ts,pid,...},
|
|
311
|
+
# overwritten every ~15s by the consumer (lib/cadence-bus.mjs writeHealth). It
|
|
312
|
+
# lives under the symlinked state/ tree, so it is readable at a fixed path no
|
|
313
|
+
# matter where the daemon LOG went — the path-independent liveness signal the
|
|
314
|
+
# gate falls back to when the resolved daemon log cannot be read.
|
|
315
|
+
HEALTH_JSON="$AGENT_DIR/state/cadence-bus/health.json"
|
|
316
|
+
HEALTH_FRESH_S="${MAESTRO_AUTOUPDATE_HEALTH_FRESH_S:-120}" # 8× the 15s heartbeat cadence, well under the 300s org-offline bar
|
|
267
317
|
HEALTH_WAIT="${MAESTRO_AUTOUPDATE_HEALTH_WAIT:-30}"
|
|
268
318
|
STABLE_S="${MAESTRO_AUTOUPDATE_STABLE_S:-90}"
|
|
269
319
|
STABLE_GAP_S="${MAESTRO_AUTOUPDATE_STABLE_GAP_S:-$STABLE_S}" # the gap between the two pid samples; the test zeroes it (its stub changes per CALL)
|
|
@@ -311,8 +361,22 @@ pid_uptime_s(){ # $1 = pid → seconds since it started, "" when ps cannot say
|
|
|
311
361
|
day_of_epoch(){ # $1 = epoch seconds → YYYY-MM-DD in local time (the wrapper's `date +%Y-%m-%d` basis)
|
|
312
362
|
date -r "$1" +%Y-%m-%d 2>/dev/null || date -d "@$1" +%Y-%m-%d 2>/dev/null
|
|
313
363
|
}
|
|
314
|
-
daemon_log_for(){ # $1 = uptime seconds → the log file the wrapper opened for THIS incarnation (named for its start day)
|
|
315
|
-
|
|
364
|
+
daemon_log_for(){ # $1 = uptime seconds → the log file the wrapper opened for THIS incarnation (named for its start day, in the dir it actually writes — SSD or $AGENT_DIR)
|
|
365
|
+
local day; day="$(day_of_epoch $(( $(date +%s) - $1 )))"
|
|
366
|
+
echo "$(daemon_log_base "$day")/daemon-$day.log"
|
|
367
|
+
}
|
|
368
|
+
cadence_heartbeat_stale(){ # prints WHY the cadence heartbeat is not a fresh proof of a live daemon loop, or nothing when it is (state/cadence-bus/health.json, path-independent)
|
|
369
|
+
node -e '
|
|
370
|
+
const [file, freshS] = process.argv.slice(1);
|
|
371
|
+
const out = (s) => process.stdout.write(s);
|
|
372
|
+
let j;
|
|
373
|
+
try { j = JSON.parse(require("fs").readFileSync(file, "utf8")); }
|
|
374
|
+
catch (e) { out(`daemon log unreadable at the resolved path and no cadence heartbeat (${e && e.code === "ENOENT" ? "state/cadence-bus/health.json absent" : "state/cadence-bus/health.json unreadable"})`); process.exit(0); }
|
|
375
|
+
const at = Date.parse(j && j.ts);
|
|
376
|
+
if (!Number.isFinite(at)) { out("daemon log unreadable at the resolved path and cadence health.json carries no parseable ts"); process.exit(0); }
|
|
377
|
+
const ageS = Math.round((Date.now() - at) / 1000);
|
|
378
|
+
if (ageS > Number(freshS)) out(`daemon log unreadable at the resolved path and the cadence heartbeat is ${ageS}s old (> ${freshS}s) — the daemon loop is not cycling`);
|
|
379
|
+
' "$HEALTH_JSON" "$HEALTH_FRESH_S" 2>/dev/null
|
|
316
380
|
}
|
|
317
381
|
first_fatal_after(){ # $1 = log file, $2 = line offset (lines up to it predate this daemon) → the first fatal line, or nothing
|
|
318
382
|
[ -f "$1" ] || return 0
|
|
@@ -424,11 +488,26 @@ is_healthy(){ # (a)+(b)+(c) above. Sets HEALTH_REASON on failure.
|
|
|
424
488
|
[ -n "$oldest" ] && [ "$oldest" -ge "$up" ] || { oldest="$up"; oldest_pid="$pid"; }
|
|
425
489
|
done
|
|
426
490
|
dlog="$(daemon_log_for "$oldest")"
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
491
|
+
if [ -f "$dlog" ]; then
|
|
492
|
+
# The daemon log is readable (at $AGENT_DIR or the SSD path the wrapper
|
|
493
|
+
# redirected it to): scan it for a fatal line since this daemon booted.
|
|
494
|
+
offset=0; [ "$dlog" = "$PRE_LOG" ] && offset="$PRE_LOG_LINES"
|
|
495
|
+
boot="$(boot_offset "$dlog" "$oldest_pid")"
|
|
496
|
+
[ "$boot" -gt "$offset" ] && offset="$boot"
|
|
497
|
+
fatal="$(first_fatal_after "$dlog" "$offset")"
|
|
498
|
+
[ -z "$fatal" ] || { HEALTH_REASON="fatal in $(basename "$dlog") since this daemon booted (scanned from line $(( offset + 1 ))): $(printf '%s' "$fatal" | tr -d '"\\' | cut -c1-160)"; return 1; }
|
|
499
|
+
else
|
|
500
|
+
# The daemon log is not readable even at the SSD-resolved path (a redirect
|
|
501
|
+
# this gate could not resolve, or a brand-new start-day file the daemon has
|
|
502
|
+
# not written yet). Do NOT fail-open blindly (that silently drops check (b)
|
|
503
|
+
# on every such seat) and do NOT roll a healthy release back on a path
|
|
504
|
+
# artefact. Fall back to the path-independent cadence heartbeat: fresh →
|
|
505
|
+
# the daemon's main loop is demonstrably cycling, so the missing log is a
|
|
506
|
+
# path artefact, not death → pass; stale/absent → no log AND no heartbeat is
|
|
507
|
+
# genuine unhealth → fail.
|
|
508
|
+
why="$(cadence_heartbeat_stale)"
|
|
509
|
+
[ -z "$why" ] || { HEALTH_REASON="$why"; return 1; }
|
|
510
|
+
fi
|
|
432
511
|
if org_enrolled; then
|
|
433
512
|
why="$(beat_not_acknowledged $(( $(date +%s) - oldest - 2 )) "$(installed_version)")"
|
|
434
513
|
[ -z "$why" ] || { HEALTH_REASON="$why"; return 1; }
|
|
@@ -463,8 +542,9 @@ label_loaded(){ # $1 = label — measured in the gui domain (right from ssh too)
|
|
|
463
542
|
restart_daemon(){
|
|
464
543
|
resolve_labels
|
|
465
544
|
# Mark "since the restart" for the fatal scan: the new process appends to
|
|
466
|
-
# today's file (the wrapper names it for its start day), after these lines
|
|
467
|
-
|
|
545
|
+
# today's file (the wrapper names it for its start day), after these lines —
|
|
546
|
+
# in the dir the wrapper actually writes (SSD when redirected, else $AGENT_DIR).
|
|
547
|
+
PRE_LOG="$(daemon_log_base "$(date +%Y-%m-%d)")/daemon-$(date +%Y-%m-%d).log"
|
|
468
548
|
PRE_LOG_LINES=0
|
|
469
549
|
[ -f "$PRE_LOG" ] && PRE_LOG_LINES="$(wc -l < "$PRE_LOG" 2>/dev/null | tr -d ' ')"
|
|
470
550
|
PRE_LOG_LINES="${PRE_LOG_LINES:-0}"
|
|
@@ -619,7 +699,7 @@ revive_daemon(){ # the daemon-DOWN backstop. A daemon cannot restart ITSELF —
|
|
|
619
699
|
# saw: the launchctl list line and the tail of the daemon's own log.
|
|
620
700
|
local lc_line dlog_tail evidence
|
|
621
701
|
lc_line="$(launchctl list 2>/dev/null | awk -v l="$DAEMON_LABEL" 'NR>1 && $3==l { print; exit }')"
|
|
622
|
-
dlog_tail="$(tail -n 3 "$
|
|
702
|
+
dlog_tail="$(tail -n 3 "$(daemon_log_base "$(date +%Y-%m-%d)")/daemon-$(date +%Y-%m-%d).log" 2>/dev/null | tr '\n' '|')"
|
|
623
703
|
evidence="assert=${ok:-unknown} samples=${samples} launchctl=[${lc_line:-none}] dlog=[${dlog_tail:-none}]"
|
|
624
704
|
log "daemon $DAEMON_LABEL kickstart did NOT revive it (${ok:-unknown}) — a helper's exit code is not a revive; this seat needs a person (failure to escalate). ${evidence}"
|
|
625
705
|
write_revive_note "daemon-revive-failed" "$DAEMON_LABEL" "$evidence"
|