@inerrata-corporation/errata 2.0.2-dev.1130 → 2.0.2-dev.1166
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/errata.mjs +351 -15
- package/package.json +1 -1
package/errata.mjs
CHANGED
|
@@ -22976,6 +22976,14 @@ function renderSnapshot(s) {
|
|
|
22976
22976
|
);
|
|
22977
22977
|
lines.push("");
|
|
22978
22978
|
}
|
|
22979
|
+
if (s.daemonDistress) {
|
|
22980
|
+
const mins = Math.max(1, Math.round((Date.now() - s.daemonDistress.sinceTs) / 6e4));
|
|
22981
|
+
lines.push("### \u{1F6A8} errata daemon distress");
|
|
22982
|
+
lines.push(
|
|
22983
|
+
`${s.daemonDistress.body} (ongoing ~${mins} min). TELL YOUR USER about this now \u2014 the daemon cannot fix everything itself, and this line may be its only witness. \`errata status\` has detail; a daemon restart (\`errata stop\` then \`errata start\`) is safe mid-session.`
|
|
22984
|
+
);
|
|
22985
|
+
lines.push("");
|
|
22986
|
+
}
|
|
22979
22987
|
if (s.needsRevisit.length > 0) {
|
|
22980
22988
|
lines.push("### \u26A0\uFE0F Needs revisit \u2014 a fact these rested on changed");
|
|
22981
22989
|
lines.push(
|
|
@@ -23288,6 +23296,7 @@ function assembleAgentContext(opts) {
|
|
|
23288
23296
|
});
|
|
23289
23297
|
if (opts.edgeElicitation) snapshot.edgeElicitation = opts.edgeElicitation;
|
|
23290
23298
|
if (opts.pendingUpdate) snapshot.pendingUpdate = opts.pendingUpdate;
|
|
23299
|
+
if (opts.daemonDistress) snapshot.daemonDistress = opts.daemonDistress;
|
|
23291
23300
|
if (opts.reviewItems?.length) snapshot.reviewItems = [...opts.reviewItems];
|
|
23292
23301
|
if (opts.statedIntent) snapshot.statedIntent = opts.statedIntent;
|
|
23293
23302
|
if (opts.remote && opts.remote.length > 0) {
|
|
@@ -24650,6 +24659,7 @@ function defaultConfig() {
|
|
|
24650
24659
|
// user opts into sync at all.
|
|
24651
24660
|
consent: { sync: false, telemetry: false, contributePackages: true },
|
|
24652
24661
|
notifications: true,
|
|
24662
|
+
opsNotices: false,
|
|
24653
24663
|
onboardedAt: null,
|
|
24654
24664
|
machineId: null,
|
|
24655
24665
|
updateChannel: "dev",
|
|
@@ -56427,7 +56437,8 @@ var TITLES = {
|
|
|
56427
56437
|
"hotspot-problem": "Errata \xB7 high-impact problem",
|
|
56428
56438
|
"index-started": "Errata \xB7 indexing\u2026",
|
|
56429
56439
|
"index-completed": "Errata \xB7 index ready \u2713",
|
|
56430
|
-
"mechanism-stalled": "Errata \xB7 a mechanism is not working"
|
|
56440
|
+
"mechanism-stalled": "Errata \xB7 a mechanism is not working",
|
|
56441
|
+
"daemon-distress": "Errata \xB7 daemon health \u26A0"
|
|
56431
56442
|
};
|
|
56432
56443
|
var lastFired = /* @__PURE__ */ new Map();
|
|
56433
56444
|
var THROTTLE_MS = 3e4;
|
|
@@ -56443,6 +56454,14 @@ function notifyEvent(kind, body2, opts = {}) {
|
|
|
56443
56454
|
} catch {
|
|
56444
56455
|
}
|
|
56445
56456
|
}
|
|
56457
|
+
function notifyOpsEvent(kind, body2, opts = {}) {
|
|
56458
|
+
try {
|
|
56459
|
+
if (!loadConfig().opsNotices) return;
|
|
56460
|
+
} catch {
|
|
56461
|
+
return;
|
|
56462
|
+
}
|
|
56463
|
+
notifyEvent(kind, body2, opts);
|
|
56464
|
+
}
|
|
56446
56465
|
function notifyTick(delta) {
|
|
56447
56466
|
const { problemsResolved, reviewsTriggered } = delta;
|
|
56448
56467
|
if (problemsResolved > 0)
|
|
@@ -56532,11 +56551,30 @@ function startLoopLagMonitor(thresholdMs = 1e3) {
|
|
|
56532
56551
|
}
|
|
56533
56552
|
|
|
56534
56553
|
// src/watch-census.ts
|
|
56535
|
-
var census = {
|
|
56554
|
+
var census = {
|
|
56555
|
+
dispatched: 0,
|
|
56556
|
+
emitted: 0,
|
|
56557
|
+
logged: 0,
|
|
56558
|
+
rejected: 0,
|
|
56559
|
+
since: Date.now(),
|
|
56560
|
+
breakerRecycles: 0,
|
|
56561
|
+
breakerQuarantined: [],
|
|
56562
|
+
topOffender: null
|
|
56563
|
+
};
|
|
56564
|
+
var QUARANTINE_LIST_CAP = 8;
|
|
56536
56565
|
var noteWatchDispatched = () => void census.dispatched++;
|
|
56537
56566
|
var noteWatchEmitted = () => void census.emitted++;
|
|
56538
56567
|
var noteWatchLogged = () => void census.logged++;
|
|
56539
56568
|
var noteWatchRejected = () => void census.rejected++;
|
|
56569
|
+
var noteBreakerRecycle = (_path) => void census.breakerRecycles++;
|
|
56570
|
+
var noteBreakerQuarantine = (path2) => {
|
|
56571
|
+
if (!census.breakerQuarantined.includes(path2) && census.breakerQuarantined.length < QUARANTINE_LIST_CAP) {
|
|
56572
|
+
census.breakerQuarantined.push(path2);
|
|
56573
|
+
}
|
|
56574
|
+
};
|
|
56575
|
+
var noteWindowOffender = (path2, count, windowMs) => {
|
|
56576
|
+
census.topOffender = { path: path2, count, windowMs, at: Date.now() };
|
|
56577
|
+
};
|
|
56540
56578
|
function watchCensus() {
|
|
56541
56579
|
return { ...census };
|
|
56542
56580
|
}
|
|
@@ -56544,9 +56582,14 @@ function watchCensusLine(c = watchCensus()) {
|
|
|
56544
56582
|
if (c.dispatched === 0 && c.emitted === 0) return null;
|
|
56545
56583
|
const mins = Math.max(1, Math.round((Date.now() - c.since) / 6e4));
|
|
56546
56584
|
const perMin = Math.round(c.dispatched / mins);
|
|
56547
|
-
|
|
56548
|
-
if (c.
|
|
56549
|
-
|
|
56585
|
+
let base = `${c.dispatched} OS event(s) \u2192 ${c.logged} logged (${perMin}/min, ${c.rejected} rejected here)`;
|
|
56586
|
+
if (c.breakerRecycles > 0 || c.breakerQuarantined.length > 0) {
|
|
56587
|
+
const q = c.breakerQuarantined.length > 0 ? `, ${c.breakerQuarantined.length} quarantined` : "";
|
|
56588
|
+
base += ` \xB7 breaker: ${c.breakerRecycles} recycled${q}`;
|
|
56589
|
+
}
|
|
56590
|
+
if (c.dispatched > 500 && c.dispatched / Math.max(c.logged, 1) > 1e4) {
|
|
56591
|
+
const top = c.topOffender ? ` \u2014 top: ${c.topOffender.path} (${c.topOffender.count}/${Math.round(c.topOffender.windowMs / 1e3)}s)` : "";
|
|
56592
|
+
return `${base} \u26A0 watcher is doing work with NO output \u2014 a watch target likely contains an ignored tree${top}`;
|
|
56550
56593
|
}
|
|
56551
56594
|
return base;
|
|
56552
56595
|
}
|
|
@@ -56588,6 +56631,11 @@ function createLivenessWatch(deps) {
|
|
|
56588
56631
|
watchEmitted: watch2.emitted,
|
|
56589
56632
|
watchLogged: watch2.logged,
|
|
56590
56633
|
watchRejected: watch2.rejected,
|
|
56634
|
+
// HZ-watch-breaker: a recycle count trending upward across hours is a
|
|
56635
|
+
// handle that keeps wedging — the restart-survivable trace the 8-16
|
|
56636
|
+
// phantom-rename storm never left.
|
|
56637
|
+
watchBreakerRecycles: watch2.breakerRecycles,
|
|
56638
|
+
watchBreakerQuarantined: watch2.breakerQuarantined.length,
|
|
56591
56639
|
// Concurrency and the longest single holder — the two readings that
|
|
56592
56640
|
// named the 6017s zombie pass. A pass legitimately running for hours
|
|
56593
56641
|
// and one wedged forever look identical without a trend.
|
|
@@ -56612,8 +56660,108 @@ function createLivenessWatch(deps) {
|
|
|
56612
56660
|
};
|
|
56613
56661
|
}
|
|
56614
56662
|
|
|
56663
|
+
// src/watch-breaker.ts
|
|
56664
|
+
function numEnv3(key, fallback) {
|
|
56665
|
+
const n = Number(process.env[key]);
|
|
56666
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
56667
|
+
}
|
|
56668
|
+
var BREAKER_WINDOW_MS = numEnv3("ERRATA_WATCH_BREAKER_WINDOW_MS", 1e4);
|
|
56669
|
+
var BREAKER_TRIP_SELF_EVENTS = numEnv3("ERRATA_WATCH_BREAKER_TRIP", 1e3);
|
|
56670
|
+
var BREAKER_MAX_RECYCLES = numEnv3("ERRATA_WATCH_BREAKER_RECYCLES", 2);
|
|
56671
|
+
var BREAKER_REWEDGE_TTL_MS = numEnv3("ERRATA_WATCH_BREAKER_TTL_MS", 6e5);
|
|
56672
|
+
var BREAKER_READD_DELAY_MS = numEnv3("ERRATA_WATCH_BREAKER_READD_MS", 250);
|
|
56673
|
+
var OFFENDER_FLOOR = 100;
|
|
56674
|
+
function normalizeWatchPath(p) {
|
|
56675
|
+
let s = p.startsWith("\\\\?\\") ? p.slice(4) : p;
|
|
56676
|
+
s = s.replace(/\\/g, "/");
|
|
56677
|
+
if (s.length > 1 && s.endsWith("/")) s = s.slice(0, -1);
|
|
56678
|
+
return process.platform === "win32" ? s.toLowerCase() : s;
|
|
56679
|
+
}
|
|
56680
|
+
function createWatchBreaker(deps) {
|
|
56681
|
+
const now = deps.now ?? Date.now;
|
|
56682
|
+
const say = deps.onAction ?? (() => {
|
|
56683
|
+
});
|
|
56684
|
+
const windows2 = /* @__PURE__ */ new Map();
|
|
56685
|
+
let windowStart = now();
|
|
56686
|
+
const recycleHistory = /* @__PURE__ */ new Map();
|
|
56687
|
+
const quarantined = /* @__PURE__ */ new Set();
|
|
56688
|
+
const pendingReadds = /* @__PURE__ */ new Set();
|
|
56689
|
+
const rotate = (t) => {
|
|
56690
|
+
let top = null;
|
|
56691
|
+
for (const [path2, w] of windows2) {
|
|
56692
|
+
if (w.dispatched >= OFFENDER_FLOOR && (!top || w.dispatched > top.count)) {
|
|
56693
|
+
top = { path: path2, count: w.dispatched };
|
|
56694
|
+
}
|
|
56695
|
+
}
|
|
56696
|
+
if (top) noteWindowOffender(top.path, top.count, BREAKER_WINDOW_MS);
|
|
56697
|
+
windows2.clear();
|
|
56698
|
+
windowStart = t;
|
|
56699
|
+
};
|
|
56700
|
+
const trip = (path2, t) => {
|
|
56701
|
+
const history = (recycleHistory.get(path2) ?? []).filter(
|
|
56702
|
+
(ts) => t - ts < BREAKER_REWEDGE_TTL_MS
|
|
56703
|
+
);
|
|
56704
|
+
if (history.length >= BREAKER_MAX_RECYCLES) {
|
|
56705
|
+
quarantined.add(path2);
|
|
56706
|
+
recycleHistory.delete(path2);
|
|
56707
|
+
try {
|
|
56708
|
+
deps.unwatch(path2);
|
|
56709
|
+
} catch {
|
|
56710
|
+
}
|
|
56711
|
+
noteBreakerQuarantine(path2);
|
|
56712
|
+
say(
|
|
56713
|
+
`QUARANTINED ${path2} \u2014 re-wedged after ${history.length} recycle(s); unwatched for this session (reconcile covers its edits)`
|
|
56714
|
+
);
|
|
56715
|
+
return;
|
|
56716
|
+
}
|
|
56717
|
+
history.push(t);
|
|
56718
|
+
recycleHistory.set(path2, history);
|
|
56719
|
+
try {
|
|
56720
|
+
deps.unwatch(path2);
|
|
56721
|
+
} catch {
|
|
56722
|
+
}
|
|
56723
|
+
const timer = setTimeout(() => {
|
|
56724
|
+
pendingReadds.delete(timer);
|
|
56725
|
+
try {
|
|
56726
|
+
deps.readd(path2);
|
|
56727
|
+
} catch {
|
|
56728
|
+
}
|
|
56729
|
+
}, BREAKER_READD_DELAY_MS);
|
|
56730
|
+
timer.unref?.();
|
|
56731
|
+
pendingReadds.add(timer);
|
|
56732
|
+
noteBreakerRecycle(path2);
|
|
56733
|
+
say(
|
|
56734
|
+
`recycled ${path2} \u2014 ${BREAKER_TRIP_SELF_EVENTS}+ phantom self-events in ${Math.round(BREAKER_WINDOW_MS / 1e3)}s; handle closed + re-created`
|
|
56735
|
+
);
|
|
56736
|
+
};
|
|
56737
|
+
return {
|
|
56738
|
+
onRaw(event, evPath, watchedPath) {
|
|
56739
|
+
if (typeof watchedPath !== "string" || watchedPath.length === 0) return;
|
|
56740
|
+
const t = now();
|
|
56741
|
+
if (t - windowStart >= BREAKER_WINDOW_MS) rotate(t);
|
|
56742
|
+
let w = windows2.get(watchedPath);
|
|
56743
|
+
if (!w) {
|
|
56744
|
+
w = { dispatched: 0, self: 0, tripped: false };
|
|
56745
|
+
windows2.set(watchedPath, w);
|
|
56746
|
+
}
|
|
56747
|
+
w.dispatched++;
|
|
56748
|
+
const self = evPath == null || evPath === "" || typeof evPath === "string" && normalizeWatchPath(evPath) === normalizeWatchPath(watchedPath);
|
|
56749
|
+
if (!self) return;
|
|
56750
|
+
w.self++;
|
|
56751
|
+
if (!w.tripped && !quarantined.has(watchedPath) && w.self >= BREAKER_TRIP_SELF_EVENTS) {
|
|
56752
|
+
w.tripped = true;
|
|
56753
|
+
trip(watchedPath, t);
|
|
56754
|
+
}
|
|
56755
|
+
},
|
|
56756
|
+
stop() {
|
|
56757
|
+
for (const timer of pendingReadds) clearTimeout(timer);
|
|
56758
|
+
pendingReadds.clear();
|
|
56759
|
+
}
|
|
56760
|
+
};
|
|
56761
|
+
}
|
|
56762
|
+
|
|
56615
56763
|
// src/engine.ts
|
|
56616
|
-
var DAEMON_VERSION = true ? "2.0.2-dev.
|
|
56764
|
+
var DAEMON_VERSION = true ? "2.0.2-dev.1166" : "2.0.0-alpha.0";
|
|
56617
56765
|
var IGNORED_PATH = /[\\/](?:\.git|node_modules|\.errata|\.claude|\.codex|\.cursor|\.turbo|\.next|dist|coverage|test-results|playwright-report|__pycache__)(?:[\\/]|$)/;
|
|
56618
56766
|
var IGNORED_NOISE = /castalia\.db|eventlog\.sqlite|turn-cursors/;
|
|
56619
56767
|
var GIT_OP_MUTE_MS = 4e3;
|
|
@@ -56745,6 +56893,7 @@ function createWorkspaceEngine(opts) {
|
|
|
56745
56893
|
const cloud = opts.cloud;
|
|
56746
56894
|
const telemetry = new TelemetryRecorder(profile.id, DAEMON_VERSION);
|
|
56747
56895
|
let watcher = null;
|
|
56896
|
+
let watchBreaker = null;
|
|
56748
56897
|
let lastActivityTs = Date.now();
|
|
56749
56898
|
let stopGit = null;
|
|
56750
56899
|
let flushTimer = null;
|
|
@@ -56757,7 +56906,23 @@ function createWorkspaceEngine(opts) {
|
|
|
56757
56906
|
ignoreInitial: true,
|
|
56758
56907
|
persistent: true
|
|
56759
56908
|
});
|
|
56760
|
-
|
|
56909
|
+
watchBreaker = createWatchBreaker({
|
|
56910
|
+
unwatch: (p) => void watcher?.unwatch(p),
|
|
56911
|
+
readd: (p) => void watcher?.add(p),
|
|
56912
|
+
// Toast AND log: the autostarted daemon's stdout is a discarded stream
|
|
56913
|
+
// (the 8-01 unobservability finding), so console alone would make every
|
|
56914
|
+
// production breaker action invisible — the exact gauge-with-no-alarm
|
|
56915
|
+
// shape this mechanism exists to end.
|
|
56916
|
+
onAction: (line) => {
|
|
56917
|
+
console.warn(`[errata] watch-breaker: ${line}`);
|
|
56918
|
+
notifyOpsEvent("daemon-distress", `watch-breaker: ${line}`, { key: `breaker:${profile.id}` });
|
|
56919
|
+
}
|
|
56920
|
+
});
|
|
56921
|
+
watcher.on("raw", (event, evPath, opts2) => {
|
|
56922
|
+
noteWatchDispatched();
|
|
56923
|
+
const watchedPath = opts2 && typeof opts2 === "object" && "watchedPath" in opts2 ? opts2.watchedPath : void 0;
|
|
56924
|
+
watchBreaker?.onRaw(event, evPath, watchedPath);
|
|
56925
|
+
});
|
|
56761
56926
|
const onFs = (kind) => (path2) => {
|
|
56762
56927
|
noteWatchEmitted();
|
|
56763
56928
|
if (IGNORED_PATH.test(path2) || IGNORED_NOISE.test(path2)) {
|
|
@@ -57052,6 +57217,7 @@ function createWorkspaceEngine(opts) {
|
|
|
57052
57217
|
}) : null;
|
|
57053
57218
|
const doneRender = prebuilt ? null : markPass(`context-render-inline:${profile.name}`);
|
|
57054
57219
|
const pendingUpdate = opts.pendingUpdate?.() ?? null;
|
|
57220
|
+
const daemonDistress = opts.daemonDistress?.() ?? null;
|
|
57055
57221
|
const reviewItems = pickReviewItems();
|
|
57056
57222
|
const liveIntent = intents.mostRecent(Date.now());
|
|
57057
57223
|
maybeRefreshIntentPriors(liveIntent ?? null);
|
|
@@ -57069,6 +57235,7 @@ function createWorkspaceEngine(opts) {
|
|
|
57069
57235
|
...elicit ? { edgeElicitation: { instruction: PRIOR_TAG_INSTRUCTION } } : {},
|
|
57070
57236
|
...prebuilt ? { snapshot: prebuilt } : {},
|
|
57071
57237
|
...pendingUpdate ? { pendingUpdate } : {},
|
|
57238
|
+
...daemonDistress ? { daemonDistress } : {},
|
|
57072
57239
|
...reviewItems.length ? { reviewItems } : {},
|
|
57073
57240
|
...liveIntent ? {
|
|
57074
57241
|
statedIntent: {
|
|
@@ -58586,6 +58753,7 @@ function createWorkspaceEngine(opts) {
|
|
|
58586
58753
|
async stop() {
|
|
58587
58754
|
if (stopGit) stopGit();
|
|
58588
58755
|
if (passWorker) await passWorker.stop();
|
|
58756
|
+
if (watchBreaker) watchBreaker.stop();
|
|
58589
58757
|
if (watcher) await watcher.close();
|
|
58590
58758
|
if (flushTimer) clearTimeout(flushTimer);
|
|
58591
58759
|
for (const tmr of diagTimers.values()) clearTimeout(tmr);
|
|
@@ -59955,6 +60123,137 @@ function createRootAdopter(deps) {
|
|
|
59955
60123
|
};
|
|
59956
60124
|
}
|
|
59957
60125
|
|
|
60126
|
+
// src/health-sentinel.ts
|
|
60127
|
+
function numEnv4(key, fallback) {
|
|
60128
|
+
const n = Number(process.env[key]);
|
|
60129
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
60130
|
+
}
|
|
60131
|
+
var SENTINEL_TICK_MS = numEnv4("ERRATA_SENTINEL_TICK_MS", 3e4);
|
|
60132
|
+
var SENTINEL_WINDOW_TICKS = numEnv4("ERRATA_SENTINEL_WINDOW_TICKS", 6);
|
|
60133
|
+
var SENTINEL_BURN_CORES = numEnv4("ERRATA_SENTINEL_BURN_CORES", 1.2);
|
|
60134
|
+
var SENTINEL_STORM_RATE = numEnv4("ERRATA_SENTINEL_STORM_RATE", 1e3);
|
|
60135
|
+
var SENTINEL_STORM_TICKS = numEnv4("ERRATA_SENTINEL_STORM_TICKS", 4);
|
|
60136
|
+
var SENTINEL_RENOTIFY_MS = numEnv4("ERRATA_SENTINEL_RENOTIFY_MS", 18e5);
|
|
60137
|
+
var SENTINEL_CALM_TICKS = numEnv4("ERRATA_SENTINEL_CALM_TICKS", 2);
|
|
60138
|
+
var SENTINEL_TOAST_COOLDOWN_MS = numEnv4("ERRATA_SENTINEL_TOAST_COOLDOWN_MS", 6e5);
|
|
60139
|
+
function createHealthSentinel(deps = {}) {
|
|
60140
|
+
const now = deps.now ?? Date.now;
|
|
60141
|
+
const cpuUsage = deps.cpuUsage ?? (() => process.cpuUsage());
|
|
60142
|
+
const readCensus = deps.readCensus ?? (() => {
|
|
60143
|
+
const c = watchCensus();
|
|
60144
|
+
return {
|
|
60145
|
+
dispatched: c.dispatched,
|
|
60146
|
+
logged: c.logged,
|
|
60147
|
+
topOffender: c.topOffender ? { path: c.topOffender.path, count: c.topOffender.count } : null
|
|
60148
|
+
};
|
|
60149
|
+
});
|
|
60150
|
+
const activePasses = deps.activePasses ?? (() => passState().active.map((p) => p.name));
|
|
60151
|
+
const rssMb = deps.rssMb ?? (() => Math.round(process.memoryUsage.rss() / 1048576));
|
|
60152
|
+
const toast = deps.notify ?? ((body2, key) => notifyOpsEvent("daemon-distress", body2, { key }));
|
|
60153
|
+
let prevCpu = cpuUsage();
|
|
60154
|
+
let prevTs = now();
|
|
60155
|
+
let prevDispatched = null;
|
|
60156
|
+
let prevLogged = 0;
|
|
60157
|
+
const coreSamples = [];
|
|
60158
|
+
let stormTicks = 0;
|
|
60159
|
+
let calmTicks = 0;
|
|
60160
|
+
let distress = null;
|
|
60161
|
+
let lastNotifiedAt = 0;
|
|
60162
|
+
const lastToastByKind = /* @__PURE__ */ new Map();
|
|
60163
|
+
const attribution = () => {
|
|
60164
|
+
const parts2 = [];
|
|
60165
|
+
const passes = activePasses();
|
|
60166
|
+
if (passes.length > 0) parts2.push(`active pass: ${passes.join(", ")}`);
|
|
60167
|
+
const c = readCensus();
|
|
60168
|
+
if (c.topOffender) parts2.push(`hottest watch path: ${c.topOffender.path}`);
|
|
60169
|
+
parts2.push(`rss ${rssMb()}MB`);
|
|
60170
|
+
return parts2.join(" \xB7 ");
|
|
60171
|
+
};
|
|
60172
|
+
const maybeToast = (d, t) => {
|
|
60173
|
+
if (t - (lastToastByKind.get(d.kind) ?? 0) < SENTINEL_TOAST_COOLDOWN_MS) return;
|
|
60174
|
+
lastToastByKind.set(d.kind, t);
|
|
60175
|
+
toast(d.headline, d.kind);
|
|
60176
|
+
lastNotifiedAt = t;
|
|
60177
|
+
};
|
|
60178
|
+
const transition = (next) => {
|
|
60179
|
+
const t = now();
|
|
60180
|
+
if (next && !distress) {
|
|
60181
|
+
distress = next;
|
|
60182
|
+
deps.ledger?.("enter", next);
|
|
60183
|
+
maybeToast(next, t);
|
|
60184
|
+
deps.onChange?.(next);
|
|
60185
|
+
} else if (next && distress) {
|
|
60186
|
+
distress.body = next.body;
|
|
60187
|
+
distress.headline = next.headline;
|
|
60188
|
+
if (t - lastNotifiedAt >= SENTINEL_RENOTIFY_MS) maybeToast(distress, t);
|
|
60189
|
+
} else if (!next && distress) {
|
|
60190
|
+
deps.ledger?.("clear", distress);
|
|
60191
|
+
distress = null;
|
|
60192
|
+
deps.onChange?.(null);
|
|
60193
|
+
}
|
|
60194
|
+
};
|
|
60195
|
+
const evaluate = () => {
|
|
60196
|
+
const t = now();
|
|
60197
|
+
const wallMs = Math.max(1, t - prevTs);
|
|
60198
|
+
const cpu = cpuUsage();
|
|
60199
|
+
const cores = (cpu.user + cpu.system - prevCpu.user - prevCpu.system) / 1e3 / wallMs;
|
|
60200
|
+
prevCpu = cpu;
|
|
60201
|
+
prevTs = t;
|
|
60202
|
+
coreSamples.push(cores);
|
|
60203
|
+
if (coreSamples.length > SENTINEL_WINDOW_TICKS) coreSamples.shift();
|
|
60204
|
+
const c = readCensus();
|
|
60205
|
+
const dispatchRate = prevDispatched === null ? 0 : (c.dispatched - prevDispatched) / wallMs * 1e3;
|
|
60206
|
+
const survived = c.logged - prevLogged;
|
|
60207
|
+
prevDispatched = c.dispatched;
|
|
60208
|
+
prevLogged = c.logged;
|
|
60209
|
+
stormTicks = dispatchRate > SENTINEL_STORM_RATE && survived === 0 ? stormTicks + 1 : 0;
|
|
60210
|
+
let wanted = null;
|
|
60211
|
+
if (stormTicks >= SENTINEL_STORM_TICKS) {
|
|
60212
|
+
wanted = {
|
|
60213
|
+
kind: "watch-storm",
|
|
60214
|
+
headline: `errata's file watcher is stuck busy (~${Math.round(dispatchRate)} events/sec with nothing changing) \u2014 run \`errata status\` for detail`,
|
|
60215
|
+
body: `filesystem watcher is processing ~${Math.round(dispatchRate)} events/sec with NOTHING surviving \u2014 a wedged or ignored-tree watch is burning CPU for no signal \xB7 ${attribution()}`,
|
|
60216
|
+
sinceTs: distress?.kind === "watch-storm" ? distress.sinceTs : t
|
|
60217
|
+
};
|
|
60218
|
+
} else {
|
|
60219
|
+
const windowFull = coreSamples.length >= SENTINEL_WINDOW_TICKS;
|
|
60220
|
+
const avgCores = coreSamples.reduce((a, b) => a + b, 0) / Math.max(1, coreSamples.length);
|
|
60221
|
+
if (windowFull && avgCores >= SENTINEL_BURN_CORES && cores >= SENTINEL_BURN_CORES && activePasses().length === 0) {
|
|
60222
|
+
wanted = {
|
|
60223
|
+
kind: "cpu-burn",
|
|
60224
|
+
headline: `errata is using more CPU than expected (${avgCores.toFixed(1)} cores with no work running) \u2014 run \`errata status\` for detail`,
|
|
60225
|
+
body: `sustained CPU with no pass running \u2014 avg ${avgCores.toFixed(1)} cores \xB7 ${attribution()}`,
|
|
60226
|
+
sinceTs: distress?.kind === "cpu-burn" ? distress.sinceTs : t
|
|
60227
|
+
};
|
|
60228
|
+
}
|
|
60229
|
+
}
|
|
60230
|
+
if (wanted) {
|
|
60231
|
+
calmTicks = 0;
|
|
60232
|
+
transition(wanted);
|
|
60233
|
+
} else if (distress) {
|
|
60234
|
+
calmTicks++;
|
|
60235
|
+
if (calmTicks >= SENTINEL_CALM_TICKS) {
|
|
60236
|
+
calmTicks = 0;
|
|
60237
|
+
transition(null);
|
|
60238
|
+
}
|
|
60239
|
+
}
|
|
60240
|
+
};
|
|
60241
|
+
const interval = deps.scheduleTicks === false ? null : setInterval(() => {
|
|
60242
|
+
try {
|
|
60243
|
+
evaluate();
|
|
60244
|
+
} catch {
|
|
60245
|
+
}
|
|
60246
|
+
}, SENTINEL_TICK_MS);
|
|
60247
|
+
interval?.unref?.();
|
|
60248
|
+
return {
|
|
60249
|
+
current: () => distress ? { ...distress } : null,
|
|
60250
|
+
tick: evaluate,
|
|
60251
|
+
stop: () => {
|
|
60252
|
+
if (interval) clearInterval(interval);
|
|
60253
|
+
}
|
|
60254
|
+
};
|
|
60255
|
+
}
|
|
60256
|
+
|
|
59958
60257
|
// src/multi.ts
|
|
59959
60258
|
var projectSymbolSalts = /* @__PURE__ */ new Map();
|
|
59960
60259
|
async function resolveProjectSymbolSalt(client, projectId) {
|
|
@@ -60099,6 +60398,20 @@ async function startMultiDaemon(opts = {}) {
|
|
|
60099
60398
|
for (const r of records) r.engine.markContextDirty();
|
|
60100
60399
|
}
|
|
60101
60400
|
}) : null;
|
|
60401
|
+
const healthSentinel = createHealthSentinel({
|
|
60402
|
+
onChange: () => {
|
|
60403
|
+
for (const r of records) r.engine.markContextDirty();
|
|
60404
|
+
},
|
|
60405
|
+
ledger: (event, d) => {
|
|
60406
|
+
const dir = records[0]?.engine.paths.configDir;
|
|
60407
|
+
if (dir) {
|
|
60408
|
+
appendPassLedger(dir, "health-sentinel", 0, {
|
|
60409
|
+
[`distress:${event}:${d.kind}`]: 1,
|
|
60410
|
+
distressSinceTs: d.sinceTs
|
|
60411
|
+
});
|
|
60412
|
+
}
|
|
60413
|
+
}
|
|
60414
|
+
});
|
|
60102
60415
|
let serverPending = null;
|
|
60103
60416
|
const boundaryListeners = [];
|
|
60104
60417
|
let boundaryFlushing = false;
|
|
@@ -60193,6 +60506,10 @@ async function startMultiDaemon(opts = {}) {
|
|
|
60193
60506
|
reviewUrl: () => `${baseUrl}/ws/${id}/review`,
|
|
60194
60507
|
sharedStore,
|
|
60195
60508
|
pendingUpdate: () => serverPending ?? updatePoller?.current() ?? null,
|
|
60509
|
+
// Ops-gated (default OFF): the 🚨 notice reads as "errata broke my
|
|
60510
|
+
// machine" to a non-operator. Detection + ledger run regardless — only
|
|
60511
|
+
// this push surface (and the toast, gated in notifyOpsEvent) is scoped.
|
|
60512
|
+
daemonDistress: () => loadConfig().opsNotices ? healthSentinel.current() : null,
|
|
60196
60513
|
...opts.skipWatchers ? { skipWatchers: true } : {}
|
|
60197
60514
|
});
|
|
60198
60515
|
const webApp = buildWebUi({
|
|
@@ -60281,7 +60598,7 @@ async function startMultiDaemon(opts = {}) {
|
|
|
60281
60598
|
registerWorkspace(rec.engine.profile, rec.root);
|
|
60282
60599
|
} catch {
|
|
60283
60600
|
}
|
|
60284
|
-
void rec.engine.reindex({ skipEmbed: true }).catch(() => {
|
|
60601
|
+
void rec.engine.reindex({ skipEmbed: true, quiet: rec.engine.store.nodeCount() > 0 }).catch(() => {
|
|
60285
60602
|
});
|
|
60286
60603
|
console.log(`[errata] hot-registered ${rec.entry.name} (${rec.id}) \u2014 now watching live`);
|
|
60287
60604
|
return { attached: true, id: rec.id };
|
|
@@ -61199,6 +61516,7 @@ async function startMultiDaemon(opts = {}) {
|
|
|
61199
61516
|
(res) => server.close(res)
|
|
61200
61517
|
);
|
|
61201
61518
|
updatePoller?.stop();
|
|
61519
|
+
healthSentinel.stop();
|
|
61202
61520
|
for (const r of records) await r.engine.stop();
|
|
61203
61521
|
if (consolidateWorker) await consolidateWorker.stop();
|
|
61204
61522
|
try {
|
|
@@ -61954,7 +62272,7 @@ var DEFAULT_CONSOLIDATION_POLICY = {
|
|
|
61954
62272
|
quiescenceMs: 6e4
|
|
61955
62273
|
// sim preferred 60s over 90s (fresher, esp. heavy-code)
|
|
61956
62274
|
};
|
|
61957
|
-
function
|
|
62275
|
+
function numEnv5(env2, key, fallback) {
|
|
61958
62276
|
const raw2 = env2[key];
|
|
61959
62277
|
if (raw2 == null || raw2.trim() === "") return fallback;
|
|
61960
62278
|
const n = Number(raw2);
|
|
@@ -61962,12 +62280,12 @@ function numEnv3(env2, key, fallback) {
|
|
|
61962
62280
|
}
|
|
61963
62281
|
function consolidationPolicyFromEnv(env2 = process.env) {
|
|
61964
62282
|
return {
|
|
61965
|
-
baseFloorMs:
|
|
61966
|
-
dutyFactor:
|
|
61967
|
-
momentumThreshold:
|
|
61968
|
-
momentumRatio:
|
|
61969
|
-
momentumRatioCap:
|
|
61970
|
-
quiescenceMs:
|
|
62283
|
+
baseFloorMs: numEnv5(env2, "ERRATA_CONSOLIDATE_BASE_FLOOR_MS", DEFAULT_CONSOLIDATION_POLICY.baseFloorMs),
|
|
62284
|
+
dutyFactor: numEnv5(env2, "ERRATA_CONSOLIDATE_DUTY_FACTOR", DEFAULT_CONSOLIDATION_POLICY.dutyFactor),
|
|
62285
|
+
momentumThreshold: numEnv5(env2, "ERRATA_CONSOLIDATE_MOMENTUM", DEFAULT_CONSOLIDATION_POLICY.momentumThreshold),
|
|
62286
|
+
momentumRatio: numEnv5(env2, "ERRATA_CONSOLIDATE_MOMENTUM_RATIO", DEFAULT_CONSOLIDATION_POLICY.momentumRatio),
|
|
62287
|
+
momentumRatioCap: numEnv5(env2, "ERRATA_CONSOLIDATE_MOMENTUM_RATIO_CAP", DEFAULT_CONSOLIDATION_POLICY.momentumRatioCap),
|
|
62288
|
+
quiescenceMs: numEnv5(env2, "ERRATA_CONSOLIDATE_QUIESCENCE_MS", DEFAULT_CONSOLIDATION_POLICY.quiescenceMs)
|
|
61971
62289
|
};
|
|
61972
62290
|
}
|
|
61973
62291
|
function consolidationGapMs(lastPassMs, policy) {
|
|
@@ -62105,6 +62423,8 @@ async function main() {
|
|
|
62105
62423
|
return cmdConsent(rest);
|
|
62106
62424
|
case "notifications":
|
|
62107
62425
|
return cmdNotifications(rest[0] ?? "");
|
|
62426
|
+
case "ops-notices":
|
|
62427
|
+
return cmdOpsNotices(rest[0] ?? "");
|
|
62108
62428
|
case "update":
|
|
62109
62429
|
return cmdUpdate(rest);
|
|
62110
62430
|
case "projects":
|
|
@@ -62228,6 +62548,9 @@ Commands:
|
|
|
62228
62548
|
Send scrubbed feedback to the maintainers (needs login)
|
|
62229
62549
|
notifications <on|off>
|
|
62230
62550
|
Desktop toasts on problem open/resolve + review ready
|
|
62551
|
+
ops-notices <on|off>
|
|
62552
|
+
Ops-grade health surfaces (daemon-distress toast + agent
|
|
62553
|
+
notice). Default off; for people operating errata itself
|
|
62231
62554
|
|
|
62232
62555
|
Environment:
|
|
62233
62556
|
ERRATA_CLOUD_URL Cloud base URL (default ${DEFAULT_CLOUD_URL})
|
|
@@ -64753,6 +65076,19 @@ async function cmdNotifications(state) {
|
|
|
64753
65076
|
saveConfig(cfg);
|
|
64754
65077
|
console.log(`desktop notifications ${cfg.notifications ? "enabled" : "disabled"}`);
|
|
64755
65078
|
}
|
|
65079
|
+
async function cmdOpsNotices(state) {
|
|
65080
|
+
const s = state.toLowerCase();
|
|
65081
|
+
if (s !== "on" && s !== "off") {
|
|
65082
|
+
console.error("usage: errata ops-notices <on|off>");
|
|
65083
|
+
process.exit(2);
|
|
65084
|
+
}
|
|
65085
|
+
const cfg = loadConfig();
|
|
65086
|
+
cfg.opsNotices = s === "on";
|
|
65087
|
+
saveConfig(cfg);
|
|
65088
|
+
console.log(
|
|
65089
|
+
`ops health notices ${cfg.opsNotices ? "enabled \u2014 distress conditions will toast and reach the agent context" : "disabled \u2014 detection and `errata status` still run"}`
|
|
65090
|
+
);
|
|
65091
|
+
}
|
|
64756
65092
|
async function cmdFeedback(args2) {
|
|
64757
65093
|
const cfg = loadConfig();
|
|
64758
65094
|
if (!hasCloudCredential(cfg)) {
|