@crewhaus/runtime-core 0.1.8 β 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +138 -0
- package/dist/alert-watchdog.js +290 -0
- package/dist/incident-collector.d.ts +42 -0
- package/dist/incident-collector.js +125 -0
- package/dist/index.d.ts +286 -4
- package/dist/index.js +733 -30
- package/dist/janitor.d.ts +88 -0
- package/dist/janitor.js +287 -0
- package/dist/observability.d.ts +178 -2
- package/dist/observability.js +466 -2
- package/dist/project-memory.d.ts +17 -1
- package/dist/project-memory.js +42 -0
- package/dist/slo-monitor.d.ts +221 -0
- package/dist/slo-monitor.js +397 -0
- package/package.json +40 -39
package/dist/observability.js
CHANGED
|
@@ -2,6 +2,8 @@ import { createCostTracker, formatUsdMicros } from "@crewhaus/cost-tracker";
|
|
|
2
2
|
import { attachIfEnvSet as attachMetricsIfEnvSet, } from "@crewhaus/metrics-collector";
|
|
3
3
|
import { attachIfEnvSet as attachOtelIfEnvSet, } from "@crewhaus/otel-exporter";
|
|
4
4
|
import { attachIfEnvSet as attachPrinterIfEnvSet, } from "@crewhaus/structured-event-printer";
|
|
5
|
+
import { SessionMetricsAccumulator, appendMetricsSnapshot, deriveThresholds, detectBreaches, readMetricsHistory, } from "./alert-watchdog";
|
|
6
|
+
import { attachSloMonitor, } from "./slo-monitor";
|
|
5
7
|
/**
|
|
6
8
|
* Format a single `cost_accrual` event as a one-line CLI status:
|
|
7
9
|
* `[πΈ $0.0042 Β· 12.3k in / 1.1k out Β· model=claude-sonnet-4-6]`
|
|
@@ -14,7 +16,128 @@ export function formatCostInlineLine(ev) {
|
|
|
14
16
|
const fmtTokens = (n) => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : `${n}`);
|
|
15
17
|
return `[πΈ ${cost} Β· ${fmtTokens(ev.inputTokens)} in / ${fmtTokens(ev.outputTokens)} out Β· model=${ev.modelId}]`;
|
|
16
18
|
}
|
|
17
|
-
export
|
|
19
|
+
export function createSecurityTally() {
|
|
20
|
+
return {
|
|
21
|
+
justificationDenials: 0,
|
|
22
|
+
permissionDenials: 0,
|
|
23
|
+
egressWarned: 0,
|
|
24
|
+
egressBlocked: 0,
|
|
25
|
+
injectionRedactions: 0,
|
|
26
|
+
injectionWarnings: 0,
|
|
27
|
+
injectionRuleHits: {},
|
|
28
|
+
hookDenials: 0,
|
|
29
|
+
circuitOpens: 0,
|
|
30
|
+
circuitRecoveries: 0,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Fold one TraceEvent into the tally. Returns true when the event counted.
|
|
35
|
+
* Classification rules (kept mutually exclusive so nothing double-counts):
|
|
36
|
+
* - a `permission_decision` carrying an outcome (egress-warned,
|
|
37
|
+
* egress-blocked, redacted, warned) is an egress/injection verdict,
|
|
38
|
+
* NEVER also a plain permission denial (the egress-blocked event
|
|
39
|
+
* carries decision "deny" too);
|
|
40
|
+
* - a `judgeModel` marks the justification gate β deny β justification
|
|
41
|
+
* denial;
|
|
42
|
+
* - remaining decision "deny" events are plain policy denials. "ask" is
|
|
43
|
+
* not tallied β a prompt is not an incident.
|
|
44
|
+
*/
|
|
45
|
+
export function tallySecurityEvent(tally, event) {
|
|
46
|
+
switch (event.kind) {
|
|
47
|
+
case "permission_decision": {
|
|
48
|
+
if (event.outcome !== undefined) {
|
|
49
|
+
if (event.outcome === "egress-warned")
|
|
50
|
+
tally.egressWarned += 1;
|
|
51
|
+
else if (event.outcome === "egress-blocked")
|
|
52
|
+
tally.egressBlocked += 1;
|
|
53
|
+
else if (event.outcome === "redacted")
|
|
54
|
+
tally.injectionRedactions += 1;
|
|
55
|
+
else if (event.outcome === "warned")
|
|
56
|
+
tally.injectionWarnings += 1;
|
|
57
|
+
else
|
|
58
|
+
return false; // "egress-passed" β clean, not security-relevant
|
|
59
|
+
if (event.outcome === "redacted" || event.outcome === "warned") {
|
|
60
|
+
for (const rule of event.rules ?? []) {
|
|
61
|
+
tally.injectionRuleHits[rule] = (tally.injectionRuleHits[rule] ?? 0) + 1;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
if (event.decision !== "deny")
|
|
67
|
+
return false;
|
|
68
|
+
if (event.judgeModel !== undefined)
|
|
69
|
+
tally.justificationDenials += 1;
|
|
70
|
+
else
|
|
71
|
+
tally.permissionDenials += 1;
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
case "hook_fired": {
|
|
75
|
+
if (event.allowed)
|
|
76
|
+
return false;
|
|
77
|
+
tally.hookDenials += 1;
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
case "circuit_state_changed": {
|
|
81
|
+
if (event.toState === "open") {
|
|
82
|
+
tally.circuitOpens += 1;
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
if (event.toState === "closed" && event.fromState !== "closed") {
|
|
86
|
+
tally.circuitRecoveries += 1;
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
default:
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/** Sum of every tallied counter β 0 means "print nothing at end of run". */
|
|
96
|
+
export function securityTallyTotal(tally) {
|
|
97
|
+
return (tally.justificationDenials +
|
|
98
|
+
tally.permissionDenials +
|
|
99
|
+
tally.egressWarned +
|
|
100
|
+
tally.egressBlocked +
|
|
101
|
+
tally.injectionRedactions +
|
|
102
|
+
tally.injectionWarnings +
|
|
103
|
+
tally.hookDenials +
|
|
104
|
+
tally.circuitOpens +
|
|
105
|
+
tally.circuitRecoveries);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Format the end-of-run summary line. Only non-zero segments render, e.g.
|
|
109
|
+
* `[security] 2 justification denial(s) Β· 1 egress block(s) Β· 1 injection
|
|
110
|
+
* redaction(s) [rules: exfil-url Γ2] β run crewhaus security digest β¦`
|
|
111
|
+
* Exported so target bundles (and tests) can render the identical format,
|
|
112
|
+
* mirroring `formatCostInlineLine`.
|
|
113
|
+
*/
|
|
114
|
+
export function formatSecurityTallyLine(tally) {
|
|
115
|
+
const parts = [];
|
|
116
|
+
if (tally.justificationDenials > 0)
|
|
117
|
+
parts.push(`${tally.justificationDenials} justification denial(s)`);
|
|
118
|
+
if (tally.permissionDenials > 0)
|
|
119
|
+
parts.push(`${tally.permissionDenials} permission denial(s)`);
|
|
120
|
+
if (tally.egressWarned > 0)
|
|
121
|
+
parts.push(`${tally.egressWarned} egress warn(s)`);
|
|
122
|
+
if (tally.egressBlocked > 0)
|
|
123
|
+
parts.push(`${tally.egressBlocked} egress block(s)`);
|
|
124
|
+
if (tally.injectionRedactions > 0)
|
|
125
|
+
parts.push(`${tally.injectionRedactions} injection redaction(s)`);
|
|
126
|
+
if (tally.injectionWarnings > 0)
|
|
127
|
+
parts.push(`${tally.injectionWarnings} injection warning(s)`);
|
|
128
|
+
if (tally.hookDenials > 0)
|
|
129
|
+
parts.push(`${tally.hookDenials} hook denial(s)`);
|
|
130
|
+
if (tally.circuitOpens > 0)
|
|
131
|
+
parts.push(`${tally.circuitOpens} circuit open(s)`);
|
|
132
|
+
if (tally.circuitRecoveries > 0)
|
|
133
|
+
parts.push(`${tally.circuitRecoveries} circuit recovery(-ies)`);
|
|
134
|
+
const rules = Object.entries(tally.injectionRuleHits)
|
|
135
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
136
|
+
.map(([rule, n]) => `${rule} Γ${n}`)
|
|
137
|
+
.join(", ");
|
|
138
|
+
return `[security] ${parts.join(" Β· ")}${rules === "" ? "" : ` [rules: ${rules}]`} β run \`crewhaus security digest\` for the windowed rollup`;
|
|
139
|
+
}
|
|
140
|
+
export async function attachDefaultSubscribers(bus, runContext, env = process.env, options = {}) {
|
|
18
141
|
const printer = attachPrinterIfEnvSet(bus, env);
|
|
19
142
|
const metrics = await attachMetricsIfEnvSet(bus, env);
|
|
20
143
|
const otel = attachOtelIfEnvSet(bus, env);
|
|
@@ -40,6 +163,43 @@ export async function attachDefaultSubscribers(bus, runContext, env = process.en
|
|
|
40
163
|
};
|
|
41
164
|
costInlineUnsubscribe = bus.subscribe(handler);
|
|
42
165
|
}
|
|
166
|
+
// Item 48 β per-run security tally, opt-in via CREWHAUS_SECURITY_DIGEST=1.
|
|
167
|
+
// Counts security-relevant events during the run and prints exactly one
|
|
168
|
+
// end-of-run summary line to STDERR from flushAll when anything was
|
|
169
|
+
// tallied (flushAll is the orchestrator's `finally` hook; the printed
|
|
170
|
+
// guard keeps a double flush from double-printing).
|
|
171
|
+
const securityTallyEnabled = env["CREWHAUS_SECURITY_DIGEST"] === "1" || env["CREWHAUS_SECURITY_DIGEST"] === "true";
|
|
172
|
+
const securityTally = securityTallyEnabled ? createSecurityTally() : undefined;
|
|
173
|
+
let securityTallyUnsubscribe;
|
|
174
|
+
let securityTallyPrinted = false;
|
|
175
|
+
if (securityTally !== undefined) {
|
|
176
|
+
securityTallyUnsubscribe = bus.subscribe((event) => {
|
|
177
|
+
tallySecurityEvent(securityTally, event);
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const printSecurityTallyOnce = () => {
|
|
181
|
+
if (securityTally === undefined || securityTallyPrinted)
|
|
182
|
+
return;
|
|
183
|
+
if (securityTallyTotal(securityTally) === 0)
|
|
184
|
+
return;
|
|
185
|
+
securityTallyPrinted = true;
|
|
186
|
+
process.stderr.write(`${formatSecurityTallyLine(securityTally)}\n`);
|
|
187
|
+
};
|
|
188
|
+
// Ops item 31 β alert watchdog, opt-in via CREWHAUS_ALERTS=1. Folds the run's
|
|
189
|
+
// events into a per-session metrics snapshot; at session end it persists the
|
|
190
|
+
// snapshot (building the history baselines are derived from), derives
|
|
191
|
+
// baseline thresholds (trailing p95 Γ headroom) from prior snapshots, and
|
|
192
|
+
// raises an alert per breach (trace event + injected audit/hook sinks).
|
|
193
|
+
const watchdog = attachAlertWatchdog(bus, runContext, env, options);
|
|
194
|
+
// Ops item 37 β production-signal SLO monitor, opt-in via CREWHAUS_SLO=1 AND a
|
|
195
|
+
// lowered `observability.slo` block. Folds events into rolling windows; a
|
|
196
|
+
// periodic timer evaluates them and walks the declared mitigation ladder
|
|
197
|
+
// (alert β pause-intake β rollback) on a sustained breach via the injected
|
|
198
|
+
// sink. No targets / env off β undefined (zero overhead).
|
|
199
|
+
const sloMonitor = attachSloMonitor(bus, runContext, env, {
|
|
200
|
+
...(options.sloTargets !== undefined ? { targets: options.sloTargets } : {}),
|
|
201
|
+
...(options.sloSink !== undefined ? { sink: options.sloSink } : {}),
|
|
202
|
+
});
|
|
43
203
|
const flushAll = async () => {
|
|
44
204
|
printer?.finalize();
|
|
45
205
|
const tasks = [];
|
|
@@ -49,6 +209,21 @@ export async function attachDefaultSubscribers(bus, runContext, env = process.en
|
|
|
49
209
|
tasks.push(otel.flush().catch((err) => logFlushError(runContext, "otel", err)));
|
|
50
210
|
tasks.push(bus.flush().catch((err) => logFlushError(runContext, "bus", err)));
|
|
51
211
|
await Promise.all(tasks);
|
|
212
|
+
// AFTER the bus flush so every in-flight event has reached the tally.
|
|
213
|
+
printSecurityTallyOnce();
|
|
214
|
+
// AFTER the flush so the snapshot sees every event; a watchdog failure
|
|
215
|
+
// must never break flushAll, so its own finalize swallows/logs internally.
|
|
216
|
+
if (watchdog !== undefined) {
|
|
217
|
+
await watchdog.finalize().catch((err) => logFlushError(runContext, "alert-watchdog", err));
|
|
218
|
+
}
|
|
219
|
+
// A final SLO evaluation at flush so a breach that formed late in a short
|
|
220
|
+
// run (below the periodic timer's first tick) still walks the ladder, then
|
|
221
|
+
// stop the timer so it never outlives the run.
|
|
222
|
+
if (sloMonitor !== undefined) {
|
|
223
|
+
await sloMonitor.evaluate().catch((err) => logFlushError(runContext, "slo-monitor", err));
|
|
224
|
+
sloMonitor.stop();
|
|
225
|
+
sloMonitor.unsubscribe();
|
|
226
|
+
}
|
|
52
227
|
};
|
|
53
228
|
const shutdownAll = async () => {
|
|
54
229
|
printer?.unsubscribe();
|
|
@@ -60,10 +235,299 @@ export async function attachDefaultSubscribers(bus, runContext, env = process.en
|
|
|
60
235
|
}
|
|
61
236
|
costTracker?.unsubscribe();
|
|
62
237
|
costInlineUnsubscribe?.();
|
|
238
|
+
// A caller that shuts down without flushing still gets its one line.
|
|
239
|
+
printSecurityTallyOnce();
|
|
240
|
+
securityTallyUnsubscribe?.();
|
|
241
|
+
if (watchdog !== undefined) {
|
|
242
|
+
await watchdog.finalize().catch((err) => logFlushError(runContext, "alert-watchdog", err));
|
|
243
|
+
watchdog.unsubscribe();
|
|
244
|
+
}
|
|
245
|
+
if (sloMonitor !== undefined) {
|
|
246
|
+
sloMonitor.stop();
|
|
247
|
+
sloMonitor.unsubscribe();
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
return {
|
|
251
|
+
printer,
|
|
252
|
+
metrics,
|
|
253
|
+
otel,
|
|
254
|
+
costTracker,
|
|
255
|
+
costInlineUnsubscribe,
|
|
256
|
+
securityTally,
|
|
257
|
+
flushAll,
|
|
258
|
+
shutdownAll,
|
|
63
259
|
};
|
|
64
|
-
return { printer, metrics, otel, costTracker, costInlineUnsubscribe, flushAll, shutdownAll };
|
|
65
260
|
}
|
|
66
261
|
function logFlushError(runContext, name, err) {
|
|
67
262
|
const message = err instanceof Error ? err.message : String(err);
|
|
68
263
|
runContext.logger.error("observability.flush_failed", { name, message });
|
|
69
264
|
}
|
|
265
|
+
/**
|
|
266
|
+
* Ops item 31 β attach the alert watchdog when CREWHAUS_ALERTS is set. Folds
|
|
267
|
+
* events into a per-session {@link SessionMetricsAccumulator}; `finalize()`
|
|
268
|
+
* (called from flush/shutdown) persists the snapshot, derives baseline
|
|
269
|
+
* thresholds from the accrued history, and β per breach β publishes an
|
|
270
|
+
* `alert_raised` trace event, appends an audit record, and fires the alert
|
|
271
|
+
* hook/webhook via the injected {@link AlertSink}. Returns undefined (a no-op)
|
|
272
|
+
* when the env gate is off, so it adds zero overhead by default.
|
|
273
|
+
*
|
|
274
|
+
* NOTE (spec gate): the primary gate is the CREWHAUS_ALERTS env var, matching
|
|
275
|
+
* every other subscriber in this file (attachDefaultSubscribers only receives
|
|
276
|
+
* env). A spec-level `observability.alerts` block would be lowered by the
|
|
277
|
+
* compiler into an env stamp / options flag on the emitted bundle; that
|
|
278
|
+
* lowering is the follow-up β the runtime seam here already honours it.
|
|
279
|
+
*/
|
|
280
|
+
export function attachAlertWatchdog(bus, runContext, env, options = {}) {
|
|
281
|
+
const gate = env["CREWHAUS_ALERTS"];
|
|
282
|
+
if (gate !== "1" && gate !== "true")
|
|
283
|
+
return undefined;
|
|
284
|
+
const acc = new SessionMetricsAccumulator();
|
|
285
|
+
const unsubscribe = bus.subscribe((event) => {
|
|
286
|
+
acc.fold(event);
|
|
287
|
+
});
|
|
288
|
+
let finalized = false;
|
|
289
|
+
const finalize = async () => {
|
|
290
|
+
if (finalized)
|
|
291
|
+
return;
|
|
292
|
+
finalized = true;
|
|
293
|
+
const snapshot = acc.snapshot(runContext.sessionId);
|
|
294
|
+
// Derive thresholds from PRIOR history (before this session's snapshot is
|
|
295
|
+
// appended) so a session is never graded against itself.
|
|
296
|
+
const history = readMetricsHistory(options.metricsDir);
|
|
297
|
+
const thresholds = deriveThresholds(history);
|
|
298
|
+
// Persist AFTER reading history so the next session benefits from this one.
|
|
299
|
+
appendMetricsSnapshot(snapshot, options.metricsDir);
|
|
300
|
+
const breaches = detectBreaches(snapshot, thresholds);
|
|
301
|
+
const sink = options.alertSink;
|
|
302
|
+
for (const breach of breaches) {
|
|
303
|
+
// Live observable surface.
|
|
304
|
+
const env2 = bus.envelope();
|
|
305
|
+
bus.publish({
|
|
306
|
+
...env2,
|
|
307
|
+
kind: "alert_raised",
|
|
308
|
+
metric: breach.metric,
|
|
309
|
+
observed: breach.observed,
|
|
310
|
+
threshold: breach.threshold,
|
|
311
|
+
baselineSessions: thresholds.baselineSessions,
|
|
312
|
+
detail: breach.detail,
|
|
313
|
+
});
|
|
314
|
+
const payload = {
|
|
315
|
+
sessionId: runContext.sessionId,
|
|
316
|
+
metric: breach.metric,
|
|
317
|
+
observed: breach.observed,
|
|
318
|
+
threshold: breach.threshold,
|
|
319
|
+
baselineSessions: thresholds.baselineSessions,
|
|
320
|
+
detail: breach.detail,
|
|
321
|
+
};
|
|
322
|
+
// Durable + off-box delivery β each best-effort so one failing sink never
|
|
323
|
+
// blocks the others (or a turn). Errors are logged, not thrown.
|
|
324
|
+
if (sink?.appendAudit !== undefined) {
|
|
325
|
+
await sink
|
|
326
|
+
.appendAudit(payload)
|
|
327
|
+
.catch((err) => logFlushError(runContext, "alert-audit", err));
|
|
328
|
+
}
|
|
329
|
+
if (sink?.fireAlertHook !== undefined) {
|
|
330
|
+
await sink
|
|
331
|
+
.fireAlertHook(payload)
|
|
332
|
+
.catch((err) => logFlushError(runContext, "alert-hook", err));
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
};
|
|
336
|
+
return { finalize, unsubscribe };
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Cheap in-run thresholds for the digest tally. Keep in sync with
|
|
340
|
+
* `DEFAULT_ADVICE_THRESHOLDS` in `apps/cli/src/advise-rules.ts` β the rule
|
|
341
|
+
* library owns the canonical values, but runtime-core cannot import from
|
|
342
|
+
* apps/cli, so the digest mirrors them here (a drift only ever means the
|
|
343
|
+
* nudge line appears/hides slightly off the report; the report itself is
|
|
344
|
+
* always authoritative).
|
|
345
|
+
*/
|
|
346
|
+
const DIGEST_THRESHOLDS = {
|
|
347
|
+
toolFailureMinCalls: 5,
|
|
348
|
+
toolFailureRate: 0.5,
|
|
349
|
+
truncationContinues: 2,
|
|
350
|
+
compactions: 3,
|
|
351
|
+
asksPerTool: 3,
|
|
352
|
+
stopMinResponses: 4,
|
|
353
|
+
stopAnomalyRate: 0.25,
|
|
354
|
+
};
|
|
355
|
+
/** Stop reasons that are part of healthy operation; everything else
|
|
356
|
+
* (max_tokens, refusal, pause_turn, β¦) counts as anomalous. */
|
|
357
|
+
const CLEAN_STOP_REASONS = new Set(["end_turn", "tool_use", "stop_sequence"]);
|
|
358
|
+
/**
|
|
359
|
+
* Advisor groundwork (items 14/15/17) β mirror the runtime signals that were
|
|
360
|
+
* previously trace-bus-only into the session JSONL, through the SAME
|
|
361
|
+
* `event-log` append path the run already writes its transcript with:
|
|
362
|
+
*
|
|
363
|
+
* error_recovered β `recovery` { errorName, action, depth }
|
|
364
|
+
* tool_call_end β `tool_stats` { toolName, durationMs, isError }
|
|
365
|
+
* permission_decision β `permission` { toolName, decision, askOutcome }
|
|
366
|
+
* model_response β `model_meta` { stopReason, model }
|
|
367
|
+
*
|
|
368
|
+
* Unlike the other `attach*IfEnvSet` subscribers this one is DEFAULT-ON:
|
|
369
|
+
* the lines are tiny (see the granularity note on the `tool_stats` kind in
|
|
370
|
+
* `@crewhaus/event-log`) and they are the food `crewhaus advise` mines, so
|
|
371
|
+
* opting in would starve the advisor on exactly the runs that need it.
|
|
372
|
+
* Disable with `CREWHAUS_ADVISOR_EVENTS=0` (or `false`).
|
|
373
|
+
*
|
|
374
|
+
* Permission mapping detail: a `decision: "ask"` event WITHOUT `askOutcome`
|
|
375
|
+
* is the pre-prompt publish β skipped here so each ask persists exactly once,
|
|
376
|
+
* as its resolution (`askOutcome: "approved" | "denied"`). Allow/deny
|
|
377
|
+
* decisions (including the justification-gate and egress verdicts, which
|
|
378
|
+
* publish additional `permission_decision` events per call) persist with
|
|
379
|
+
* `askOutcome: null`.
|
|
380
|
+
*
|
|
381
|
+
* `append()` failures must never abort a turn, so β like the cost_accrual
|
|
382
|
+
* persistence path β the promise result is logged rather than thrown.
|
|
383
|
+
*/
|
|
384
|
+
export function attachAdvisorPersistence(bus, eventLog, runContext, env = process.env) {
|
|
385
|
+
const gate = env["CREWHAUS_ADVISOR_EVENTS"];
|
|
386
|
+
if (gate === "0" || gate === "false")
|
|
387
|
+
return undefined;
|
|
388
|
+
const persist = (kind, payload) => {
|
|
389
|
+
void eventLog.append({ kind, payload }).catch((err) => {
|
|
390
|
+
runContext.logger.error("advisor_event.persist_failed", {
|
|
391
|
+
kind,
|
|
392
|
+
message: err instanceof Error ? err.message : String(err),
|
|
393
|
+
});
|
|
394
|
+
});
|
|
395
|
+
};
|
|
396
|
+
// In-run digest counters (item 14). Bumped inline in the same handler so
|
|
397
|
+
// the digest costs nothing beyond the persistence pass itself.
|
|
398
|
+
const toolTally = new Map();
|
|
399
|
+
const askTally = new Map();
|
|
400
|
+
let truncationContinues = 0;
|
|
401
|
+
let compactions = 0;
|
|
402
|
+
let responses = 0;
|
|
403
|
+
let anomalousStops = 0;
|
|
404
|
+
const unsubscribe = bus.subscribe((event) => {
|
|
405
|
+
switch (event.kind) {
|
|
406
|
+
case "error_recovered": {
|
|
407
|
+
if (event.action === "continue")
|
|
408
|
+
truncationContinues += 1;
|
|
409
|
+
persist("recovery", {
|
|
410
|
+
errorName: event.errorName,
|
|
411
|
+
action: event.action,
|
|
412
|
+
depth: event.depth,
|
|
413
|
+
});
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
case "tool_call_end": {
|
|
417
|
+
const tally = toolTally.get(event.toolName) ?? { calls: 0, errors: 0 };
|
|
418
|
+
tally.calls += 1;
|
|
419
|
+
if (event.isError)
|
|
420
|
+
tally.errors += 1;
|
|
421
|
+
toolTally.set(event.toolName, tally);
|
|
422
|
+
persist("tool_stats", {
|
|
423
|
+
toolName: event.toolName,
|
|
424
|
+
// Whole milliseconds β performance.now() floats add line bytes
|
|
425
|
+
// without adding advisory signal.
|
|
426
|
+
durationMs: Math.round(event.durationMs),
|
|
427
|
+
isError: event.isError,
|
|
428
|
+
});
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
case "permission_decision": {
|
|
432
|
+
// Pre-prompt ask publish β the resolution (askOutcome set) follows.
|
|
433
|
+
if (event.decision === "ask" && event.askOutcome === undefined)
|
|
434
|
+
return;
|
|
435
|
+
if (event.decision === "ask" && event.askOutcome !== undefined) {
|
|
436
|
+
const ask = askTally.get(event.toolName) ?? { asks: 0, approved: 0 };
|
|
437
|
+
ask.asks += 1;
|
|
438
|
+
if (event.askOutcome === "approved")
|
|
439
|
+
ask.approved += 1;
|
|
440
|
+
askTally.set(event.toolName, ask);
|
|
441
|
+
}
|
|
442
|
+
persist("permission", {
|
|
443
|
+
toolName: event.toolName,
|
|
444
|
+
decision: event.decision,
|
|
445
|
+
askOutcome: event.askOutcome ?? null,
|
|
446
|
+
});
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
case "model_response": {
|
|
450
|
+
responses += 1;
|
|
451
|
+
if (!CLEAN_STOP_REASONS.has(event.stopReason))
|
|
452
|
+
anomalousStops += 1;
|
|
453
|
+
persist("model_meta", { stopReason: event.stopReason, model: event.model });
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
case "compaction_fired": {
|
|
457
|
+
// Already persisted by the runtime as the `compaction` event-log
|
|
458
|
+
// kind β tallied here for the digest only.
|
|
459
|
+
compactions += 1;
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
default:
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
});
|
|
466
|
+
const digestLine = () => {
|
|
467
|
+
const t = DIGEST_THRESHOLDS;
|
|
468
|
+
let findings = 0;
|
|
469
|
+
for (const { calls, errors } of toolTally.values()) {
|
|
470
|
+
if (calls >= t.toolFailureMinCalls && errors / calls >= t.toolFailureRate)
|
|
471
|
+
findings += 1;
|
|
472
|
+
}
|
|
473
|
+
for (const { asks, approved } of askTally.values()) {
|
|
474
|
+
if (asks >= t.asksPerTool && approved === asks)
|
|
475
|
+
findings += 1;
|
|
476
|
+
}
|
|
477
|
+
if (truncationContinues >= t.truncationContinues)
|
|
478
|
+
findings += 1;
|
|
479
|
+
if (compactions >= t.compactions)
|
|
480
|
+
findings += 1;
|
|
481
|
+
if (responses >= t.stopMinResponses && anomalousStops / responses >= t.stopAnomalyRate) {
|
|
482
|
+
findings += 1;
|
|
483
|
+
}
|
|
484
|
+
if (findings === 0)
|
|
485
|
+
return undefined;
|
|
486
|
+
return `advisor: ${findings} finding${findings === 1 ? "" : "s"} β run \`crewhaus advise --session ${runContext.sessionId}\``;
|
|
487
|
+
};
|
|
488
|
+
return { unsubscribe, digestLine };
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* Ops item 38 β mirror the trace-bus-only `mcp_call_end` events into the same
|
|
492
|
+
* session JSONL the transcript already writes to, as the durable `mcp_stats`
|
|
493
|
+
* kind, so `crewhaus mcp doctor` can score per-server MCP health OFFLINE across
|
|
494
|
+
* sessions. `mcp_call_start`/`mcp_call_end` are trace-bus-only today β nothing
|
|
495
|
+
* durably records per-server error-rate / latency β so this subscriber is the
|
|
496
|
+
* durable history the report reads.
|
|
497
|
+
*
|
|
498
|
+
* Payload per line: `{ server, toolName, durationMs, isError }` β the same
|
|
499
|
+
* granularity decision the advisor's `tool_stats` made: per-call, not a per-turn
|
|
500
|
+
* aggregate, so the report keeps the full latency/error distribution. Whole
|
|
501
|
+
* milliseconds only (performance.now() floats add bytes without adding signal).
|
|
502
|
+
*
|
|
503
|
+
* Shares the DEFAULT-ON gate with the advisor events (disable with
|
|
504
|
+
* CREWHAUS_ADVISOR_EVENTS=0): the lines are tiny and they are exactly what the
|
|
505
|
+
* MCP-health report mines, so opting in would starve the doctor on the runs
|
|
506
|
+
* that used MCP servers. `append()` failures are logged, never thrown β a
|
|
507
|
+
* persistence hiccup must not abort a turn.
|
|
508
|
+
*/
|
|
509
|
+
export function attachMcpStatsPersistence(bus, eventLog, runContext, env = process.env) {
|
|
510
|
+
const gate = env["CREWHAUS_ADVISOR_EVENTS"];
|
|
511
|
+
if (gate === "0" || gate === "false")
|
|
512
|
+
return undefined;
|
|
513
|
+
const unsubscribe = bus.subscribe((event) => {
|
|
514
|
+
if (event.kind !== "mcp_call_end")
|
|
515
|
+
return;
|
|
516
|
+
void eventLog
|
|
517
|
+
.append({
|
|
518
|
+
kind: "mcp_stats",
|
|
519
|
+
payload: {
|
|
520
|
+
server: event.server,
|
|
521
|
+
toolName: event.toolName,
|
|
522
|
+
durationMs: Math.round(event.durationMs),
|
|
523
|
+
isError: event.isError,
|
|
524
|
+
},
|
|
525
|
+
})
|
|
526
|
+
.catch((err) => {
|
|
527
|
+
runContext.logger.error("mcp_stats.persist_failed", {
|
|
528
|
+
message: err instanceof Error ? err.message : String(err),
|
|
529
|
+
});
|
|
530
|
+
});
|
|
531
|
+
});
|
|
532
|
+
return { unsubscribe };
|
|
533
|
+
}
|
package/dist/project-memory.d.ts
CHANGED
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
export declare const CANONICAL_MEMORY_FILES: readonly ["AGENTS.md", "CLAUDE.md", "CODE-COMPANION.md", "AGENT.md"];
|
|
1
|
+
export declare const CANONICAL_MEMORY_FILES: readonly ["AGENTS.md", "CLAUDE.md", "CODE-COMPANION.md", "AGENT.md", "LESSONS.md"];
|
|
2
|
+
/**
|
|
3
|
+
* #56 F2 β canonical files whose auto-load is gated on a crewhaus-generated
|
|
4
|
+
* marker. These filenames collide with common human-authored conventions, so
|
|
5
|
+
* only the crewhaus-managed variant (which carries the marker `crewhaus`
|
|
6
|
+
* writes) is auto-injected. Maps filename β the exact marker substring that
|
|
7
|
+
* must be present. `LESSONS.md`'s marker mirrors `apps/cli/src/lessons.ts`'s
|
|
8
|
+
* `LESSONS_MARKER` β the same string `crewhaus lessons update` writes.
|
|
9
|
+
*/
|
|
10
|
+
export declare const MARKER_GATED_FILES: Readonly<Record<string, string>>;
|
|
2
11
|
export declare const DEFAULT_PROJECT_MEMORY_CAP_BYTES: number;
|
|
3
12
|
export interface ProjectMemoryLoadOptions {
|
|
4
13
|
/** Override cwd. Defaults to `process.cwd()`. */
|
|
@@ -7,6 +16,13 @@ export interface ProjectMemoryLoadOptions {
|
|
|
7
16
|
readonly filenames?: readonly string[];
|
|
8
17
|
/** Maximum total bytes across all loaded files. Defaults to 64 KB. */
|
|
9
18
|
readonly capBytes?: number;
|
|
19
|
+
/**
|
|
20
|
+
* Item #56 β absolute path(s) to per-user preference files (outside cwd, e.g.
|
|
21
|
+
* `.crewhaus/preferences/<rater>.md`). Loaded + injected exactly like the
|
|
22
|
+
* canonical files when they exist, so the current user's prefs reach the
|
|
23
|
+
* system prompt. Absent/nonexistent β no-op.
|
|
24
|
+
*/
|
|
25
|
+
readonly extraFiles?: readonly string[];
|
|
10
26
|
}
|
|
11
27
|
export interface LoadedMemoryFile {
|
|
12
28
|
readonly filename: string;
|
package/dist/project-memory.js
CHANGED
|
@@ -40,7 +40,28 @@ export const CANONICAL_MEMORY_FILES = [
|
|
|
40
40
|
"CLAUDE.md",
|
|
41
41
|
"CODE-COMPANION.md",
|
|
42
42
|
"AGENT.md",
|
|
43
|
+
// Item #56 β auto-maintained lessons file. `crewhaus lessons update` mines
|
|
44
|
+
// corrections + recurring failureβfix patterns into this deduped file; the
|
|
45
|
+
// runtime auto-loads it at run start exactly like the other memory files.
|
|
46
|
+
//
|
|
47
|
+
// #56 F2 β GATED: unlike the other canonical files, a project's `LESSONS.md`
|
|
48
|
+
// is a common human-authored convention, so we ONLY auto-inject one that
|
|
49
|
+
// carries the crewhaus marker (`MARKER_GATED_FILES` below). A marker-less
|
|
50
|
+
// human LESSONS.md is skipped β we never silently prepend an unrelated file
|
|
51
|
+
// to the agent's system prompt.
|
|
52
|
+
"LESSONS.md",
|
|
43
53
|
];
|
|
54
|
+
/**
|
|
55
|
+
* #56 F2 β canonical files whose auto-load is gated on a crewhaus-generated
|
|
56
|
+
* marker. These filenames collide with common human-authored conventions, so
|
|
57
|
+
* only the crewhaus-managed variant (which carries the marker `crewhaus`
|
|
58
|
+
* writes) is auto-injected. Maps filename β the exact marker substring that
|
|
59
|
+
* must be present. `LESSONS.md`'s marker mirrors `apps/cli/src/lessons.ts`'s
|
|
60
|
+
* `LESSONS_MARKER` β the same string `crewhaus lessons update` writes.
|
|
61
|
+
*/
|
|
62
|
+
export const MARKER_GATED_FILES = {
|
|
63
|
+
"LESSONS.md": "<!-- crewhaus:lessons -->",
|
|
64
|
+
};
|
|
44
65
|
export const DEFAULT_PROJECT_MEMORY_CAP_BYTES = 64 * 1024;
|
|
45
66
|
/**
|
|
46
67
|
* Scan cwd for project-memory files and return the rendered prompt block.
|
|
@@ -64,6 +85,20 @@ export async function loadProjectMemory(opts = {}) {
|
|
|
64
85
|
}
|
|
65
86
|
catch { }
|
|
66
87
|
}
|
|
88
|
+
// Item #56 β extra per-user preference files (absolute paths outside cwd).
|
|
89
|
+
for (const fullPath of opts.extraFiles ?? []) {
|
|
90
|
+
if (!existsSync(fullPath))
|
|
91
|
+
continue;
|
|
92
|
+
try {
|
|
93
|
+
const st = await stat(fullPath);
|
|
94
|
+
if (!st.isFile())
|
|
95
|
+
continue;
|
|
96
|
+
// Use the basename as the `file=` label in the wrapper.
|
|
97
|
+
const filename = fullPath.split(/[/\\]/).pop() ?? fullPath;
|
|
98
|
+
matched.push({ filename, fullPath, size: st.size });
|
|
99
|
+
}
|
|
100
|
+
catch { }
|
|
101
|
+
}
|
|
67
102
|
if (matched.length === 0) {
|
|
68
103
|
return { files: [], totalBytes: 0, prompt: "" };
|
|
69
104
|
}
|
|
@@ -79,6 +114,13 @@ export async function loadProjectMemory(opts = {}) {
|
|
|
79
114
|
catch {
|
|
80
115
|
continue;
|
|
81
116
|
}
|
|
117
|
+
// #56 F2 β marker-gated files (e.g. LESSONS.md) only auto-inject when they
|
|
118
|
+
// carry the crewhaus marker; a human-authored file of the same name (no
|
|
119
|
+
// marker) is skipped so we never silently prepend it to the system prompt.
|
|
120
|
+
const requiredMarker = MARKER_GATED_FILES[m.filename];
|
|
121
|
+
if (requiredMarker !== undefined && !content.includes(requiredMarker)) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
82
124
|
let truncated = false;
|
|
83
125
|
const contentBytes = Buffer.byteLength(content, "utf8");
|
|
84
126
|
if (contentBytes > remaining) {
|