@crewhaus/runtime-core 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.js +7 -0
- package/dist/exit-rating.d.ts +92 -0
- package/dist/exit-rating.js +163 -0
- package/dist/index.d.ts +33 -1
- package/dist/index.js +95 -10
- package/dist/observability.d.ts +20 -0
- package/dist/observability.js +97 -1
- package/package.json +43 -43
package/dist/alert-watchdog.js
CHANGED
|
@@ -133,6 +133,13 @@ export class SessionMetricsAccumulator {
|
|
|
133
133
|
this.permissionDenials += 1;
|
|
134
134
|
return;
|
|
135
135
|
default:
|
|
136
|
+
// NEW-E-2 is PARTIAL by decision, not by omission: `eval_graded`,
|
|
137
|
+
// `judge_verdict` and `response_rated` fold into metrics-collector
|
|
138
|
+
// (crewhaus_eval_verdicts_total / _score / crewhaus_response_ratings_total)
|
|
139
|
+
// but deliberately NOT into this accumulator, so there is no
|
|
140
|
+
// "judge score dropped below baseline" breach yet. Adding one means
|
|
141
|
+
// new persisted snapshot fields + a `detectBreaches` rule; see the
|
|
142
|
+
// Wave 5 amendments in EVALS-CAMPAIGN-0.4.md before closing the gap.
|
|
136
143
|
return;
|
|
137
144
|
}
|
|
138
145
|
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/** Opt-out env for the exit rating prompt. */
|
|
2
|
+
export declare const NO_EXIT_RATING_ENV = "CREWHAUS_NO_EXIT_RATING";
|
|
3
|
+
export declare const EXIT_RATING_PROMPT = "rate this session? [g]ood / [b]ad / [enter] skip: ";
|
|
4
|
+
export declare const EXIT_RATING_TIMEOUT_MS = 10000;
|
|
5
|
+
/** The event-log EventKind a rating is persisted under (mirrors the CLI's
|
|
6
|
+
* `FEEDBACK_EVENT_KIND` and the channel target's reaction codegen). */
|
|
7
|
+
export declare const FEEDBACK_EVENT_KIND: "user_feedback";
|
|
8
|
+
/**
|
|
9
|
+
* The subset of the spec's `feedback:` block (IR `IrFeedback`) the runtime
|
|
10
|
+
* needs. Presence of the block is the opt-in; `enabled: false` disables the
|
|
11
|
+
* whole contract and `exitPrompt: false` disables just the prompt.
|
|
12
|
+
*/
|
|
13
|
+
export type ExitRatingFeedback = {
|
|
14
|
+
readonly enabled?: boolean;
|
|
15
|
+
readonly exitPrompt?: boolean;
|
|
16
|
+
};
|
|
17
|
+
export type ExitRatingDecision = {
|
|
18
|
+
readonly prompt: boolean;
|
|
19
|
+
readonly reason: string;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* The pure gate. Prompts only when ALL hold: the spec has a feedback block
|
|
23
|
+
* (presence opts in), it isn't disabled (`enabled: false`) or prompt-opted-out
|
|
24
|
+
* (`exitPrompt: false`), `CREWHAUS_NO_EXIT_RATING` is not set, the loop exited
|
|
25
|
+
* cleanly (an aborted or crashed run is not a session anyone wants to rate),
|
|
26
|
+
* stdin is a real TTY (NEVER in piped/CI mode), and the session produced at
|
|
27
|
+
* least one assistant answer.
|
|
28
|
+
*/
|
|
29
|
+
export declare function shouldPromptExitRating(opts: {
|
|
30
|
+
readonly stdinIsTTY: boolean;
|
|
31
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
32
|
+
readonly feedback: ExitRatingFeedback | undefined;
|
|
33
|
+
readonly assistantTurns: number;
|
|
34
|
+
readonly cleanExit: boolean;
|
|
35
|
+
}): ExitRatingDecision;
|
|
36
|
+
export type ExitRatingChoice = "up" | "down" | "skip";
|
|
37
|
+
/** Map the raw keystroke to a rating: g/G → good (thumbs up), b/B → bad
|
|
38
|
+
* (thumbs down), anything else — enter, timeout (undefined), Ctrl-C/D —
|
|
39
|
+
* skips. One keystroke, no confirmation, zero-cost to ignore. */
|
|
40
|
+
export declare function parseExitRatingKey(key: string | undefined): ExitRatingChoice;
|
|
41
|
+
/**
|
|
42
|
+
* The `user_feedback` payload for an exit rating — byte-compatible with the
|
|
43
|
+
* record `crewhaus rate --thumbs` builds, so one distill reads both.
|
|
44
|
+
*/
|
|
45
|
+
export declare function buildExitRatingRecord(input: {
|
|
46
|
+
readonly sessionId: string;
|
|
47
|
+
readonly turnNumber: number;
|
|
48
|
+
readonly thumbs: "up" | "down";
|
|
49
|
+
readonly id?: string;
|
|
50
|
+
readonly ts?: string;
|
|
51
|
+
}): {
|
|
52
|
+
schemaVersion: 1;
|
|
53
|
+
id: string;
|
|
54
|
+
sessionId: string;
|
|
55
|
+
turnNumber: number;
|
|
56
|
+
modality: "binary";
|
|
57
|
+
rating: {
|
|
58
|
+
thumbs: "up" | "down";
|
|
59
|
+
};
|
|
60
|
+
source: "cli";
|
|
61
|
+
ts: string;
|
|
62
|
+
};
|
|
63
|
+
export type ExitRatingRecord = ReturnType<typeof buildExitRatingRecord>;
|
|
64
|
+
/**
|
|
65
|
+
* The whole prompt, with every IO edge injected so the decision + append are
|
|
66
|
+
* unit-testable without a TTY. Returns the recorded choice, or `undefined`
|
|
67
|
+
* when the gate declined to prompt.
|
|
68
|
+
*
|
|
69
|
+
* Best-effort by contract: `readKey`/`append` failures are surfaced to the
|
|
70
|
+
* caller's `onError` (the runtime logs a warning) and never propagate — a
|
|
71
|
+
* rating prompt must not turn a clean session into a failed one.
|
|
72
|
+
*/
|
|
73
|
+
export declare function runExitRating(opts: {
|
|
74
|
+
readonly feedback: ExitRatingFeedback | undefined;
|
|
75
|
+
readonly stdinIsTTY: boolean;
|
|
76
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
77
|
+
readonly assistantTurns: number;
|
|
78
|
+
readonly cleanExit: boolean;
|
|
79
|
+
readonly sessionId: string;
|
|
80
|
+
readonly turnNumber: number;
|
|
81
|
+
/** Reads one keystroke; defaults to the raw-mode {@link readExitRatingKey}. */
|
|
82
|
+
readonly readKey?: () => Promise<string | undefined>;
|
|
83
|
+
readonly append: (record: ExitRatingRecord) => Promise<void>;
|
|
84
|
+
readonly write: (line: string) => void;
|
|
85
|
+
readonly onError?: (err: unknown) => void;
|
|
86
|
+
}): Promise<ExitRatingChoice | undefined>;
|
|
87
|
+
/**
|
|
88
|
+
* One raw-mode keystroke with a timeout (undefined on timeout or when stdin
|
|
89
|
+
* is unusable). Thin IO by design — the gate and the key mapping above are
|
|
90
|
+
* the unit-tested pure functions.
|
|
91
|
+
*/
|
|
92
|
+
export declare function readExitRatingKey(timeoutMs?: number): Promise<string | undefined>;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item 1 — the one-keystroke exit rating prompt, owned by the RUNTIME so both
|
|
3
|
+
* cli surfaces get it.
|
|
4
|
+
*
|
|
5
|
+
* The spec's `feedback:` block declares that a harness collects human ratings
|
|
6
|
+
* on responses. Until now only `crewhaus run` honoured it: the prompt lived in
|
|
7
|
+
* the CLI's own post-session teardown, and the cli emitter dropped the block
|
|
8
|
+
* entirely — a compiled bundle had no rating prompt, so the improvement
|
|
9
|
+
* contract the spec advertised silently vanished from the shipped artifact.
|
|
10
|
+
* The prompt belongs where the REPL is, and the REPL is `runChatLoop`, so it
|
|
11
|
+
* lives here and both surfaces reach it by threading `feedback` into the loop.
|
|
12
|
+
*
|
|
13
|
+
* On a CLEAN REPL exit (the loop returned — not aborted, not thrown) with at
|
|
14
|
+
* least one assistant answer, a TTY stdin, and no opt-out, the runtime asks
|
|
15
|
+
*
|
|
16
|
+
* rate this session? [g]ood / [b]ad / [enter] skip:
|
|
17
|
+
*
|
|
18
|
+
* and appends the answer as the SAME durable `user_feedback` event-log record
|
|
19
|
+
* `crewhaus rate` writes (`source: "cli"`, rating the last turn) — so
|
|
20
|
+
* `crewhaus rate`/`feedback`/`distill` and `optimize --ratings` read a
|
|
21
|
+
* compiled bundle's ratings exactly as they read an interpreted run's.
|
|
22
|
+
*
|
|
23
|
+
* Opt out with `CREWHAUS_NO_EXIT_RATING=1` or spec `feedback.exitPrompt:
|
|
24
|
+
* false`. NEVER prompts in non-TTY/piped mode, so scripts and CI are unaffected.
|
|
25
|
+
*
|
|
26
|
+
* Everything decidable is a pure function here (unit-tested); the raw-mode
|
|
27
|
+
* keystroke read is the one thin IO helper.
|
|
28
|
+
*/
|
|
29
|
+
import { randomBytes } from "node:crypto";
|
|
30
|
+
/** Opt-out env for the exit rating prompt. */
|
|
31
|
+
export const NO_EXIT_RATING_ENV = "CREWHAUS_NO_EXIT_RATING";
|
|
32
|
+
export const EXIT_RATING_PROMPT = "rate this session? [g]ood / [b]ad / [enter] skip: ";
|
|
33
|
+
export const EXIT_RATING_TIMEOUT_MS = 10_000;
|
|
34
|
+
/** The event-log EventKind a rating is persisted under (mirrors the CLI's
|
|
35
|
+
* `FEEDBACK_EVENT_KIND` and the channel target's reaction codegen). */
|
|
36
|
+
export const FEEDBACK_EVENT_KIND = "user_feedback";
|
|
37
|
+
/**
|
|
38
|
+
* The pure gate. Prompts only when ALL hold: the spec has a feedback block
|
|
39
|
+
* (presence opts in), it isn't disabled (`enabled: false`) or prompt-opted-out
|
|
40
|
+
* (`exitPrompt: false`), `CREWHAUS_NO_EXIT_RATING` is not set, the loop exited
|
|
41
|
+
* cleanly (an aborted or crashed run is not a session anyone wants to rate),
|
|
42
|
+
* stdin is a real TTY (NEVER in piped/CI mode), and the session produced at
|
|
43
|
+
* least one assistant answer.
|
|
44
|
+
*/
|
|
45
|
+
export function shouldPromptExitRating(opts) {
|
|
46
|
+
if (opts.feedback === undefined)
|
|
47
|
+
return { prompt: false, reason: "spec has no feedback block" };
|
|
48
|
+
if (opts.feedback.enabled === false) {
|
|
49
|
+
return { prompt: false, reason: "feedback block is disabled (enabled: false)" };
|
|
50
|
+
}
|
|
51
|
+
if (opts.feedback.exitPrompt === false) {
|
|
52
|
+
return { prompt: false, reason: "feedback.exitPrompt is false" };
|
|
53
|
+
}
|
|
54
|
+
const optOut = opts.env[NO_EXIT_RATING_ENV];
|
|
55
|
+
if (optOut !== undefined && optOut !== "" && optOut !== "0") {
|
|
56
|
+
return { prompt: false, reason: `${NO_EXIT_RATING_ENV} is set` };
|
|
57
|
+
}
|
|
58
|
+
if (!opts.cleanExit)
|
|
59
|
+
return { prompt: false, reason: "run did not exit cleanly" };
|
|
60
|
+
if (!opts.stdinIsTTY)
|
|
61
|
+
return { prompt: false, reason: "stdin is not a TTY" };
|
|
62
|
+
if (opts.assistantTurns < 1) {
|
|
63
|
+
return { prompt: false, reason: "session has no assistant turns to rate" };
|
|
64
|
+
}
|
|
65
|
+
return { prompt: true, reason: "feedback block present, TTY, rated-able session" };
|
|
66
|
+
}
|
|
67
|
+
/** Map the raw keystroke to a rating: g/G → good (thumbs up), b/B → bad
|
|
68
|
+
* (thumbs down), anything else — enter, timeout (undefined), Ctrl-C/D —
|
|
69
|
+
* skips. One keystroke, no confirmation, zero-cost to ignore. */
|
|
70
|
+
export function parseExitRatingKey(key) {
|
|
71
|
+
if (key === undefined || key.length === 0)
|
|
72
|
+
return "skip";
|
|
73
|
+
const first = key[0];
|
|
74
|
+
if (first === "g" || first === "G")
|
|
75
|
+
return "up";
|
|
76
|
+
if (first === "b" || first === "B")
|
|
77
|
+
return "down";
|
|
78
|
+
return "skip";
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The `user_feedback` payload for an exit rating — byte-compatible with the
|
|
82
|
+
* record `crewhaus rate --thumbs` builds, so one distill reads both.
|
|
83
|
+
*/
|
|
84
|
+
export function buildExitRatingRecord(input) {
|
|
85
|
+
return {
|
|
86
|
+
schemaVersion: 1,
|
|
87
|
+
id: input.id ?? `fb_${randomBytes(6).toString("hex")}`,
|
|
88
|
+
sessionId: input.sessionId,
|
|
89
|
+
turnNumber: input.turnNumber,
|
|
90
|
+
modality: "binary",
|
|
91
|
+
rating: { thumbs: input.thumbs },
|
|
92
|
+
source: "cli",
|
|
93
|
+
ts: input.ts ?? new Date().toISOString(),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* The whole prompt, with every IO edge injected so the decision + append are
|
|
98
|
+
* unit-testable without a TTY. Returns the recorded choice, or `undefined`
|
|
99
|
+
* when the gate declined to prompt.
|
|
100
|
+
*
|
|
101
|
+
* Best-effort by contract: `readKey`/`append` failures are surfaced to the
|
|
102
|
+
* caller's `onError` (the runtime logs a warning) and never propagate — a
|
|
103
|
+
* rating prompt must not turn a clean session into a failed one.
|
|
104
|
+
*/
|
|
105
|
+
export async function runExitRating(opts) {
|
|
106
|
+
const decision = shouldPromptExitRating({
|
|
107
|
+
stdinIsTTY: opts.stdinIsTTY,
|
|
108
|
+
env: opts.env,
|
|
109
|
+
feedback: opts.feedback,
|
|
110
|
+
assistantTurns: opts.assistantTurns,
|
|
111
|
+
cleanExit: opts.cleanExit,
|
|
112
|
+
});
|
|
113
|
+
if (!decision.prompt)
|
|
114
|
+
return undefined;
|
|
115
|
+
try {
|
|
116
|
+
const choice = parseExitRatingKey(await (opts.readKey ?? readExitRatingKey)());
|
|
117
|
+
if (choice === "skip")
|
|
118
|
+
return "skip";
|
|
119
|
+
await opts.append(buildExitRatingRecord({
|
|
120
|
+
sessionId: opts.sessionId,
|
|
121
|
+
turnNumber: opts.turnNumber,
|
|
122
|
+
thumbs: choice,
|
|
123
|
+
}));
|
|
124
|
+
opts.write(`[feedback] recorded ${choice === "up" ? "good" : "bad"} on ${opts.sessionId} turn ${opts.turnNumber}\n`);
|
|
125
|
+
return choice;
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
opts.onError?.(err);
|
|
129
|
+
return undefined;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* One raw-mode keystroke with a timeout (undefined on timeout or when stdin
|
|
134
|
+
* is unusable). Thin IO by design — the gate and the key mapping above are
|
|
135
|
+
* the unit-tested pure functions.
|
|
136
|
+
*/
|
|
137
|
+
export async function readExitRatingKey(timeoutMs = EXIT_RATING_TIMEOUT_MS) {
|
|
138
|
+
const stdin = process.stdin;
|
|
139
|
+
if (stdin.isTTY !== true || stdin.destroyed)
|
|
140
|
+
return undefined;
|
|
141
|
+
process.stdout.write(EXIT_RATING_PROMPT);
|
|
142
|
+
return await new Promise((resolveKey) => {
|
|
143
|
+
let done = false;
|
|
144
|
+
const finish = (v) => {
|
|
145
|
+
if (done)
|
|
146
|
+
return;
|
|
147
|
+
done = true;
|
|
148
|
+
clearTimeout(timer);
|
|
149
|
+
stdin.off("data", onData);
|
|
150
|
+
if (typeof stdin.setRawMode === "function")
|
|
151
|
+
stdin.setRawMode(false);
|
|
152
|
+
stdin.pause();
|
|
153
|
+
process.stdout.write("\n");
|
|
154
|
+
resolveKey(v);
|
|
155
|
+
};
|
|
156
|
+
const timer = setTimeout(() => finish(undefined), timeoutMs);
|
|
157
|
+
const onData = (chunk) => finish(chunk.toString());
|
|
158
|
+
if (typeof stdin.setRawMode === "function")
|
|
159
|
+
stdin.setRawMode(true);
|
|
160
|
+
stdin.resume();
|
|
161
|
+
stdin.on("data", onData);
|
|
162
|
+
});
|
|
163
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { type SkillRef } from "@crewhaus/skills-registry";
|
|
|
16
16
|
import { type SlashCommand } from "@crewhaus/slash-commands";
|
|
17
17
|
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
18
18
|
import { type ModelUsage } from "@crewhaus/trace-event-bus";
|
|
19
|
+
import { type ExitRatingFeedback } from "./exit-rating";
|
|
19
20
|
import { type AlertSink } from "./observability";
|
|
20
21
|
import type { SloMitigationSink, SloTargets } from "./slo-monitor";
|
|
21
22
|
/**
|
|
@@ -124,10 +125,11 @@ export { JANITOR_BUILTIN_STEPS } from "./janitor";
|
|
|
124
125
|
export type { CreateJanitorOptions, Janitor, JanitorBuiltinStepName, JanitorReservationStore, JanitorRunResult, JanitorStep, JanitorStepName, JanitorStepOutcome, JanitorStepResult, JanitorStepStatus, } from "./janitor";
|
|
125
126
|
export { AGENT_IDENTITY_SCHEMA_VERSION, DEFAULT_IDENTITY_DIR, IDENTITY_FILENAME, agentFingerprint, loadOrCreateAgentIdentity, } from "./identity";
|
|
126
127
|
export type { AgentIdentityFile } from "./identity";
|
|
127
|
-
export { type ApprovalDecision, type PendingApproval, type PendingApprovalStore, type PendingApprovalStoreOptions, createPendingApprovalStore, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
128
|
+
export { type ApprovalDecision, type PendingApproval, type PendingApprovalStore, type PendingApprovalStoreOptions, createPendingApprovalStore, evictExpiredApprovals, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
128
129
|
export { type AlertBreachPayload, type AlertSink, type AttachDefaultSubscribersOptions, attachDefaultSubscribers, } from "./observability";
|
|
129
130
|
export { type AttachedSloMonitor, type AttachSloMonitorOptions, type SloBreach, type SloMitigationEvent, type SloMitigationRung, type SloMitigationSink, type SloTargets, type SloWindowMetrics, DEFAULT_SLO_WINDOW_MS, MIN_SLO_SAMPLES, SloWindow, attachSloMonitor, detectSloBreaches, } from "./slo-monitor";
|
|
130
131
|
export { type AttachIncidentCollectorOptions, type IncidentCapture, type IncidentTriggerKind, attachIncidentCollector, classifyIncidentTrigger, } from "./incident-collector";
|
|
132
|
+
export { type ExitRatingChoice, type ExitRatingDecision, type ExitRatingFeedback, type ExitRatingRecord, EXIT_RATING_PROMPT, EXIT_RATING_TIMEOUT_MS, NO_EXIT_RATING_ENV, buildExitRatingRecord, parseExitRatingKey, runExitRating, shouldPromptExitRating, } from "./exit-rating";
|
|
131
133
|
/**
|
|
132
134
|
* Reconcile a message history by dropping "orphan" `tool_use` blocks — a
|
|
133
135
|
* `tool_use` with no answering `tool_result` anywhere later in the
|
|
@@ -393,6 +395,20 @@ export type RunChatLoopOptions = {
|
|
|
393
395
|
* `singleTurn` and injected-`input` runs never spin.
|
|
394
396
|
*/
|
|
395
397
|
spinner?: boolean;
|
|
398
|
+
/**
|
|
399
|
+
* Sink for the loop's HUMAN-facing output — the `agent> ` prefix, streamed
|
|
400
|
+
* assistant text, `[tool: X]` headers, the `[memory]` notices, abort/timeout
|
|
401
|
+
* notes. Defaults to `process.stdout`.
|
|
402
|
+
*
|
|
403
|
+
* The seam exists because concurrent runs share one stdout: an eval or
|
|
404
|
+
* optimizer pass runs N samples at once, and with every loop writing raw
|
|
405
|
+
* token deltas to the same file descriptor the terminal splices sentences
|
|
406
|
+
* from different samples together mid-word (audit item 20). Callers that
|
|
407
|
+
* fan out give each run its own sink — see eval-runner's per-sample
|
|
408
|
+
* line-buffered writer — so every line stays whole and attributable. A
|
|
409
|
+
* single interactive run passes nothing and is byte-identical to before.
|
|
410
|
+
*/
|
|
411
|
+
stdout?: (chunk: string) => void;
|
|
396
412
|
/**
|
|
397
413
|
* Override the directory under which session metadata `.json` files and
|
|
398
414
|
* event-log `.jsonl` files live. Defaults to `.crewhaus/sessions`.
|
|
@@ -442,6 +458,22 @@ export type RunChatLoopOptions = {
|
|
|
442
458
|
* `pre-slash` first; deny falls through to the original input.
|
|
443
459
|
*/
|
|
444
460
|
slashCommands?: ReadonlyMap<string, SlashCommand>;
|
|
461
|
+
/**
|
|
462
|
+
* Item 1 — the spec's `feedback:` block (the `IrFeedback` subset the runtime
|
|
463
|
+
* consumes). Presence opts the REPL into the one-keystroke exit rating
|
|
464
|
+
* prompt at clean exit; the answer is appended as the same durable
|
|
465
|
+
* `user_feedback` event `crewhaus rate` writes, so `distill` /
|
|
466
|
+
* `optimize --ratings` read it whichever surface produced it.
|
|
467
|
+
*
|
|
468
|
+
* Owned here rather than in the caller so a COMPILED bundle gets the prompt
|
|
469
|
+
* too: it used to live in the CLI's own teardown, and the cli emitter
|
|
470
|
+
* dropped the block, so a compiled harness silently had no rating capture at
|
|
471
|
+
* all. `enabled: false` disables the block; `exitPrompt: false` disables
|
|
472
|
+
* just the prompt; `CREWHAUS_NO_EXIT_RATING=1` is the per-run opt-out.
|
|
473
|
+
* Never prompts in non-TTY/piped mode, nor in `singleTurn` mode (the prompt
|
|
474
|
+
* is a REPL affordance, not a scripting one).
|
|
475
|
+
*/
|
|
476
|
+
feedback?: ExitRatingFeedback;
|
|
445
477
|
/**
|
|
446
478
|
* Feature #53 — cross-session memory auto-recall/auto-capture wiring. When
|
|
447
479
|
* `store` is supplied (the caller builds it from `createMemoryStore`), the
|
package/dist/index.js
CHANGED
|
@@ -11,7 +11,7 @@ import { snip } from "@crewhaus/compaction-snip";
|
|
|
11
11
|
import { DEFAULT_PRICING, computeCostMicros, createCostTracker, resolvePricing, } from "@crewhaus/cost-tracker";
|
|
12
12
|
import { classifyEgress, summarizeEgress, } from "@crewhaus/egress-classifier";
|
|
13
13
|
import { ConfigError, EXIT_CODES, RunFailedError, RuntimeError, isRunFailedError, } from "@crewhaus/errors";
|
|
14
|
-
import { openEventLog } from "@crewhaus/event-log";
|
|
14
|
+
import { DEFAULT_ROOT_DIR, openEventLog } from "@crewhaus/event-log";
|
|
15
15
|
import { aggregateDecisions, runHooks } from "@crewhaus/hooks-engine";
|
|
16
16
|
import { createFailoverChain, createPolicyRouter, createTierRouter, parseModelString, resolveModel, } from "@crewhaus/model-router";
|
|
17
17
|
import { BUILTIN_DEFAULT_RULES, emptyRuleSet, evaluateJustification, evaluateWithReason, ruleBasedJustificationJudge, } from "@crewhaus/permission-engine";
|
|
@@ -21,7 +21,7 @@ import { createRateLimiter } from "@crewhaus/rate-limiter";
|
|
|
21
21
|
import { advanceState, initialRecoveryState, recover, } from "@crewhaus/recovery-engine";
|
|
22
22
|
import { computeReward, openScoreboard, } from "@crewhaus/routing-store";
|
|
23
23
|
import { createRunContext, tagContent } from "@crewhaus/run-context";
|
|
24
|
-
import { createSessionStore, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
24
|
+
import { createSessionStore, evictExpiredApprovals, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
25
25
|
import { formatSkillsForPrompt } from "@crewhaus/skills-registry";
|
|
26
26
|
import { expand as expandSlash } from "@crewhaus/slash-commands";
|
|
27
27
|
import { createStore } from "@crewhaus/state-store";
|
|
@@ -36,8 +36,9 @@ import { TraceEventBus, } from "@crewhaus/trace-event-bus";
|
|
|
36
36
|
import { initialState, transition, } from "@crewhaus/turn-state-machine";
|
|
37
37
|
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
38
38
|
import { createCliMarkdownRenderer, isCliMarkdownEnabled, } from "./cli-markdown";
|
|
39
|
+
import { FEEDBACK_EVENT_KIND, runExitRating } from "./exit-rating";
|
|
39
40
|
import { attachIncidentCollector } from "./incident-collector";
|
|
40
|
-
import { attachAdvisorPersistence, attachDefaultSubscribers, attachMcpStatsPersistence, } from "./observability";
|
|
41
|
+
import { attachAdvisorPersistence, attachDefaultSubscribers, attachMcpStatsPersistence, attachWatchmeCapture, } from "./observability";
|
|
41
42
|
import { loadProjectMemory } from "./project-memory";
|
|
42
43
|
import { createCliOutput, isSpinnerEnabled } from "./spinner";
|
|
43
44
|
/**
|
|
@@ -383,7 +384,7 @@ export { AGENT_IDENTITY_SCHEMA_VERSION, DEFAULT_IDENTITY_DIR, IDENTITY_FILENAME,
|
|
|
383
384
|
// owned by `@crewhaus/session-store`; re-exported here so the CLI/codegen can
|
|
384
385
|
// build the store + input hash it passes as `RunChatLoopOptions.approvals`
|
|
385
386
|
// without depending on session-store directly.
|
|
386
|
-
export { createPendingApprovalStore, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
387
|
+
export { createPendingApprovalStore, evictExpiredApprovals, generateApprovalId, hashApprovalInput, } from "@crewhaus/session-store";
|
|
387
388
|
// Ops item 31 — alert watchdog seams re-exported so the CLI/codegen can build
|
|
388
389
|
// the durable + off-box alert sink (audit append + settings.json alert hook /
|
|
389
390
|
// webhook) it passes as `RunChatLoopOptions.alertSink`.
|
|
@@ -395,6 +396,10 @@ export { DEFAULT_SLO_WINDOW_MS, MIN_SLO_SAMPLES, SloWindow, attachSloMonitor, de
|
|
|
395
396
|
// Ops item 32 — incident collector seams re-exported so the CLI can share the
|
|
396
397
|
// raw-capture shape + trigger classifier with `incident collect`.
|
|
397
398
|
export { attachIncidentCollector, classifyIncidentTrigger, } from "./incident-collector";
|
|
399
|
+
// Item 1 — the exit rating prompt now lives in the runtime (so a COMPILED
|
|
400
|
+
// bundle gets it too, not just `crewhaus run`). Re-exported so the CLI can
|
|
401
|
+
// document the same opt-out env and share the record shape.
|
|
402
|
+
export { EXIT_RATING_PROMPT, EXIT_RATING_TIMEOUT_MS, NO_EXIT_RATING_ENV, buildExitRatingRecord, parseExitRatingKey, runExitRating, shouldPromptExitRating, } from "./exit-rating";
|
|
398
403
|
/**
|
|
399
404
|
* Reconcile a message history by dropping "orphan" `tool_use` blocks — a
|
|
400
405
|
* `tool_use` with no answering `tool_result` anywhere later in the
|
|
@@ -898,6 +903,21 @@ export async function runChatLoop(opts) {
|
|
|
898
903
|
// Housekeeping side-effect: evicts any session whose mtime is older than
|
|
899
904
|
// the TTL (default 30 days). The returned list is intentionally discarded.
|
|
900
905
|
await sessionStore.list();
|
|
906
|
+
// Loop contract 0.4 (G11) — the same sweep for the approvals log sitting
|
|
907
|
+
// beside those sessions. It has no other unattended pruner: only `list()`
|
|
908
|
+
// compacts it, and `list()`'s only production callers are the human-invoked
|
|
909
|
+
// `crewhaus approvals` verbs — so on a daemon or a cron'd run the file grew
|
|
910
|
+
// forever, carrying the raw tool input of every park.
|
|
911
|
+
//
|
|
912
|
+
// Deliberately NOT routed through `opts.approvals.store`: that option is
|
|
913
|
+
// narrowed to `persist | get | resolve` so callers can inject minimal twins,
|
|
914
|
+
// and widening the Pick would break them. Unconditional (not gated on
|
|
915
|
+
// `opts.approvals`) so a harness that has since switched to `ask_mode: deny`
|
|
916
|
+
// still gets its old parks pruned; it costs one ENOENT when the file does
|
|
917
|
+
// not exist. Best-effort, exactly like the session sweep above.
|
|
918
|
+
await evictExpiredApprovals({
|
|
919
|
+
...(sessionRootDir !== undefined ? { rootDir: sessionRootDir } : {}),
|
|
920
|
+
}).catch(() => undefined);
|
|
901
921
|
// Adaptive model routing (spec `agent.model_pool`). Resolve every candidate
|
|
902
922
|
// adapter ONCE at boot (mirroring the tier router), open the durable reward
|
|
903
923
|
// scoreboard beside the sessions dir, and build the PolicyRouter. The
|
|
@@ -1179,9 +1199,14 @@ export async function runChatLoop(opts) {
|
|
|
1179
1199
|
// byte-identical to before. `out.write` is the spinner-aware stdout sink:
|
|
1180
1200
|
// every interactive write that can land while the animation is live goes
|
|
1181
1201
|
// through it so the two never collide. `opts.spinner` overrides the auto
|
|
1182
|
-
// decision.
|
|
1202
|
+
// decision. `opts.stdout` redirects the whole human-facing surface (item 20
|
|
1203
|
+
// — concurrent eval samples each get their own sink instead of splicing
|
|
1204
|
+
// token deltas into one shared stdout).
|
|
1183
1205
|
const spinnerEnabled = opts.spinner ?? (opts.singleTurn !== true && opts.input === undefined && isSpinnerEnabled());
|
|
1184
|
-
const out = createCliOutput({
|
|
1206
|
+
const out = createCliOutput({
|
|
1207
|
+
enabled: spinnerEnabled,
|
|
1208
|
+
...(opts.stdout !== undefined ? { write: opts.stdout } : {}),
|
|
1209
|
+
});
|
|
1185
1210
|
// M2.1 — CLI markdown renderer (env-gated). When CREWHAUS_CLI_MARKDOWN=1,
|
|
1186
1211
|
// text-delta chunks flow through a streaming markdown → ANSI transform so
|
|
1187
1212
|
// the user sees rendered bold/italic/headers/lists/code-fences instead of
|
|
@@ -1263,6 +1288,11 @@ export async function runChatLoop(opts) {
|
|
|
1263
1288
|
// score per-server MCP health offline. Shares the advisor's DEFAULT-ON gate
|
|
1264
1289
|
// (CREWHAUS_ADVISOR_EVENTS=0 disables both). See observability.ts.
|
|
1265
1290
|
const mcpStatsPersist = attachMcpStatsPersistence(bus, eventLog, runContext);
|
|
1291
|
+
// "Watch me" — live capture tap (CREWHAUS_WATCHME-gated, off by default):
|
|
1292
|
+
// append the bus-only TraceEvent kinds to the `sessions/<id>.events.jsonl`
|
|
1293
|
+
// sibling that `sessions export` already reads. Mirrored/ephemeral kinds are
|
|
1294
|
+
// skipped, so the sibling stays metadata-grade. See observability.ts.
|
|
1295
|
+
const watchmeCapture = attachWatchmeCapture(bus, sessionRootDir ?? DEFAULT_ROOT_DIR, sessionId);
|
|
1266
1296
|
// Per-run state container — coordination surface for hooks/skills/tools.
|
|
1267
1297
|
// Shipped as plumbing in Section 10; v0.3.0 Goal 1 (§2.5) lands its first
|
|
1268
1298
|
// consumer: plan-mutating tools set `PLAN_DIRTY_STATE_KEY` here (via the
|
|
@@ -1320,6 +1350,34 @@ export async function runChatLoop(opts) {
|
|
|
1320
1350
|
runContext.logger.warn("memory auto-capture failed", { error: err.message });
|
|
1321
1351
|
}
|
|
1322
1352
|
};
|
|
1353
|
+
// Item 1 — how many turns produced a non-empty assistant answer. The exit
|
|
1354
|
+
// rating prompt's "this session actually said something worth rating" floor.
|
|
1355
|
+
let assistantAnswers = 0;
|
|
1356
|
+
/**
|
|
1357
|
+
* Item 1 — the exit rating prompt (spec `feedback:`). Runs in the REPL
|
|
1358
|
+
* teardown AFTER readline has released stdin (the read is raw-mode) and
|
|
1359
|
+
* BEFORE the event log closes, so the rating lands in this session's own
|
|
1360
|
+
* transcript. Best-effort by contract: a prompt/append failure must never
|
|
1361
|
+
* turn a clean session into a non-zero exit.
|
|
1362
|
+
*/
|
|
1363
|
+
const maybeExitRating = async (cleanExit) => {
|
|
1364
|
+
await runExitRating({
|
|
1365
|
+
feedback: opts.feedback,
|
|
1366
|
+
// A caller-supplied `input` stream means the runtime does not own the
|
|
1367
|
+
// terminal (tests, piped hosts), so it is never a prompt target.
|
|
1368
|
+
stdinIsTTY: opts.input === undefined && process.stdin.isTTY === true,
|
|
1369
|
+
env: process.env,
|
|
1370
|
+
assistantTurns: assistantAnswers,
|
|
1371
|
+
cleanExit,
|
|
1372
|
+
sessionId,
|
|
1373
|
+
turnNumber: runContext.turnNumber,
|
|
1374
|
+
append: async (record) => {
|
|
1375
|
+
await eventLog.append({ kind: FEEDBACK_EVENT_KIND, payload: record });
|
|
1376
|
+
},
|
|
1377
|
+
write: (line) => out.write(line),
|
|
1378
|
+
onError: (err) => runContext.logger.warn("exit rating prompt failed", { error: err.message }),
|
|
1379
|
+
});
|
|
1380
|
+
};
|
|
1323
1381
|
// -------------------------------------------------------------------------
|
|
1324
1382
|
// v0.3.0 Goal 1 — continuity wiring (§2.3 ledger, §2.5 tail, §2.8 handoff).
|
|
1325
1383
|
// Everything below is inert when `opts.continuity` is absent: no tail
|
|
@@ -1544,7 +1602,9 @@ export async function runChatLoop(opts) {
|
|
|
1544
1602
|
]
|
|
1545
1603
|
: [];
|
|
1546
1604
|
if (projectMemory.files.length > 0) {
|
|
1547
|
-
|
|
1605
|
+
// Human-facing loop output → the same sink as everything else, so a
|
|
1606
|
+
// fanned-out caller (eval samples) can attribute the line (item 20).
|
|
1607
|
+
out.write(`[memory] loaded ${projectMemory.files.length} project memory file(s): ${projectMemory.files
|
|
1548
1608
|
.map((f) => f.filename)
|
|
1549
1609
|
.join(", ")}\n`);
|
|
1550
1610
|
}
|
|
@@ -1574,7 +1634,7 @@ export async function runChatLoop(opts) {
|
|
|
1574
1634
|
const text = await renderRecalledMemory(lines);
|
|
1575
1635
|
if (text !== undefined) {
|
|
1576
1636
|
memoryRecallBlock = [{ type: "text", text, cache_control: { type: "ephemeral" } }];
|
|
1577
|
-
|
|
1637
|
+
out.write(`[memory] recalled ${lines.length} memory(ies) into the prompt\n`);
|
|
1578
1638
|
}
|
|
1579
1639
|
}
|
|
1580
1640
|
catch (err) {
|
|
@@ -1609,7 +1669,7 @@ export async function runChatLoop(opts) {
|
|
|
1609
1669
|
const text = await renderRecalledMemory(lines);
|
|
1610
1670
|
volatileRecallBlocks = text !== undefined ? [{ type: "text", text }] : [];
|
|
1611
1671
|
if (lines.length > 0) {
|
|
1612
|
-
|
|
1672
|
+
out.write(`[memory] refreshed ${lines.length} recalled memory(ies) for this turn\n`);
|
|
1613
1673
|
}
|
|
1614
1674
|
}
|
|
1615
1675
|
catch (err) {
|
|
@@ -2284,6 +2344,13 @@ export async function runChatLoop(opts) {
|
|
|
2284
2344
|
...(skills.length > 0 ? { skills } : {}),
|
|
2285
2345
|
...(opts.failureTaxonomy !== undefined ? { failureTaxonomy: opts.failureTaxonomy } : {}),
|
|
2286
2346
|
...(bridgeContinuitySeam !== undefined ? { continuity: bridgeContinuitySeam } : {}),
|
|
2347
|
+
// Loop contract 0.4 (G11) — the approval seam, so a Task child's
|
|
2348
|
+
// headless `ask` parks against the SAME store the parent parks against
|
|
2349
|
+
// instead of collapsing to a denial. `askMode` is spread only when the
|
|
2350
|
+
// caller set it, so an unset parent leaves the child on this loop's own
|
|
2351
|
+
// `"pause"` default — identical resolution, no double-defaulting.
|
|
2352
|
+
...(opts.askMode !== undefined ? { askMode: opts.askMode } : {}),
|
|
2353
|
+
...(opts.approvals !== undefined ? { approvals: opts.approvals } : {}),
|
|
2287
2354
|
};
|
|
2288
2355
|
// Spin "running <tool>…" for exactly the execution window — started after
|
|
2289
2356
|
// every gate (permission/justification/egress) so it never overlaps the
|
|
@@ -3555,6 +3622,10 @@ export async function runChatLoop(opts) {
|
|
|
3555
3622
|
verdict,
|
|
3556
3623
|
graderType: evaluation.graderType,
|
|
3557
3624
|
retryIndex,
|
|
3625
|
+
// The cap is only knowable HERE. Without it a downstream reader
|
|
3626
|
+
// (`dataset mine`'s eval-fail signal) cannot distinguish a ladder that
|
|
3627
|
+
// was fully spent from one cut short, and over-claims "exhausted".
|
|
3628
|
+
maxRetries,
|
|
3558
3629
|
});
|
|
3559
3630
|
if (verdict === "pass" || evaluation.onFail === "note")
|
|
3560
3631
|
return attempt;
|
|
@@ -3692,6 +3763,7 @@ export async function runChatLoop(opts) {
|
|
|
3692
3763
|
budgetMeter?.unsubscribe();
|
|
3693
3764
|
advisorPersist?.unsubscribe();
|
|
3694
3765
|
mcpStatsPersist?.unsubscribe();
|
|
3766
|
+
watchmeCapture?.unsubscribe();
|
|
3695
3767
|
printAdvisorDigest(advisorPersist);
|
|
3696
3768
|
await sessionStore
|
|
3697
3769
|
.update(sessionId, { lastTurnIndex: runContext.turnNumber })
|
|
@@ -3794,6 +3866,10 @@ export async function runChatLoop(opts) {
|
|
|
3794
3866
|
if (shouldInstallSigint) {
|
|
3795
3867
|
process.on("SIGINT", sigintHandler);
|
|
3796
3868
|
}
|
|
3869
|
+
// Item 1 — set once the REPL loop has run to completion without throwing.
|
|
3870
|
+
// The exit rating prompt fires only on such a clean exit: an aborted or
|
|
3871
|
+
// crashed session is not one anyone wants to be asked to rate.
|
|
3872
|
+
let replExitedCleanly = false;
|
|
3797
3873
|
try {
|
|
3798
3874
|
while (true) {
|
|
3799
3875
|
if (runAbort.signal.aborted)
|
|
@@ -3902,7 +3978,11 @@ export async function runChatLoop(opts) {
|
|
|
3902
3978
|
// RunFailedError, which the catch rethrows.
|
|
3903
3979
|
armTurnTimer();
|
|
3904
3980
|
try {
|
|
3905
|
-
await runEvaluatedTurn(messages);
|
|
3981
|
+
const { terminalContent } = await runEvaluatedTurn(messages);
|
|
3982
|
+
// Item 1 — the exit rating prompt's "worth rating" floor: count only
|
|
3983
|
+
// turns that ended in a real assistant answer.
|
|
3984
|
+
if (terminalText(terminalContent).trim() !== "")
|
|
3985
|
+
assistantAnswers += 1;
|
|
3906
3986
|
}
|
|
3907
3987
|
catch (err) {
|
|
3908
3988
|
// Defensive: tear down any animation a thrown turn left spinning.
|
|
@@ -3939,6 +4019,7 @@ export async function runChatLoop(opts) {
|
|
|
3939
4019
|
if (runTimeout !== undefined) {
|
|
3940
4020
|
await throwTimeout(runTimeout);
|
|
3941
4021
|
}
|
|
4022
|
+
replExitedCleanly = !runAbort.signal.aborted;
|
|
3942
4023
|
}
|
|
3943
4024
|
finally {
|
|
3944
4025
|
clearAllTimers();
|
|
@@ -3959,11 +4040,15 @@ export async function runChatLoop(opts) {
|
|
|
3959
4040
|
budgetMeter?.unsubscribe();
|
|
3960
4041
|
advisorPersist?.unsubscribe();
|
|
3961
4042
|
mcpStatsPersist?.unsubscribe();
|
|
4043
|
+
watchmeCapture?.unsubscribe();
|
|
3962
4044
|
printAdvisorDigest(advisorPersist);
|
|
3963
4045
|
if (shouldInstallSigint) {
|
|
3964
4046
|
process.removeListener("SIGINT", sigintHandler);
|
|
3965
4047
|
}
|
|
3966
4048
|
rl.close();
|
|
4049
|
+
// Item 1 — after readline released stdin (the rating read is raw-mode) and
|
|
4050
|
+
// before the event log closes (the rating is appended to this session).
|
|
4051
|
+
await maybeExitRating(replExitedCleanly);
|
|
3967
4052
|
await sessionStore.update(sessionId, { lastTurnIndex: runContext.turnNumber }).catch((err) => {
|
|
3968
4053
|
runContext.logger.warn("session-store: lastTurnIndex update failed", {
|
|
3969
4054
|
error: err.message,
|
package/dist/observability.d.ts
CHANGED
|
@@ -212,3 +212,23 @@ export type AttachedMcpStatsPersistence = {
|
|
|
212
212
|
* persistence hiccup must not abort a turn.
|
|
213
213
|
*/
|
|
214
214
|
export declare function attachMcpStatsPersistence(bus: TraceEventBus, eventLog: EventLog, runContext: RunContext, env?: NodeJS.ProcessEnv): AttachedMcpStatsPersistence | undefined;
|
|
215
|
+
export declare const WATCHME_MIRRORED_KINDS: ReadonlySet<string>;
|
|
216
|
+
export type AttachedWatchmeCapture = {
|
|
217
|
+
unsubscribe(): void;
|
|
218
|
+
};
|
|
219
|
+
/**
|
|
220
|
+
* "Watch me" (§6.1) — the live capture tap behind `crewhaus watchme`. Appends
|
|
221
|
+
* every BUS-ONLY TraceEvent (envelope included — `formatJsonLine` shape) as
|
|
222
|
+
* one JSON line to `<sessionsDir>/<sessionId>.events.jsonl`, the sibling
|
|
223
|
+
* `sessions export` already reads. Skipped: the ephemeral stream kinds and
|
|
224
|
+
* every durably mirrored kind ({@link WATCHME_MIRRORED_KINDS}), so content
|
|
225
|
+
* stays solely in the session `.jsonl` while the sibling carries exact
|
|
226
|
+
* per-turn model attribution via the envelope `turnNumber` on
|
|
227
|
+
* `model_response`.
|
|
228
|
+
*
|
|
229
|
+
* Gated on CREWHAUS_WATCHME=1|true (stamped by `crewhaus run` /
|
|
230
|
+
* the compiled-bundle preambles); returns undefined otherwise. Appends are
|
|
231
|
+
* synchronous (`appendFileSync`, mode 0600, one line < PIPE_BUF ⇒ atomic) and
|
|
232
|
+
* every failure is swallowed — capture can never crash a run.
|
|
233
|
+
*/
|
|
234
|
+
export declare function attachWatchmeCapture(bus: TraceEventBus, sessionsDir: string, sessionId: string, env?: NodeJS.ProcessEnv): AttachedWatchmeCapture | undefined;
|
package/dist/observability.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { appendFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
1
3
|
import { createCostTracker, formatUsdMicros } from "@crewhaus/cost-tracker";
|
|
2
4
|
import { attachIfEnvSet as attachMetricsIfEnvSet, } from "@crewhaus/metrics-collector";
|
|
3
5
|
import { attachIfEnvSet as attachOtelIfEnvSet, } from "@crewhaus/otel-exporter";
|
|
4
|
-
import { attachIfEnvSet as attachPrinterIfEnvSet, } from "@crewhaus/structured-event-printer";
|
|
6
|
+
import { attachIfEnvSet as attachPrinterIfEnvSet, formatJsonLine, } from "@crewhaus/structured-event-printer";
|
|
5
7
|
import { SessionMetricsAccumulator, appendMetricsSnapshot, deriveThresholds, detectBreaches, readMetricsHistory, } from "./alert-watchdog";
|
|
6
8
|
import { attachSloMonitor, } from "./slo-monitor";
|
|
7
9
|
/**
|
|
@@ -531,3 +533,97 @@ export function attachMcpStatsPersistence(bus, eventLog, runContext, env = proce
|
|
|
531
533
|
});
|
|
532
534
|
return { unsubscribe };
|
|
533
535
|
}
|
|
536
|
+
// -------- "watch me" live capture tap (design/watch-me.md §6.1) --------
|
|
537
|
+
/**
|
|
538
|
+
* Reconciliation with the SHIPPED G26 `observability:` block (watch-me §4.5 —
|
|
539
|
+
* decided, superseding all "when G26 lands" language): `watchme:` is a SIBLING
|
|
540
|
+
* spec block, not an `observability:` sub-key. `observability.trace.level`
|
|
541
|
+
* controls the ring buffer + printers ONLY; the bus always exists, so this
|
|
542
|
+
* capture subscriber attaches independently of that knob, gated solely on
|
|
543
|
+
* CREWHAUS_WATCHME. The env itself is stamped at the same junctions the G26
|
|
544
|
+
* stamps already use (`applyRunObservabilityEnv` on the interpreter path, the
|
|
545
|
+
* target bundles' boot-stamp emitters), so precedence semantics stay in one
|
|
546
|
+
* place per path.
|
|
547
|
+
*/
|
|
548
|
+
/** Stream kinds published per token/chunk — pure progress signal, never captured. */
|
|
549
|
+
const WATCHME_EPHEMERAL_KINDS = new Set([
|
|
550
|
+
"model_stream_token",
|
|
551
|
+
"tool_stream_chunk",
|
|
552
|
+
]);
|
|
553
|
+
/**
|
|
554
|
+
* The session `.jsonl` mirror vocabulary — every kind the event log persists
|
|
555
|
+
* durably: the transcript kinds the runtime writes itself plus the mirrored
|
|
556
|
+
* kinds emitted by the cost/advisor/mcp-stats subscribers in this file. The
|
|
557
|
+
* capture tap skips these, which keeps the `.events.jsonl` sibling
|
|
558
|
+
* metadata-grade BY CONSTRUCTION (all content-carrying kinds are durable ones)
|
|
559
|
+
* and makes the two files' kind sets disjoint — a pinned invariant. Typed as
|
|
560
|
+
* an exhaustive Record over event-log's `EventKind` so a new durable kind
|
|
561
|
+
* cannot land without this set learning about it.
|
|
562
|
+
*/
|
|
563
|
+
const WATCHME_MIRRORED_KIND_RECORD = {
|
|
564
|
+
user_message: true,
|
|
565
|
+
assistant_message: true,
|
|
566
|
+
tool_use: true,
|
|
567
|
+
tool_result: true,
|
|
568
|
+
error: true,
|
|
569
|
+
run_failed: true,
|
|
570
|
+
context_evicted: true,
|
|
571
|
+
compaction: true,
|
|
572
|
+
sub_agent_start: true,
|
|
573
|
+
sub_agent_end: true,
|
|
574
|
+
role_start: true,
|
|
575
|
+
role_end: true,
|
|
576
|
+
handoff: true,
|
|
577
|
+
a2a_message: true,
|
|
578
|
+
a2a_turn_start: true,
|
|
579
|
+
a2a_turn_end: true,
|
|
580
|
+
crew_done: true,
|
|
581
|
+
cost_accrual: true,
|
|
582
|
+
user_feedback: true,
|
|
583
|
+
recovery: true,
|
|
584
|
+
tool_stats: true,
|
|
585
|
+
permission: true,
|
|
586
|
+
model_meta: true,
|
|
587
|
+
mcp_stats: true,
|
|
588
|
+
model_route: true,
|
|
589
|
+
wiki_write: true,
|
|
590
|
+
plan_update: true,
|
|
591
|
+
goal_update: true,
|
|
592
|
+
action_proof: true,
|
|
593
|
+
dream_run: true,
|
|
594
|
+
};
|
|
595
|
+
export const WATCHME_MIRRORED_KINDS = new Set(Object.keys(WATCHME_MIRRORED_KIND_RECORD));
|
|
596
|
+
/**
|
|
597
|
+
* "Watch me" (§6.1) — the live capture tap behind `crewhaus watchme`. Appends
|
|
598
|
+
* every BUS-ONLY TraceEvent (envelope included — `formatJsonLine` shape) as
|
|
599
|
+
* one JSON line to `<sessionsDir>/<sessionId>.events.jsonl`, the sibling
|
|
600
|
+
* `sessions export` already reads. Skipped: the ephemeral stream kinds and
|
|
601
|
+
* every durably mirrored kind ({@link WATCHME_MIRRORED_KINDS}), so content
|
|
602
|
+
* stays solely in the session `.jsonl` while the sibling carries exact
|
|
603
|
+
* per-turn model attribution via the envelope `turnNumber` on
|
|
604
|
+
* `model_response`.
|
|
605
|
+
*
|
|
606
|
+
* Gated on CREWHAUS_WATCHME=1|true (stamped by `crewhaus run` /
|
|
607
|
+
* the compiled-bundle preambles); returns undefined otherwise. Appends are
|
|
608
|
+
* synchronous (`appendFileSync`, mode 0600, one line < PIPE_BUF ⇒ atomic) and
|
|
609
|
+
* every failure is swallowed — capture can never crash a run.
|
|
610
|
+
*/
|
|
611
|
+
export function attachWatchmeCapture(bus, sessionsDir, sessionId, env = process.env) {
|
|
612
|
+
const gate = env["CREWHAUS_WATCHME"];
|
|
613
|
+
if (gate !== "1" && gate !== "true")
|
|
614
|
+
return undefined;
|
|
615
|
+
const path = join(sessionsDir, `${sessionId}.events.jsonl`);
|
|
616
|
+
const unsubscribe = bus.subscribe((event) => {
|
|
617
|
+
if (WATCHME_EPHEMERAL_KINDS.has(event.kind))
|
|
618
|
+
return;
|
|
619
|
+
if (WATCHME_MIRRORED_KINDS.has(event.kind))
|
|
620
|
+
return;
|
|
621
|
+
try {
|
|
622
|
+
appendFileSync(path, formatJsonLine(event), { mode: 0o600 });
|
|
623
|
+
}
|
|
624
|
+
catch {
|
|
625
|
+
// Swallowed by design — a capture hiccup must never abort a turn.
|
|
626
|
+
}
|
|
627
|
+
});
|
|
628
|
+
return { unsubscribe };
|
|
629
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/runtime-core",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Minimal runtime imported by generated harness bundles",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -16,51 +16,51 @@
|
|
|
16
16
|
},
|
|
17
17
|
"dependencies": {
|
|
18
18
|
"@anthropic-ai/sdk": "^0.96.0",
|
|
19
|
-
"@crewhaus/abort-controller": "0.4.
|
|
20
|
-
"@crewhaus/adapter-anthropic": "0.4.
|
|
21
|
-
"@crewhaus/agent-context-isolation": "0.4.
|
|
22
|
-
"@crewhaus/boundary-classifier": "0.4.
|
|
23
|
-
"@crewhaus/circuit-breaker": "0.4.
|
|
24
|
-
"@crewhaus/compaction-autocompact": "0.4.
|
|
25
|
-
"@crewhaus/compaction-curator": "0.4.
|
|
26
|
-
"@crewhaus/compaction-snip": "0.4.
|
|
27
|
-
"@crewhaus/cost-tracker": "0.4.
|
|
28
|
-
"@crewhaus/egress-classifier": "0.4.
|
|
29
|
-
"@crewhaus/errors": "0.4.
|
|
30
|
-
"@crewhaus/event-log": "0.4.
|
|
31
|
-
"@crewhaus/hooks-engine": "0.4.
|
|
32
|
-
"@crewhaus/model-router": "0.4.
|
|
33
|
-
"@crewhaus/permission-engine": "0.4.
|
|
34
|
-
"@crewhaus/prompt-cache-manager": "0.4.
|
|
35
|
-
"@crewhaus/prompt-injection-detector": "0.4.
|
|
36
|
-
"@crewhaus/rate-limiter": "0.4.
|
|
37
|
-
"@crewhaus/recovery-engine": "0.4.
|
|
38
|
-
"@crewhaus/routing-store": "0.4.
|
|
39
|
-
"@crewhaus/run-context": "0.4.
|
|
40
|
-
"@crewhaus/session-store": "0.4.
|
|
41
|
-
"@crewhaus/skills-registry": "0.4.
|
|
42
|
-
"@crewhaus/slash-commands": "0.4.
|
|
43
|
-
"@crewhaus/state-store": "0.4.
|
|
44
|
-
"@crewhaus/tenancy": "0.4.
|
|
45
|
-
"@crewhaus/token-budget": "0.4.
|
|
46
|
-
"@crewhaus/tool-catalog": "0.4.
|
|
47
|
-
"@crewhaus/tool-executor": "0.4.
|
|
48
|
-
"@crewhaus/tool-loop-detection": "0.4.
|
|
49
|
-
"@crewhaus/tool-orchestrator": "0.4.
|
|
50
|
-
"@crewhaus/tool-result-store": "0.4.
|
|
51
|
-
"@crewhaus/streaming-tool-executor": "0.4.
|
|
52
|
-
"@crewhaus/turn-state-machine": "0.4.
|
|
53
|
-
"@crewhaus/trace-event-bus": "0.4.
|
|
54
|
-
"@crewhaus/worker-runtime": "0.4.
|
|
55
|
-
"@crewhaus/structured-event-printer": "0.4.
|
|
56
|
-
"@crewhaus/metrics-collector": "0.4.
|
|
57
|
-
"@crewhaus/otel-exporter": "0.4.
|
|
19
|
+
"@crewhaus/abort-controller": "0.4.2",
|
|
20
|
+
"@crewhaus/adapter-anthropic": "0.4.2",
|
|
21
|
+
"@crewhaus/agent-context-isolation": "0.4.2",
|
|
22
|
+
"@crewhaus/boundary-classifier": "0.4.2",
|
|
23
|
+
"@crewhaus/circuit-breaker": "0.4.2",
|
|
24
|
+
"@crewhaus/compaction-autocompact": "0.4.2",
|
|
25
|
+
"@crewhaus/compaction-curator": "0.4.2",
|
|
26
|
+
"@crewhaus/compaction-snip": "0.4.2",
|
|
27
|
+
"@crewhaus/cost-tracker": "0.4.2",
|
|
28
|
+
"@crewhaus/egress-classifier": "0.4.2",
|
|
29
|
+
"@crewhaus/errors": "0.4.2",
|
|
30
|
+
"@crewhaus/event-log": "0.4.2",
|
|
31
|
+
"@crewhaus/hooks-engine": "0.4.2",
|
|
32
|
+
"@crewhaus/model-router": "0.4.2",
|
|
33
|
+
"@crewhaus/permission-engine": "0.4.2",
|
|
34
|
+
"@crewhaus/prompt-cache-manager": "0.4.2",
|
|
35
|
+
"@crewhaus/prompt-injection-detector": "0.4.2",
|
|
36
|
+
"@crewhaus/rate-limiter": "0.4.2",
|
|
37
|
+
"@crewhaus/recovery-engine": "0.4.2",
|
|
38
|
+
"@crewhaus/routing-store": "0.4.2",
|
|
39
|
+
"@crewhaus/run-context": "0.4.2",
|
|
40
|
+
"@crewhaus/session-store": "0.4.2",
|
|
41
|
+
"@crewhaus/skills-registry": "0.4.2",
|
|
42
|
+
"@crewhaus/slash-commands": "0.4.2",
|
|
43
|
+
"@crewhaus/state-store": "0.4.2",
|
|
44
|
+
"@crewhaus/tenancy": "0.4.2",
|
|
45
|
+
"@crewhaus/token-budget": "0.4.2",
|
|
46
|
+
"@crewhaus/tool-catalog": "0.4.2",
|
|
47
|
+
"@crewhaus/tool-executor": "0.4.2",
|
|
48
|
+
"@crewhaus/tool-loop-detection": "0.4.2",
|
|
49
|
+
"@crewhaus/tool-orchestrator": "0.4.2",
|
|
50
|
+
"@crewhaus/tool-result-store": "0.4.2",
|
|
51
|
+
"@crewhaus/streaming-tool-executor": "0.4.2",
|
|
52
|
+
"@crewhaus/turn-state-machine": "0.4.2",
|
|
53
|
+
"@crewhaus/trace-event-bus": "0.4.2",
|
|
54
|
+
"@crewhaus/worker-runtime": "0.4.2",
|
|
55
|
+
"@crewhaus/structured-event-printer": "0.4.2",
|
|
56
|
+
"@crewhaus/metrics-collector": "0.4.2",
|
|
57
|
+
"@crewhaus/otel-exporter": "0.4.2",
|
|
58
58
|
"zod-to-json-schema": "^3.23.5"
|
|
59
59
|
},
|
|
60
60
|
"devDependencies": {
|
|
61
|
-
"@crewhaus/logging": "0.4.
|
|
62
|
-
"@crewhaus/tool-builder": "0.4.
|
|
63
|
-
"@crewhaus/tool-plan": "0.4.
|
|
61
|
+
"@crewhaus/logging": "0.4.2",
|
|
62
|
+
"@crewhaus/tool-builder": "0.4.2",
|
|
63
|
+
"@crewhaus/tool-plan": "0.4.2",
|
|
64
64
|
"zod": "^3.23.8"
|
|
65
65
|
},
|
|
66
66
|
"license": "Apache-2.0",
|