crewhaus 0.1.8 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/dist/advice-apply.d.ts +182 -0
- package/dist/advice-apply.js +286 -0
- package/dist/advise-rules.d.ts +348 -0
- package/dist/advise-rules.js +905 -0
- package/dist/alert-sink.d.ts +48 -0
- package/dist/alert-sink.js +86 -0
- package/dist/approval-gate.d.ts +127 -0
- package/dist/approval-gate.js +254 -0
- package/dist/audit-verify.d.ts +69 -0
- package/dist/audit-verify.js +97 -0
- package/dist/autodistill.d.ts +113 -0
- package/dist/autodistill.js +256 -0
- package/dist/channel-provision.d.ts +360 -0
- package/dist/channel-provision.js +881 -0
- package/dist/ci-scaffold.d.ts +31 -0
- package/dist/ci-scaffold.js +343 -0
- package/dist/compile-check.d.ts +90 -0
- package/dist/compile-check.js +285 -0
- package/dist/compliance-schedule.d.ts +35 -0
- package/dist/compliance-schedule.js +36 -0
- package/dist/context-pressure.d.ts +80 -0
- package/dist/context-pressure.js +166 -0
- package/dist/dataset-mine.d.ts +173 -0
- package/dist/dataset-mine.js +404 -0
- package/dist/datasets.d.ts +124 -0
- package/dist/datasets.js +260 -0
- package/dist/deploy-canary.d.ts +83 -0
- package/dist/deploy-canary.js +87 -0
- package/dist/doctor-checks.d.ts +33 -0
- package/dist/doctor-checks.js +92 -0
- package/dist/doctor-detect.d.ts +108 -0
- package/dist/doctor-detect.js +214 -0
- package/dist/doctor-fix.d.ts +81 -0
- package/dist/doctor-fix.js +164 -0
- package/dist/egress-triage.d.ts +121 -0
- package/dist/egress-triage.js +261 -0
- package/dist/eval-bridge.d.ts +114 -0
- package/dist/eval-bridge.js +158 -0
- package/dist/eval-coverage.d.ts +140 -0
- package/dist/eval-coverage.js +428 -0
- package/dist/eval-history.d.ts +48 -0
- package/dist/eval-history.js +157 -0
- package/dist/eval-matrix.d.ts +80 -0
- package/dist/eval-matrix.js +182 -0
- package/dist/eval-sentinel.d.ts +65 -0
- package/dist/eval-sentinel.js +132 -0
- package/dist/faq.d.ts +68 -0
- package/dist/faq.js +168 -0
- package/dist/feedback.d.ts +9 -2
- package/dist/feedback.js +17 -7
- package/dist/fewshot.d.ts +83 -0
- package/dist/fewshot.js +158 -0
- package/dist/fleet.d.ts +207 -0
- package/dist/fleet.js +488 -0
- package/dist/flywheel.d.ts +193 -0
- package/dist/flywheel.js +519 -0
- package/dist/graders-suggest.d.ts +186 -0
- package/dist/graders-suggest.js +658 -0
- package/dist/incident.d.ts +99 -0
- package/dist/incident.js +217 -0
- package/dist/index.d.ts +9 -1
- package/dist/index.js +11664 -963
- package/dist/init-interactive.d.ts +105 -0
- package/dist/init-interactive.js +208 -0
- package/dist/intents.d.ts +105 -0
- package/dist/intents.js +292 -0
- package/dist/judge-calibrate.d.ts +137 -0
- package/dist/judge-calibrate.js +247 -0
- package/dist/justification-calibrate.d.ts +150 -0
- package/dist/justification-calibrate.js +262 -0
- package/dist/justification-gate.d.ts +27 -6
- package/dist/justification-gate.js +30 -6
- package/dist/knowledge-sync.d.ts +179 -0
- package/dist/knowledge-sync.js +551 -0
- package/dist/lessons.d.ts +87 -0
- package/dist/lessons.js +207 -0
- package/dist/lint.d.ts +127 -0
- package/dist/lint.js +226 -0
- package/dist/loadtest.d.ts +114 -0
- package/dist/loadtest.js +196 -0
- package/dist/marketplace-cli.d.ts +110 -0
- package/dist/marketplace-cli.js +250 -0
- package/dist/mcp-doctor.d.ts +121 -0
- package/dist/mcp-doctor.js +249 -0
- package/dist/model-scan.d.ts +116 -0
- package/dist/model-scan.js +226 -0
- package/dist/onchain-tune.d.ts +164 -0
- package/dist/onchain-tune.js +346 -0
- package/dist/permissions-suggest.d.ts +126 -0
- package/dist/permissions-suggest.js +333 -0
- package/dist/pii-tune.d.ts +107 -0
- package/dist/pii-tune.js +122 -0
- package/dist/propose.d.ts +117 -0
- package/dist/propose.js +184 -0
- package/dist/refresh-goldens.d.ts +82 -0
- package/dist/refresh-goldens.js +221 -0
- package/dist/regression-pin.d.ts +160 -0
- package/dist/regression-pin.js +281 -0
- package/dist/retention.d.ts +193 -0
- package/dist/retention.js +607 -0
- package/dist/retire.d.ts +118 -0
- package/dist/retire.js +291 -0
- package/dist/right-size.d.ts +100 -0
- package/dist/right-size.js +123 -0
- package/dist/route.d.ts +19 -0
- package/dist/route.js +90 -0
- package/dist/scaffold-evals.d.ts +138 -0
- package/dist/scaffold-evals.js +410 -0
- package/dist/scope-audit-drift.d.ts +139 -0
- package/dist/scope-audit-drift.js +260 -0
- package/dist/security-corpus.d.ts +237 -0
- package/dist/security-corpus.js +516 -0
- package/dist/security-digest.d.ts +174 -0
- package/dist/security-digest.js +656 -0
- package/dist/sessions-index.d.ts +27 -0
- package/dist/sessions-index.js +51 -0
- package/dist/slo-doctor.d.ts +67 -0
- package/dist/slo-doctor.js +119 -0
- package/dist/slo-sink.d.ts +96 -0
- package/dist/slo-sink.js +107 -0
- package/dist/spec-changelog.d.ts +102 -0
- package/dist/spec-changelog.js +237 -0
- package/dist/state-backup.d.ts +223 -0
- package/dist/state-backup.js +648 -0
- package/dist/tools-cli.d.ts +170 -0
- package/dist/tools-cli.js +300 -0
- package/dist/triage.d.ts +202 -0
- package/dist/triage.js +403 -0
- package/dist/upgrade.d.ts +57 -0
- package/dist/upgrade.js +113 -0
- package/dist/version.d.ts +6 -0
- package/dist/version.js +27 -0
- package/dist/voice-eval.d.ts +138 -0
- package/dist/voice-eval.js +309 -0
- package/dist/watch.d.ts +58 -0
- package/dist/watch.js +97 -0
- package/package.json +90 -65
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item 33 — `crewhaus compile --check` core (shape assertion → dep install →
|
|
3
|
+
* liveness boot), in a side-effect-free module so it is unit-testable (the
|
|
4
|
+
* CLI entry file runs an argv switch on import). The compile path calls
|
|
5
|
+
* `runCompileCheck()` after writing the bundle; every subprocess goes
|
|
6
|
+
* through the injectable `CheckStepRunner` so tests never hit the network.
|
|
7
|
+
*
|
|
8
|
+
* The three steps, in degradation order:
|
|
9
|
+
* 1. assertion — the target shape's `ShapeAssertion` from
|
|
10
|
+
* @crewhaus/smoke-harness applied generically (fixture-only anchors
|
|
11
|
+
* skipped). Offline + deterministic; skipped only when the target
|
|
12
|
+
* ships no assertion.
|
|
13
|
+
* 2. install — `bun install` in the out-dir (bounded by a 120s timeout so
|
|
14
|
+
* a hung registry cannot hang the compile forever). When the emitter
|
|
15
|
+
* didn't produce a package.json (only the cf-worker shapes do), a
|
|
16
|
+
* minimal one is synthesized from the bundle's @crewhaus imports,
|
|
17
|
+
* pinned to this CLI's own version (the packages publish in lockstep,
|
|
18
|
+
* so that is exactly the contract this CLI's bundles run against).
|
|
19
|
+
* 3. boot — spawn the bundle's entrypoint once, CREDENTIAL-FREE (env
|
|
20
|
+
* scrubbed to PATH/HOME + the credential-free proxy/CA vars), with
|
|
21
|
+
* `doctor --liveness` semantics: booting far enough to reach the
|
|
22
|
+
* shape's own credential/input gate IS the signal (see
|
|
23
|
+
* BOOT_GATE_PATTERNS — derived empirically by booting every fixture
|
|
24
|
+
* bundle key-less). Scrubbing is deliberate: with real credentials in
|
|
25
|
+
* the env, autonomous shapes (workflow, batch, …) would EXECUTE their
|
|
26
|
+
* agent on boot — paid model calls from a compile flag. Bun would
|
|
27
|
+
* quietly defeat that scrub by auto-loading a `.env` colocated with
|
|
28
|
+
* the bundle, so the boot passes `--env-file=<empty file>` to disable
|
|
29
|
+
* the auto-load (see bootArgvFor). Shapes whose boot needs live
|
|
30
|
+
* credentials/servers therefore degrade to "gated" (green) and the
|
|
31
|
+
* verdict reports which step ran; a structural break (SyntaxError,
|
|
32
|
+
* unresolved import) matches no gate and stays red.
|
|
33
|
+
*/
|
|
34
|
+
import { mkdtempSync, writeFileSync } from "node:fs";
|
|
35
|
+
import { tmpdir } from "node:os";
|
|
36
|
+
import { join } from "node:path";
|
|
37
|
+
import { SHAPE_ASSERTIONS, assertBundleAgainstShape, assertionForTarget, } from "@crewhaus/smoke-harness";
|
|
38
|
+
import { cliVersion } from "./version";
|
|
39
|
+
const DEFAULT_BOOT_TIMEOUT_MS = 10_000;
|
|
40
|
+
// A hung registry must not hang `compile --check` forever — the install step
|
|
41
|
+
// is bounded (override via CompileCheckOptions.installTimeoutMs).
|
|
42
|
+
const DEFAULT_INSTALL_TIMEOUT_MS = 120_000;
|
|
43
|
+
/**
|
|
44
|
+
* Env vars copied through the credential scrub. PATH/HOME are needed to find
|
|
45
|
+
* bun and its caches; the proxy/CA vars carry no credentials but are required
|
|
46
|
+
* for `bun install` (and any gated boot probe) to reach the network at all in
|
|
47
|
+
* proxied/corporate environments.
|
|
48
|
+
*/
|
|
49
|
+
const SCRUB_PASSTHROUGH_VARS = [
|
|
50
|
+
"PATH",
|
|
51
|
+
"HOME",
|
|
52
|
+
"HTTPS_PROXY",
|
|
53
|
+
"HTTP_PROXY",
|
|
54
|
+
"NO_PROXY",
|
|
55
|
+
"SSL_CERT_FILE",
|
|
56
|
+
"NODE_EXTRA_CA_CERTS",
|
|
57
|
+
];
|
|
58
|
+
/** Credential-free child env: only {@link SCRUB_PASSTHROUGH_VARS} survive. */
|
|
59
|
+
export function scrubbedEnv(baseEnv) {
|
|
60
|
+
const env = {};
|
|
61
|
+
for (const name of SCRUB_PASSTHROUGH_VARS) {
|
|
62
|
+
const value = baseEnv[name];
|
|
63
|
+
if (value !== undefined)
|
|
64
|
+
env[name] = value;
|
|
65
|
+
}
|
|
66
|
+
return env;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Lazily-created empty env file for `bun --env-file=…`. Without the flag Bun
|
|
70
|
+
* AUTO-LOADS `.env` from the child's cwd — the out-dir — so a colocated .env
|
|
71
|
+
* with real keys would silently defeat the credential scrub and let an
|
|
72
|
+
* autonomous shape execute paid agent runs from a verification flag. An
|
|
73
|
+
* explicit `--env-file` replaces the auto-load entirely (verified on Bun
|
|
74
|
+
* 1.3.14: the child sees none of the .env's vars). An empty temp file is
|
|
75
|
+
* used rather than /dev/null so the flag also works on Windows.
|
|
76
|
+
*/
|
|
77
|
+
let cachedEmptyEnvFile;
|
|
78
|
+
export function emptyEnvFile() {
|
|
79
|
+
if (cachedEmptyEnvFile === undefined) {
|
|
80
|
+
const path = join(mkdtempSync(join(tmpdir(), "crewhaus-check-env-")), "empty.env");
|
|
81
|
+
writeFileSync(path, "");
|
|
82
|
+
cachedEmptyEnvFile = path;
|
|
83
|
+
}
|
|
84
|
+
return cachedEmptyEnvFile;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* The exact argv `compile --check` boots a bundle entrypoint with. Exported
|
|
88
|
+
* so the F4 regression tests exercise the REAL construction (flag presence
|
|
89
|
+
* with an injected runner, and .env invisibility with a live subprocess).
|
|
90
|
+
*/
|
|
91
|
+
export function bootArgvFor(entry) {
|
|
92
|
+
return ["bun", `--env-file=${emptyEnvFile()}`, entry];
|
|
93
|
+
}
|
|
94
|
+
/** Collect the @crewhaus/* packages the emitted bundle imports (sorted, deduped). */
|
|
95
|
+
export function collectCrewhausDeps(files) {
|
|
96
|
+
const deps = new Set();
|
|
97
|
+
const re = /["'](@crewhaus\/[a-z0-9-]+)["']/g;
|
|
98
|
+
for (const file of files) {
|
|
99
|
+
if (!file.path.endsWith(".ts") && !file.path.endsWith(".js"))
|
|
100
|
+
continue;
|
|
101
|
+
for (const m of file.content.matchAll(re)) {
|
|
102
|
+
const dep = m[1];
|
|
103
|
+
if (dep !== undefined)
|
|
104
|
+
deps.add(dep);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return [...deps].sort();
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Minimal manifest for bundles whose emitter ships no package.json.
|
|
111
|
+
* Versions pin to the CLI's OWN version: the @crewhaus/* packages publish in
|
|
112
|
+
* lockstep (scripts/publish-workspace.ts stamps them all with one version),
|
|
113
|
+
* so `<cliVersion>` is exactly the published contract this CLI's emitters
|
|
114
|
+
* were released against — where "latest" would silently verify against
|
|
115
|
+
* whatever shipped since, making the check's verdict depend on the day it
|
|
116
|
+
* runs. "latest" remains only as the fallback when no version is resolvable
|
|
117
|
+
* (a broken installation). (In a dev checkout the pinned publish can lag
|
|
118
|
+
* unreleased emitter features; the boot step then reports the mismatch,
|
|
119
|
+
* which is a real signal, not a false positive.)
|
|
120
|
+
*/
|
|
121
|
+
export function buildBundlePackageJson(deps, version) {
|
|
122
|
+
const pin = version ?? "latest";
|
|
123
|
+
const dependencies = {};
|
|
124
|
+
for (const dep of deps)
|
|
125
|
+
dependencies[dep] = pin;
|
|
126
|
+
return `${JSON.stringify({ name: "crewhaus-compiled-bundle", private: true, type: "module", dependencies }, null, 2)}\n`;
|
|
127
|
+
}
|
|
128
|
+
/** daemon.ts (daemon shapes) beats agent.ts; undefined = nothing bootable here. */
|
|
129
|
+
export function resolveBootEntry(files) {
|
|
130
|
+
const paths = new Set(files.map((f) => f.path));
|
|
131
|
+
if (paths.has("daemon.ts"))
|
|
132
|
+
return "daemon.ts";
|
|
133
|
+
if (paths.has("agent.ts"))
|
|
134
|
+
return "agent.ts";
|
|
135
|
+
return undefined;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Known first-boot gates: the message a shape prints when it boots far
|
|
139
|
+
* enough to demand its live inputs. Reaching a gate is the liveness signal
|
|
140
|
+
* (`doctor --liveness` semantics — "booting far enough to parse argv IS the
|
|
141
|
+
* signal"). Derived empirically from booting every fixture bundle with a
|
|
142
|
+
* scrubbed env; keep patterns narrow so structural breakage stays red.
|
|
143
|
+
*/
|
|
144
|
+
export const BOOT_GATE_PATTERNS = [
|
|
145
|
+
// runtime-core resolveAuth / provider adapters (cli, workflow, graph, …).
|
|
146
|
+
{ pattern: /no Anthropic credentials found|ProviderAuthError/, gate: "provider credentials" },
|
|
147
|
+
// Env-ref rewriting: `process.env["X"] ?? throw` (channel secrets, onchain RPC).
|
|
148
|
+
{ pattern: /missing required env var/, gate: "spec env refs" },
|
|
149
|
+
// crew daemon reads its kickoff input from stdin.
|
|
150
|
+
{ pattern: /no input on stdin/, gate: "stdin input" },
|
|
151
|
+
// voice daemon's v0 headless smoke path.
|
|
152
|
+
{ pattern: /no --smoke/, gate: "a --smoke pcm fixture" },
|
|
153
|
+
// browser driver needs a prompt to open a session.
|
|
154
|
+
{ pattern: /no prompt \(pass --prompt/, gate: "an initial --prompt" },
|
|
155
|
+
// eval bundle resolves its dataset from the local registry.
|
|
156
|
+
{ pattern: /dataset "[^"]*" not found/, gate: "a registered eval dataset" },
|
|
157
|
+
];
|
|
158
|
+
export function classifyBootOutcome(result) {
|
|
159
|
+
if (result.timedOut) {
|
|
160
|
+
return { status: "ok", detail: "daemon stayed alive for the liveness window" };
|
|
161
|
+
}
|
|
162
|
+
if (result.exitCode === 0) {
|
|
163
|
+
return { status: "ok", detail: "booted and exited cleanly" };
|
|
164
|
+
}
|
|
165
|
+
// Gates are matched against BOTH streams: most shapes print their gate to
|
|
166
|
+
// stderr, but nothing guarantees it — a stdout-printed gate must not be
|
|
167
|
+
// misclassified as a structural failure.
|
|
168
|
+
const output = `${result.stdout}\n${result.stderr}`;
|
|
169
|
+
for (const { pattern, gate } of BOOT_GATE_PATTERNS) {
|
|
170
|
+
if (pattern.test(output)) {
|
|
171
|
+
return {
|
|
172
|
+
status: "gated",
|
|
173
|
+
detail: `boot reached its ${gate} gate — full boot needs live ${gate}`,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
const tail = result.stderr.trim().split("\n").slice(-3).join(" | ").slice(0, 400);
|
|
178
|
+
return { status: "failed", detail: `boot exited ${result.exitCode}: ${tail}` };
|
|
179
|
+
}
|
|
180
|
+
/** Fold the steps into the single green/red verdict line. Only "failed" is red. */
|
|
181
|
+
export function buildVerdict(steps) {
|
|
182
|
+
const green = steps.every((s) => s.status !== "failed");
|
|
183
|
+
const parts = steps.map((s) => `${s.step} ${s.status}${s.detail !== undefined ? ` (${s.detail})` : ""}`);
|
|
184
|
+
return { green, line: `compile --check: ${green ? "GREEN" : "RED"} — ${parts.join("; ")}` };
|
|
185
|
+
}
|
|
186
|
+
export const defaultCheckRunner = async ({ argv, cwd, env, timeoutMs }) => {
|
|
187
|
+
const head = argv[0] ?? "bun";
|
|
188
|
+
const proc = Bun.spawn([head, ...argv.slice(1)], {
|
|
189
|
+
cwd,
|
|
190
|
+
env: env ?? { PATH: process.env["PATH"] ?? "", HOME: process.env["HOME"] ?? "" },
|
|
191
|
+
stdin: "ignore",
|
|
192
|
+
stdout: "pipe",
|
|
193
|
+
stderr: "pipe",
|
|
194
|
+
});
|
|
195
|
+
let timedOut = false;
|
|
196
|
+
const timer = timeoutMs !== undefined
|
|
197
|
+
? setTimeout(() => {
|
|
198
|
+
timedOut = true;
|
|
199
|
+
proc.kill("SIGKILL");
|
|
200
|
+
}, timeoutMs)
|
|
201
|
+
: undefined;
|
|
202
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
203
|
+
new Response(proc.stdout).text(),
|
|
204
|
+
new Response(proc.stderr).text(),
|
|
205
|
+
proc.exited,
|
|
206
|
+
]);
|
|
207
|
+
if (timer !== undefined)
|
|
208
|
+
clearTimeout(timer);
|
|
209
|
+
return { exitCode, timedOut, stdout, stderr };
|
|
210
|
+
};
|
|
211
|
+
export async function runCompileCheck(opts) {
|
|
212
|
+
const runner = opts.runner ?? defaultCheckRunner;
|
|
213
|
+
const steps = [];
|
|
214
|
+
// 1 — shape assertion (offline, deterministic).
|
|
215
|
+
const assertion = assertionForTarget(opts.target, opts.assertions ?? SHAPE_ASSERTIONS);
|
|
216
|
+
if (assertion === undefined) {
|
|
217
|
+
steps.push({
|
|
218
|
+
step: "assertion",
|
|
219
|
+
status: "skipped",
|
|
220
|
+
detail: `no smoke assertion for target "${opts.target}"`,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
else {
|
|
224
|
+
const failures = assertBundleAgainstShape(assertion, opts.bundle);
|
|
225
|
+
const applied = assertion.anchors.filter((a) => a.fixtureOnly !== true).length;
|
|
226
|
+
steps.push(failures.length === 0
|
|
227
|
+
? { step: "assertion", status: "ok", detail: `${applied} anchors` }
|
|
228
|
+
: { step: "assertion", status: "failed", detail: failures.join("; ") });
|
|
229
|
+
}
|
|
230
|
+
// 2 — install the bundle's deps in the out-dir. Scrubbed env (proxy/CA
|
|
231
|
+
// vars pass through — bun needs them to reach the registry) + a timeout so
|
|
232
|
+
// a hung registry cannot hang the compile forever.
|
|
233
|
+
const baseEnv = opts.env ?? process.env;
|
|
234
|
+
const childEnv = scrubbedEnv(baseEnv);
|
|
235
|
+
if (!opts.bundle.files.some((f) => f.path === "package.json")) {
|
|
236
|
+
writeFileSync(join(opts.outDir, "package.json"), buildBundlePackageJson(collectCrewhausDeps(opts.bundle.files), cliVersion()));
|
|
237
|
+
}
|
|
238
|
+
const installTimeoutMs = opts.installTimeoutMs ?? DEFAULT_INSTALL_TIMEOUT_MS;
|
|
239
|
+
const install = await runner({
|
|
240
|
+
argv: ["bun", "install"],
|
|
241
|
+
cwd: opts.outDir,
|
|
242
|
+
env: childEnv,
|
|
243
|
+
timeoutMs: installTimeoutMs,
|
|
244
|
+
});
|
|
245
|
+
const installOk = !install.timedOut && install.exitCode === 0;
|
|
246
|
+
steps.push(installOk
|
|
247
|
+
? { step: "install", status: "ok" }
|
|
248
|
+
: {
|
|
249
|
+
step: "install",
|
|
250
|
+
status: "failed",
|
|
251
|
+
detail: install.timedOut
|
|
252
|
+
? `bun install timed out after ${installTimeoutMs}ms (registry unreachable/hung?)`
|
|
253
|
+
: `bun install exited ${install.exitCode}: ${install.stderr
|
|
254
|
+
.trim()
|
|
255
|
+
.split("\n")
|
|
256
|
+
.slice(-2)
|
|
257
|
+
.join(" | ")
|
|
258
|
+
.slice(0, 300)}`,
|
|
259
|
+
});
|
|
260
|
+
// 3 — liveness boot (credential-free by design; see module doc). The
|
|
261
|
+
// --env-file flag in bootArgvFor keeps Bun from auto-loading a `.env`
|
|
262
|
+
// colocated with the bundle, which would defeat the scrub.
|
|
263
|
+
const entry = resolveBootEntry(opts.bundle.files);
|
|
264
|
+
if (entry === undefined) {
|
|
265
|
+
steps.push({
|
|
266
|
+
step: "boot",
|
|
267
|
+
status: "skipped",
|
|
268
|
+
detail: "no agent.ts/daemon.ts entrypoint (bundle boots via its own tooling)",
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
else if (!installOk) {
|
|
272
|
+
steps.push({ step: "boot", status: "skipped", detail: "install failed" });
|
|
273
|
+
}
|
|
274
|
+
else {
|
|
275
|
+
const boot = await runner({
|
|
276
|
+
argv: bootArgvFor(entry),
|
|
277
|
+
cwd: opts.outDir,
|
|
278
|
+
env: childEnv,
|
|
279
|
+
timeoutMs: opts.bootTimeoutMs ?? DEFAULT_BOOT_TIMEOUT_MS,
|
|
280
|
+
});
|
|
281
|
+
const outcome = classifyBootOutcome(boot);
|
|
282
|
+
steps.push({ step: "boot", status: outcome.status, detail: outcome.detail });
|
|
283
|
+
}
|
|
284
|
+
return { ...buildVerdict(steps), steps };
|
|
285
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item 34 — scheduling ergonomics for `crewhaus compliance evidence`,
|
|
3
|
+
* factored out of the entry file `index.ts` (which runs a top-level argv
|
|
4
|
+
* switch and so cannot be imported by a test without executing the CLI).
|
|
5
|
+
* Side-effect-free and directly unit-testable, mirroring `scope-audit.ts` /
|
|
6
|
+
* `audit-verify.ts`.
|
|
7
|
+
*
|
|
8
|
+
* A cron job cannot hardcode `--period 2026-Q3` — it would silently collect
|
|
9
|
+
* the wrong quarter after a boundary. `--period current` resolves the
|
|
10
|
+
* current quarter from the clock in the same `YYYY-QN` label format the
|
|
11
|
+
* command already accepts (see `runCompliance`'s usage example), using UTC
|
|
12
|
+
* to match the audit log's UTC day rotation.
|
|
13
|
+
*/
|
|
14
|
+
/** `YYYY-QN` label for the quarter containing `date`, in UTC. */
|
|
15
|
+
export declare function currentQuarterPeriod(date: Date): string;
|
|
16
|
+
/**
|
|
17
|
+
* Resolve the raw `--period` flag value: the literal `current` becomes the
|
|
18
|
+
* clock's current quarter; any other label passes through verbatim (the
|
|
19
|
+
* collector's `writeBundle` still path-validates it downstream).
|
|
20
|
+
*/
|
|
21
|
+
export declare function resolvePeriodFlag(value: string, now?: () => Date): string;
|
|
22
|
+
/** The slice of an EvidenceBundle the empty-evidence gate needs. */
|
|
23
|
+
export type BundleCount = {
|
|
24
|
+
readonly frameworkId: string;
|
|
25
|
+
readonly controlId: string;
|
|
26
|
+
readonly recordCount: number;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* `framework/control` ids of every collected bundle with zero records — the
|
|
30
|
+
* evidence gaps that fail a scheduled run (a control that collected nothing
|
|
31
|
+
* is a control an auditor cannot be shown; exiting 0 would hide that until
|
|
32
|
+
* audit time). Order-preserving so the failure message mirrors collection
|
|
33
|
+
* order.
|
|
34
|
+
*/
|
|
35
|
+
export declare function findEmptyControls(bundles: ReadonlyArray<BundleCount>): ReadonlyArray<string>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item 34 — scheduling ergonomics for `crewhaus compliance evidence`,
|
|
3
|
+
* factored out of the entry file `index.ts` (which runs a top-level argv
|
|
4
|
+
* switch and so cannot be imported by a test without executing the CLI).
|
|
5
|
+
* Side-effect-free and directly unit-testable, mirroring `scope-audit.ts` /
|
|
6
|
+
* `audit-verify.ts`.
|
|
7
|
+
*
|
|
8
|
+
* A cron job cannot hardcode `--period 2026-Q3` — it would silently collect
|
|
9
|
+
* the wrong quarter after a boundary. `--period current` resolves the
|
|
10
|
+
* current quarter from the clock in the same `YYYY-QN` label format the
|
|
11
|
+
* command already accepts (see `runCompliance`'s usage example), using UTC
|
|
12
|
+
* to match the audit log's UTC day rotation.
|
|
13
|
+
*/
|
|
14
|
+
/** `YYYY-QN` label for the quarter containing `date`, in UTC. */
|
|
15
|
+
export function currentQuarterPeriod(date) {
|
|
16
|
+
const quarter = Math.floor(date.getUTCMonth() / 3) + 1;
|
|
17
|
+
return `${date.getUTCFullYear()}-Q${quarter}`;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Resolve the raw `--period` flag value: the literal `current` becomes the
|
|
21
|
+
* clock's current quarter; any other label passes through verbatim (the
|
|
22
|
+
* collector's `writeBundle` still path-validates it downstream).
|
|
23
|
+
*/
|
|
24
|
+
export function resolvePeriodFlag(value, now = () => new Date()) {
|
|
25
|
+
return value === "current" ? currentQuarterPeriod(now()) : value;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* `framework/control` ids of every collected bundle with zero records — the
|
|
29
|
+
* evidence gaps that fail a scheduled run (a control that collected nothing
|
|
30
|
+
* is a control an auditor cannot be shown; exiting 0 would hide that until
|
|
31
|
+
* audit time). Order-preserving so the failure message mirrors collection
|
|
32
|
+
* order.
|
|
33
|
+
*/
|
|
34
|
+
export function findEmptyControls(bundles) {
|
|
35
|
+
return bundles.filter((b) => b.recordCount === 0).map((b) => `${b.frameworkId}/${b.controlId}`);
|
|
36
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item 17 (completion) — `crewhaus doctor --context-pressure`: the
|
|
3
|
+
* telemetry-driven tuning loop's report face. The advise rules already
|
|
4
|
+
* turn recovery/compaction telemetry into SpecPatches (the detection
|
|
5
|
+
* half); this module reads the same persisted session events and prints
|
|
6
|
+
* WHERE the pressure is — truncation recoveries, compaction fires per
|
|
7
|
+
* session, the snip-vs-autocompact split — next to the spec's current
|
|
8
|
+
* knobs, and, when the advise thresholds trip, the exact advise/optimize
|
|
9
|
+
* commands that close the loop. A report, not a gate: doctor exits 0.
|
|
10
|
+
*
|
|
11
|
+
* Event vocabulary (runtime-core's `logEvent` kinds):
|
|
12
|
+
* - `recovery` with `action: "continue"` — emitted exclusively for
|
|
13
|
+
* max_output_tokens truncations (recovery-engine's taxonomy), so the
|
|
14
|
+
* continue count IS the truncation-pressure count;
|
|
15
|
+
* - `compaction` with `kind: "snip" | "autocompact" | "reactive"` —
|
|
16
|
+
* snips are free (head/tail trim), autocompact + reactive summarize
|
|
17
|
+
* via a model call. The ratio is only derivable when kind-tagged
|
|
18
|
+
* events exist (the kind field ships with the events themselves, so
|
|
19
|
+
* any persisted compaction carries it; "unknown" tolerates foreign
|
|
20
|
+
* or hand-edited logs).
|
|
21
|
+
*
|
|
22
|
+
* Kept in a side-effect-free module (the CLI entry file runs an argv
|
|
23
|
+
* switch on import) following the doctor-checks.ts pattern: pure builders
|
|
24
|
+
* over caller-supplied `SessionEvents`, directly unit-testable.
|
|
25
|
+
*/
|
|
26
|
+
import type { Spec } from "@crewhaus/spec";
|
|
27
|
+
import { type AdviceThresholds, type SessionEvents } from "./advise-rules";
|
|
28
|
+
/** Default number of most-recent sessions the doctor report scans. */
|
|
29
|
+
export declare const DEFAULT_CONTEXT_PRESSURE_SESSIONS = 20;
|
|
30
|
+
export type CompactionKindCounts = {
|
|
31
|
+
readonly snip: number;
|
|
32
|
+
readonly autocompact: number;
|
|
33
|
+
readonly reactive: number;
|
|
34
|
+
/** Compaction events without a recognized `kind` (foreign logs). */
|
|
35
|
+
readonly unknown: number;
|
|
36
|
+
};
|
|
37
|
+
export type ContextPressureReport = {
|
|
38
|
+
readonly sessionCount: number;
|
|
39
|
+
/** `recovery` events with `action: "continue"` — max_output_tokens
|
|
40
|
+
* truncations by construction. */
|
|
41
|
+
readonly truncationContinues: number;
|
|
42
|
+
readonly compactionTotal: number;
|
|
43
|
+
readonly sessionsWithCompaction: number;
|
|
44
|
+
/** Mean over ALL scanned sessions (a quiet session counts as 0). */
|
|
45
|
+
readonly avgCompactionsPerSession: number;
|
|
46
|
+
readonly maxCompactionsPerSession: number;
|
|
47
|
+
readonly maxCompactionSessionId?: string;
|
|
48
|
+
readonly compactionKinds: CompactionKindCounts;
|
|
49
|
+
/** snip / (snip + model-backed fires); undefined when no kind-tagged
|
|
50
|
+
* compaction event was seen (ratio not derivable). */
|
|
51
|
+
readonly snipRatio?: number;
|
|
52
|
+
readonly spec: {
|
|
53
|
+
/** False when no parseable crewhaus.yaml was supplied. */
|
|
54
|
+
readonly present: boolean;
|
|
55
|
+
readonly maxTokens?: number;
|
|
56
|
+
readonly compactionCurate?: boolean;
|
|
57
|
+
readonly compactionDedupeThreshold?: number;
|
|
58
|
+
readonly compactionRelevanceTopK?: number;
|
|
59
|
+
};
|
|
60
|
+
/** Advise-threshold trips: "truncation-pressure" / "compaction-thrash". */
|
|
61
|
+
readonly tripped: ReadonlyArray<string>;
|
|
62
|
+
/** The exact commands to run when any threshold tripped (empty otherwise). */
|
|
63
|
+
readonly commands: ReadonlyArray<string>;
|
|
64
|
+
};
|
|
65
|
+
/** The commands the report prints when pressure trips — the nightly
|
|
66
|
+
* advisor composition (advise emits the patches, optimize --from-advice
|
|
67
|
+
* eval-gates and applies them). */
|
|
68
|
+
export declare const CONTEXT_PRESSURE_COMMANDS: ReadonlyArray<string>;
|
|
69
|
+
/**
|
|
70
|
+
* Fold the persisted session events into the context-pressure report.
|
|
71
|
+
* Every field access is defensive (missing payloads, wrong-typed fields,
|
|
72
|
+
* old-vintage logs all pass through cleanly), matching
|
|
73
|
+
* `buildAdviceContext`'s posture.
|
|
74
|
+
*/
|
|
75
|
+
export declare function buildContextPressureReport(sessions: ReadonlyArray<SessionEvents>, opts?: {
|
|
76
|
+
readonly spec?: Spec;
|
|
77
|
+
readonly thresholds?: Partial<AdviceThresholds>;
|
|
78
|
+
}): ContextPressureReport;
|
|
79
|
+
/** One-screen report, printed by `doctor --context-pressure`. */
|
|
80
|
+
export declare function formatContextPressureLines(report: ContextPressureReport): string[];
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { DEFAULT_ADVICE_THRESHOLDS, payloadOf, } from "./advise-rules";
|
|
2
|
+
/** Default number of most-recent sessions the doctor report scans. */
|
|
3
|
+
export const DEFAULT_CONTEXT_PRESSURE_SESSIONS = 20;
|
|
4
|
+
/** The commands the report prints when pressure trips — the nightly
|
|
5
|
+
* advisor composition (advise emits the patches, optimize --from-advice
|
|
6
|
+
* eval-gates and applies them). */
|
|
7
|
+
export const CONTEXT_PRESSURE_COMMANDS = Object.freeze([
|
|
8
|
+
"crewhaus advise --all -o .",
|
|
9
|
+
"crewhaus optimize crewhaus.yaml --from-advice suggestions.json --write-back --dataset eval/dataset.jsonl --graders eval/graders.yaml",
|
|
10
|
+
]);
|
|
11
|
+
// Narrow accessors over the Spec union (mirrors advise-rules): the fields
|
|
12
|
+
// exist on most (not all) targets, so read through an index signature.
|
|
13
|
+
function specField(spec, key) {
|
|
14
|
+
if (spec === undefined)
|
|
15
|
+
return undefined;
|
|
16
|
+
return spec[key];
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Fold the persisted session events into the context-pressure report.
|
|
20
|
+
* Every field access is defensive (missing payloads, wrong-typed fields,
|
|
21
|
+
* old-vintage logs all pass through cleanly), matching
|
|
22
|
+
* `buildAdviceContext`'s posture.
|
|
23
|
+
*/
|
|
24
|
+
export function buildContextPressureReport(sessions, opts = {}) {
|
|
25
|
+
const thresholds = { ...DEFAULT_ADVICE_THRESHOLDS, ...(opts.thresholds ?? {}) };
|
|
26
|
+
let truncationContinues = 0;
|
|
27
|
+
let snip = 0;
|
|
28
|
+
let autocompact = 0;
|
|
29
|
+
let reactive = 0;
|
|
30
|
+
let unknown = 0;
|
|
31
|
+
const compactionsBySession = new Map();
|
|
32
|
+
for (const session of sessions) {
|
|
33
|
+
for (const obj of session.objects) {
|
|
34
|
+
const recovery = payloadOf(obj, "recovery");
|
|
35
|
+
if (recovery !== undefined) {
|
|
36
|
+
if (recovery["action"] === "continue")
|
|
37
|
+
truncationContinues += 1;
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
const compaction = payloadOf(obj, "compaction");
|
|
41
|
+
if (compaction !== undefined) {
|
|
42
|
+
compactionsBySession.set(session.sessionId, (compactionsBySession.get(session.sessionId) ?? 0) + 1);
|
|
43
|
+
switch (compaction["kind"]) {
|
|
44
|
+
case "snip":
|
|
45
|
+
snip += 1;
|
|
46
|
+
break;
|
|
47
|
+
case "autocompact":
|
|
48
|
+
autocompact += 1;
|
|
49
|
+
break;
|
|
50
|
+
case "reactive":
|
|
51
|
+
reactive += 1;
|
|
52
|
+
break;
|
|
53
|
+
default:
|
|
54
|
+
unknown += 1;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
let compactionTotal = 0;
|
|
60
|
+
let maxCompactionsPerSession = 0;
|
|
61
|
+
let maxCompactionSessionId;
|
|
62
|
+
for (const [sessionId, count] of compactionsBySession) {
|
|
63
|
+
compactionTotal += count;
|
|
64
|
+
if (count > maxCompactionsPerSession) {
|
|
65
|
+
maxCompactionsPerSession = count;
|
|
66
|
+
maxCompactionSessionId = sessionId;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const avgCompactionsPerSession = sessions.length === 0 ? 0 : compactionTotal / sessions.length;
|
|
70
|
+
// Ratio of free snips to ALL kind-tagged fires (autocompact + reactive
|
|
71
|
+
// both summarize via a model call). Underivable without tagged events.
|
|
72
|
+
const tagged = snip + autocompact + reactive;
|
|
73
|
+
const snipRatio = tagged > 0 ? snip / tagged : undefined;
|
|
74
|
+
const spec = opts.spec;
|
|
75
|
+
const agent = specField(spec, "agent");
|
|
76
|
+
const maxTokens = typeof agent?.["max_tokens"] === "number" ? agent["max_tokens"] : undefined;
|
|
77
|
+
const compactionBlock = specField(spec, "compaction");
|
|
78
|
+
const curate = typeof compactionBlock?.["curate"] === "boolean" ? compactionBlock["curate"] : undefined;
|
|
79
|
+
const dedupe = typeof compactionBlock?.["dedupeThreshold"] === "number"
|
|
80
|
+
? compactionBlock["dedupeThreshold"]
|
|
81
|
+
: undefined;
|
|
82
|
+
const topK = typeof compactionBlock?.["relevanceTopK"] === "number"
|
|
83
|
+
? compactionBlock["relevanceTopK"]
|
|
84
|
+
: undefined;
|
|
85
|
+
// The SAME thresholds the advise rules fire on — when the report says
|
|
86
|
+
// "tripped", `crewhaus advise` will emit a patch for it.
|
|
87
|
+
const tripped = [];
|
|
88
|
+
if (truncationContinues >= thresholds.truncationContinues)
|
|
89
|
+
tripped.push("truncation-pressure");
|
|
90
|
+
if (maxCompactionsPerSession >= thresholds.compactionsPerSession) {
|
|
91
|
+
tripped.push("compaction-thrash");
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
sessionCount: sessions.length,
|
|
95
|
+
truncationContinues,
|
|
96
|
+
compactionTotal,
|
|
97
|
+
sessionsWithCompaction: compactionsBySession.size,
|
|
98
|
+
avgCompactionsPerSession,
|
|
99
|
+
maxCompactionsPerSession,
|
|
100
|
+
...(maxCompactionSessionId !== undefined ? { maxCompactionSessionId } : {}),
|
|
101
|
+
compactionKinds: { snip, autocompact, reactive, unknown },
|
|
102
|
+
...(snipRatio !== undefined ? { snipRatio } : {}),
|
|
103
|
+
spec: {
|
|
104
|
+
present: spec !== undefined,
|
|
105
|
+
...(maxTokens !== undefined ? { maxTokens } : {}),
|
|
106
|
+
...(curate !== undefined ? { compactionCurate: curate } : {}),
|
|
107
|
+
...(dedupe !== undefined ? { compactionDedupeThreshold: dedupe } : {}),
|
|
108
|
+
...(topK !== undefined ? { compactionRelevanceTopK: topK } : {}),
|
|
109
|
+
},
|
|
110
|
+
tripped,
|
|
111
|
+
commands: tripped.length > 0 ? [...CONTEXT_PRESSURE_COMMANDS] : [],
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/** One-screen report, printed by `doctor --context-pressure`. */
|
|
115
|
+
export function formatContextPressureLines(report) {
|
|
116
|
+
const lines = ["── context-pressure report ──────────────────────"];
|
|
117
|
+
lines.push(`sessions scanned: ${report.sessionCount} (most recent)`);
|
|
118
|
+
lines.push(`truncation recoveries: ${report.truncationContinues} max_output_tokens continue(s)`);
|
|
119
|
+
const maxNote = report.maxCompactionSessionId !== undefined
|
|
120
|
+
? `, max ${report.maxCompactionsPerSession} in ${report.maxCompactionSessionId}`
|
|
121
|
+
: "";
|
|
122
|
+
lines.push(`compaction fires: ${report.compactionTotal} across ${report.sessionsWithCompaction} session(s) ` +
|
|
123
|
+
`(avg ${report.avgCompactionsPerSession.toFixed(1)}/session${maxNote})`);
|
|
124
|
+
if (report.snipRatio !== undefined) {
|
|
125
|
+
const k = report.compactionKinds;
|
|
126
|
+
lines.push(`snip vs autocompact: ${k.snip} snip / ${k.autocompact} autocompact / ${k.reactive} reactive ` +
|
|
127
|
+
`(${(report.snipRatio * 100).toFixed(0)}% free snips)`);
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
lines.push("snip vs autocompact: not derivable (no kind-tagged compaction events)");
|
|
131
|
+
}
|
|
132
|
+
if (!report.spec.present) {
|
|
133
|
+
lines.push("spec: no parseable crewhaus.yaml in cwd");
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
lines.push(`spec agent.max_tokens: ${report.spec.maxTokens !== undefined
|
|
137
|
+
? report.spec.maxTokens
|
|
138
|
+
: "not set (runtime default 8192)"}`);
|
|
139
|
+
const curate = report.spec.compactionCurate === true
|
|
140
|
+
? "on"
|
|
141
|
+
: report.spec.compactionCurate === false
|
|
142
|
+
? "off (explicit)"
|
|
143
|
+
: "off (default)";
|
|
144
|
+
const curatorKnobs = [
|
|
145
|
+
...(report.spec.compactionDedupeThreshold !== undefined
|
|
146
|
+
? [`dedupeThreshold ${report.spec.compactionDedupeThreshold}`]
|
|
147
|
+
: []),
|
|
148
|
+
...(report.spec.compactionRelevanceTopK !== undefined
|
|
149
|
+
? [`relevanceTopK ${report.spec.compactionRelevanceTopK}`]
|
|
150
|
+
: []),
|
|
151
|
+
];
|
|
152
|
+
lines.push(`spec compaction.curate: ${curate}${curatorKnobs.length > 0 ? ` (${curatorKnobs.join(", ")})` : ""}`);
|
|
153
|
+
}
|
|
154
|
+
if (report.tripped.length === 0) {
|
|
155
|
+
lines.push("pressure: healthy — no advise thresholds tripped");
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
lines.push(`pressure: ${report.tripped.join(", ")} tripped`);
|
|
159
|
+
lines.push("next:");
|
|
160
|
+
for (const cmd of report.commands) {
|
|
161
|
+
lines.push(` ${cmd}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
lines.push("─────────────────────────────────────────────────");
|
|
165
|
+
return lines;
|
|
166
|
+
}
|