crewhaus 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +10 -3
  2. package/dist/advice-apply.d.ts +182 -0
  3. package/dist/advice-apply.js +286 -0
  4. package/dist/advise-rules.d.ts +348 -0
  5. package/dist/advise-rules.js +905 -0
  6. package/dist/alert-sink.d.ts +48 -0
  7. package/dist/alert-sink.js +86 -0
  8. package/dist/approval-gate.d.ts +127 -0
  9. package/dist/approval-gate.js +254 -0
  10. package/dist/audit-verify.d.ts +69 -0
  11. package/dist/audit-verify.js +97 -0
  12. package/dist/autodistill.d.ts +113 -0
  13. package/dist/autodistill.js +256 -0
  14. package/dist/channel-provision.d.ts +360 -0
  15. package/dist/channel-provision.js +881 -0
  16. package/dist/ci-scaffold.d.ts +31 -0
  17. package/dist/ci-scaffold.js +343 -0
  18. package/dist/compile-check.d.ts +90 -0
  19. package/dist/compile-check.js +285 -0
  20. package/dist/compliance-schedule.d.ts +35 -0
  21. package/dist/compliance-schedule.js +36 -0
  22. package/dist/context-pressure.d.ts +80 -0
  23. package/dist/context-pressure.js +166 -0
  24. package/dist/dataset-mine.d.ts +172 -0
  25. package/dist/dataset-mine.js +403 -0
  26. package/dist/datasets.d.ts +124 -0
  27. package/dist/datasets.js +260 -0
  28. package/dist/deploy-canary.d.ts +83 -0
  29. package/dist/deploy-canary.js +87 -0
  30. package/dist/doctor-checks.d.ts +33 -0
  31. package/dist/doctor-checks.js +92 -0
  32. package/dist/doctor-detect.d.ts +108 -0
  33. package/dist/doctor-detect.js +214 -0
  34. package/dist/doctor-fix.d.ts +81 -0
  35. package/dist/doctor-fix.js +164 -0
  36. package/dist/egress-triage.d.ts +121 -0
  37. package/dist/egress-triage.js +261 -0
  38. package/dist/eval-bridge.d.ts +114 -0
  39. package/dist/eval-bridge.js +158 -0
  40. package/dist/eval-coverage.d.ts +140 -0
  41. package/dist/eval-coverage.js +428 -0
  42. package/dist/eval-history.d.ts +48 -0
  43. package/dist/eval-history.js +157 -0
  44. package/dist/eval-matrix.d.ts +80 -0
  45. package/dist/eval-matrix.js +182 -0
  46. package/dist/eval-sentinel.d.ts +65 -0
  47. package/dist/eval-sentinel.js +132 -0
  48. package/dist/faq.d.ts +68 -0
  49. package/dist/faq.js +168 -0
  50. package/dist/feedback.d.ts +246 -0
  51. package/dist/feedback.js +612 -0
  52. package/dist/fewshot.d.ts +83 -0
  53. package/dist/fewshot.js +158 -0
  54. package/dist/fleet.d.ts +207 -0
  55. package/dist/fleet.js +488 -0
  56. package/dist/flywheel.d.ts +193 -0
  57. package/dist/flywheel.js +519 -0
  58. package/dist/graders-suggest.d.ts +186 -0
  59. package/dist/graders-suggest.js +658 -0
  60. package/dist/incident.d.ts +99 -0
  61. package/dist/incident.js +217 -0
  62. package/dist/index.d.ts +9 -1
  63. package/dist/index.js +11727 -740
  64. package/dist/init-interactive.d.ts +105 -0
  65. package/dist/init-interactive.js +208 -0
  66. package/dist/intents.d.ts +105 -0
  67. package/dist/intents.js +292 -0
  68. package/dist/judge-calibrate.d.ts +137 -0
  69. package/dist/judge-calibrate.js +247 -0
  70. package/dist/justification-calibrate.d.ts +150 -0
  71. package/dist/justification-calibrate.js +262 -0
  72. package/dist/justification-gate.d.ts +27 -6
  73. package/dist/justification-gate.js +30 -6
  74. package/dist/knowledge-sync.d.ts +179 -0
  75. package/dist/knowledge-sync.js +551 -0
  76. package/dist/lessons.d.ts +87 -0
  77. package/dist/lessons.js +207 -0
  78. package/dist/lint.d.ts +127 -0
  79. package/dist/lint.js +226 -0
  80. package/dist/loadtest.d.ts +114 -0
  81. package/dist/loadtest.js +196 -0
  82. package/dist/marketplace-cli.d.ts +110 -0
  83. package/dist/marketplace-cli.js +250 -0
  84. package/dist/mcp-doctor.d.ts +121 -0
  85. package/dist/mcp-doctor.js +249 -0
  86. package/dist/model-scan.d.ts +116 -0
  87. package/dist/model-scan.js +226 -0
  88. package/dist/onchain-tune.d.ts +164 -0
  89. package/dist/onchain-tune.js +346 -0
  90. package/dist/permissions-suggest.d.ts +126 -0
  91. package/dist/permissions-suggest.js +333 -0
  92. package/dist/pii-tune.d.ts +107 -0
  93. package/dist/pii-tune.js +122 -0
  94. package/dist/propose.d.ts +117 -0
  95. package/dist/propose.js +184 -0
  96. package/dist/refresh-goldens.d.ts +82 -0
  97. package/dist/refresh-goldens.js +221 -0
  98. package/dist/regression-pin.d.ts +160 -0
  99. package/dist/regression-pin.js +281 -0
  100. package/dist/retention.d.ts +193 -0
  101. package/dist/retention.js +607 -0
  102. package/dist/retire.d.ts +118 -0
  103. package/dist/retire.js +291 -0
  104. package/dist/right-size.d.ts +100 -0
  105. package/dist/right-size.js +123 -0
  106. package/dist/scaffold-evals.d.ts +138 -0
  107. package/dist/scaffold-evals.js +410 -0
  108. package/dist/scope-audit-drift.d.ts +139 -0
  109. package/dist/scope-audit-drift.js +260 -0
  110. package/dist/security-corpus.d.ts +237 -0
  111. package/dist/security-corpus.js +516 -0
  112. package/dist/security-digest.d.ts +173 -0
  113. package/dist/security-digest.js +650 -0
  114. package/dist/sessions-index.d.ts +27 -0
  115. package/dist/sessions-index.js +51 -0
  116. package/dist/slo-doctor.d.ts +67 -0
  117. package/dist/slo-doctor.js +119 -0
  118. package/dist/slo-sink.d.ts +96 -0
  119. package/dist/slo-sink.js +107 -0
  120. package/dist/spec-changelog.d.ts +102 -0
  121. package/dist/spec-changelog.js +237 -0
  122. package/dist/state-backup.d.ts +223 -0
  123. package/dist/state-backup.js +648 -0
  124. package/dist/tools-cli.d.ts +170 -0
  125. package/dist/tools-cli.js +298 -0
  126. package/dist/triage.d.ts +202 -0
  127. package/dist/triage.js +403 -0
  128. package/dist/upgrade.d.ts +57 -0
  129. package/dist/upgrade.js +113 -0
  130. package/dist/version.d.ts +6 -0
  131. package/dist/version.js +27 -0
  132. package/dist/voice-eval.d.ts +138 -0
  133. package/dist/voice-eval.js +309 -0
  134. package/dist/watch.d.ts +58 -0
  135. package/dist/watch.js +97 -0
  136. package/package.json +89 -64
@@ -0,0 +1,150 @@
1
+ import { type JustificationJudge } from "@crewhaus/permission-engine";
2
+ import type { SessionEvents } from "./advise-rules";
3
+ /**
4
+ * AUTOMATION-OPPORTUNITIES.md item 52 — `crewhaus justification calibrate` +
5
+ * `justification preflight` core: replay the intent gate over historical
6
+ * permission-justification records and compare the judge's verdict to the
7
+ * ACTUAL outcome, computing allow/deny agreement + a false-block estimate,
8
+ * proposing a tuned confidence threshold, and flagging high-disagreement
9
+ * prompts. Side-effect-free so it is unit-testable (all filesystem access in
10
+ * index.ts), mirroring `advise-rules.ts` / `egress-triage.ts`.
11
+ *
12
+ * DURABILITY (verified 2026-07):
13
+ *
14
+ * - The judge's verdicts are durable: the intent gate appends one
15
+ * `permission_justification_evaluated` audit record per evaluated call
16
+ * `{ toolName, justification, verdict: "allow"|"deny", reason, judgeModel,
17
+ * confidence? }` to `.crewhaus/audit`.
18
+ *
19
+ * - The ACTUAL outcome proxy is durable too, but at PER-TOOL granularity:
20
+ * the audit record does NOT carry a call id linking it to a specific
21
+ * `tool_result`, and a DENIED call never runs (so it has no outcome at
22
+ * all). What IS cleanly joinable is the session `tool_stats` stream
23
+ * (`{ toolName, isError, durationMs }`) the advisor already persists —
24
+ * the same signal `crewhaus advise` mines. So calibration joins the
25
+ * judge's per-tool ALLOW verdicts to that tool's observed error rate: the
26
+ * proxy for "the allowed call was ultimately fine" is "this tool's
27
+ * tool_stats calls mostly did NOT error". This is aggregate, not per-call
28
+ * — the honest granularity the durable data supports, and the report says
29
+ * so. (A NEUTRAL/"never followed by an error" per-call proxy would need a
30
+ * call-id the audit record does not store; adding one is a separate,
31
+ * schema-touching change deliberately out of scope here.)
32
+ *
33
+ * - `sessionGoal` (the spec's `agent.instructions`) is NOT stored in the
34
+ * record, so a faithful RULE-BASED replay (used by `preflight`) needs the
35
+ * goal supplied — the caller passes the cwd spec's instructions. Without
36
+ * it, the rule-based judge only exercises its length check and the report
37
+ * flags the degraded fidelity.
38
+ */
39
+ export type JustificationRecord = {
40
+ readonly toolName: string;
41
+ readonly justification: string;
42
+ readonly verdict: "allow" | "deny";
43
+ readonly reason?: string;
44
+ readonly judgeModel: string;
45
+ readonly confidence?: number;
46
+ };
47
+ /** Extract `permission_justification_evaluated` records from parsed audit
48
+ * objects, defensively (malformed/other kinds skipped). */
49
+ export declare function extractJustificationRecords(auditObjects: ReadonlyArray<unknown>): JustificationRecord[];
50
+ export type ToolOutcome = {
51
+ readonly calls: number;
52
+ readonly errors: number;
53
+ };
54
+ /** Fold session `tool_stats` lines into per-tool { calls, errors } — the same
55
+ * aggregate `crewhaus advise` uses, reused here as the outcome proxy. */
56
+ export declare function buildToolOutcomes(sessions: ReadonlyArray<SessionEvents>): ReadonlyMap<string, ToolOutcome>;
57
+ export type CalibrationThresholds = {
58
+ /** A tool's error rate at/above which an ALLOW is judged "outcome-bad". */
59
+ readonly badOutcomeRate: number;
60
+ /** Minimum tool_stats calls before a tool's outcome is trusted. */
61
+ readonly minCalls: number;
62
+ };
63
+ export declare const DEFAULT_CALIBRATION_THRESHOLDS: CalibrationThresholds;
64
+ export type PerToolCalibration = {
65
+ readonly toolName: string;
66
+ readonly allowed: number;
67
+ readonly denied: number;
68
+ /** Mean judge confidence over this tool's ALLOW verdicts (null if none). */
69
+ readonly meanAllowConfidence: number | null;
70
+ /** Actual outcome from tool_stats (undefined when the tool never ran —
71
+ * e.g. every call was denied, so there is no observed outcome). */
72
+ readonly outcome?: ToolOutcome;
73
+ /** outcome.errors / outcome.calls, when observed. */
74
+ readonly errorRate: number | null;
75
+ /**
76
+ * Agreement class for this tool's ALLOW verdicts vs the outcome proxy:
77
+ * - "agree" allowed AND the tool mostly succeeded
78
+ * - "over-allow" allowed BUT the tool mostly errored (judge too lenient)
79
+ * - "unknown" allowed but no/too-few observed outcomes to judge
80
+ * - "n/a" no allow verdicts for this tool
81
+ */
82
+ readonly allowAgreement: "agree" | "over-allow" | "unknown" | "n/a";
83
+ };
84
+ export type CalibrationResult = {
85
+ readonly evaluated: number;
86
+ readonly allowed: number;
87
+ readonly denied: number;
88
+ /** denied / evaluated. */
89
+ readonly denyRate: number | null;
90
+ readonly perTool: ReadonlyArray<PerToolCalibration>;
91
+ /** ALLOW verdicts whose tool mostly succeeded / total ALLOWs with a trusted
92
+ * observed outcome — the allow-agreement rate. Null when none observed. */
93
+ readonly allowAgreementRate: number | null;
94
+ /**
95
+ * FALSE-BLOCK ESTIMATE. A denied call never runs, so its true outcome is
96
+ * unobservable; we estimate the false-block rate as the fraction of DENY
97
+ * verdicts that are LOW-confidence (< the proposed threshold) — a
98
+ * low-confidence deny is the judge's own signal that it was unsure, the best
99
+ * durable proxy for "might have been fine". Reported as an ESTIMATE.
100
+ */
101
+ readonly estimatedFalseBlockRate: number | null;
102
+ /** Proposed confidence threshold — see `proposeThreshold`. */
103
+ readonly proposedThreshold: number;
104
+ /** Tools whose ALLOW verdicts disagree with the outcome (over-allow). */
105
+ readonly highDisagreementTools: ReadonlyArray<string>;
106
+ };
107
+ /**
108
+ * Propose a confidence threshold that best separates ALLOW verdicts whose
109
+ * tool succeeded (should stay allow) from ALLOW verdicts whose tool errored
110
+ * (should have been denied / re-justified). Sweeps candidate cutpoints over
111
+ * the observed allow-confidences and picks the one maximizing balanced
112
+ * accuracy (good allows kept above, bad allows pushed below). Falls back to
113
+ * 0.5 when there is no confidence signal to learn from.
114
+ */
115
+ export declare function proposeThreshold(samples: ReadonlyArray<{
116
+ confidence: number;
117
+ outcomeBad: boolean;
118
+ }>): number;
119
+ /**
120
+ * Calibrate the judge over historical records + per-tool outcomes. Aggregate
121
+ * per tool: join each tool's ALLOW confidence to its observed error rate,
122
+ * derive the allow-agreement, propose a threshold, and estimate the false-
123
+ * block rate from low-confidence denies.
124
+ */
125
+ export declare function calibrateJustification(records: ReadonlyArray<JustificationRecord>, outcomes: ReadonlyMap<string, ToolOutcome>, thresholds?: CalibrationThresholds): CalibrationResult;
126
+ export type PreflightResult = {
127
+ readonly sampled: number;
128
+ readonly wouldAllow: number;
129
+ readonly wouldDeny: number;
130
+ /** Records where the replayed verdict DIFFERS from the stored verdict. */
131
+ readonly flips: ReadonlyArray<{
132
+ readonly toolName: string;
133
+ readonly stored: "allow" | "deny";
134
+ readonly replayed: "allow" | "deny";
135
+ }>;
136
+ /** True when no sessionGoal was supplied (rule-based fidelity is degraded —
137
+ * only the length check runs). */
138
+ readonly degraded: boolean;
139
+ };
140
+ /**
141
+ * Preflight: dry-run a judge over a sample of historical justifications BEFORE
142
+ * deploy. Defaults to the deterministic `ruleBasedJustificationJudge` (offline,
143
+ * no credentials — a clean explanation for the LLM-judge path is that it needs
144
+ * a live model, which preflight does not spin up). `sessionGoal` is the spec's
145
+ * `agent.instructions`; without it the rule-based judge runs degraded (length
146
+ * check only) and `degraded` is true.
147
+ */
148
+ export declare function preflightJustification(records: ReadonlyArray<JustificationRecord>, sessionGoal: string, judge?: JustificationJudge): Promise<PreflightResult>;
149
+ export declare function renderCalibrationLines(r: CalibrationResult): ReadonlyArray<string>;
150
+ export declare function renderPreflightLines(r: PreflightResult): ReadonlyArray<string>;
@@ -0,0 +1,262 @@
1
+ import { ruleBasedJustificationJudge } from "@crewhaus/permission-engine";
2
+ import { payloadOf } from "./advise-rules";
3
+ function asObject(v) {
4
+ return v !== null && typeof v === "object" && !Array.isArray(v)
5
+ ? v
6
+ : undefined;
7
+ }
8
+ /** Extract `permission_justification_evaluated` records from parsed audit
9
+ * objects, defensively (malformed/other kinds skipped). */
10
+ export function extractJustificationRecords(auditObjects) {
11
+ const out = [];
12
+ for (const obj of auditObjects) {
13
+ const o = asObject(obj);
14
+ if (o === undefined || o["kind"] !== "permission_justification_evaluated")
15
+ continue;
16
+ const p = asObject(o["payload"]);
17
+ if (p === undefined)
18
+ continue;
19
+ const toolName = typeof p["toolName"] === "string" ? p["toolName"] : undefined;
20
+ const justification = typeof p["justification"] === "string" ? p["justification"] : undefined;
21
+ const verdict = p["verdict"];
22
+ const judgeModel = typeof p["judgeModel"] === "string" ? p["judgeModel"] : "(unknown)";
23
+ if (toolName === undefined || justification === undefined)
24
+ continue;
25
+ if (verdict !== "allow" && verdict !== "deny")
26
+ continue;
27
+ out.push({
28
+ toolName,
29
+ justification,
30
+ verdict,
31
+ ...(typeof p["reason"] === "string" ? { reason: p["reason"] } : {}),
32
+ judgeModel,
33
+ ...(typeof p["confidence"] === "number" ? { confidence: p["confidence"] } : {}),
34
+ });
35
+ }
36
+ return out;
37
+ }
38
+ /** Fold session `tool_stats` lines into per-tool { calls, errors } — the same
39
+ * aggregate `crewhaus advise` uses, reused here as the outcome proxy. */
40
+ export function buildToolOutcomes(sessions) {
41
+ const acc = new Map();
42
+ for (const session of sessions) {
43
+ for (const obj of session.objects) {
44
+ const tool = payloadOf(obj, "tool_stats");
45
+ if (tool === undefined || typeof tool["toolName"] !== "string")
46
+ continue;
47
+ const name = tool["toolName"];
48
+ const t = acc.get(name) ?? { calls: 0, errors: 0 };
49
+ t.calls += 1;
50
+ if (tool["isError"] === true)
51
+ t.errors += 1;
52
+ acc.set(name, t);
53
+ }
54
+ }
55
+ return acc;
56
+ }
57
+ export const DEFAULT_CALIBRATION_THRESHOLDS = {
58
+ badOutcomeRate: 0.5,
59
+ minCalls: 3,
60
+ };
61
+ /**
62
+ * Propose a confidence threshold that best separates ALLOW verdicts whose
63
+ * tool succeeded (should stay allow) from ALLOW verdicts whose tool errored
64
+ * (should have been denied / re-justified). Sweeps candidate cutpoints over
65
+ * the observed allow-confidences and picks the one maximizing balanced
66
+ * accuracy (good allows kept above, bad allows pushed below). Falls back to
67
+ * 0.5 when there is no confidence signal to learn from.
68
+ */
69
+ export function proposeThreshold(samples) {
70
+ const withConf = samples.filter((s) => Number.isFinite(s.confidence));
71
+ if (withConf.length === 0)
72
+ return 0.5;
73
+ const good = withConf.filter((s) => !s.outcomeBad);
74
+ const bad = withConf.filter((s) => s.outcomeBad);
75
+ if (good.length === 0 || bad.length === 0) {
76
+ // No contrast to learn a boundary from — keep the neutral default.
77
+ return 0.5;
78
+ }
79
+ const cuts = [...new Set(withConf.map((s) => s.confidence))].sort((a, b) => a - b);
80
+ // Candidate thresholds sit between observed confidences (and at the extremes).
81
+ const candidates = new Set([0.5]);
82
+ for (let i = 0; i < cuts.length; i++) {
83
+ candidates.add(cuts[i]);
84
+ if (i + 1 < cuts.length)
85
+ candidates.add((cuts[i] + cuts[i + 1]) / 2);
86
+ }
87
+ let best = 0.5;
88
+ let bestScore = -1;
89
+ for (const thr of [...candidates].sort((a, b) => a - b)) {
90
+ // good allows should be ABOVE thr (kept), bad allows BELOW (re-gated).
91
+ const goodKept = good.filter((s) => s.confidence >= thr).length / good.length;
92
+ const badReGated = bad.filter((s) => s.confidence < thr).length / bad.length;
93
+ const balanced = (goodKept + badReGated) / 2;
94
+ if (balanced > bestScore) {
95
+ bestScore = balanced;
96
+ best = thr;
97
+ }
98
+ }
99
+ return Number(best.toFixed(3));
100
+ }
101
+ /**
102
+ * Calibrate the judge over historical records + per-tool outcomes. Aggregate
103
+ * per tool: join each tool's ALLOW confidence to its observed error rate,
104
+ * derive the allow-agreement, propose a threshold, and estimate the false-
105
+ * block rate from low-confidence denies.
106
+ */
107
+ export function calibrateJustification(records, outcomes, thresholds = DEFAULT_CALIBRATION_THRESHOLDS) {
108
+ const byTool = new Map();
109
+ let allowed = 0;
110
+ let denied = 0;
111
+ for (const r of records) {
112
+ const a = byTool.get(r.toolName) ?? {
113
+ allowed: 0,
114
+ denied: 0,
115
+ allowConfidences: [],
116
+ denyConfidences: [],
117
+ };
118
+ if (r.verdict === "allow") {
119
+ a.allowed += 1;
120
+ allowed += 1;
121
+ if (typeof r.confidence === "number")
122
+ a.allowConfidences.push(r.confidence);
123
+ }
124
+ else {
125
+ a.denied += 1;
126
+ denied += 1;
127
+ if (typeof r.confidence === "number")
128
+ a.denyConfidences.push(r.confidence);
129
+ }
130
+ byTool.set(r.toolName, a);
131
+ }
132
+ const mean = (xs) => xs.length > 0 ? xs.reduce((s, x) => s + x, 0) / xs.length : null;
133
+ // Threshold samples: one per allow-with-confidence whose tool has a trusted
134
+ // observed outcome.
135
+ const thresholdSamples = [];
136
+ const perTool = [];
137
+ const highDisagreement = [];
138
+ for (const [toolName, a] of [...byTool.entries()].sort((x, y) => x[0].localeCompare(y[0]))) {
139
+ const outcome = outcomes.get(toolName);
140
+ const errorRate = outcome !== undefined && outcome.calls > 0 ? outcome.errors / outcome.calls : null;
141
+ const trusted = outcome !== undefined && outcome.calls >= thresholds.minCalls && errorRate !== null;
142
+ let allowAgreement;
143
+ if (a.allowed === 0) {
144
+ allowAgreement = "n/a";
145
+ }
146
+ else if (!trusted) {
147
+ allowAgreement = "unknown";
148
+ }
149
+ else if (errorRate >= thresholds.badOutcomeRate) {
150
+ allowAgreement = "over-allow";
151
+ highDisagreement.push(toolName);
152
+ }
153
+ else {
154
+ allowAgreement = "agree";
155
+ }
156
+ if (a.allowed > 0 && trusted) {
157
+ const outcomeBad = errorRate >= thresholds.badOutcomeRate;
158
+ for (const c of a.allowConfidences)
159
+ thresholdSamples.push({ confidence: c, outcomeBad });
160
+ }
161
+ perTool.push({
162
+ toolName,
163
+ allowed: a.allowed,
164
+ denied: a.denied,
165
+ meanAllowConfidence: mean(a.allowConfidences),
166
+ ...(outcome !== undefined ? { outcome } : {}),
167
+ errorRate,
168
+ allowAgreement,
169
+ });
170
+ }
171
+ const proposedThreshold = proposeThreshold(thresholdSamples);
172
+ // Allow-agreement rate: agree / (agree + over-allow) over trusted allows.
173
+ const trustedAllowTools = perTool.filter((t) => t.allowAgreement === "agree" || t.allowAgreement === "over-allow");
174
+ const agreeCount = trustedAllowTools.filter((t) => t.allowAgreement === "agree").length;
175
+ const allowAgreementRate = trustedAllowTools.length > 0 ? agreeCount / trustedAllowTools.length : null;
176
+ // False-block estimate: fraction of DENY verdicts whose confidence is below
177
+ // the proposed threshold (the judge's own low-confidence denies).
178
+ const denyConfidences = records
179
+ .filter((r) => r.verdict === "deny" && typeof r.confidence === "number")
180
+ .map((r) => r.confidence);
181
+ const estimatedFalseBlockRate = denyConfidences.length > 0
182
+ ? denyConfidences.filter((c) => c < proposedThreshold).length / denyConfidences.length
183
+ : null;
184
+ return {
185
+ evaluated: records.length,
186
+ allowed,
187
+ denied,
188
+ denyRate: records.length > 0 ? denied / records.length : null,
189
+ perTool,
190
+ allowAgreementRate,
191
+ estimatedFalseBlockRate,
192
+ proposedThreshold,
193
+ highDisagreementTools: highDisagreement.sort(),
194
+ };
195
+ }
196
+ /**
197
+ * Preflight: dry-run a judge over a sample of historical justifications BEFORE
198
+ * deploy. Defaults to the deterministic `ruleBasedJustificationJudge` (offline,
199
+ * no credentials — a clean explanation for the LLM-judge path is that it needs
200
+ * a live model, which preflight does not spin up). `sessionGoal` is the spec's
201
+ * `agent.instructions`; without it the rule-based judge runs degraded (length
202
+ * check only) and `degraded` is true.
203
+ */
204
+ export async function preflightJustification(records, sessionGoal, judge = ruleBasedJustificationJudge) {
205
+ let wouldAllow = 0;
206
+ let wouldDeny = 0;
207
+ const flips = [];
208
+ const flipsMut = flips;
209
+ for (const r of records) {
210
+ const verdict = await judge({
211
+ toolName: r.toolName,
212
+ justification: r.justification,
213
+ sessionGoal,
214
+ input: null,
215
+ });
216
+ const replayed = verdict.allow ? "allow" : "deny";
217
+ if (verdict.allow)
218
+ wouldAllow += 1;
219
+ else
220
+ wouldDeny += 1;
221
+ if (replayed !== r.verdict) {
222
+ flipsMut.push({ toolName: r.toolName, stored: r.verdict, replayed });
223
+ }
224
+ }
225
+ return {
226
+ sampled: records.length,
227
+ wouldAllow,
228
+ wouldDeny,
229
+ flips,
230
+ degraded: sessionGoal.length === 0,
231
+ };
232
+ }
233
+ // -------- renderers --------
234
+ export function renderCalibrationLines(r) {
235
+ const lines = [];
236
+ const pct = (x) => (x === null ? "n/a" : `${(x * 100).toFixed(0)}%`);
237
+ lines.push(`${r.evaluated} justification(s) evaluated: ${r.allowed} allow, ${r.denied} deny (deny rate ${pct(r.denyRate)})`);
238
+ lines.push(`allow-agreement (allowed tools that succeeded / trusted allows): ${pct(r.allowAgreementRate)}`);
239
+ lines.push(`estimated false-block rate (low-confidence denies): ${pct(r.estimatedFalseBlockRate)}`);
240
+ lines.push(`proposed confidence threshold: ${r.proposedThreshold}`);
241
+ for (const t of r.perTool) {
242
+ const conf = t.meanAllowConfidence !== null ? t.meanAllowConfidence.toFixed(2) : "n/a";
243
+ const err = t.errorRate !== null ? pct(t.errorRate) : "no outcome observed";
244
+ lines.push(` ${t.allowAgreement === "over-allow" ? "✗" : "•"} ${t.toolName}: ${t.allowed} allow / ${t.denied} deny, mean allow-confidence ${conf}, actual error rate ${err} → ${t.allowAgreement}`);
245
+ }
246
+ if (r.highDisagreementTools.length > 0) {
247
+ lines.push(`high-disagreement tools (allowed but mostly errored — tighten the justification prompt or lower the threshold): ${r.highDisagreementTools.join(", ")}`);
248
+ }
249
+ lines.push(" ~ outcome is a PER-TOOL proxy from tool_stats error rate; a denied call never runs, so the false-block figure is an ESTIMATE from low-confidence denies");
250
+ return lines;
251
+ }
252
+ export function renderPreflightLines(r) {
253
+ const lines = [];
254
+ lines.push(`preflight: ${r.sampled} historical justification(s) replayed → ${r.wouldAllow} allow, ${r.wouldDeny} deny, ${r.flips.length} flip(s) vs the stored verdict`);
255
+ if (r.degraded) {
256
+ lines.push(" ~ no session goal supplied (spec agent.instructions) — rule-based replay ran DEGRADED (length check only); pass a spec for full-fidelity replay");
257
+ }
258
+ for (const f of r.flips.slice(0, 20)) {
259
+ lines.push(` ↔ ${f.toolName}: stored ${f.stored} → would ${f.replayed}`);
260
+ }
261
+ return lines;
262
+ }
@@ -1,5 +1,6 @@
1
+ import type { AuditLog } from "@crewhaus/audit-log";
1
2
  import type { JustificationJudge } from "@crewhaus/permission-engine";
2
- import type { JustificationAuditSink } from "@crewhaus/runtime-core";
3
+ import type { EgressAuditSink, JustificationAuditSink } from "@crewhaus/runtime-core";
3
4
  /**
4
5
  * FR-004 — Pillar 3 intent-gate wiring for `crewhaus run`, factored out of
5
6
  * the entry file `index.ts` (which runs a top-level argv switch and so cannot
@@ -75,10 +76,30 @@ export type OpenJustificationAuditSinkOptions = {
75
76
  readonly enabled?: boolean;
76
77
  };
77
78
  /**
78
- * Open the durable `JustificationAuditSink` the gate appends to. Returns
79
- * `undefined` when disabled (so the caller spreads nothing into
80
- * `runChatLoop` and the durable path is a no-op). The returned object is a
81
- * real `@crewhaus/audit-log` `AuditLog`, which structurally satisfies the
82
- * `JustificationAuditSink` seam (its `append({ kind, payload })`).
79
+ * Open the durable audit log the Pillar 3 gates append to, rooted at
80
+ * `<cwd>/.crewhaus/audit`. Returns `undefined` when disabled (so the caller
81
+ * spreads nothing into `runChatLoop` and the durable path is a no-op).
82
+ *
83
+ * The returned object is a real `@crewhaus/audit-log` `AuditLog`, which
84
+ * structurally satisfies BOTH the `JustificationAuditSink`
85
+ * (`permission_justification_evaluated`) and the `EgressAuditSink`
86
+ * (`egress_decision`) seams — its `append({ kind, payload })`. A single log
87
+ * instance is deliberately shared across both gates so their records land on
88
+ * ONE hash-chained, gapless chain (item 20): two separate logs would each own
89
+ * their own `_chain-tail.json` and interleave inconsistently.
90
+ */
91
+ export declare function openSecurityAuditSink(opts: OpenJustificationAuditSinkOptions): Promise<AuditLog | undefined>;
92
+ /**
93
+ * Back-compat alias — the justification gate's original opener. Returns the
94
+ * shared `AuditLog` typed down to the `JustificationAuditSink` seam. Prefer
95
+ * {@link openSecurityAuditSink} when a caller also needs the egress sink from
96
+ * the same chain.
83
97
  */
84
98
  export declare function openJustificationAuditSink(opts: OpenJustificationAuditSinkOptions): Promise<JustificationAuditSink | undefined>;
99
+ /**
100
+ * Narrow a shared security audit log to the `EgressAuditSink` seam. The same
101
+ * `AuditLog` instance already satisfies `EgressAuditSink` structurally; this
102
+ * is a typed pass-through so the run path can thread one log into both
103
+ * `runChatLoop({ justificationAuditSink, egressAuditSink })` fields.
104
+ */
105
+ export declare function asEgressAuditSink(sink: AuditLog | undefined): EgressAuditSink | undefined;
@@ -58,15 +58,39 @@ export function justificationAuditDir(cwd) {
58
58
  return join(cwd, ".crewhaus", "audit");
59
59
  }
60
60
  /**
61
- * Open the durable `JustificationAuditSink` the gate appends to. Returns
62
- * `undefined` when disabled (so the caller spreads nothing into
63
- * `runChatLoop` and the durable path is a no-op). The returned object is a
64
- * real `@crewhaus/audit-log` `AuditLog`, which structurally satisfies the
65
- * `JustificationAuditSink` seam (its `append({ kind, payload })`).
61
+ * Open the durable audit log the Pillar 3 gates append to, rooted at
62
+ * `<cwd>/.crewhaus/audit`. Returns `undefined` when disabled (so the caller
63
+ * spreads nothing into `runChatLoop` and the durable path is a no-op).
64
+ *
65
+ * The returned object is a real `@crewhaus/audit-log` `AuditLog`, which
66
+ * structurally satisfies BOTH the `JustificationAuditSink`
67
+ * (`permission_justification_evaluated`) and the `EgressAuditSink`
68
+ * (`egress_decision`) seams — its `append({ kind, payload })`. A single log
69
+ * instance is deliberately shared across both gates so their records land on
70
+ * ONE hash-chained, gapless chain (item 20): two separate logs would each own
71
+ * their own `_chain-tail.json` and interleave inconsistently.
66
72
  */
67
- export async function openJustificationAuditSink(opts) {
73
+ export async function openSecurityAuditSink(opts) {
68
74
  if (opts.enabled === false)
69
75
  return undefined;
70
76
  const { openAuditLog } = await import("@crewhaus/audit-log");
71
77
  return await openAuditLog({ rootDir: justificationAuditDir(opts.cwd) });
72
78
  }
79
+ /**
80
+ * Back-compat alias — the justification gate's original opener. Returns the
81
+ * shared `AuditLog` typed down to the `JustificationAuditSink` seam. Prefer
82
+ * {@link openSecurityAuditSink} when a caller also needs the egress sink from
83
+ * the same chain.
84
+ */
85
+ export async function openJustificationAuditSink(opts) {
86
+ return await openSecurityAuditSink(opts);
87
+ }
88
+ /**
89
+ * Narrow a shared security audit log to the `EgressAuditSink` seam. The same
90
+ * `AuditLog` instance already satisfies `EgressAuditSink` structurally; this
91
+ * is a typed pass-through so the run path can thread one log into both
92
+ * `runChatLoop({ justificationAuditSink, egressAuditSink })` fields.
93
+ */
94
+ export function asEgressAuditSink(sink) {
95
+ return sink;
96
+ }