crewhaus 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +10 -3
  2. package/dist/advice-apply.d.ts +182 -0
  3. package/dist/advice-apply.js +286 -0
  4. package/dist/advise-rules.d.ts +348 -0
  5. package/dist/advise-rules.js +905 -0
  6. package/dist/alert-sink.d.ts +48 -0
  7. package/dist/alert-sink.js +86 -0
  8. package/dist/approval-gate.d.ts +127 -0
  9. package/dist/approval-gate.js +254 -0
  10. package/dist/audit-verify.d.ts +69 -0
  11. package/dist/audit-verify.js +97 -0
  12. package/dist/autodistill.d.ts +113 -0
  13. package/dist/autodistill.js +256 -0
  14. package/dist/channel-provision.d.ts +360 -0
  15. package/dist/channel-provision.js +881 -0
  16. package/dist/ci-scaffold.d.ts +31 -0
  17. package/dist/ci-scaffold.js +343 -0
  18. package/dist/compile-check.d.ts +90 -0
  19. package/dist/compile-check.js +285 -0
  20. package/dist/compliance-schedule.d.ts +35 -0
  21. package/dist/compliance-schedule.js +36 -0
  22. package/dist/context-pressure.d.ts +80 -0
  23. package/dist/context-pressure.js +166 -0
  24. package/dist/dataset-mine.d.ts +172 -0
  25. package/dist/dataset-mine.js +403 -0
  26. package/dist/datasets.d.ts +124 -0
  27. package/dist/datasets.js +260 -0
  28. package/dist/deploy-canary.d.ts +83 -0
  29. package/dist/deploy-canary.js +87 -0
  30. package/dist/doctor-checks.d.ts +33 -0
  31. package/dist/doctor-checks.js +92 -0
  32. package/dist/doctor-detect.d.ts +108 -0
  33. package/dist/doctor-detect.js +214 -0
  34. package/dist/doctor-fix.d.ts +81 -0
  35. package/dist/doctor-fix.js +164 -0
  36. package/dist/egress-triage.d.ts +121 -0
  37. package/dist/egress-triage.js +261 -0
  38. package/dist/eval-bridge.d.ts +114 -0
  39. package/dist/eval-bridge.js +158 -0
  40. package/dist/eval-coverage.d.ts +140 -0
  41. package/dist/eval-coverage.js +428 -0
  42. package/dist/eval-history.d.ts +48 -0
  43. package/dist/eval-history.js +157 -0
  44. package/dist/eval-matrix.d.ts +80 -0
  45. package/dist/eval-matrix.js +182 -0
  46. package/dist/eval-sentinel.d.ts +65 -0
  47. package/dist/eval-sentinel.js +132 -0
  48. package/dist/faq.d.ts +68 -0
  49. package/dist/faq.js +168 -0
  50. package/dist/feedback.d.ts +9 -2
  51. package/dist/feedback.js +17 -7
  52. package/dist/fewshot.d.ts +83 -0
  53. package/dist/fewshot.js +158 -0
  54. package/dist/fleet.d.ts +207 -0
  55. package/dist/fleet.js +488 -0
  56. package/dist/flywheel.d.ts +193 -0
  57. package/dist/flywheel.js +519 -0
  58. package/dist/graders-suggest.d.ts +186 -0
  59. package/dist/graders-suggest.js +658 -0
  60. package/dist/incident.d.ts +99 -0
  61. package/dist/incident.js +217 -0
  62. package/dist/index.d.ts +9 -1
  63. package/dist/index.js +11601 -964
  64. package/dist/init-interactive.d.ts +105 -0
  65. package/dist/init-interactive.js +208 -0
  66. package/dist/intents.d.ts +105 -0
  67. package/dist/intents.js +292 -0
  68. package/dist/judge-calibrate.d.ts +137 -0
  69. package/dist/judge-calibrate.js +247 -0
  70. package/dist/justification-calibrate.d.ts +150 -0
  71. package/dist/justification-calibrate.js +262 -0
  72. package/dist/justification-gate.d.ts +27 -6
  73. package/dist/justification-gate.js +30 -6
  74. package/dist/knowledge-sync.d.ts +179 -0
  75. package/dist/knowledge-sync.js +551 -0
  76. package/dist/lessons.d.ts +87 -0
  77. package/dist/lessons.js +207 -0
  78. package/dist/lint.d.ts +127 -0
  79. package/dist/lint.js +226 -0
  80. package/dist/loadtest.d.ts +114 -0
  81. package/dist/loadtest.js +196 -0
  82. package/dist/marketplace-cli.d.ts +110 -0
  83. package/dist/marketplace-cli.js +250 -0
  84. package/dist/mcp-doctor.d.ts +121 -0
  85. package/dist/mcp-doctor.js +249 -0
  86. package/dist/model-scan.d.ts +116 -0
  87. package/dist/model-scan.js +226 -0
  88. package/dist/onchain-tune.d.ts +164 -0
  89. package/dist/onchain-tune.js +346 -0
  90. package/dist/permissions-suggest.d.ts +126 -0
  91. package/dist/permissions-suggest.js +333 -0
  92. package/dist/pii-tune.d.ts +107 -0
  93. package/dist/pii-tune.js +122 -0
  94. package/dist/propose.d.ts +117 -0
  95. package/dist/propose.js +184 -0
  96. package/dist/refresh-goldens.d.ts +82 -0
  97. package/dist/refresh-goldens.js +221 -0
  98. package/dist/regression-pin.d.ts +160 -0
  99. package/dist/regression-pin.js +281 -0
  100. package/dist/retention.d.ts +193 -0
  101. package/dist/retention.js +607 -0
  102. package/dist/retire.d.ts +118 -0
  103. package/dist/retire.js +291 -0
  104. package/dist/right-size.d.ts +100 -0
  105. package/dist/right-size.js +123 -0
  106. package/dist/scaffold-evals.d.ts +138 -0
  107. package/dist/scaffold-evals.js +410 -0
  108. package/dist/scope-audit-drift.d.ts +139 -0
  109. package/dist/scope-audit-drift.js +260 -0
  110. package/dist/security-corpus.d.ts +237 -0
  111. package/dist/security-corpus.js +516 -0
  112. package/dist/security-digest.d.ts +173 -0
  113. package/dist/security-digest.js +650 -0
  114. package/dist/sessions-index.d.ts +27 -0
  115. package/dist/sessions-index.js +51 -0
  116. package/dist/slo-doctor.d.ts +67 -0
  117. package/dist/slo-doctor.js +119 -0
  118. package/dist/slo-sink.d.ts +96 -0
  119. package/dist/slo-sink.js +107 -0
  120. package/dist/spec-changelog.d.ts +102 -0
  121. package/dist/spec-changelog.js +237 -0
  122. package/dist/state-backup.d.ts +223 -0
  123. package/dist/state-backup.js +648 -0
  124. package/dist/tools-cli.d.ts +170 -0
  125. package/dist/tools-cli.js +298 -0
  126. package/dist/triage.d.ts +202 -0
  127. package/dist/triage.js +403 -0
  128. package/dist/upgrade.d.ts +57 -0
  129. package/dist/upgrade.js +113 -0
  130. package/dist/version.d.ts +6 -0
  131. package/dist/version.js +27 -0
  132. package/dist/voice-eval.d.ts +138 -0
  133. package/dist/voice-eval.js +309 -0
  134. package/dist/watch.d.ts +58 -0
  135. package/dist/watch.js +97 -0
  136. package/package.json +89 -65
@@ -0,0 +1,207 @@
1
+ /**
2
+ * Item #56 — auto-maintained LESSONS.md + per-user preference files.
3
+ *
4
+ * `crewhaus lessons update` mines two durable signals:
5
+ * - CORRECTION lessons: a `user_feedback` record carrying a `correction`
6
+ * (the user's better answer) OR a low-rated turn with a comment — the
7
+ * "here's what went wrong / do this instead" signal.
8
+ * - FAILURE→FIX lessons: recurring negative signals from `dataset-mine`'s
9
+ * `mineSession` (tool-error spikes, runtime errors, loop nudges, retries)
10
+ * paired to the triggering turn — "when X happens, watch out."
11
+ * These fold into a deduped, append-with-merge LESSONS.md (like the project-
12
+ * memory files the runtime already auto-loads — LESSONS.md is added to the
13
+ * canonical set, so it is injected into the system prompt at run start).
14
+ *
15
+ * Per-user preferences: when feedback carries a `rater` identity, ratings +
16
+ * comments for that rater fold into a per-user prefs file
17
+ * (`.crewhaus/preferences/<rater>.md`) injected when that user is known.
18
+ *
19
+ * Pure + deterministic (stable order, normalized-text dedupe → idempotent
20
+ * re-runs); FS + redaction wiring lives in `apps/cli/src/index.ts`. Lesson +
21
+ * preference text is model/user output → PII/secret-redacted by the caller.
22
+ */
23
+ import { mergeFeedback, normalizeRating } from "./feedback";
24
+ function normalizeKey(s) {
25
+ return s.toLowerCase().replace(/\s+/g, " ").trim();
26
+ }
27
+ function clip(s, n) {
28
+ const t = s.replace(/\s+/g, " ").trim();
29
+ return t.length > n ? `${t.slice(0, n - 1).trimEnd()}…` : t;
30
+ }
31
+ /**
32
+ * Mine lessons from turns + feedback + pre-mined failure signals. Returns a
33
+ * deterministically-ordered, deduped list. `failureSignals` is the output of
34
+ * `mineSession` across the scanned sessions (each `{ input, reason }`), so the
35
+ * failure→fix lessons reuse the same negative-signal detection the dataset
36
+ * miner uses rather than reinventing it.
37
+ */
38
+ export async function mineLessons(turns, feedback, failureSignals, opts = {}) {
39
+ const lowScore = opts.lowScore ?? 0.5;
40
+ const redact = opts.redact ?? (async (t) => t);
41
+ const turnByKey = new Map();
42
+ for (const t of turns)
43
+ turnByKey.set(`${t.sessionId}#${t.turnNumber}`, t);
44
+ const byKey = new Map();
45
+ const add = (text, kind) => {
46
+ const clean = clip(text, 300);
47
+ if (clean === "")
48
+ return;
49
+ const key = normalizeKey(clean);
50
+ if (!byKey.has(key))
51
+ byKey.set(key, { key, text: clean, kind });
52
+ };
53
+ // Correction lessons from feedback (merged so one per turn). #56 F3 — the
54
+ // echoed question prefix is a raw user turn (may carry a pasted credential),
55
+ // so it is redacted too, not just the correction/comment body.
56
+ for (const fb of mergeFeedback(feedback)) {
57
+ const turn = turnByKey.get(`${fb.sessionId}#${fb.turnNumber}`);
58
+ const question = turn?.input?.trim();
59
+ const score = normalizeRating(fb);
60
+ if (fb.correction !== undefined && fb.correction.trim() !== "") {
61
+ const q = question ? `For "${await redact(clip(question, 80))}": ` : "";
62
+ add(`${q}prefer this answer — ${await redact(fb.correction)}`, "correction");
63
+ }
64
+ else if (fb.comment !== undefined &&
65
+ fb.comment.trim() !== "" &&
66
+ score !== undefined &&
67
+ score <= lowScore) {
68
+ const q = question ? `For "${await redact(clip(question, 80))}": ` : "";
69
+ add(`${q}avoid — ${await redact(fb.comment)}`, "correction");
70
+ }
71
+ }
72
+ // Failure→fix lessons from the mined negative signals. #56 F3 — BOTH the
73
+ // echoed user input AND the failure reason (raw error messages carry
74
+ // connection strings / tokens) are redacted before landing in an
75
+ // auto-injected lesson.
76
+ for (const sig of failureSignals) {
77
+ const redactedInput = sig.input.trim() !== "" ? await redact(clip(sig.input, 80)) : "";
78
+ const q = redactedInput !== "" ? ` on "${redactedInput}"` : "";
79
+ add(`Watch out${q}: ${await redact(sig.reason)}.`, "failure-fix");
80
+ }
81
+ // Stable order: corrections first (higher-signal), then failure-fixes; each
82
+ // group alphabetized by key so re-runs produce byte-identical output.
83
+ return [...byKey.values()].sort((a, b) => (a.kind === b.kind ? 0 : a.kind === "correction" ? -1 : 1) || a.key.localeCompare(b.key));
84
+ }
85
+ /**
86
+ * Merge fresh lessons into existing ones, deduping by normalized key so a
87
+ * re-run never duplicates a lesson (append-with-merge idempotency). Existing
88
+ * lessons are preserved; new keys appended; ordering re-stabilized.
89
+ */
90
+ export function mergeLessons(existing, fresh) {
91
+ const byKey = new Map();
92
+ for (const l of existing)
93
+ byKey.set(l.key, l);
94
+ for (const l of fresh)
95
+ if (!byKey.has(l.key))
96
+ byKey.set(l.key, l);
97
+ return [...byKey.values()].sort((a, b) => (a.kind === b.kind ? 0 : a.kind === "correction" ? -1 : 1) || a.key.localeCompare(b.key));
98
+ }
99
+ const LESSONS_HEADER = "# LESSONS";
100
+ const LESSONS_NOTE = "<!-- Auto-maintained by `crewhaus lessons update`. Deduped + idempotent; hand-edits above the marker are preserved. -->";
101
+ const LESSONS_MARKER = "<!-- crewhaus:lessons -->";
102
+ /**
103
+ * Render LESSONS.md. Everything BELOW the `crewhaus:lessons` marker is machine-
104
+ * managed (the sorted lesson list); anything a human wrote ABOVE it is
105
+ * preserved verbatim by `parseLessonsMd` on the next update.
106
+ */
107
+ export function renderLessonsMd(lessons, preamble = "") {
108
+ const corrections = lessons.filter((l) => l.kind === "correction");
109
+ const failures = lessons.filter((l) => l.kind === "failure-fix");
110
+ const lines = [];
111
+ const pre = preamble.trimEnd();
112
+ lines.push(pre === "" ? `${LESSONS_HEADER}\n\n${LESSONS_NOTE}` : pre);
113
+ lines.push("");
114
+ lines.push(LESSONS_MARKER);
115
+ if (corrections.length > 0) {
116
+ lines.push("");
117
+ lines.push("## Corrections & preferences");
118
+ lines.push("");
119
+ for (const l of corrections)
120
+ lines.push(`- ${l.text}`);
121
+ }
122
+ if (failures.length > 0) {
123
+ lines.push("");
124
+ lines.push("## Known failure patterns");
125
+ lines.push("");
126
+ for (const l of failures)
127
+ lines.push(`- ${l.text}`);
128
+ }
129
+ return `${lines.join("\n")}\n`;
130
+ }
131
+ /**
132
+ * Parse an existing LESSONS.md back into `{ preamble, lessons }`. The preamble
133
+ * is everything before the `crewhaus:lessons` marker (preserved on re-render);
134
+ * lessons are the `- ` bullets under the two managed sections, tagged by which
135
+ * section they came from. Files without the marker are treated as all-preamble
136
+ * (a human-authored file), so we never clobber hand-written lessons.
137
+ */
138
+ export function parseLessonsMd(content) {
139
+ const markerIdx = content.indexOf(LESSONS_MARKER);
140
+ if (markerIdx === -1)
141
+ return { preamble: content, lessons: [] };
142
+ const preamble = content.slice(0, markerIdx).trimEnd();
143
+ const managed = content.slice(markerIdx + LESSONS_MARKER.length);
144
+ const lessons = [];
145
+ let kind = "correction";
146
+ for (const rawLine of managed.split("\n")) {
147
+ const line = rawLine.trim();
148
+ if (line.startsWith("## Known failure"))
149
+ kind = "failure-fix";
150
+ else if (line.startsWith("## Corrections"))
151
+ kind = "correction";
152
+ else if (line.startsWith("- ")) {
153
+ const text = line.slice(2).trim();
154
+ if (text !== "")
155
+ lessons.push({ key: normalizeKey(text), text, kind });
156
+ }
157
+ }
158
+ return { preamble, lessons };
159
+ }
160
+ /**
161
+ * Fold feedback carrying a `rater` into per-user preference notes. A rater's
162
+ * comments (on any rating) and corrections become their preference bullets,
163
+ * deduped + stably ordered. Returns one entry per rater, sorted by rater id.
164
+ */
165
+ export async function minePreferences(feedback, opts = {}) {
166
+ const redact = opts.redact ?? (async (t) => t);
167
+ const byRater = new Map();
168
+ for (const fb of mergeFeedback(feedback)) {
169
+ if (fb.rater === undefined || fb.rater.trim() === "")
170
+ continue;
171
+ const bucket = byRater.get(fb.rater) ?? { keys: new Set(), notes: [] };
172
+ const candidates = [];
173
+ if (fb.correction !== undefined && fb.correction.trim() !== "") {
174
+ candidates.push(`prefers answers like: ${await redact(fb.correction)}`);
175
+ }
176
+ if (fb.comment !== undefined && fb.comment.trim() !== "") {
177
+ candidates.push(await redact(fb.comment));
178
+ }
179
+ for (const c of candidates) {
180
+ const clean = clip(c, 240);
181
+ const key = normalizeKey(clean);
182
+ if (clean !== "" && !bucket.keys.has(key)) {
183
+ bucket.keys.add(key);
184
+ bucket.notes.push(clean);
185
+ }
186
+ }
187
+ byRater.set(fb.rater, bucket);
188
+ }
189
+ return [...byRater.entries()]
190
+ .filter(([, b]) => b.notes.length > 0)
191
+ .map(([rater, b]) => ({ rater, notes: b.notes }))
192
+ .sort((a, b) => a.rater.localeCompare(b.rater));
193
+ }
194
+ /** Render a per-user preferences markdown file (a project-memory-style block). */
195
+ export function renderPreferencesMd(prefs) {
196
+ const lines = [
197
+ `# Preferences — ${prefs.rater}`,
198
+ "",
199
+ "<!-- Auto-maintained by `crewhaus lessons update`. -->",
200
+ "",
201
+ "When serving this user, honour these preferences learned from their feedback:",
202
+ "",
203
+ ];
204
+ for (const n of prefs.notes)
205
+ lines.push(`- ${n}`);
206
+ return `${lines.join("\n")}\n`;
207
+ }
package/dist/lint.d.ts ADDED
@@ -0,0 +1,127 @@
1
+ import { type IrNode } from "@crewhaus/compiler";
2
+ import { type IrPass } from "@crewhaus/ir-passes";
3
+ import { type Spec } from "@crewhaus/spec";
4
+ import { auditToolScopes } from "@crewhaus/tool-builder";
5
+ import type { RegisteredTool } from "@crewhaus/tool-catalog";
6
+ /**
7
+ * Item 41 — `crewhaus lint`. A check-only command: `parseSpec` +
8
+ * `compile({ applyIrPasses: true })` (the §47 chain / graph-crew
9
+ * well-formedness passes) + `auditToolScopes`, WITHOUT emitting a bundle.
10
+ *
11
+ * WHY THIS EXISTS: the CLI compile path never applied the ir-passes today, so
12
+ * §47 referential-integrity and graph/crew well-formedness checks silently
13
+ * skipped for CLI users — a spec with a dangling `wallets[].chainId` or an
14
+ * unreachable graph node compiled clean. `lint` runs those passes (discarding
15
+ * the result) so authoring bugs surface without a build.
16
+ *
17
+ * COLLECT-ALL vs FAIL-FAST: the IR passes throw on the FIRST violation. For
18
+ * `--format text` that fail-fast is fine (one error, fix, re-run). For
19
+ * `--format json` (editors/CI) we want as many findings as possible, so we run
20
+ * each pass INDEPENDENTLY and catch-and-continue — collecting at most one
21
+ * finding per pass. That is the honest limit of an exception-based pass API
22
+ * (a single pass still stops at its own first violation); it is documented
23
+ * here and surfaced as `severity: "error"` findings with a stable `path`.
24
+ *
25
+ * Side-effect-free: `runLint` is a pure function over the spec text plus an
26
+ * injected tool resolver (so tests don't import the heavy tool packages). The
27
+ * CLI wrapper owns file IO and process exit.
28
+ */
29
+ export type LintSeverity = "error" | "warning";
30
+ /** One structured finding. `path` is a dot-joined spec/IR location (or a
31
+ * synthetic id for whole-spec failures); shaped for editor/CI consumption. */
32
+ export type LintFinding = {
33
+ readonly message: string;
34
+ readonly path: string;
35
+ readonly severity: LintSeverity;
36
+ /** Stable rule id (parse | ir-pass:<name> | scope | …) for grouping. */
37
+ readonly rule: string;
38
+ };
39
+ export type LintResult = {
40
+ readonly ok: boolean;
41
+ readonly findings: readonly LintFinding[];
42
+ /** The parsed spec when parse succeeded (so --fix can operate on it). */
43
+ readonly spec?: Spec;
44
+ /** The lowered IR when lowering succeeded. */
45
+ readonly ir?: IrNode;
46
+ };
47
+ /**
48
+ * Run the lint pipeline. `resolveTool` maps a tool NAME (either the camelCase
49
+ * spec key or its registered PascalCase name) to a `RegisteredTool`, or
50
+ * undefined — injected so this stays pure. `passes` defaults to the published
51
+ * `DEFAULT_PIPELINE` and is overridable for tests.
52
+ */
53
+ export declare function runLint(yamlText: string, resolveTool: (name: string) => RegisteredTool | undefined, passes?: ReadonlyArray<IrPass>): LintResult;
54
+ /** Re-exported for the CLI wrapper's philosophy-alignment parity note. */
55
+ export { auditToolScopes };
56
+ /** A single mechanical fix suggestion for a lint finding. */
57
+ export type LintFixSuggestion = {
58
+ readonly kind: "unknown-tool" | "secret-typo" | "safe-name";
59
+ /** The offending token as written in the spec. */
60
+ readonly from: string;
61
+ /** The suggested replacement. */
62
+ readonly to: string;
63
+ /** Human-readable one-liner. */
64
+ readonly note: string;
65
+ };
66
+ /** Case-insensitive Levenshtein distance — the nearest-match metric. */
67
+ export declare function levenshtein(a: string, b: string): number;
68
+ /**
69
+ * The outcome of a nearest-match lookup:
70
+ * - `undefined` — no candidate close enough (genuinely unknown, not a typo),
71
+ * or the name is already legal.
72
+ * - `{ kind: "match", name }` — a single unambiguous nearest candidate;
73
+ * safe to auto-apply.
74
+ * - `{ kind: "ambiguous", candidates }` — two or more equally-near
75
+ * candidates whose I/O capability DIFFERS (e.g. a read-only tool vs a
76
+ * mutating one), so auto-applying could silently cross a capability
77
+ * boundary. Callers should surface this as a suggestion, not a fix.
78
+ */
79
+ export type NearestToolMatch = {
80
+ readonly kind: "match";
81
+ readonly name: string;
82
+ } | {
83
+ readonly kind: "ambiguous";
84
+ readonly candidates: readonly string[];
85
+ };
86
+ /**
87
+ * Nearest legal name(s) for a mistyped tool name. `candidates` is the union of
88
+ * the legal spellings — the camelCase BUILTIN_TOOL_MAP keys AND the registered
89
+ * PascalCase names (both forms are legal in a spec: top-level `tools:` uses the
90
+ * camelCase key; a sub-agent `tools:` uses the PascalCase registered name). An
91
+ * exact match returns undefined (nothing to fix). Otherwise the closest
92
+ * candidate(s) within `maxDistance` (default 3) are considered; undefined
93
+ * means nothing is close enough (a genuinely unknown tool, not a typo).
94
+ *
95
+ * `getReadOnly` is an optional injected lookup from candidate name → the
96
+ * resolved tool's `readOnly` flag (undefined when the candidate can't be
97
+ * resolved to a RegisteredTool, e.g. an unregistered custom-tool name — in
98
+ * that case its capability is unknown and it is never treated as crossing a
99
+ * boundary against another candidate). When ALL of the closest candidates
100
+ * (same minimum distance) agree on `readOnly` — including the degenerate
101
+ * single-candidate case — this returns a `"match"` for the first of them
102
+ * (stable, deterministic). When they DISAGREE — a typo equidistant from a
103
+ * read-only tool (e.g. `Read`) and a mutating one (e.g. `Edit`) — this
104
+ * returns `"ambiguous"` with the full tied set rather than silently picking
105
+ * one, because auto-applying would cross a read-only/mutating capability
106
+ * boundary the author never asked for.
107
+ */
108
+ export declare function nearestToolName(name: string, candidates: readonly string[], maxDistance?: number, getReadOnly?: (candidateName: string) => boolean | undefined): NearestToolMatch | undefined;
109
+ /**
110
+ * Suggest a `$UPPER_SNAKE_CASE` correction for a credential value that looks
111
+ * like an env reference but isn't a valid one — the same class the compiler's
112
+ * `lowerCredential` rejects (`$slack_token`, `${SLACK}`, `$1PASSWORD`).
113
+ * Returns the normalised form, or undefined when `value` is not a
114
+ * malformed-env-ref (a genuine literal, or already valid). Pure string logic.
115
+ */
116
+ export declare function suggestSecretFix(value: string): string | undefined;
117
+ /**
118
+ * Normalise a name that violates the `safeName` charset (letters, digits,
119
+ * spaces, `_ . - :`) by replacing every illegal character with `-` and
120
+ * collapsing runs. Returns undefined when `name` is already safe. Pure.
121
+ */
122
+ export declare function suggestSafeName(name: string): string | undefined;
123
+ /** Render the lint findings as the human-readable `text` report. Returns the
124
+ * block plus whether it is clean, so the CLI can pick the exit code. */
125
+ export declare function formatLintText(result: LintResult): string;
126
+ /** Render the lint findings as `{ findings: LintFinding[] }` JSON for editors/CI. */
127
+ export declare function formatLintJson(result: LintResult): string;
package/dist/lint.js ADDED
@@ -0,0 +1,226 @@
1
+ import { lower } from "@crewhaus/compiler";
2
+ import { CrewhausError } from "@crewhaus/errors";
3
+ import { DEFAULT_PIPELINE } from "@crewhaus/ir-passes";
4
+ import { SpecParseError, parseSpec } from "@crewhaus/spec";
5
+ import { auditToolScopes } from "@crewhaus/tool-builder";
6
+ import { auditSpecToolNames, collectToolNames } from "./scope-audit";
7
+ /**
8
+ * Run the lint pipeline. `resolveTool` maps a tool NAME (either the camelCase
9
+ * spec key or its registered PascalCase name) to a `RegisteredTool`, or
10
+ * undefined — injected so this stays pure. `passes` defaults to the published
11
+ * `DEFAULT_PIPELINE` and is overridable for tests.
12
+ */
13
+ export function runLint(yamlText, resolveTool, passes = DEFAULT_PIPELINE) {
14
+ const findings = [];
15
+ // Stage 1 — parse. A parse failure is terminal: without a spec there is
16
+ // nothing to lower or audit.
17
+ let spec;
18
+ try {
19
+ spec = parseSpec(yamlText);
20
+ }
21
+ catch (err) {
22
+ const message = err instanceof SpecParseError ? err.message : err.message;
23
+ findings.push({ message, path: "<spec>", severity: "error", rule: "parse" });
24
+ return { ok: false, findings };
25
+ }
26
+ // Stage 2 — lower. lower() can throw CompilerError (e.g. a malformed
27
+ // credential env-ref); treat as terminal for the IR-pass stage.
28
+ let ir;
29
+ try {
30
+ ir = lower(spec);
31
+ }
32
+ catch (err) {
33
+ const message = err instanceof CrewhausError ? err.message : err.message;
34
+ findings.push({ message, path: "<lower>", severity: "error", rule: "lower" });
35
+ return { ok: false, findings, spec };
36
+ }
37
+ // Stage 3 — IR passes, COLLECT-ALL. Run each pass independently so one pass's
38
+ // violation doesn't hide the others'. Validating passes
39
+ // (transactionPolicyEnforcement, wellFormednessCheck) throw IrPassError; the
40
+ // rewriting passes are pure and never throw. We discard rewritten IR (lint
41
+ // never emits) — only the thrown violations matter.
42
+ for (const pass of passes) {
43
+ try {
44
+ pass(ir);
45
+ }
46
+ catch (err) {
47
+ const message = err instanceof CrewhausError ? err.message : err.message;
48
+ findings.push({
49
+ message,
50
+ path: `ir-pass:${pass.name || "anonymous"}`,
51
+ severity: "error",
52
+ rule: `ir-pass:${pass.name || "anonymous"}`,
53
+ });
54
+ }
55
+ }
56
+ // Stage 4 — tool-scope audit over the IR's tool names, sharing the exact
57
+ // gate `compile --strict` uses. A resolvable built-in is audited by
58
+ // capability/scope; an outward-by-name sink that resolves to no external
59
+ // tool is a finding.
60
+ const toolNames = collectToolNames(ir);
61
+ const scopeFindings = auditSpecToolNames(toolNames, resolveTool);
62
+ for (const f of scopeFindings) {
63
+ findings.push({
64
+ message: `tool "${f.toolName}" ${f.reason}`,
65
+ path: `tools.${f.toolName}`,
66
+ severity: "error",
67
+ rule: "scope",
68
+ });
69
+ }
70
+ return { ok: findings.length === 0, findings, spec, ir };
71
+ }
72
+ /** Re-exported for the CLI wrapper's philosophy-alignment parity note. */
73
+ export { auditToolScopes };
74
+ /** Case-insensitive Levenshtein distance — the nearest-match metric. */
75
+ export function levenshtein(a, b) {
76
+ const s = a.toLowerCase();
77
+ const t = b.toLowerCase();
78
+ const m = s.length;
79
+ const n = t.length;
80
+ if (m === 0)
81
+ return n;
82
+ if (n === 0)
83
+ return m;
84
+ let prev = Array.from({ length: n + 1 }, (_, i) => i);
85
+ let curr = new Array(n + 1);
86
+ for (let i = 1; i <= m; i++) {
87
+ curr[0] = i;
88
+ for (let j = 1; j <= n; j++) {
89
+ const cost = s[i - 1] === t[j - 1] ? 0 : 1;
90
+ curr[j] = Math.min((prev[j] ?? 0) + 1, (curr[j - 1] ?? 0) + 1, (prev[j - 1] ?? 0) + cost);
91
+ }
92
+ [prev, curr] = [curr, prev];
93
+ }
94
+ return prev[n] ?? 0;
95
+ }
96
+ /**
97
+ * Nearest legal name(s) for a mistyped tool name. `candidates` is the union of
98
+ * the legal spellings — the camelCase BUILTIN_TOOL_MAP keys AND the registered
99
+ * PascalCase names (both forms are legal in a spec: top-level `tools:` uses the
100
+ * camelCase key; a sub-agent `tools:` uses the PascalCase registered name). An
101
+ * exact match returns undefined (nothing to fix). Otherwise the closest
102
+ * candidate(s) within `maxDistance` (default 3) are considered; undefined
103
+ * means nothing is close enough (a genuinely unknown tool, not a typo).
104
+ *
105
+ * `getReadOnly` is an optional injected lookup from candidate name → the
106
+ * resolved tool's `readOnly` flag (undefined when the candidate can't be
107
+ * resolved to a RegisteredTool, e.g. an unregistered custom-tool name — in
108
+ * that case its capability is unknown and it is never treated as crossing a
109
+ * boundary against another candidate). When ALL of the closest candidates
110
+ * (same minimum distance) agree on `readOnly` — including the degenerate
111
+ * single-candidate case — this returns a `"match"` for the first of them
112
+ * (stable, deterministic). When they DISAGREE — a typo equidistant from a
113
+ * read-only tool (e.g. `Read`) and a mutating one (e.g. `Edit`) — this
114
+ * returns `"ambiguous"` with the full tied set rather than silently picking
115
+ * one, because auto-applying would cross a read-only/mutating capability
116
+ * boundary the author never asked for.
117
+ */
118
+ export function nearestToolName(name, candidates, maxDistance = 3, getReadOnly) {
119
+ if (candidates.includes(name))
120
+ return undefined;
121
+ let bestDist;
122
+ let tied = [];
123
+ for (const cand of candidates) {
124
+ const dist = levenshtein(name, cand);
125
+ if (bestDist === undefined || dist < bestDist) {
126
+ bestDist = dist;
127
+ tied = [cand];
128
+ }
129
+ else if (dist === bestDist) {
130
+ tied.push(cand);
131
+ }
132
+ }
133
+ if (bestDist === undefined || bestDist > maxDistance)
134
+ return undefined;
135
+ if (tied.length === 1) {
136
+ const only = tied[0];
137
+ if (only === undefined)
138
+ return undefined;
139
+ return { kind: "match", name: only };
140
+ }
141
+ // Multiple equally-near candidates. If a capability lookup was injected and
142
+ // the tied set spans more than one `readOnly` value, that is a genuine
143
+ // cross-capability ambiguity — refuse to pick one. Candidates with unknown
144
+ // capability (getReadOnly returns undefined) don't by themselves create
145
+ // ambiguity: they only conflict when they disagree with a KNOWN capability
146
+ // among the tied set.
147
+ if (getReadOnly !== undefined) {
148
+ const knownCapabilities = new Set(tied.map((c) => getReadOnly(c)).filter((v) => v !== undefined));
149
+ if (knownCapabilities.size > 1) {
150
+ return { kind: "ambiguous", candidates: tied };
151
+ }
152
+ }
153
+ else {
154
+ // No capability signal available — a plain tie is still ambiguous rather
155
+ // than silently guessing via iteration order.
156
+ return { kind: "ambiguous", candidates: tied };
157
+ }
158
+ const first = tied[0];
159
+ if (first === undefined)
160
+ return undefined;
161
+ return { kind: "match", name: first };
162
+ }
163
+ /**
164
+ * Suggest a `$UPPER_SNAKE_CASE` correction for a credential value that looks
165
+ * like an env reference but isn't a valid one — the same class the compiler's
166
+ * `lowerCredential` rejects (`$slack_token`, `${SLACK}`, `$1PASSWORD`).
167
+ * Returns the normalised form, or undefined when `value` is not a
168
+ * malformed-env-ref (a genuine literal, or already valid). Pure string logic.
169
+ */
170
+ export function suggestSecretFix(value) {
171
+ if (!value.startsWith("$"))
172
+ return undefined;
173
+ // Already valid $UPPER_SNAKE_CASE — nothing to fix.
174
+ if (/^\$[A-Z_][A-Z0-9_]*$/.test(value))
175
+ return undefined;
176
+ // Strip a ${...} brace wrapper, then normalise the inner token.
177
+ const inner = value.replace(/^\$\{?/, "").replace(/\}$/, "");
178
+ if (inner === "")
179
+ return undefined;
180
+ // UPPER_SNAKE_CASE it: non-alnum → _, then uppercase, then ensure it does
181
+ // not start with a digit (prefix `_`).
182
+ let normalised = inner.replace(/[^A-Za-z0-9]+/g, "_").toUpperCase();
183
+ if (/^[0-9]/.test(normalised))
184
+ normalised = `_${normalised}`;
185
+ if (!/^[A-Z_][A-Z0-9_]*$/.test(normalised))
186
+ return undefined;
187
+ return `$${normalised}`;
188
+ }
189
+ /** The single-line-safe name charset the spec's `safeName` enforces. */
190
+ const SAFE_NAME_RE = /^[\w .:-]+$/;
191
+ /**
192
+ * Normalise a name that violates the `safeName` charset (letters, digits,
193
+ * spaces, `_ . - :`) by replacing every illegal character with `-` and
194
+ * collapsing runs. Returns undefined when `name` is already safe. Pure.
195
+ */
196
+ export function suggestSafeName(name) {
197
+ if (SAFE_NAME_RE.test(name) && name.length >= 1)
198
+ return undefined;
199
+ const fixed = name
200
+ .replace(/[^\w .:-]+/g, "-")
201
+ .replace(/-+/g, "-")
202
+ .replace(/^-|-$/g, "")
203
+ .trim();
204
+ if (fixed === "" || !SAFE_NAME_RE.test(fixed))
205
+ return undefined;
206
+ return fixed;
207
+ }
208
+ /** Render the lint findings as the human-readable `text` report. Returns the
209
+ * block plus whether it is clean, so the CLI can pick the exit code. */
210
+ export function formatLintText(result) {
211
+ if (result.findings.length === 0)
212
+ return "lint: clean — no findings.\n";
213
+ const lines = [];
214
+ for (const f of result.findings) {
215
+ const marker = f.severity === "error" ? "✗" : "~";
216
+ lines.push(`${marker} [${f.rule}] ${f.path}: ${f.message}`);
217
+ }
218
+ const errorCount = result.findings.filter((f) => f.severity === "error").length;
219
+ lines.push("");
220
+ lines.push(`lint: ${errorCount} error(s), ${result.findings.length - errorCount} warning(s).`);
221
+ return `${lines.join("\n")}\n`;
222
+ }
223
+ /** Render the lint findings as `{ findings: LintFinding[] }` JSON for editors/CI. */
224
+ export function formatLintJson(result) {
225
+ return `${JSON.stringify({ ok: result.ok, findings: result.findings }, null, 2)}\n`;
226
+ }
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Item 68 — `crewhaus loadtest <spec>`: drive a daemon-shape harness (managed
3
+ * gateway / channel-bot / batch) under concurrent load and measure throughput,
4
+ * p50/p95/p99 latency, error rate, and cost/req — with a `--gate` mode that
5
+ * fails (exit 1) when p95 latency or error rate exceed declared thresholds (a
6
+ * pre-deploy gate).
7
+ *
8
+ * Load is driven through an INJECTED `LoadDriver` — one async call per request
9
+ * that returns a `RequestOutcome`. That seam makes the runner deterministic +
10
+ * credential-free in tests (a stub driver returns canned latencies), while the
11
+ * CLI wires a real driver: the managed gateway's in-process `handle()` behind a
12
+ * stub/echo model, or the batch queue's consumer. No live server or provider
13
+ * key is needed for the runner itself.
14
+ *
15
+ * Everything here is side-effect-free (the CLI entry file constructs the driver
16
+ * + writes the report). The aggregation is pure: same outcomes → same report.
17
+ */
18
+ /** Thrown on bad flags / driver misuse. The CLI routes it through `die()`. */
19
+ export declare class LoadtestError extends Error {
20
+ readonly name = "LoadtestError";
21
+ }
22
+ /** The result of one driven request. */
23
+ export type RequestOutcome = {
24
+ readonly ok: boolean;
25
+ readonly latencyMs: number;
26
+ /** Token usage, when the driver knows it (for cost/req). */
27
+ readonly tokens?: {
28
+ input: number;
29
+ output: number;
30
+ };
31
+ /** A short classification when `ok` is false (e.g. "429", "timeout"). */
32
+ readonly errorKind?: string;
33
+ };
34
+ /** The injected load driver: given a 0-based request index, drive ONE request
35
+ * and resolve with its outcome. Must not throw — a thrown error is caught and
36
+ * recorded as an errored request (errorKind "threw"). */
37
+ export type LoadDriver = (reqIndex: number) => Promise<RequestOutcome>;
38
+ export type LoadtestConfig = {
39
+ /** Concurrent in-flight requests. Default 10. */
40
+ readonly concurrency?: number;
41
+ /** Total requests to send. Mutually informative with `durationMs`; when both
42
+ * are set, whichever bound is hit first ends the run. Default 100. */
43
+ readonly requests?: number;
44
+ /** Wall-clock budget (ms). When set, the run stops issuing new requests once
45
+ * elapsed ≥ durationMs (in-flight requests still drain). */
46
+ readonly durationMs?: number;
47
+ /** Target requests-per-second. When set, the runner paces request dispatch to
48
+ * approximately this rate (open-loop). Default: unthrottled (closed-loop). */
49
+ readonly rps?: number;
50
+ /** Price per 1M input / output tokens (USD) for the cost/req estimate. */
51
+ readonly pricePerMInput?: number;
52
+ readonly pricePerMOutput?: number;
53
+ /** A monotonic clock (ms) — injected for deterministic tests. Default performance.now. */
54
+ readonly now?: () => number;
55
+ /** A sleep(ms) primitive — injected for deterministic tests (rps pacing). */
56
+ readonly sleep?: (ms: number) => Promise<void>;
57
+ };
58
+ /** Gate thresholds — a run fails when any is exceeded. */
59
+ export type GateThresholds = {
60
+ readonly maxP95LatencyMs?: number;
61
+ readonly maxErrorRate?: number;
62
+ };
63
+ export type LoadtestReport = {
64
+ readonly requests: number;
65
+ readonly succeeded: number;
66
+ readonly failed: number;
67
+ readonly errorRate: number;
68
+ readonly wallClockMs: number;
69
+ readonly throughputRps: number;
70
+ readonly latency: {
71
+ readonly p50: number;
72
+ readonly p95: number;
73
+ readonly p99: number;
74
+ readonly min: number;
75
+ readonly max: number;
76
+ readonly mean: number;
77
+ };
78
+ readonly tokens: {
79
+ input: number;
80
+ output: number;
81
+ };
82
+ /** USD per request (0 when pricing not supplied). */
83
+ readonly costPerReqUsd: number;
84
+ readonly totalCostUsd: number;
85
+ /** Breakdown of error kinds → count. */
86
+ readonly errorKinds: Readonly<Record<string, number>>;
87
+ };
88
+ /**
89
+ * Run a concurrent load test against the injected driver. Dispatches up to
90
+ * `concurrency` requests in flight until the request budget (or duration) is
91
+ * exhausted, then drains. Deterministic given a deterministic driver + clock.
92
+ */
93
+ export declare function runLoadtest(driver: LoadDriver, config?: LoadtestConfig): Promise<LoadtestReport>;
94
+ /**
95
+ * Aggregate raw outcomes into a report. Pure — exported so tests can assert on
96
+ * percentile math without driving a run, and so the CLI can re-aggregate a
97
+ * persisted outcome set.
98
+ */
99
+ export declare function aggregateLoadtest(outcomes: readonly RequestOutcome[], opts: {
100
+ wallClockMs: number;
101
+ pricePerMInput?: number;
102
+ pricePerMOutput?: number;
103
+ }): LoadtestReport;
104
+ /** The gate verdict — which thresholds (if any) were breached. */
105
+ export type GateVerdict = {
106
+ readonly passed: boolean;
107
+ readonly breaches: readonly string[];
108
+ };
109
+ /** Evaluate a report against gate thresholds. Empty thresholds → always passes. */
110
+ export declare function evaluateGate(report: LoadtestReport, thresholds: GateThresholds): GateVerdict;
111
+ /** Render a plain-text loadtest report. */
112
+ export declare function renderLoadtestText(report: LoadtestReport, verdict?: GateVerdict): string;
113
+ /** Render an HTML loadtest report (self-contained). */
114
+ export declare function renderLoadtestHtml(report: LoadtestReport, verdict?: GateVerdict): string;