crewhaus 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/dist/advice-apply.d.ts +182 -0
- package/dist/advice-apply.js +286 -0
- package/dist/advise-rules.d.ts +348 -0
- package/dist/advise-rules.js +905 -0
- package/dist/alert-sink.d.ts +48 -0
- package/dist/alert-sink.js +86 -0
- package/dist/approval-gate.d.ts +127 -0
- package/dist/approval-gate.js +254 -0
- package/dist/audit-verify.d.ts +69 -0
- package/dist/audit-verify.js +97 -0
- package/dist/autodistill.d.ts +113 -0
- package/dist/autodistill.js +256 -0
- package/dist/channel-provision.d.ts +360 -0
- package/dist/channel-provision.js +881 -0
- package/dist/ci-scaffold.d.ts +31 -0
- package/dist/ci-scaffold.js +343 -0
- package/dist/compile-check.d.ts +90 -0
- package/dist/compile-check.js +285 -0
- package/dist/compliance-schedule.d.ts +35 -0
- package/dist/compliance-schedule.js +36 -0
- package/dist/context-pressure.d.ts +80 -0
- package/dist/context-pressure.js +166 -0
- package/dist/dataset-mine.d.ts +172 -0
- package/dist/dataset-mine.js +403 -0
- package/dist/datasets.d.ts +124 -0
- package/dist/datasets.js +260 -0
- package/dist/deploy-canary.d.ts +83 -0
- package/dist/deploy-canary.js +87 -0
- package/dist/doctor-checks.d.ts +33 -0
- package/dist/doctor-checks.js +92 -0
- package/dist/doctor-detect.d.ts +108 -0
- package/dist/doctor-detect.js +214 -0
- package/dist/doctor-fix.d.ts +81 -0
- package/dist/doctor-fix.js +164 -0
- package/dist/egress-triage.d.ts +121 -0
- package/dist/egress-triage.js +261 -0
- package/dist/eval-bridge.d.ts +114 -0
- package/dist/eval-bridge.js +158 -0
- package/dist/eval-coverage.d.ts +140 -0
- package/dist/eval-coverage.js +428 -0
- package/dist/eval-history.d.ts +48 -0
- package/dist/eval-history.js +157 -0
- package/dist/eval-matrix.d.ts +80 -0
- package/dist/eval-matrix.js +182 -0
- package/dist/eval-sentinel.d.ts +65 -0
- package/dist/eval-sentinel.js +132 -0
- package/dist/faq.d.ts +68 -0
- package/dist/faq.js +168 -0
- package/dist/feedback.d.ts +246 -0
- package/dist/feedback.js +612 -0
- package/dist/fewshot.d.ts +83 -0
- package/dist/fewshot.js +158 -0
- package/dist/fleet.d.ts +207 -0
- package/dist/fleet.js +488 -0
- package/dist/flywheel.d.ts +193 -0
- package/dist/flywheel.js +519 -0
- package/dist/graders-suggest.d.ts +186 -0
- package/dist/graders-suggest.js +658 -0
- package/dist/incident.d.ts +99 -0
- package/dist/incident.js +217 -0
- package/dist/index.d.ts +9 -1
- package/dist/index.js +11727 -740
- package/dist/init-interactive.d.ts +105 -0
- package/dist/init-interactive.js +208 -0
- package/dist/intents.d.ts +105 -0
- package/dist/intents.js +292 -0
- package/dist/judge-calibrate.d.ts +137 -0
- package/dist/judge-calibrate.js +247 -0
- package/dist/justification-calibrate.d.ts +150 -0
- package/dist/justification-calibrate.js +262 -0
- package/dist/justification-gate.d.ts +27 -6
- package/dist/justification-gate.js +30 -6
- package/dist/knowledge-sync.d.ts +179 -0
- package/dist/knowledge-sync.js +551 -0
- package/dist/lessons.d.ts +87 -0
- package/dist/lessons.js +207 -0
- package/dist/lint.d.ts +127 -0
- package/dist/lint.js +226 -0
- package/dist/loadtest.d.ts +114 -0
- package/dist/loadtest.js +196 -0
- package/dist/marketplace-cli.d.ts +110 -0
- package/dist/marketplace-cli.js +250 -0
- package/dist/mcp-doctor.d.ts +121 -0
- package/dist/mcp-doctor.js +249 -0
- package/dist/model-scan.d.ts +116 -0
- package/dist/model-scan.js +226 -0
- package/dist/onchain-tune.d.ts +164 -0
- package/dist/onchain-tune.js +346 -0
- package/dist/permissions-suggest.d.ts +126 -0
- package/dist/permissions-suggest.js +333 -0
- package/dist/pii-tune.d.ts +107 -0
- package/dist/pii-tune.js +122 -0
- package/dist/propose.d.ts +117 -0
- package/dist/propose.js +184 -0
- package/dist/refresh-goldens.d.ts +82 -0
- package/dist/refresh-goldens.js +221 -0
- package/dist/regression-pin.d.ts +160 -0
- package/dist/regression-pin.js +281 -0
- package/dist/retention.d.ts +193 -0
- package/dist/retention.js +607 -0
- package/dist/retire.d.ts +118 -0
- package/dist/retire.js +291 -0
- package/dist/right-size.d.ts +100 -0
- package/dist/right-size.js +123 -0
- package/dist/scaffold-evals.d.ts +138 -0
- package/dist/scaffold-evals.js +410 -0
- package/dist/scope-audit-drift.d.ts +139 -0
- package/dist/scope-audit-drift.js +260 -0
- package/dist/security-corpus.d.ts +237 -0
- package/dist/security-corpus.js +516 -0
- package/dist/security-digest.d.ts +173 -0
- package/dist/security-digest.js +650 -0
- package/dist/sessions-index.d.ts +27 -0
- package/dist/sessions-index.js +51 -0
- package/dist/slo-doctor.d.ts +67 -0
- package/dist/slo-doctor.js +119 -0
- package/dist/slo-sink.d.ts +96 -0
- package/dist/slo-sink.js +107 -0
- package/dist/spec-changelog.d.ts +102 -0
- package/dist/spec-changelog.js +237 -0
- package/dist/state-backup.d.ts +223 -0
- package/dist/state-backup.js +648 -0
- package/dist/tools-cli.d.ts +170 -0
- package/dist/tools-cli.js +298 -0
- package/dist/triage.d.ts +202 -0
- package/dist/triage.js +403 -0
- package/dist/upgrade.d.ts +57 -0
- package/dist/upgrade.js +113 -0
- package/dist/version.d.ts +6 -0
- package/dist/version.js +27 -0
- package/dist/voice-eval.d.ts +138 -0
- package/dist/voice-eval.js +309 -0
- package/dist/watch.d.ts +58 -0
- package/dist/watch.js +97 -0
- package/package.json +89 -64
package/dist/lessons.js
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item #56 — auto-maintained LESSONS.md + per-user preference files.
|
|
3
|
+
*
|
|
4
|
+
* `crewhaus lessons update` mines two durable signals:
|
|
5
|
+
* - CORRECTION lessons: a `user_feedback` record carrying a `correction`
|
|
6
|
+
* (the user's better answer) OR a low-rated turn with a comment — the
|
|
7
|
+
* "here's what went wrong / do this instead" signal.
|
|
8
|
+
* - FAILURE→FIX lessons: recurring negative signals from `dataset-mine`'s
|
|
9
|
+
* `mineSession` (tool-error spikes, runtime errors, loop nudges, retries)
|
|
10
|
+
* paired to the triggering turn — "when X happens, watch out."
|
|
11
|
+
* These fold into a deduped, append-with-merge LESSONS.md (like the project-
|
|
12
|
+
* memory files the runtime already auto-loads — LESSONS.md is added to the
|
|
13
|
+
* canonical set, so it is injected into the system prompt at run start).
|
|
14
|
+
*
|
|
15
|
+
* Per-user preferences: when feedback carries a `rater` identity, ratings +
|
|
16
|
+
* comments for that rater fold into a per-user prefs file
|
|
17
|
+
* (`.crewhaus/preferences/<rater>.md`) injected when that user is known.
|
|
18
|
+
*
|
|
19
|
+
* Pure + deterministic (stable order, normalized-text dedupe → idempotent
|
|
20
|
+
* re-runs); FS + redaction wiring lives in `apps/cli/src/index.ts`. Lesson +
|
|
21
|
+
* preference text is model/user output → PII/secret-redacted by the caller.
|
|
22
|
+
*/
|
|
23
|
+
import { mergeFeedback, normalizeRating } from "./feedback";
|
|
24
|
+
function normalizeKey(s) {
|
|
25
|
+
return s.toLowerCase().replace(/\s+/g, " ").trim();
|
|
26
|
+
}
|
|
27
|
+
function clip(s, n) {
|
|
28
|
+
const t = s.replace(/\s+/g, " ").trim();
|
|
29
|
+
return t.length > n ? `${t.slice(0, n - 1).trimEnd()}…` : t;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Mine lessons from turns + feedback + pre-mined failure signals. Returns a
|
|
33
|
+
* deterministically-ordered, deduped list. `failureSignals` is the output of
|
|
34
|
+
* `mineSession` across the scanned sessions (each `{ input, reason }`), so the
|
|
35
|
+
* failure→fix lessons reuse the same negative-signal detection the dataset
|
|
36
|
+
* miner uses rather than reinventing it.
|
|
37
|
+
*/
|
|
38
|
+
export async function mineLessons(turns, feedback, failureSignals, opts = {}) {
|
|
39
|
+
const lowScore = opts.lowScore ?? 0.5;
|
|
40
|
+
const redact = opts.redact ?? (async (t) => t);
|
|
41
|
+
const turnByKey = new Map();
|
|
42
|
+
for (const t of turns)
|
|
43
|
+
turnByKey.set(`${t.sessionId}#${t.turnNumber}`, t);
|
|
44
|
+
const byKey = new Map();
|
|
45
|
+
const add = (text, kind) => {
|
|
46
|
+
const clean = clip(text, 300);
|
|
47
|
+
if (clean === "")
|
|
48
|
+
return;
|
|
49
|
+
const key = normalizeKey(clean);
|
|
50
|
+
if (!byKey.has(key))
|
|
51
|
+
byKey.set(key, { key, text: clean, kind });
|
|
52
|
+
};
|
|
53
|
+
// Correction lessons from feedback (merged so one per turn). #56 F3 — the
|
|
54
|
+
// echoed question prefix is a raw user turn (may carry a pasted credential),
|
|
55
|
+
// so it is redacted too, not just the correction/comment body.
|
|
56
|
+
for (const fb of mergeFeedback(feedback)) {
|
|
57
|
+
const turn = turnByKey.get(`${fb.sessionId}#${fb.turnNumber}`);
|
|
58
|
+
const question = turn?.input?.trim();
|
|
59
|
+
const score = normalizeRating(fb);
|
|
60
|
+
if (fb.correction !== undefined && fb.correction.trim() !== "") {
|
|
61
|
+
const q = question ? `For "${await redact(clip(question, 80))}": ` : "";
|
|
62
|
+
add(`${q}prefer this answer — ${await redact(fb.correction)}`, "correction");
|
|
63
|
+
}
|
|
64
|
+
else if (fb.comment !== undefined &&
|
|
65
|
+
fb.comment.trim() !== "" &&
|
|
66
|
+
score !== undefined &&
|
|
67
|
+
score <= lowScore) {
|
|
68
|
+
const q = question ? `For "${await redact(clip(question, 80))}": ` : "";
|
|
69
|
+
add(`${q}avoid — ${await redact(fb.comment)}`, "correction");
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// Failure→fix lessons from the mined negative signals. #56 F3 — BOTH the
|
|
73
|
+
// echoed user input AND the failure reason (raw error messages carry
|
|
74
|
+
// connection strings / tokens) are redacted before landing in an
|
|
75
|
+
// auto-injected lesson.
|
|
76
|
+
for (const sig of failureSignals) {
|
|
77
|
+
const redactedInput = sig.input.trim() !== "" ? await redact(clip(sig.input, 80)) : "";
|
|
78
|
+
const q = redactedInput !== "" ? ` on "${redactedInput}"` : "";
|
|
79
|
+
add(`Watch out${q}: ${await redact(sig.reason)}.`, "failure-fix");
|
|
80
|
+
}
|
|
81
|
+
// Stable order: corrections first (higher-signal), then failure-fixes; each
|
|
82
|
+
// group alphabetized by key so re-runs produce byte-identical output.
|
|
83
|
+
return [...byKey.values()].sort((a, b) => (a.kind === b.kind ? 0 : a.kind === "correction" ? -1 : 1) || a.key.localeCompare(b.key));
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Merge fresh lessons into existing ones, deduping by normalized key so a
|
|
87
|
+
* re-run never duplicates a lesson (append-with-merge idempotency). Existing
|
|
88
|
+
* lessons are preserved; new keys appended; ordering re-stabilized.
|
|
89
|
+
*/
|
|
90
|
+
export function mergeLessons(existing, fresh) {
|
|
91
|
+
const byKey = new Map();
|
|
92
|
+
for (const l of existing)
|
|
93
|
+
byKey.set(l.key, l);
|
|
94
|
+
for (const l of fresh)
|
|
95
|
+
if (!byKey.has(l.key))
|
|
96
|
+
byKey.set(l.key, l);
|
|
97
|
+
return [...byKey.values()].sort((a, b) => (a.kind === b.kind ? 0 : a.kind === "correction" ? -1 : 1) || a.key.localeCompare(b.key));
|
|
98
|
+
}
|
|
99
|
+
const LESSONS_HEADER = "# LESSONS";
|
|
100
|
+
const LESSONS_NOTE = "<!-- Auto-maintained by `crewhaus lessons update`. Deduped + idempotent; hand-edits above the marker are preserved. -->";
|
|
101
|
+
const LESSONS_MARKER = "<!-- crewhaus:lessons -->";
|
|
102
|
+
/**
|
|
103
|
+
* Render LESSONS.md. Everything BELOW the `crewhaus:lessons` marker is machine-
|
|
104
|
+
* managed (the sorted lesson list); anything a human wrote ABOVE it is
|
|
105
|
+
* preserved verbatim by `parseLessonsMd` on the next update.
|
|
106
|
+
*/
|
|
107
|
+
export function renderLessonsMd(lessons, preamble = "") {
|
|
108
|
+
const corrections = lessons.filter((l) => l.kind === "correction");
|
|
109
|
+
const failures = lessons.filter((l) => l.kind === "failure-fix");
|
|
110
|
+
const lines = [];
|
|
111
|
+
const pre = preamble.trimEnd();
|
|
112
|
+
lines.push(pre === "" ? `${LESSONS_HEADER}\n\n${LESSONS_NOTE}` : pre);
|
|
113
|
+
lines.push("");
|
|
114
|
+
lines.push(LESSONS_MARKER);
|
|
115
|
+
if (corrections.length > 0) {
|
|
116
|
+
lines.push("");
|
|
117
|
+
lines.push("## Corrections & preferences");
|
|
118
|
+
lines.push("");
|
|
119
|
+
for (const l of corrections)
|
|
120
|
+
lines.push(`- ${l.text}`);
|
|
121
|
+
}
|
|
122
|
+
if (failures.length > 0) {
|
|
123
|
+
lines.push("");
|
|
124
|
+
lines.push("## Known failure patterns");
|
|
125
|
+
lines.push("");
|
|
126
|
+
for (const l of failures)
|
|
127
|
+
lines.push(`- ${l.text}`);
|
|
128
|
+
}
|
|
129
|
+
return `${lines.join("\n")}\n`;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Parse an existing LESSONS.md back into `{ preamble, lessons }`. The preamble
|
|
133
|
+
* is everything before the `crewhaus:lessons` marker (preserved on re-render);
|
|
134
|
+
* lessons are the `- ` bullets under the two managed sections, tagged by which
|
|
135
|
+
* section they came from. Files without the marker are treated as all-preamble
|
|
136
|
+
* (a human-authored file), so we never clobber hand-written lessons.
|
|
137
|
+
*/
|
|
138
|
+
export function parseLessonsMd(content) {
|
|
139
|
+
const markerIdx = content.indexOf(LESSONS_MARKER);
|
|
140
|
+
if (markerIdx === -1)
|
|
141
|
+
return { preamble: content, lessons: [] };
|
|
142
|
+
const preamble = content.slice(0, markerIdx).trimEnd();
|
|
143
|
+
const managed = content.slice(markerIdx + LESSONS_MARKER.length);
|
|
144
|
+
const lessons = [];
|
|
145
|
+
let kind = "correction";
|
|
146
|
+
for (const rawLine of managed.split("\n")) {
|
|
147
|
+
const line = rawLine.trim();
|
|
148
|
+
if (line.startsWith("## Known failure"))
|
|
149
|
+
kind = "failure-fix";
|
|
150
|
+
else if (line.startsWith("## Corrections"))
|
|
151
|
+
kind = "correction";
|
|
152
|
+
else if (line.startsWith("- ")) {
|
|
153
|
+
const text = line.slice(2).trim();
|
|
154
|
+
if (text !== "")
|
|
155
|
+
lessons.push({ key: normalizeKey(text), text, kind });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return { preamble, lessons };
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Fold feedback carrying a `rater` into per-user preference notes. A rater's
|
|
162
|
+
* comments (on any rating) and corrections become their preference bullets,
|
|
163
|
+
* deduped + stably ordered. Returns one entry per rater, sorted by rater id.
|
|
164
|
+
*/
|
|
165
|
+
export async function minePreferences(feedback, opts = {}) {
|
|
166
|
+
const redact = opts.redact ?? (async (t) => t);
|
|
167
|
+
const byRater = new Map();
|
|
168
|
+
for (const fb of mergeFeedback(feedback)) {
|
|
169
|
+
if (fb.rater === undefined || fb.rater.trim() === "")
|
|
170
|
+
continue;
|
|
171
|
+
const bucket = byRater.get(fb.rater) ?? { keys: new Set(), notes: [] };
|
|
172
|
+
const candidates = [];
|
|
173
|
+
if (fb.correction !== undefined && fb.correction.trim() !== "") {
|
|
174
|
+
candidates.push(`prefers answers like: ${await redact(fb.correction)}`);
|
|
175
|
+
}
|
|
176
|
+
if (fb.comment !== undefined && fb.comment.trim() !== "") {
|
|
177
|
+
candidates.push(await redact(fb.comment));
|
|
178
|
+
}
|
|
179
|
+
for (const c of candidates) {
|
|
180
|
+
const clean = clip(c, 240);
|
|
181
|
+
const key = normalizeKey(clean);
|
|
182
|
+
if (clean !== "" && !bucket.keys.has(key)) {
|
|
183
|
+
bucket.keys.add(key);
|
|
184
|
+
bucket.notes.push(clean);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
byRater.set(fb.rater, bucket);
|
|
188
|
+
}
|
|
189
|
+
return [...byRater.entries()]
|
|
190
|
+
.filter(([, b]) => b.notes.length > 0)
|
|
191
|
+
.map(([rater, b]) => ({ rater, notes: b.notes }))
|
|
192
|
+
.sort((a, b) => a.rater.localeCompare(b.rater));
|
|
193
|
+
}
|
|
194
|
+
/** Render a per-user preferences markdown file (a project-memory-style block). */
|
|
195
|
+
export function renderPreferencesMd(prefs) {
|
|
196
|
+
const lines = [
|
|
197
|
+
`# Preferences — ${prefs.rater}`,
|
|
198
|
+
"",
|
|
199
|
+
"<!-- Auto-maintained by `crewhaus lessons update`. -->",
|
|
200
|
+
"",
|
|
201
|
+
"When serving this user, honour these preferences learned from their feedback:",
|
|
202
|
+
"",
|
|
203
|
+
];
|
|
204
|
+
for (const n of prefs.notes)
|
|
205
|
+
lines.push(`- ${n}`);
|
|
206
|
+
return `${lines.join("\n")}\n`;
|
|
207
|
+
}
|
package/dist/lint.d.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { type IrNode } from "@crewhaus/compiler";
|
|
2
|
+
import { type IrPass } from "@crewhaus/ir-passes";
|
|
3
|
+
import { type Spec } from "@crewhaus/spec";
|
|
4
|
+
import { auditToolScopes } from "@crewhaus/tool-builder";
|
|
5
|
+
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
6
|
+
/**
|
|
7
|
+
* Item 41 — `crewhaus lint`. A check-only command: `parseSpec` +
|
|
8
|
+
* `compile({ applyIrPasses: true })` (the §47 chain / graph-crew
|
|
9
|
+
* well-formedness passes) + `auditToolScopes`, WITHOUT emitting a bundle.
|
|
10
|
+
*
|
|
11
|
+
* WHY THIS EXISTS: the CLI compile path never applied the ir-passes today, so
|
|
12
|
+
* §47 referential-integrity and graph/crew well-formedness checks silently
|
|
13
|
+
* skipped for CLI users — a spec with a dangling `wallets[].chainId` or an
|
|
14
|
+
* unreachable graph node compiled clean. `lint` runs those passes (discarding
|
|
15
|
+
* the result) so authoring bugs surface without a build.
|
|
16
|
+
*
|
|
17
|
+
* COLLECT-ALL vs FAIL-FAST: the IR passes throw on the FIRST violation. For
|
|
18
|
+
* `--format text` that fail-fast is fine (one error, fix, re-run). For
|
|
19
|
+
* `--format json` (editors/CI) we want as many findings as possible, so we run
|
|
20
|
+
* each pass INDEPENDENTLY and catch-and-continue — collecting at most one
|
|
21
|
+
* finding per pass. That is the honest limit of an exception-based pass API
|
|
22
|
+
* (a single pass still stops at its own first violation); it is documented
|
|
23
|
+
* here and surfaced as `severity: "error"` findings with a stable `path`.
|
|
24
|
+
*
|
|
25
|
+
* Side-effect-free: `runLint` is a pure function over the spec text plus an
|
|
26
|
+
* injected tool resolver (so tests don't import the heavy tool packages). The
|
|
27
|
+
* CLI wrapper owns file IO and process exit.
|
|
28
|
+
*/
|
|
29
|
+
export type LintSeverity = "error" | "warning";
|
|
30
|
+
/** One structured finding. `path` is a dot-joined spec/IR location (or a
|
|
31
|
+
* synthetic id for whole-spec failures); shaped for editor/CI consumption. */
|
|
32
|
+
export type LintFinding = {
|
|
33
|
+
readonly message: string;
|
|
34
|
+
readonly path: string;
|
|
35
|
+
readonly severity: LintSeverity;
|
|
36
|
+
/** Stable rule id (parse | ir-pass:<name> | scope | …) for grouping. */
|
|
37
|
+
readonly rule: string;
|
|
38
|
+
};
|
|
39
|
+
export type LintResult = {
|
|
40
|
+
readonly ok: boolean;
|
|
41
|
+
readonly findings: readonly LintFinding[];
|
|
42
|
+
/** The parsed spec when parse succeeded (so --fix can operate on it). */
|
|
43
|
+
readonly spec?: Spec;
|
|
44
|
+
/** The lowered IR when lowering succeeded. */
|
|
45
|
+
readonly ir?: IrNode;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Run the lint pipeline. `resolveTool` maps a tool NAME (either the camelCase
|
|
49
|
+
* spec key or its registered PascalCase name) to a `RegisteredTool`, or
|
|
50
|
+
* undefined — injected so this stays pure. `passes` defaults to the published
|
|
51
|
+
* `DEFAULT_PIPELINE` and is overridable for tests.
|
|
52
|
+
*/
|
|
53
|
+
export declare function runLint(yamlText: string, resolveTool: (name: string) => RegisteredTool | undefined, passes?: ReadonlyArray<IrPass>): LintResult;
|
|
54
|
+
/** Re-exported for the CLI wrapper's philosophy-alignment parity note. */
|
|
55
|
+
export { auditToolScopes };
|
|
56
|
+
/** A single mechanical fix suggestion for a lint finding. */
|
|
57
|
+
export type LintFixSuggestion = {
|
|
58
|
+
readonly kind: "unknown-tool" | "secret-typo" | "safe-name";
|
|
59
|
+
/** The offending token as written in the spec. */
|
|
60
|
+
readonly from: string;
|
|
61
|
+
/** The suggested replacement. */
|
|
62
|
+
readonly to: string;
|
|
63
|
+
/** Human-readable one-liner. */
|
|
64
|
+
readonly note: string;
|
|
65
|
+
};
|
|
66
|
+
/** Case-insensitive Levenshtein distance — the nearest-match metric. */
|
|
67
|
+
export declare function levenshtein(a: string, b: string): number;
|
|
68
|
+
/**
|
|
69
|
+
* The outcome of a nearest-match lookup:
|
|
70
|
+
* - `undefined` — no candidate close enough (genuinely unknown, not a typo),
|
|
71
|
+
* or the name is already legal.
|
|
72
|
+
* - `{ kind: "match", name }` — a single unambiguous nearest candidate;
|
|
73
|
+
* safe to auto-apply.
|
|
74
|
+
* - `{ kind: "ambiguous", candidates }` — two or more equally-near
|
|
75
|
+
* candidates whose I/O capability DIFFERS (e.g. a read-only tool vs a
|
|
76
|
+
* mutating one), so auto-applying could silently cross a capability
|
|
77
|
+
* boundary. Callers should surface this as a suggestion, not a fix.
|
|
78
|
+
*/
|
|
79
|
+
export type NearestToolMatch = {
|
|
80
|
+
readonly kind: "match";
|
|
81
|
+
readonly name: string;
|
|
82
|
+
} | {
|
|
83
|
+
readonly kind: "ambiguous";
|
|
84
|
+
readonly candidates: readonly string[];
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* Nearest legal name(s) for a mistyped tool name. `candidates` is the union of
|
|
88
|
+
* the legal spellings — the camelCase BUILTIN_TOOL_MAP keys AND the registered
|
|
89
|
+
* PascalCase names (both forms are legal in a spec: top-level `tools:` uses the
|
|
90
|
+
* camelCase key; a sub-agent `tools:` uses the PascalCase registered name). An
|
|
91
|
+
* exact match returns undefined (nothing to fix). Otherwise the closest
|
|
92
|
+
* candidate(s) within `maxDistance` (default 3) are considered; undefined
|
|
93
|
+
* means nothing is close enough (a genuinely unknown tool, not a typo).
|
|
94
|
+
*
|
|
95
|
+
* `getReadOnly` is an optional injected lookup from candidate name → the
|
|
96
|
+
* resolved tool's `readOnly` flag (undefined when the candidate can't be
|
|
97
|
+
* resolved to a RegisteredTool, e.g. an unregistered custom-tool name — in
|
|
98
|
+
* that case its capability is unknown and it is never treated as crossing a
|
|
99
|
+
* boundary against another candidate). When ALL of the closest candidates
|
|
100
|
+
* (same minimum distance) agree on `readOnly` — including the degenerate
|
|
101
|
+
* single-candidate case — this returns a `"match"` for the first of them
|
|
102
|
+
* (stable, deterministic). When they DISAGREE — a typo equidistant from a
|
|
103
|
+
* read-only tool (e.g. `Read`) and a mutating one (e.g. `Edit`) — this
|
|
104
|
+
* returns `"ambiguous"` with the full tied set rather than silently picking
|
|
105
|
+
* one, because auto-applying would cross a read-only/mutating capability
|
|
106
|
+
* boundary the author never asked for.
|
|
107
|
+
*/
|
|
108
|
+
export declare function nearestToolName(name: string, candidates: readonly string[], maxDistance?: number, getReadOnly?: (candidateName: string) => boolean | undefined): NearestToolMatch | undefined;
|
|
109
|
+
/**
|
|
110
|
+
* Suggest a `$UPPER_SNAKE_CASE` correction for a credential value that looks
|
|
111
|
+
* like an env reference but isn't a valid one — the same class the compiler's
|
|
112
|
+
* `lowerCredential` rejects (`$slack_token`, `${SLACK}`, `$1PASSWORD`).
|
|
113
|
+
* Returns the normalised form, or undefined when `value` is not a
|
|
114
|
+
* malformed-env-ref (a genuine literal, or already valid). Pure string logic.
|
|
115
|
+
*/
|
|
116
|
+
export declare function suggestSecretFix(value: string): string | undefined;
|
|
117
|
+
/**
|
|
118
|
+
* Normalise a name that violates the `safeName` charset (letters, digits,
|
|
119
|
+
* spaces, `_ . - :`) by replacing every illegal character with `-` and
|
|
120
|
+
* collapsing runs. Returns undefined when `name` is already safe. Pure.
|
|
121
|
+
*/
|
|
122
|
+
export declare function suggestSafeName(name: string): string | undefined;
|
|
123
|
+
/** Render the lint findings as the human-readable `text` report. Returns the
|
|
124
|
+
* block plus whether it is clean, so the CLI can pick the exit code. */
|
|
125
|
+
export declare function formatLintText(result: LintResult): string;
|
|
126
|
+
/** Render the lint findings as `{ findings: LintFinding[] }` JSON for editors/CI. */
|
|
127
|
+
export declare function formatLintJson(result: LintResult): string;
|
package/dist/lint.js
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import { lower } from "@crewhaus/compiler";
|
|
2
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
3
|
+
import { DEFAULT_PIPELINE } from "@crewhaus/ir-passes";
|
|
4
|
+
import { SpecParseError, parseSpec } from "@crewhaus/spec";
|
|
5
|
+
import { auditToolScopes } from "@crewhaus/tool-builder";
|
|
6
|
+
import { auditSpecToolNames, collectToolNames } from "./scope-audit";
|
|
7
|
+
/**
|
|
8
|
+
* Run the lint pipeline. `resolveTool` maps a tool NAME (either the camelCase
|
|
9
|
+
* spec key or its registered PascalCase name) to a `RegisteredTool`, or
|
|
10
|
+
* undefined — injected so this stays pure. `passes` defaults to the published
|
|
11
|
+
* `DEFAULT_PIPELINE` and is overridable for tests.
|
|
12
|
+
*/
|
|
13
|
+
export function runLint(yamlText, resolveTool, passes = DEFAULT_PIPELINE) {
|
|
14
|
+
const findings = [];
|
|
15
|
+
// Stage 1 — parse. A parse failure is terminal: without a spec there is
|
|
16
|
+
// nothing to lower or audit.
|
|
17
|
+
let spec;
|
|
18
|
+
try {
|
|
19
|
+
spec = parseSpec(yamlText);
|
|
20
|
+
}
|
|
21
|
+
catch (err) {
|
|
22
|
+
const message = err instanceof SpecParseError ? err.message : err.message;
|
|
23
|
+
findings.push({ message, path: "<spec>", severity: "error", rule: "parse" });
|
|
24
|
+
return { ok: false, findings };
|
|
25
|
+
}
|
|
26
|
+
// Stage 2 — lower. lower() can throw CompilerError (e.g. a malformed
|
|
27
|
+
// credential env-ref); treat as terminal for the IR-pass stage.
|
|
28
|
+
let ir;
|
|
29
|
+
try {
|
|
30
|
+
ir = lower(spec);
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
const message = err instanceof CrewhausError ? err.message : err.message;
|
|
34
|
+
findings.push({ message, path: "<lower>", severity: "error", rule: "lower" });
|
|
35
|
+
return { ok: false, findings, spec };
|
|
36
|
+
}
|
|
37
|
+
// Stage 3 — IR passes, COLLECT-ALL. Run each pass independently so one pass's
|
|
38
|
+
// violation doesn't hide the others'. Validating passes
|
|
39
|
+
// (transactionPolicyEnforcement, wellFormednessCheck) throw IrPassError; the
|
|
40
|
+
// rewriting passes are pure and never throw. We discard rewritten IR (lint
|
|
41
|
+
// never emits) — only the thrown violations matter.
|
|
42
|
+
for (const pass of passes) {
|
|
43
|
+
try {
|
|
44
|
+
pass(ir);
|
|
45
|
+
}
|
|
46
|
+
catch (err) {
|
|
47
|
+
const message = err instanceof CrewhausError ? err.message : err.message;
|
|
48
|
+
findings.push({
|
|
49
|
+
message,
|
|
50
|
+
path: `ir-pass:${pass.name || "anonymous"}`,
|
|
51
|
+
severity: "error",
|
|
52
|
+
rule: `ir-pass:${pass.name || "anonymous"}`,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// Stage 4 — tool-scope audit over the IR's tool names, sharing the exact
|
|
57
|
+
// gate `compile --strict` uses. A resolvable built-in is audited by
|
|
58
|
+
// capability/scope; an outward-by-name sink that resolves to no external
|
|
59
|
+
// tool is a finding.
|
|
60
|
+
const toolNames = collectToolNames(ir);
|
|
61
|
+
const scopeFindings = auditSpecToolNames(toolNames, resolveTool);
|
|
62
|
+
for (const f of scopeFindings) {
|
|
63
|
+
findings.push({
|
|
64
|
+
message: `tool "${f.toolName}" ${f.reason}`,
|
|
65
|
+
path: `tools.${f.toolName}`,
|
|
66
|
+
severity: "error",
|
|
67
|
+
rule: "scope",
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return { ok: findings.length === 0, findings, spec, ir };
|
|
71
|
+
}
|
|
72
|
+
/** Re-exported for the CLI wrapper's philosophy-alignment parity note. */
|
|
73
|
+
export { auditToolScopes };
|
|
74
|
+
/** Case-insensitive Levenshtein distance — the nearest-match metric. */
|
|
75
|
+
export function levenshtein(a, b) {
|
|
76
|
+
const s = a.toLowerCase();
|
|
77
|
+
const t = b.toLowerCase();
|
|
78
|
+
const m = s.length;
|
|
79
|
+
const n = t.length;
|
|
80
|
+
if (m === 0)
|
|
81
|
+
return n;
|
|
82
|
+
if (n === 0)
|
|
83
|
+
return m;
|
|
84
|
+
let prev = Array.from({ length: n + 1 }, (_, i) => i);
|
|
85
|
+
let curr = new Array(n + 1);
|
|
86
|
+
for (let i = 1; i <= m; i++) {
|
|
87
|
+
curr[0] = i;
|
|
88
|
+
for (let j = 1; j <= n; j++) {
|
|
89
|
+
const cost = s[i - 1] === t[j - 1] ? 0 : 1;
|
|
90
|
+
curr[j] = Math.min((prev[j] ?? 0) + 1, (curr[j - 1] ?? 0) + 1, (prev[j - 1] ?? 0) + cost);
|
|
91
|
+
}
|
|
92
|
+
[prev, curr] = [curr, prev];
|
|
93
|
+
}
|
|
94
|
+
return prev[n] ?? 0;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Nearest legal name(s) for a mistyped tool name. `candidates` is the union of
|
|
98
|
+
* the legal spellings — the camelCase BUILTIN_TOOL_MAP keys AND the registered
|
|
99
|
+
* PascalCase names (both forms are legal in a spec: top-level `tools:` uses the
|
|
100
|
+
* camelCase key; a sub-agent `tools:` uses the PascalCase registered name). An
|
|
101
|
+
* exact match returns undefined (nothing to fix). Otherwise the closest
|
|
102
|
+
* candidate(s) within `maxDistance` (default 3) are considered; undefined
|
|
103
|
+
* means nothing is close enough (a genuinely unknown tool, not a typo).
|
|
104
|
+
*
|
|
105
|
+
* `getReadOnly` is an optional injected lookup from candidate name → the
|
|
106
|
+
* resolved tool's `readOnly` flag (undefined when the candidate can't be
|
|
107
|
+
* resolved to a RegisteredTool, e.g. an unregistered custom-tool name — in
|
|
108
|
+
* that case its capability is unknown and it is never treated as crossing a
|
|
109
|
+
* boundary against another candidate). When ALL of the closest candidates
|
|
110
|
+
* (same minimum distance) agree on `readOnly` — including the degenerate
|
|
111
|
+
* single-candidate case — this returns a `"match"` for the first of them
|
|
112
|
+
* (stable, deterministic). When they DISAGREE — a typo equidistant from a
|
|
113
|
+
* read-only tool (e.g. `Read`) and a mutating one (e.g. `Edit`) — this
|
|
114
|
+
* returns `"ambiguous"` with the full tied set rather than silently picking
|
|
115
|
+
* one, because auto-applying would cross a read-only/mutating capability
|
|
116
|
+
* boundary the author never asked for.
|
|
117
|
+
*/
|
|
118
|
+
export function nearestToolName(name, candidates, maxDistance = 3, getReadOnly) {
|
|
119
|
+
if (candidates.includes(name))
|
|
120
|
+
return undefined;
|
|
121
|
+
let bestDist;
|
|
122
|
+
let tied = [];
|
|
123
|
+
for (const cand of candidates) {
|
|
124
|
+
const dist = levenshtein(name, cand);
|
|
125
|
+
if (bestDist === undefined || dist < bestDist) {
|
|
126
|
+
bestDist = dist;
|
|
127
|
+
tied = [cand];
|
|
128
|
+
}
|
|
129
|
+
else if (dist === bestDist) {
|
|
130
|
+
tied.push(cand);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (bestDist === undefined || bestDist > maxDistance)
|
|
134
|
+
return undefined;
|
|
135
|
+
if (tied.length === 1) {
|
|
136
|
+
const only = tied[0];
|
|
137
|
+
if (only === undefined)
|
|
138
|
+
return undefined;
|
|
139
|
+
return { kind: "match", name: only };
|
|
140
|
+
}
|
|
141
|
+
// Multiple equally-near candidates. If a capability lookup was injected and
|
|
142
|
+
// the tied set spans more than one `readOnly` value, that is a genuine
|
|
143
|
+
// cross-capability ambiguity — refuse to pick one. Candidates with unknown
|
|
144
|
+
// capability (getReadOnly returns undefined) don't by themselves create
|
|
145
|
+
// ambiguity: they only conflict when they disagree with a KNOWN capability
|
|
146
|
+
// among the tied set.
|
|
147
|
+
if (getReadOnly !== undefined) {
|
|
148
|
+
const knownCapabilities = new Set(tied.map((c) => getReadOnly(c)).filter((v) => v !== undefined));
|
|
149
|
+
if (knownCapabilities.size > 1) {
|
|
150
|
+
return { kind: "ambiguous", candidates: tied };
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
// No capability signal available — a plain tie is still ambiguous rather
|
|
155
|
+
// than silently guessing via iteration order.
|
|
156
|
+
return { kind: "ambiguous", candidates: tied };
|
|
157
|
+
}
|
|
158
|
+
const first = tied[0];
|
|
159
|
+
if (first === undefined)
|
|
160
|
+
return undefined;
|
|
161
|
+
return { kind: "match", name: first };
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Suggest a `$UPPER_SNAKE_CASE` correction for a credential value that looks
|
|
165
|
+
* like an env reference but isn't a valid one — the same class the compiler's
|
|
166
|
+
* `lowerCredential` rejects (`$slack_token`, `${SLACK}`, `$1PASSWORD`).
|
|
167
|
+
* Returns the normalised form, or undefined when `value` is not a
|
|
168
|
+
* malformed-env-ref (a genuine literal, or already valid). Pure string logic.
|
|
169
|
+
*/
|
|
170
|
+
export function suggestSecretFix(value) {
|
|
171
|
+
if (!value.startsWith("$"))
|
|
172
|
+
return undefined;
|
|
173
|
+
// Already valid $UPPER_SNAKE_CASE — nothing to fix.
|
|
174
|
+
if (/^\$[A-Z_][A-Z0-9_]*$/.test(value))
|
|
175
|
+
return undefined;
|
|
176
|
+
// Strip a ${...} brace wrapper, then normalise the inner token.
|
|
177
|
+
const inner = value.replace(/^\$\{?/, "").replace(/\}$/, "");
|
|
178
|
+
if (inner === "")
|
|
179
|
+
return undefined;
|
|
180
|
+
// UPPER_SNAKE_CASE it: non-alnum → _, then uppercase, then ensure it does
|
|
181
|
+
// not start with a digit (prefix `_`).
|
|
182
|
+
let normalised = inner.replace(/[^A-Za-z0-9]+/g, "_").toUpperCase();
|
|
183
|
+
if (/^[0-9]/.test(normalised))
|
|
184
|
+
normalised = `_${normalised}`;
|
|
185
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(normalised))
|
|
186
|
+
return undefined;
|
|
187
|
+
return `$${normalised}`;
|
|
188
|
+
}
|
|
189
|
+
/** The single-line-safe name charset the spec's `safeName` enforces. */
|
|
190
|
+
const SAFE_NAME_RE = /^[\w .:-]+$/;
|
|
191
|
+
/**
|
|
192
|
+
* Normalise a name that violates the `safeName` charset (letters, digits,
|
|
193
|
+
* spaces, `_ . - :`) by replacing every illegal character with `-` and
|
|
194
|
+
* collapsing runs. Returns undefined when `name` is already safe. Pure.
|
|
195
|
+
*/
|
|
196
|
+
export function suggestSafeName(name) {
|
|
197
|
+
if (SAFE_NAME_RE.test(name) && name.length >= 1)
|
|
198
|
+
return undefined;
|
|
199
|
+
const fixed = name
|
|
200
|
+
.replace(/[^\w .:-]+/g, "-")
|
|
201
|
+
.replace(/-+/g, "-")
|
|
202
|
+
.replace(/^-|-$/g, "")
|
|
203
|
+
.trim();
|
|
204
|
+
if (fixed === "" || !SAFE_NAME_RE.test(fixed))
|
|
205
|
+
return undefined;
|
|
206
|
+
return fixed;
|
|
207
|
+
}
|
|
208
|
+
/** Render the lint findings as the human-readable `text` report. Returns the
|
|
209
|
+
* block plus whether it is clean, so the CLI can pick the exit code. */
|
|
210
|
+
export function formatLintText(result) {
|
|
211
|
+
if (result.findings.length === 0)
|
|
212
|
+
return "lint: clean — no findings.\n";
|
|
213
|
+
const lines = [];
|
|
214
|
+
for (const f of result.findings) {
|
|
215
|
+
const marker = f.severity === "error" ? "✗" : "~";
|
|
216
|
+
lines.push(`${marker} [${f.rule}] ${f.path}: ${f.message}`);
|
|
217
|
+
}
|
|
218
|
+
const errorCount = result.findings.filter((f) => f.severity === "error").length;
|
|
219
|
+
lines.push("");
|
|
220
|
+
lines.push(`lint: ${errorCount} error(s), ${result.findings.length - errorCount} warning(s).`);
|
|
221
|
+
return `${lines.join("\n")}\n`;
|
|
222
|
+
}
|
|
223
|
+
/** Render the lint findings as `{ findings: LintFinding[] }` JSON for editors/CI. */
|
|
224
|
+
export function formatLintJson(result) {
|
|
225
|
+
return `${JSON.stringify({ ok: result.ok, findings: result.findings }, null, 2)}\n`;
|
|
226
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item 68 — `crewhaus loadtest <spec>`: drive a daemon-shape harness (managed
|
|
3
|
+
* gateway / channel-bot / batch) under concurrent load and measure throughput,
|
|
4
|
+
* p50/p95/p99 latency, error rate, and cost/req — with a `--gate` mode that
|
|
5
|
+
* fails (exit 1) when p95 latency or error rate exceed declared thresholds (a
|
|
6
|
+
* pre-deploy gate).
|
|
7
|
+
*
|
|
8
|
+
* Load is driven through an INJECTED `LoadDriver` — one async call per request
|
|
9
|
+
* that returns a `RequestOutcome`. That seam makes the runner deterministic +
|
|
10
|
+
* credential-free in tests (a stub driver returns canned latencies), while the
|
|
11
|
+
* CLI wires a real driver: the managed gateway's in-process `handle()` behind a
|
|
12
|
+
* stub/echo model, or the batch queue's consumer. No live server or provider
|
|
13
|
+
* key is needed for the runner itself.
|
|
14
|
+
*
|
|
15
|
+
* Everything here is side-effect-free (the CLI entry file constructs the driver
|
|
16
|
+
* + writes the report). The aggregation is pure: same outcomes → same report.
|
|
17
|
+
*/
|
|
18
|
+
/** Thrown on bad flags / driver misuse. The CLI routes it through `die()`. */
|
|
19
|
+
export declare class LoadtestError extends Error {
|
|
20
|
+
readonly name = "LoadtestError";
|
|
21
|
+
}
|
|
22
|
+
/** The result of one driven request. */
|
|
23
|
+
export type RequestOutcome = {
|
|
24
|
+
readonly ok: boolean;
|
|
25
|
+
readonly latencyMs: number;
|
|
26
|
+
/** Token usage, when the driver knows it (for cost/req). */
|
|
27
|
+
readonly tokens?: {
|
|
28
|
+
input: number;
|
|
29
|
+
output: number;
|
|
30
|
+
};
|
|
31
|
+
/** A short classification when `ok` is false (e.g. "429", "timeout"). */
|
|
32
|
+
readonly errorKind?: string;
|
|
33
|
+
};
|
|
34
|
+
/** The injected load driver: given a 0-based request index, drive ONE request
|
|
35
|
+
* and resolve with its outcome. Must not throw — a thrown error is caught and
|
|
36
|
+
* recorded as an errored request (errorKind "threw"). */
|
|
37
|
+
export type LoadDriver = (reqIndex: number) => Promise<RequestOutcome>;
|
|
38
|
+
export type LoadtestConfig = {
|
|
39
|
+
/** Concurrent in-flight requests. Default 10. */
|
|
40
|
+
readonly concurrency?: number;
|
|
41
|
+
/** Total requests to send. Mutually informative with `durationMs`; when both
|
|
42
|
+
* are set, whichever bound is hit first ends the run. Default 100. */
|
|
43
|
+
readonly requests?: number;
|
|
44
|
+
/** Wall-clock budget (ms). When set, the run stops issuing new requests once
|
|
45
|
+
* elapsed ≥ durationMs (in-flight requests still drain). */
|
|
46
|
+
readonly durationMs?: number;
|
|
47
|
+
/** Target requests-per-second. When set, the runner paces request dispatch to
|
|
48
|
+
* approximately this rate (open-loop). Default: unthrottled (closed-loop). */
|
|
49
|
+
readonly rps?: number;
|
|
50
|
+
/** Price per 1M input / output tokens (USD) for the cost/req estimate. */
|
|
51
|
+
readonly pricePerMInput?: number;
|
|
52
|
+
readonly pricePerMOutput?: number;
|
|
53
|
+
/** A monotonic clock (ms) — injected for deterministic tests. Default performance.now. */
|
|
54
|
+
readonly now?: () => number;
|
|
55
|
+
/** A sleep(ms) primitive — injected for deterministic tests (rps pacing). */
|
|
56
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
57
|
+
};
|
|
58
|
+
/** Gate thresholds — a run fails when any is exceeded. */
|
|
59
|
+
export type GateThresholds = {
|
|
60
|
+
readonly maxP95LatencyMs?: number;
|
|
61
|
+
readonly maxErrorRate?: number;
|
|
62
|
+
};
|
|
63
|
+
export type LoadtestReport = {
|
|
64
|
+
readonly requests: number;
|
|
65
|
+
readonly succeeded: number;
|
|
66
|
+
readonly failed: number;
|
|
67
|
+
readonly errorRate: number;
|
|
68
|
+
readonly wallClockMs: number;
|
|
69
|
+
readonly throughputRps: number;
|
|
70
|
+
readonly latency: {
|
|
71
|
+
readonly p50: number;
|
|
72
|
+
readonly p95: number;
|
|
73
|
+
readonly p99: number;
|
|
74
|
+
readonly min: number;
|
|
75
|
+
readonly max: number;
|
|
76
|
+
readonly mean: number;
|
|
77
|
+
};
|
|
78
|
+
readonly tokens: {
|
|
79
|
+
input: number;
|
|
80
|
+
output: number;
|
|
81
|
+
};
|
|
82
|
+
/** USD per request (0 when pricing not supplied). */
|
|
83
|
+
readonly costPerReqUsd: number;
|
|
84
|
+
readonly totalCostUsd: number;
|
|
85
|
+
/** Breakdown of error kinds → count. */
|
|
86
|
+
readonly errorKinds: Readonly<Record<string, number>>;
|
|
87
|
+
};
|
|
88
|
+
/**
|
|
89
|
+
* Run a concurrent load test against the injected driver. Dispatches up to
|
|
90
|
+
* `concurrency` requests in flight until the request budget (or duration) is
|
|
91
|
+
* exhausted, then drains. Deterministic given a deterministic driver + clock.
|
|
92
|
+
*/
|
|
93
|
+
export declare function runLoadtest(driver: LoadDriver, config?: LoadtestConfig): Promise<LoadtestReport>;
|
|
94
|
+
/**
|
|
95
|
+
* Aggregate raw outcomes into a report. Pure — exported so tests can assert on
|
|
96
|
+
* percentile math without driving a run, and so the CLI can re-aggregate a
|
|
97
|
+
* persisted outcome set.
|
|
98
|
+
*/
|
|
99
|
+
export declare function aggregateLoadtest(outcomes: readonly RequestOutcome[], opts: {
|
|
100
|
+
wallClockMs: number;
|
|
101
|
+
pricePerMInput?: number;
|
|
102
|
+
pricePerMOutput?: number;
|
|
103
|
+
}): LoadtestReport;
|
|
104
|
+
/** The gate verdict — which thresholds (if any) were breached. */
|
|
105
|
+
export type GateVerdict = {
|
|
106
|
+
readonly passed: boolean;
|
|
107
|
+
readonly breaches: readonly string[];
|
|
108
|
+
};
|
|
109
|
+
/** Evaluate a report against gate thresholds. Empty thresholds → always passes. */
|
|
110
|
+
export declare function evaluateGate(report: LoadtestReport, thresholds: GateThresholds): GateVerdict;
|
|
111
|
+
/** Render a plain-text loadtest report. */
|
|
112
|
+
export declare function renderLoadtestText(report: LoadtestReport, verdict?: GateVerdict): string;
|
|
113
|
+
/** Render an HTML loadtest report (self-contained). */
|
|
114
|
+
export declare function renderLoadtestHtml(report: LoadtestReport, verdict?: GateVerdict): string;
|