crewhaus 0.1.8 → 0.2.1

package/dist/datasets.d.ts

@@ -0,0 +1,124 @@
+import { type DatasetRecord, type DatasetRegistry, type DatasetSplit } from "@crewhaus/dataset-registry";
+import type { Sample } from "@crewhaus/eval-dataset";
+/** `--dataset` values with this prefix resolve via the registry, not a file. */
+export declare const REGISTRY_PREFIX = "registry:";
+/** Thrown on a malformed ref / split-spec or an unresolvable registry lookup.
+ *  The CLI entry file catches it and routes the message through `die()`;
+ *  tests assert on `.message` without the process exiting. */
+export declare class DatasetRefError extends Error {
+    readonly name = "DatasetRefError";
+}
+export declare function isDatasetSplit(s: string): s is DatasetSplit;
+/** Registry root shared with the emitted eval-bundle harness (see
+ *  target-eval-bundle): `.crewhaus/datasets` under the cwd, or the
+ *  CREWHAUS_DATASETS_DIR override. */
+export declare function defaultDatasetsRoot(): string;
+export type RegistryRef = {
+    readonly name: string;
+    /** Omitted → resolve to the latest version at lookup time. */
+    readonly version?: string;
+    /** Omitted → the union of every split present. */
+    readonly split?: DatasetSplit;
+};
+/** Parse `<name>[@version]` (the `datasets get` positional). */
+export declare function parseNameVersion(refStr: string): {
+    name: string;
+    version?: string;
+};
+/**
+ * Parse a `--dataset` value. Not `registry:`-prefixed → undefined (the caller
+ * falls through to `loadDataset` — bare file paths keep working exactly as
+ * before). Prefixed but malformed → DatasetRefError.
+ */
+export declare function parseRegistryRef(value: string): RegistryRef | undefined;
+export type SplitSpec = {
+    readonly train: number;
+    readonly dev: number;
+    readonly test: number;
+};
+/** Default promotion split for `datasets put` / `distill --register`. */
+export declare const DEFAULT_SPLIT_SPEC: SplitSpec;
+/** Parse `--split-spec`: `train/dev` or `train/dev/test` integer percentages
+ *  summing to 100 (e.g. `70/15/15`, `80/20`). */
+export declare function parseSplitSpec(s: string): SplitSpec;
+/**
+ * Deterministic split assignment. Samples are ordered by sha256(id) (stable
+ * pseudo-shuffle, independent of input order; ties break on the raw id), then
+ * cut at the cumulative percentage boundaries: train takes
+ * `[0, floor(n·train%))`, dev the next slice, test the remainder — so the
+ * three always partition the input exactly. NOT random: the same ids map to
+ * the same splits on every run, which is what makes registry versions
+ * reproducible. Tiny inputs can leave dev/test empty (floor); consumers fall
+ * back accordingly (see the optimize `registry:` path).
+ */
+export declare function splitSamples(samples: ReadonlyArray<Sample>, spec: SplitSpec): {
+    train: Sample[];
+    dev: Sample[];
+    test: Sample[];
+};
+/**
+ * Auto-bump: `v<N+1>` where N is the highest existing `v<digits>` version
+ * (0 when there is none → `v1`). Versions outside that grammar (a
+ * hand-imported "1.0.0") are ignored rather than guessed at — the CLI's own
+ * promotion lineage always lives in the vN namespace, so this can't collide.
+ */
+export declare function nextVersion(existing: ReadonlyArray<string>): string;
+/** The splits a record actually carries (train/dev are always present keys;
+ *  test is optional), in canonical train→dev→test order. */
+export declare function splitsPresent(record: DatasetRecord): DatasetSplit[];
+/** `<name>@<version>[#split]` — the datasetName recorded in the run-history
+ *  index, so runs against different splits key to different baselines. */
+export declare function registryDatasetName(name: string, version: string, split?: DatasetSplit): string;
+/** The selected splits' samples, concatenated in canonical train→dev→test
+ *  order (deterministic regardless of the `splits` array order). */
+export declare function samplesForSplits(record: DatasetRecord, splits: ReadonlyArray<DatasetSplit>): Sample[];
+/**
+ * One stable sha256 over the per-sample content hashes of the selected
+ * splits. Split names are folded in as domain separators (train vs dev with
+ * identical samples still hash apart) and splits are visited in canonical
+ * order, so neither the caller's array order nor record key order can change
+ * the digest. Feeds RunIndexEntry.datasetHash for `registry:` datasets.
+ */
+export declare function overallDatasetHash(record: DatasetRecord, splits: ReadonlyArray<DatasetSplit>): string;
+export type ResolvedRegistryRef = {
+    readonly record: DatasetRecord;
+    readonly version: string;
+    /** The splits the ref selects: the explicit `#split`, or every split present. */
+    readonly splits: DatasetSplit[];
+    /** `<name>@<version>[#split]` — see {@link registryDatasetName}. */
+    readonly datasetName: string;
+    /** Union of the selected splits' samples (canonical train→dev→test order). */
+    readonly samples: Sample[];
+    /** Content hash of the selected splits — see {@link overallDatasetHash}. */
+    readonly datasetHash: string;
+};
+/**
+ * Resolve a parsed `registry:` ref against a registry: pick the latest
+ * version when none was pinned, load the record, and derive the run-index
+ * keying fields (datasetName + datasetHash). Registry lookups may also throw
+ * DatasetRegistryError (a CrewhausError) — callers route both through die().
+ */
+export declare function resolveRegistryRef(registry: DatasetRegistry, ref: RegistryRef): Promise<ResolvedRegistryRef>;
+export type RegisterDatasetOptions = {
+    readonly registry: DatasetRegistry;
+    readonly name: string;
+    readonly samples: ReadonlyArray<Sample>;
+    /** Percentage split (default {@link DEFAULT_SPLIT_SPEC}); ignored with `split`. */
+    readonly splitSpec?: SplitSpec;
+    /** Put every sample into this single named split instead of splitting. */
+    readonly split?: DatasetSplit;
+};
+/**
+ * Import samples as a new auto-bumped version (v1, v2, …) of `name`, either
+ * deterministically split per the spec (test key omitted when the spec gives
+ * test 0%) or wholesale into one named split. Returns the persisted record.
+ */
+export declare function registerDataset(opts: RegisterDatasetOptions): Promise<DatasetRecord>;
+/**
+ * Samples as JSONL for stdout piping. With `split` given: that split's
+ * samples verbatim (SampleSchema-valid — feed straight back into
+ * `--dataset <file>.jsonl`). Without: all splits merged, each line tagged
+ * with a top-level `split` column (SampleSchema strips unknown keys on load,
+ * so the merged form round-trips too).
+ */
+export declare function recordToJsonl(record: DatasetRecord, split?: DatasetSplit): string;

package/dist/datasets.js

@@ -0,0 +1,260 @@
+/**
+ * Item 12 — dataset-registry plumbing for the CLI: the `crewhaus datasets`
+ * subcommand family, `distill --register` versioned promotion, and the
+ * `--dataset registry:<name>[@version][#split]` shorthand shared by `eval`
+ * and `optimize`. Factored out of the entry file `index.ts` (which runs a
+ * top-level argv switch and so cannot be imported by a test without
+ * executing the CLI). Side-effect-free — all filesystem access goes through
+ * an injected `DatasetRegistry` — mirroring `eval-history.ts` / `feedback.ts`.
+ *
+ * Determinism contract (the reason none of this uses an RNG):
+ * - Split assignment orders samples by sha256(sample.id) — a stable
+ *   pseudo-shuffle independent of file order — then cuts at the cumulative
+ *   percentage boundaries. Re-importing the same ids always lands each
+ *   sample in the same split.
+ * - `overallDatasetHash` folds the registry record's per-sample content
+ *   hashes (computed at `put`) into one sha256, so the run-history index and
+ *   (spec, dataset) baselines from item 3 key on dataset CONTENT — the
+ *   registry analogue of `hashDatasetFile`.
+ */
+import { createHash } from "node:crypto";
+import { resolve } from "node:path";
+import { latestVersion, } from "@crewhaus/dataset-registry";
+/** `--dataset` values with this prefix resolve via the registry, not a file. */
+export const REGISTRY_PREFIX = "registry:";
+const SPLIT_ORDER = ["train", "dev", "test"];
+/** Thrown on a malformed ref / split-spec or an unresolvable registry lookup.
+ *  The CLI entry file catches it and routes the message through `die()`;
+ *  tests assert on `.message` without the process exiting. */
+export class DatasetRefError extends Error {
+    name = "DatasetRefError";
+}
+export function isDatasetSplit(s) {
+    return SPLIT_ORDER.includes(s);
+}
+/** Registry root shared with the emitted eval-bundle harness (see
+ *  target-eval-bundle): `.crewhaus/datasets` under the cwd, or the
+ *  CREWHAUS_DATASETS_DIR override. */
+export function defaultDatasetsRoot() {
+    return process.env["CREWHAUS_DATASETS_DIR"] ?? resolve(process.cwd(), ".crewhaus", "datasets");
+}
+/** Parse `<name>[@version]` (the `datasets get` positional). */
+export function parseNameVersion(refStr) {
+    const at = refStr.indexOf("@");
+    if (at === -1)
+        return { name: refStr };
+    const name = refStr.slice(0, at);
+    const version = refStr.slice(at + 1);
+    if (version === "")
+        throw new DatasetRefError(`empty version in dataset ref "${refStr}"`);
+    return { name, version };
+}
+/**
+ * Parse a `--dataset` value. Not `registry:`-prefixed → undefined (the caller
+ * falls through to `loadDataset` — bare file paths keep working exactly as
+ * before). Prefixed but malformed → DatasetRefError.
+ */
+export function parseRegistryRef(value) {
+    if (!value.startsWith(REGISTRY_PREFIX))
+        return undefined;
+    let rest = value.slice(REGISTRY_PREFIX.length);
+    let split;
+    const hash = rest.indexOf("#");
+    if (hash !== -1) {
+        const s = rest.slice(hash + 1);
+        rest = rest.slice(0, hash);
+        if (!isDatasetSplit(s)) {
+            throw new DatasetRefError(`invalid split "#${s}" in "${value}" — expected #train, #dev, or #test`);
+        }
+        split = s;
+    }
+    const { name, version } = parseNameVersion(rest);
+    if (name === "") {
+        throw new DatasetRefError(`missing dataset name in "${value}" — expected registry:<name>[@version][#split]`);
+    }
+    return {
+        name,
+        ...(version !== undefined ? { version } : {}),
+        ...(split !== undefined ? { split } : {}),
+    };
+}
+/** Default promotion split for `datasets put` / `distill --register`. */
+export const DEFAULT_SPLIT_SPEC = { train: 70, dev: 15, test: 15 };
+/** Parse `--split-spec`: `train/dev` or `train/dev/test` integer percentages
+ *  summing to 100 (e.g. `70/15/15`, `80/20`). */
+export function parseSplitSpec(s) {
+    const parts = s.split("/");
+    if (parts.length !== 2 && parts.length !== 3) {
+        throw new DatasetRefError(`invalid split spec "${s}" — expected train/dev or train/dev/test percentages (e.g. 70/15/15)`);
+    }
+    const nums = parts.map((p) => {
+        if (!/^\d+$/.test(p)) {
+            throw new DatasetRefError(`invalid split spec "${s}" — "${p}" is not a whole number`);
+        }
+        return Number.parseInt(p, 10);
+    });
+    const train = nums[0] ?? 0;
+    const dev = nums[1] ?? 0;
+    const test = nums[2] ?? 0;
+    if (train + dev + test !== 100) {
+        throw new DatasetRefError(`invalid split spec "${s}" — must sum to 100 (got ${train + dev + test})`);
+    }
+    return { train, dev, test };
+}
+function idHash(id) {
+    return createHash("sha256").update(id).digest("hex");
+}
+/**
+ * Deterministic split assignment. Samples are ordered by sha256(id) (stable
+ * pseudo-shuffle, independent of input order; ties break on the raw id), then
+ * cut at the cumulative percentage boundaries: train takes
+ * `[0, floor(n·train%))`, dev the next slice, test the remainder — so the
+ * three always partition the input exactly. NOT random: the same ids map to
+ * the same splits on every run, which is what makes registry versions
+ * reproducible. Tiny inputs can leave dev/test empty (floor); consumers fall
+ * back accordingly (see the optimize `registry:` path).
+ */
+export function splitSamples(samples, spec) {
+    const ordered = [...samples].sort((a, b) => {
+        const ha = idHash(a.id);
+        const hb = idHash(b.id);
+        if (ha !== hb)
+            return ha < hb ? -1 : 1;
+        return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+    });
+    const n = ordered.length;
+    const trainEnd = Math.floor((n * spec.train) / 100);
+    const devEnd = Math.floor((n * (spec.train + spec.dev)) / 100);
+    return {
+        train: ordered.slice(0, trainEnd),
+        dev: ordered.slice(trainEnd, devEnd),
+        test: ordered.slice(devEnd),
+    };
+}
+// -------- versioning --------
+/**
+ * Auto-bump: `v<N+1>` where N is the highest existing `v<digits>` version
+ * (0 when there is none → `v1`). Versions outside that grammar (a
+ * hand-imported "1.0.0") are ignored rather than guessed at — the CLI's own
+ * promotion lineage always lives in the vN namespace, so this can't collide.
+ */
+export function nextVersion(existing) {
+    let max = 0;
+    for (const v of existing) {
+        const m = /^v(\d+)$/.exec(v);
+        if (m === null)
+            continue;
+        const n = Number.parseInt(m[1], 10);
+        if (n > max)
+            max = n;
+    }
+    return `v${max + 1}`;
+}
+// -------- registry resolution (the `registry:` shorthand) --------
+/** The splits a record actually carries (train/dev are always present keys;
+ *  test is optional), in canonical train→dev→test order. */
+export function splitsPresent(record) {
+    return SPLIT_ORDER.filter((s) => record.splits[s] !== undefined);
+}
+/** `<name>@<version>[#split]` — the datasetName recorded in the run-history
+ *  index, so runs against different splits key to different baselines. */
+export function registryDatasetName(name, version, split) {
+    return `${name}@${version}${split !== undefined ? `#${split}` : ""}`;
+}
+/** The selected splits' samples, concatenated in canonical train→dev→test
+ *  order (deterministic regardless of the `splits` array order). */
+export function samplesForSplits(record, splits) {
+    const wanted = new Set(splits);
+    const out = [];
+    for (const s of SPLIT_ORDER) {
+        if (wanted.has(s))
+            out.push(...(record.splits[s] ?? []));
+    }
+    return out;
+}
+/**
+ * One stable sha256 over the per-sample content hashes of the selected
+ * splits. Split names are folded in as domain separators (train vs dev with
+ * identical samples still hash apart) and splits are visited in canonical
+ * order, so neither the caller's array order nor record key order can change
+ * the digest. Feeds RunIndexEntry.datasetHash for `registry:` datasets.
+ */
+export function overallDatasetHash(record, splits) {
+    const wanted = new Set(splits);
+    const h = createHash("sha256");
+    for (const s of SPLIT_ORDER) {
+        if (!wanted.has(s) || record.splits[s] === undefined)
+            continue;
+        h.update(`${s}:${(record.sampleHashes[s] ?? []).join(",")}\n`);
+    }
+    return h.digest("hex");
+}
+/**
+ * Resolve a parsed `registry:` ref against a registry: pick the latest
+ * version when none was pinned, load the record, and derive the run-index
+ * keying fields (datasetName + datasetHash). Registry lookups may also throw
+ * DatasetRegistryError (a CrewhausError) — callers route both through die().
+ */
+export async function resolveRegistryRef(registry, ref) {
+    const version = ref.version ?? (await latestVersion(registry, ref.name));
+    if (version === undefined) {
+        throw new DatasetRefError(`dataset "${ref.name}" has no versions in the registry — import one with \`crewhaus datasets put\``);
+    }
+    const record = await registry.getRecord(ref.name, version);
+    if (ref.split !== undefined && record.splits[ref.split] === undefined) {
+        throw new DatasetRefError(`split "${ref.split}" not present in "${ref.name}@${version}"`);
+    }
+    const splits = ref.split !== undefined ? [ref.split] : splitsPresent(record);
+    return {
+        record,
+        version,
+        splits,
+        datasetName: registryDatasetName(ref.name, version, ref.split),
+        samples: samplesForSplits(record, splits),
+        datasetHash: overallDatasetHash(record, splits),
+    };
+}
+/**
+ * Import samples as a new auto-bumped version (v1, v2, …) of `name`, either
+ * deterministically split per the spec (test key omitted when the spec gives
+ * test 0%) or wholesale into one named split. Returns the persisted record.
+ */
+export async function registerDataset(opts) {
+    const version = nextVersion(await opts.registry.list(opts.name));
+    let splits;
+    if (opts.split !== undefined) {
+        splits = {
+            train: opts.split === "train" ? [...opts.samples] : [],
+            dev: opts.split === "dev" ? [...opts.samples] : [],
+            ...(opts.split === "test" ? { test: [...opts.samples] } : {}),
+        };
+    }
+    else {
+        const spec = opts.splitSpec ?? DEFAULT_SPLIT_SPEC;
+        const { train, dev, test } = splitSamples(opts.samples, spec);
+        splits = { train, dev, ...(spec.test > 0 ? { test } : {}) };
+    }
+    return opts.registry.put({ name: opts.name, version, splits });
+}
+// -------- serialization (datasets get) --------
+/**
+ * Samples as JSONL for stdout piping. With `split` given: that split's
+ * samples verbatim (SampleSchema-valid — feed straight back into
+ * `--dataset <file>.jsonl`). Without: all splits merged, each line tagged
+ * with a top-level `split` column (SampleSchema strips unknown keys on load,
+ * so the merged form round-trips too).
+ */
+export function recordToJsonl(record, split) {
+    const lines = [];
+    if (split !== undefined) {
+        for (const s of record.splits[split] ?? [])
+            lines.push(JSON.stringify(s));
+    }
+    else {
+        for (const sp of splitsPresent(record)) {
+            for (const s of record.splits[sp] ?? [])
+                lines.push(JSON.stringify({ ...s, split: sp }));
+        }
+    }
+    return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
+}

package/dist/deploy-canary.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Item 29 — `crewhaus deploy canary <spec> <version> ...`: an eval-gated
+ * unattended ramp with auto-rollback, wiring the REAL `regression-runner`
+ * gate into the `canary-controller` (which ships dependency-inverted with an
+ * always-pass stub only for tests — see `makeRegressionGate`).
+ *
+ * This module is deliberately side-effect-free (the CLI entry file runs an
+ * argv switch on import, so it can't be imported by a test): all I/O — the
+ * per-version eval, registry pins, audit appends, and stdout — is injected.
+ * `apps/cli/src/index.ts` supplies the real implementations; the tests inject
+ * seeded `EvalRunSummary` results plus the real `gate()` and a real
+ * file-backed registry + audit log.
+ *
+ * The ramp drives the DECLARED traffic steps in order (e.g. 5,25,50,100). At
+ * each step it evals BOTH the pinned baseline version and the candidate
+ * version against the same dataset+graders, feeds the two summaries into
+ * `regression-runner.gate()` (pass-rate + p95-latency thresholds), and:
+ *   - on PASS at every step, auto-promotes the env pin to the candidate via
+ *     the deployment path (`canary-controller.evaluate` re-pins on pass) and
+ *     audit-logs a `deployment_action`;
+ *   - on the FIRST failing step, auto-rolls-back (re-pins to the baseline)
+ *     and audit-logs the regression reason, then stops the ramp.
+ *
+ * v1 CAVEAT (honest scope): `crewhaus eval` supports `target: cli`, and the
+ * canary-controller's `route()` has no serving-path consumer, so the ramp %
+ * gates eval SAMPLING/PROMOTION, not a real production traffic split. Each
+ * step evals the FULL dataset against both versions and gates promotion on
+ * the result — the percentages sequence the confidence ramp, they do not
+ * split live requests. A real traffic split matters only for gateway/managed
+ * shapes with a serving-path `route()` consumer; that is out of scope here
+ * and called out in the CLI `--help`.
+ */
+import type { RegressionGate } from "@crewhaus/canary-controller";
+import type { EvalRunSummary } from "@crewhaus/eval-runner";
+import { type GateThresholds } from "@crewhaus/regression-runner";
+/** Parse a `--traffic 5,25,50,100` list into validated, ordered steps. */
+export declare function parseTrafficSteps(value: string): number[];
+export declare class CanaryRampError extends Error {
+    readonly name = "CanaryRampError";
+}
+/**
+ * Build the {@link RegressionGate} closure the controller calls at each step:
+ * eval both versions with the injected `evalVersion`, then run the real
+ * `regression-runner.gate()` over the two summaries (baseline = prev,
+ * candidate = next). Latency gating is ON here (unlike the pass-rate/flip-only
+ * eval-history gate): a canary that regresses p95 latency is a real
+ * production signal. Thresholds default to regression-runner's own
+ * (5-point pass-rate drop, +5000ms p95) and are overridable via CLI flags.
+ */
+export declare function makeCanaryEvalGate(opts: {
+    readonly evalVersion: (version: string) => Promise<EvalRunSummary>;
+    readonly thresholds?: GateThresholds;
+    readonly write?: (line: string) => void;
+}): RegressionGate;
+export type CanaryRampStepResult = {
+    readonly trafficPercent: number;
+    readonly verdict: "pass" | "fail";
+    readonly reason?: string;
+    readonly action?: "promote" | "rollback";
+};
+export type CanaryRampResult = {
+    /** True only when every declared step passed and the candidate was promoted. */
+    readonly promoted: boolean;
+    readonly steps: ReadonlyArray<CanaryRampStepResult>;
+    /** The step that failed (and triggered rollback), when any. */
+    readonly failedAt?: number;
+};
+/**
+ * Drive the declared traffic steps. `evaluateStep` is
+ * `canary-controller.evaluate` bound to a config whose `trafficPercent` this
+ * function sets per step; on pass it promotes the env pin, on fail it
+ * rolls back. The ramp stops at the first failing step (the controller has
+ * already rolled the env pin back to the baseline by then).
+ */
+export declare function driveCanaryRamp(opts: {
+    readonly steps: ReadonlyArray<number>;
+    readonly evaluateStep: (trafficPercent: number) => Promise<{
+        verdict: "pass" | "fail";
+        reason?: string;
+        action?: "promote" | "rollback";
+    }>;
+    readonly write?: (line: string) => void;
+}): Promise<CanaryRampResult>;

package/dist/deploy-canary.js

@@ -0,0 +1,87 @@
+import { gate } from "@crewhaus/regression-runner";
+/** Parse a `--traffic 5,25,50,100` list into validated, ordered steps. */
+export function parseTrafficSteps(value) {
+    const raw = value
+        .split(",")
+        .map((s) => s.trim())
+        .filter((s) => s !== "");
+    if (raw.length === 0) {
+        throw new CanaryRampError("--traffic must list at least one percentage (e.g. 5,25,50,100)");
+    }
+    const steps = [];
+    let last = 0;
+    for (const token of raw) {
+        const n = Number(token);
+        if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0 || n > 100) {
+            throw new CanaryRampError(`--traffic step "${token}" must be an integer in 1..100 (e.g. 5,25,50,100)`);
+        }
+        if (n <= last) {
+            throw new CanaryRampError(`--traffic steps must strictly increase; "${n}" does not exceed the previous "${last}"`);
+        }
+        steps.push(n);
+        last = n;
+    }
+    return steps;
+}
+export class CanaryRampError extends Error {
+    name = "CanaryRampError";
+}
+/**
+ * Build the {@link RegressionGate} closure the controller calls at each step:
+ * eval both versions with the injected `evalVersion`, then run the real
+ * `regression-runner.gate()` over the two summaries (baseline = prev,
+ * candidate = next). Latency gating is ON here (unlike the pass-rate/flip-only
+ * eval-history gate): a canary that regresses p95 latency is a real
+ * production signal. Thresholds default to regression-runner's own
+ * (5-point pass-rate drop, +5000ms p95) and are overridable via CLI flags.
+ */
+export function makeCanaryEvalGate(opts) {
+    const write = opts.write ?? (() => { });
+    return async ({ fromVersion, toVersion }) => {
+        write(`[canary] evaluating baseline ${fromVersion} and candidate ${toVersion}`);
+        // Baseline first, then candidate — sequential so a low-TPM provider tier
+        // isn't hit by two concurrent eval runs (the same reason the eval CI
+        // scaffold pins --concurrency 1).
+        const baseline = await opts.evalVersion(fromVersion);
+        const candidate = await opts.evalVersion(toVersion);
+        const verdict = gate(baseline, candidate, opts.thresholds ?? {});
+        write(`[canary]   baseline pass_rate=${(baseline.aggregates.passRate * 100).toFixed(1)}% ` +
+            `p95=${baseline.aggregates.p95LatencyMs}ms · ` +
+            `candidate pass_rate=${(candidate.aggregates.passRate * 100).toFixed(1)}% ` +
+            `p95=${candidate.aggregates.p95LatencyMs}ms`);
+        if (verdict.verdict === "fail") {
+            return { verdict: "fail", reason: verdict.reason ?? "regression gate failed" };
+        }
+        return { verdict: "pass" };
+    };
+}
+/**
+ * Drive the declared traffic steps. `evaluateStep` is
+ * `canary-controller.evaluate` bound to a config whose `trafficPercent` this
+ * function sets per step; on pass it promotes the env pin, on fail it
+ * rolls back. The ramp stops at the first failing step (the controller has
+ * already rolled the env pin back to the baseline by then).
+ */
+export async function driveCanaryRamp(opts) {
+    const write = opts.write ?? (() => { });
+    const results = [];
+    for (const trafficPercent of opts.steps) {
+        write(`[canary] ramp step: ${trafficPercent}% traffic`);
+        const outcome = await opts.evaluateStep(trafficPercent);
+        const step = {
+            trafficPercent,
+            verdict: outcome.verdict,
+            ...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
+            ...(outcome.action !== undefined ? { action: outcome.action } : {}),
+        };
+        results.push(step);
+        if (outcome.verdict === "fail") {
+            write(`[canary] step ${trafficPercent}% FAILED — ${outcome.reason ?? "regression"}`);
+            write("[canary] auto-rolled-back to baseline; ramp aborted");
+            return { promoted: false, steps: results, failedAt: trafficPercent };
+        }
+        write(`[canary] step ${trafficPercent}% passed`);
+    }
+    write("[canary] all ramp steps passed — candidate promoted");
+    return { promoted: true, steps: results };
+}

package/dist/doctor-checks.d.ts

@@ -1,3 +1,4 @@
+import type { IrChannels } from "@crewhaus/ir";
 /**
  * Model-aware credential checks for `crewhaus doctor`, factored out of the
  * entry file `index.ts` (which runs a top-level argv switch and so cannot be
@@ -32,6 +33,18 @@ export declare function extractSpecModel(yamlText: string): string | undefined;
  * run will need. Returns undefined for unparseable strings.
  */
 export declare function selectedProvider(model: string): DoctorProviderId | undefined;
+/**
+ * Item 13 — online/offline mode selection for `crewhaus scaffold-evals`:
+ * true when the provider `model` routes to has visibly satisfied credentials
+ * (shares `statusFor` with the doctor checks, so the two never disagree).
+ * Unparseable model strings return false — the scaffold then stays on its
+ * deterministic template path instead of attempting a doomed call. Note the
+ * bedrock check is env-visible only (the AWS SDK's default chain may still
+ * work); scaffold prefers the safe offline mode in that ambiguity.
+ */
+export declare function providerCredentialsSatisfied(model: string, env: NodeJS.ProcessEnv): boolean;
+/** Env var names `doctor --fix` should stub for a selected provider. */
+export declare function providerEnvStubs(provider: DoctorProviderId): readonly string[];
 /**
  * Build the credential section of `crewhaus doctor`.
  *
@@ -43,3 +56,23 @@ export declare function selectedProvider(model: string): DoctorProviderId | unde
  * env is clearly set (the operator is plainly not on Anthropic).
  */
 export declare function buildCredentialChecks(specModel: string | undefined, env: NodeJS.ProcessEnv): DoctorCredentialCheck[];
+/**
+ * Item 61 — tolerantly extract the lowered channels block from a
+ * crewhaus.yaml text. Returns undefined when the spec doesn't parse/lower or
+ * is not a channel target, mirroring {@link extractSpecModel}'s tolerance —
+ * doctor must never fail because the cwd spec is a different shape. Lowering
+ * (not raw parsing) is deliberate: `lower()` is where `$VAR_NAME` strings
+ * become env-refs, so the check shares the compiled daemon's exact env
+ * semantics instead of re-implementing the regex.
+ */
+export declare function extractChannelIr(yamlText: string): IrChannels | undefined;
+/**
+ * Item 61 — channel-target doctor checks: one per configured platform
+ * (slack/telegram/discord), asserting that every secret env-ref the compiled
+ * daemon requires at boot (target-channel-bot's startup check exits 2 on the
+ * same set) is actually set. Returns [] for non-channel specs, so the check
+ * only appears when the cwd spec is a channel target. Offline by design —
+ * live platform probes (granted scopes, webhook state) belong to
+ * `crewhaus channel verify`, which the failure reason points at.
+ */
+export declare function buildChannelEnvChecks(yamlText: string, env: NodeJS.ProcessEnv): DoctorCredentialCheck[];