@herbertgao/pi-extensions 2026.9.8 → 2026.9.10

package/node_modules/pi-typesafe/dist/auth.js

@@ -0,0 +1,114 @@
+import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { credentialsPath, keySituation, keySourceLabel, piTypesafeDir } from "./credentials.js";
+import { TypeSafeIntegrationError } from "./errors.js";
+const AUTH_VERSION = 1;
+const CODES = new Set([
+    "configuration", "validation", "budget", "aborted", "timeout", "http", "connection", "response",
+]);
+/** Statuses that mean the key itself was refused, not that the service was busy. */
+const REJECTED_STATUSES = new Set([401, 403]);
+/** The auth-state file: one small, owner-only record that outlives the process that wrote it. */
+export function authStatePath() {
+    return join(piTypesafeDir(), "auth-state.json");
+}
+function readState(path) {
+    try {
+        const parsed = JSON.parse(readFileSync(path, "utf8"));
+        if (!parsed || typeof parsed !== "object")
+            return {};
+        const raw = parsed;
+        const verifiedAt = typeof raw.verifiedAt === "string" && raw.verifiedAt.length <= 40 ? raw.verifiedAt : undefined;
+        const failure = raw.lastFailure && typeof raw.lastFailure === "object" ? raw.lastFailure : undefined;
+        const code = typeof failure?.code === "string" && CODES.has(failure.code) ? failure.code : undefined;
+        const message = typeof failure?.message === "string" ? failure.message.slice(0, 300) : undefined;
+        const at = typeof failure?.at === "string" && failure.at.length <= 40 ? failure.at : undefined;
+        const status = typeof failure?.status === "number" && Number.isSafeInteger(failure.status) ? failure.status : undefined;
+        const lastFailure = code && message && at
+            ? { code, message, at, ...(status === undefined ? {} : { status }) }
+            : undefined;
+        return { ...(verifiedAt === undefined ? {} : { verifiedAt }), ...(lastFailure === undefined ? {} : { lastFailure }) };
+    }
+    catch {
+        return {};
+    }
+}
+/** Owner-only, atomic, and best-effort: an unwritable auth record never changes how a request behaves. */
+function writeState(path, state) {
+    const temporary = `${path}.${process.pid}.tmp`;
+    try {
+        mkdirSync(piTypesafeDir(), { recursive: true, mode: 0o700 });
+        writeFileSync(temporary, `${JSON.stringify({ version: AUTH_VERSION, ...state }, null, 2)}\n`, { mode: 0o600, flag: "w" });
+        chmodSync(temporary, 0o600);
+        renameSync(temporary, path);
+    }
+    catch {
+        try {
+            rmSync(temporary, { force: true });
+        }
+        catch { /* best-effort cleanup only */ }
+    }
+}
+/** What the key situation, the last outcome, and the clock add up to. Never throws. */
+export function authState(options = {}) {
+    const path = options.path ?? authStatePath();
+    const situation = keySituation();
+    const stored = readState(path);
+    const source = situation.kind === "environment" ? "environment" : situation.kind === "stored" ? "stored" : undefined;
+    const rejected = stored.lastFailure?.code === "http" && stored.lastFailure.status !== undefined && REJECTED_STATUSES.has(stored.lastFailure.status);
+    const usable = source !== undefined && !rejected;
+    return {
+        kind: situation.kind,
+        ...(source === undefined ? {} : { source }),
+        path: situation.kind === "unusable" ? situation.path : credentialsPath(),
+        ...(situation.kind === "unusable" ? { reason: situation.reason } : {}),
+        keyName: keySourceLabel(situation),
+        verified: stored.verifiedAt !== undefined && !rejected,
+        ...(stored.verifiedAt === undefined ? {} : { verifiedAt: stored.verifiedAt }),
+        ...(stored.lastFailure === undefined ? {} : { lastFailure: stored.lastFailure }),
+        usable,
+    };
+}
+/** Record that the key was accepted: login verification, or any successful request. Clears the last failure. */
+export function recordAuthVerified(at = new Date()) {
+    writeState(authStatePath(), { verifiedAt: at.toISOString() });
+}
+/** Record the failure that degraded TypeSafe. The verification timestamp is kept so a recovered key stays known. */
+export function recordAuthFailure(error, at = new Date()) {
+    const current = readState(authStatePath());
+    const failure = {
+        code: error.code,
+        message: error.message,
+        at: at.toISOString(),
+        ...(error.status === undefined ? {} : { status: error.status }),
+    };
+    writeState(authStatePath(), { ...(current.verifiedAt === undefined ? {} : { verifiedAt: current.verifiedAt }), lastFailure: failure });
+}
+/** Forget verification and degradation: used when the key itself changes (login or logout). */
+export function clearAuthState() {
+    try {
+        rmSync(authStatePath(), { force: true });
+    }
+    catch { /* nothing to clear */ }
+}
+/**
+ * One line plus a level, so a status command, a headless log, or a consumer's own status line can call out a degraded
+ * state instead of reporting "enabled".
+ */
+export function describeAuth(state = authState()) {
+    const since = state.lastFailure ? ` Last failure: ${state.lastFailure.message}${state.lastFailure.at ? ` (${state.lastFailure.at})` : ""}` : "";
+    if (state.kind === "missing") {
+        return { level: "error", text: `TypeSafe key: missing — every Jev judgment is skipped until a key is configured (/typesafe login or TYPESAFE_API_KEY).${since}` };
+    }
+    if (state.kind === "unusable") {
+        return { level: "error", text: `TypeSafe key: unusable (${state.reason ?? "unknown reason"}) — judgments are skipped until the key is fixed.${since}` };
+    }
+    const rejected = state.lastFailure?.code === "http" && state.lastFailure.status !== undefined && REJECTED_STATUSES.has(state.lastFailure.status);
+    if (rejected) {
+        return { level: "error", text: `TypeSafe key: ${state.keyName} was rejected.${since}` };
+    }
+    if (!state.verified) {
+        return { level: "warning", text: `TypeSafe key: ${state.keyName} (not verified yet — the first request proves it).${since}` };
+    }
+    return { level: "ok", text: `TypeSafe key: ${state.keyName} (verified${state.verifiedAt ? ` ${state.verifiedAt}` : ""}).${since}` };
+}

package/node_modules/pi-typesafe/dist/batch.d.ts

@@ -0,0 +1,74 @@
+import type { Questions, SystemOneRequest, Usage } from "@typesafe-ai/sdk";
+import type { Evaluation, TypeSafe } from "./client.js";
+/** Default number of requests in flight. TypeSafe answers in isolation, so a small pool is enough. */
+export declare const DEFAULT_CONCURRENCY = 4;
+/** One item's outcome. `skipped` marks work that was never started because of an abort or a stop rule. */
+export type Settled<T> = {
+    readonly ok: true;
+    readonly index: number;
+    readonly value: T;
+} | {
+    readonly ok: false;
+    readonly index: number;
+    readonly error: unknown;
+    readonly skipped: boolean;
+};
+export interface FanOutOptions {
+    /** Requests in flight at once. Default: DEFAULT_CONCURRENCY. */
+    concurrency?: number;
+    /** Stops starting new work once aborted; in-flight work still finishes. */
+    signal?: AbortSignal;
+    /** Stop launching new work once this returns true for a failure, e.g. a `budget` error. */
+    stopOn?: (error: unknown) => boolean;
+}
+/**
+ * Run `worker` over `items` with bounded concurrency, preserving input order. Never throws: every item comes back as a
+ * settled result. This is the pool the client's batching methods use, exported so script authors stop hand-rolling one.
+ */
+export declare function fanOut<I, O>(items: readonly I[], worker: (item: I, index: number) => Promise<O>, options?: FanOutOptions): Promise<Settled<O>[]>;
+export interface BatchOptions {
+    /** Requests in flight at once. Default: DEFAULT_CONCURRENCY. */
+    concurrency?: number;
+    signal?: AbortSignal;
+}
+/** Per-request outcomes plus the merged view callers usually want. */
+export interface BatchEvaluation<Q extends Questions = Questions> {
+    /** True when every request succeeded. */
+    readonly ok: boolean;
+    /** Per-request outcomes in input order. */
+    readonly results: readonly Settled<Evaluation<Q>>[];
+    /** Failures, including requests that were never submitted. */
+    readonly failures: number;
+    /** Work never started, because of an abort or a `budget` stop. */
+    readonly skipped: number;
+    /** Answers merged in input order; a repeated question id keeps the last answer. Empty when nothing succeeded. */
+    readonly answers: Partial<Evaluation<Q>["answers"]>;
+    /** The model of the first successful request, when there is one. */
+    readonly model?: string;
+    /** Usage summed over the requests that succeeded. */
+    readonly usage: Usage;
+    /** Wall-clock time for the whole batch. */
+    readonly elapsedMs: number;
+}
+/**
+ * Send several requests with bounded concurrency, in input order, and merge what came back. Each request passes through
+ * the same admission seam as `evaluate`, so an invalid request is one settled failure, not a thrown error. A `budget` or
+ * cancellation failure stops the rest from being submitted. Never throws.
+ */
+export declare function evaluateMany<Q extends Questions>(client: TypeSafe, requests: readonly SystemOneRequest<Q>[], options?: BatchOptions): Promise<BatchEvaluation<Q>>;
+/**
+ * Split a request that asks more questions than one request may carry into chunks of at most `maxQuestions`. Sharing
+ * one state across several questions is one request; asking more than the per-request limit is the only reason to fan
+ * out, and the state is repeated in each chunk. The order of `questions` is preserved. A pure splitter: admission still
+ * happens once per chunk, in `evaluate` or `evaluateMany`, so what one accepts the others accept.
+ */
+export declare function chunkEvaluationRequest(request: SystemOneRequest<Questions>, options?: {
+    maxQuestions?: number;
+}): SystemOneRequest<Questions>[];
+/**
+ * Ask any number of questions about one state: chunk to the per-request limit, fan out, and merge the answers, usage,
+ * and model. Use this when one coherent state carries many independent questions; use `evaluate` for one request.
+ */
+export declare function evaluateAll<Q extends Questions>(client: TypeSafe, request: SystemOneRequest<Q>, options?: BatchOptions & {
+    maxQuestions?: number;
+}): Promise<BatchEvaluation<Q>>;

package/node_modules/pi-typesafe/dist/batch.js

@@ -0,0 +1,116 @@
+import { TypeSafeIntegrationError } from "./errors.js";
+import { DEFAULT_MAX_QUESTIONS, prepareEvaluationRequest } from "./schema.js";
+/** Default number of requests in flight. TypeSafe answers in isolation, so a small pool is enough. */
+export const DEFAULT_CONCURRENCY = 4;
+/**
+ * Run `worker` over `items` with bounded concurrency, preserving input order. Never throws: every item comes back as a
+ * settled result. This is the pool the client's batching methods use, exported so script authors stop hand-rolling one.
+ */
+export async function fanOut(items, worker, options = {}) {
+    const concurrency = Math.max(1, Math.floor(options.concurrency ?? DEFAULT_CONCURRENCY));
+    const results = new Array(items.length);
+    let next = 0;
+    let stopped = false;
+    const run = async () => {
+        while (!stopped) {
+            const index = next++;
+            if (index >= items.length)
+                return;
+            if (options.signal?.aborted) {
+                stopped = true;
+                return;
+            }
+            try {
+                results[index] = { ok: true, index, value: await worker(items[index], index) };
+            }
+            catch (error) {
+                results[index] = { ok: false, index, error, skipped: false };
+                if (options.stopOn?.(error))
+                    stopped = true;
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run));
+    const reason = options.signal?.aborted ? "TypeSafe batch cancelled before this request was submitted." : "TypeSafe batch stopped after a failed request; this request was not submitted.";
+    for (let index = 0; index < items.length; index++) {
+        if (!results[index])
+            results[index] = { ok: false, index, error: new TypeSafeIntegrationError("aborted", reason), skipped: true };
+    }
+    return results;
+}
+function summarize(results, elapsedMs) {
+    const answers = {};
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let succeeded = 0;
+    let skipped = 0;
+    let model;
+    for (const result of results) {
+        if (!result.ok) {
+            if (result.skipped)
+                skipped++;
+            continue;
+        }
+        succeeded++;
+        Object.assign(answers, result.value.answers);
+        inputTokens += result.value.usage.input_tokens;
+        outputTokens += result.value.usage.output_tokens;
+        model ??= result.value.model;
+    }
+    return {
+        ok: succeeded === results.length,
+        results,
+        failures: results.length - succeeded,
+        skipped,
+        answers,
+        ...(model === undefined ? {} : { model }),
+        usage: { input_tokens: inputTokens, output_tokens: outputTokens },
+        elapsedMs,
+    };
+}
+/** A failure that is worth stopping the batch for: no more requests will be accepted, or the caller cancelled. */
+function stopsBatch(error) {
+    return error instanceof TypeSafeIntegrationError && (error.code === "budget" || error.code === "aborted");
+}
+/**
+ * Send several requests with bounded concurrency, in input order, and merge what came back. Each request passes through
+ * the same admission seam as `evaluate`, so an invalid request is one settled failure, not a thrown error. A `budget` or
+ * cancellation failure stops the rest from being submitted. Never throws.
+ */
+export async function evaluateMany(client, requests, options = {}) {
+    const start = performance.now();
+    const results = await fanOut(requests, (request) => client.evaluate(prepareEvaluationRequest(request), options.signal ? { signal: options.signal } : {}), {
+        ...(options.concurrency === undefined ? {} : { concurrency: options.concurrency }),
+        ...(options.signal === undefined ? {} : { signal: options.signal }),
+        stopOn: stopsBatch,
+    });
+    return summarize(results, Math.round(performance.now() - start));
+}
+/**
+ * Split a request that asks more questions than one request may carry into chunks of at most `maxQuestions`. Sharing
+ * one state across several questions is one request; asking more than the per-request limit is the only reason to fan
+ * out, and the state is repeated in each chunk. The order of `questions` is preserved. A pure splitter: admission still
+ * happens once per chunk, in `evaluate` or `evaluateMany`, so what one accepts the others accept.
+ */
+export function chunkEvaluationRequest(request, options = {}) {
+    const limit = Math.max(1, Math.floor(options.maxQuestions ?? DEFAULT_MAX_QUESTIONS));
+    const questions = request.questions;
+    const entries = questions && typeof questions === "object" && !Array.isArray(questions) ? Object.entries(questions) : undefined;
+    // Anything that is not a plain question map, or that already fits, is one chunk and is validated later.
+    if (!entries || entries.length <= limit)
+        return [request];
+    const chunks = [];
+    for (let index = 0; index < entries.length; index += limit) {
+        const questions = Object.fromEntries(entries.slice(index, index + limit));
+        chunks.push({ ...request, questions });
+    }
+    return chunks;
+}
+/**
+ * Ask any number of questions about one state: chunk to the per-request limit, fan out, and merge the answers, usage,
+ * and model. Use this when one coherent state carries many independent questions; use `evaluate` for one request.
+ */
+export async function evaluateAll(client, request, options = {}) {
+    const chunks = chunkEvaluationRequest(request, options.maxQuestions === undefined ? {} : { maxQuestions: options.maxQuestions });
+    return evaluateMany(client, chunks, options);
+}

package/node_modules/pi-typesafe/dist/calibrate.d.ts

@@ -0,0 +1,106 @@
+/**
+ * A judge-tuning kit: label a set of cases, score them with Jev, and read off AUC and threshold behaviour. It carries
+ * no domain knowledge — a case is anything a scorer can turn into a number — so the same toolkit fits an action guard,
+ * a triage rule, or a prose check. `scripts/calibrate-action.mjs` in pi-warden is the worked example it came from.
+ */
+/** One labelled observation: the truth about the case, and the number the judge assigned to it. */
+export interface ScoredSample {
+    readonly label: boolean;
+    readonly score: number;
+    /** Optional name; used in the missed and flagged listings. */
+    readonly id?: string;
+}
+/** Outcome counts for one threshold. `precision` and `recall` are undefined when their denominator is empty. */
+export interface ThresholdRow {
+    readonly threshold: number;
+    readonly flagged: number;
+    readonly tp: number;
+    readonly fp: number;
+    readonly fn: number;
+    readonly tn: number;
+    readonly precision?: number;
+    readonly recall?: number;
+    /** Share of all cases the threshold selects. */
+    readonly flagRate: number;
+}
+export interface Calibration {
+    readonly name: string;
+    readonly scored: number;
+    readonly positives: number;
+    readonly negatives: number;
+    readonly errors: number;
+    /** Rank-based AUC (Mann–Whitney, ties count half); undefined when one class is empty. */
+    readonly auc?: number;
+    readonly rows: readonly ThresholdRow[];
+    /** The lowest threshold meeting the requested precision and recall floors, when one exists. */
+    readonly recommended?: ThresholdRow;
+    /** Positive cases the recommended threshold misses. */
+    readonly missed: readonly ScoredSample[];
+    /** Negative cases the recommended threshold flags. */
+    readonly flagged: readonly ScoredSample[];
+}
+export interface CalibrateOptions {
+    /** Thresholds to evaluate. Default: every distinct score, ascending (at most 64 rows). */
+    thresholds?: readonly number[];
+    /** Precision floor for the recommendation. */
+    minPrecision?: number;
+    /** Recall floor for the recommendation. */
+    minRecall?: number;
+    /** Cases that could not be scored; reported and excluded from the metrics. */
+    errors?: number;
+}
+export declare function auc(samples: readonly ScoredSample[]): number | undefined;
+/** Counts at one threshold: a case is flagged when its score is at least the threshold. */
+export declare function metricsAt(samples: readonly ScoredSample[], threshold: number): ThresholdRow;
+export declare function sweep(samples: readonly ScoredSample[], thresholds: readonly number[]): ThresholdRow[];
+/** The distinct scores, ascending, as a threshold grid: every point where the counts can change. */
+export declare function defaultThresholds(samples: readonly ScoredSample[], limit?: number): number[];
+/**
+ * The lowest threshold that clears the precision and recall floors. With no floors, the best F1 among the rows;
+ * undefined when nothing clears them.
+ */
+export declare function pickThreshold(rows: readonly ThresholdRow[], options?: {
+    minPrecision?: number;
+    minRecall?: number;
+}): ThresholdRow | undefined;
+/** Label, score, and read the numbers: AUC, the threshold sweep, and one recommendation. */
+export declare function calibrate(name: string, samples: readonly ScoredSample[], options?: CalibrateOptions): Calibration;
+/** Plain text, no colour: safe to write to a report file or a log. */
+export declare function formatCalibration(calibration: Calibration): string;
+/** One labelled replay case: the truth, plus whatever the scorer needs to judge it. */
+export interface ReplayCase<T> {
+    readonly id: string;
+    readonly label: boolean;
+    readonly data: T;
+}
+export interface ReplayResult<T> {
+    readonly id: string;
+    readonly label: boolean;
+    readonly data: T;
+    /** The judge's number, absent when the case could not be scored. */
+    readonly score?: number;
+    /** The failure message, absent on success. Carries no upstream body. */
+    readonly error?: string;
+    /** True when the case was never submitted (abort or a stopped batch). */
+    readonly skipped: boolean;
+}
+export interface ReplayOptions {
+    /** Cases in flight at once. Default: DEFAULT_CONCURRENCY (4). */
+    concurrency?: number;
+    signal?: AbortSignal;
+    /** Stop launching new cases once this returns true for a failure, e.g. a `budget` error. */
+    stopOn?: (error: unknown) => boolean;
+    /** Turns a thrown scorer error into the reported message. Default: the error's own message. */
+    describeError?: (error: unknown) => string;
+}
+/**
+ * Replay labelled cases through a scorer with bounded concurrency, keeping order and capturing per-case failures.
+ * The scorer is usually one Jev question; a thrown error is recorded rather than aborting the run, so one bad case
+ * cannot destroy a long calibration. Results feed straight into `samplesOf` and `calibrate`.
+ */
+export declare function replay<T>(cases: readonly ReplayCase<T>[], score: (data: T, index: number) => Promise<number>, options?: ReplayOptions): Promise<ReplayResult<T>[]>;
+/** The scored cases of a replay, in replay order. Unscored cases are excluded and counted as errors. */
+export declare function samplesOf<T>(results: readonly ReplayResult<T>[]): {
+    samples: ScoredSample[];
+    errors: number;
+};

package/node_modules/pi-typesafe/dist/calibrate.js

@@ -0,0 +1,157 @@
+import { fanOut } from "./batch.js";
+export function auc(samples) {
+    const positives = samples.filter(sample => sample.label).map(sample => sample.score);
+    const negatives = samples.filter(sample => !sample.label).map(sample => sample.score);
+    if (!positives.length || !negatives.length)
+        return undefined;
+    let wins = 0;
+    for (const positive of positives) {
+        for (const negative of negatives)
+            wins += positive > negative ? 1 : positive === negative ? 0.5 : 0;
+    }
+    return wins / (positives.length * negatives.length);
+}
+function ratio(numerator, denominator) {
+    return denominator ? numerator / denominator : undefined;
+}
+/** Counts at one threshold: a case is flagged when its score is at least the threshold. */
+export function metricsAt(samples, threshold) {
+    let tp = 0, fp = 0, fn = 0, tn = 0;
+    for (const sample of samples) {
+        const flagged = sample.score >= threshold;
+        if (flagged && sample.label)
+            tp++;
+        else if (flagged)
+            fp++;
+        else if (sample.label)
+            fn++;
+        else
+            tn++;
+    }
+    const flagged = tp + fp;
+    const precision = ratio(tp, tp + fp);
+    const recall = ratio(tp, tp + fn);
+    return {
+        threshold,
+        flagged,
+        tp,
+        fp,
+        fn,
+        tn,
+        ...(precision === undefined ? {} : { precision }),
+        ...(recall === undefined ? {} : { recall }),
+        flagRate: samples.length ? flagged / samples.length : 0,
+    };
+}
+export function sweep(samples, thresholds) {
+    return thresholds.map(threshold => metricsAt(samples, threshold));
+}
+/** The distinct scores, ascending, as a threshold grid: every point where the counts can change. */
+export function defaultThresholds(samples, limit = 64) {
+    const distinct = [...new Set(samples.map(sample => sample.score))].sort((a, b) => a - b);
+    if (distinct.length <= limit)
+        return distinct;
+    const step = (distinct.length - 1) / (limit - 1);
+    return Array.from({ length: limit }, (_, index) => distinct[Math.round(index * step)]);
+}
+/**
+ * The lowest threshold that clears the precision and recall floors. With no floors, the best F1 among the rows;
+ * undefined when nothing clears them.
+ */
+export function pickThreshold(rows, options = {}) {
+    const { minPrecision, minRecall } = options;
+    if (minPrecision === undefined && minRecall === undefined) {
+        let best;
+        let bestF1 = -1;
+        for (const row of rows) {
+            if (row.precision === undefined || row.recall === undefined)
+                continue;
+            const f1 = row.precision + row.recall === 0 ? 0 : 2 * row.precision * row.recall / (row.precision + row.recall);
+            if (f1 > bestF1) {
+                bestF1 = f1;
+                best = row;
+            }
+        }
+        return best;
+    }
+    const candidates = rows
+        .filter(row => (minPrecision === undefined || (row.precision ?? 0) >= minPrecision) && (minRecall === undefined || (row.recall ?? 0) >= minRecall))
+        .sort((a, b) => a.threshold - b.threshold);
+    return candidates[0];
+}
+/** Label, score, and read the numbers: AUC, the threshold sweep, and one recommendation. */
+export function calibrate(name, samples, options = {}) {
+    const thresholds = options.thresholds ?? defaultThresholds(samples);
+    const rows = sweep(samples, thresholds);
+    const recommendation = pickThreshold(rows, {
+        ...(options.minPrecision === undefined ? {} : { minPrecision: options.minPrecision }),
+        ...(options.minRecall === undefined ? {} : { minRecall: options.minRecall }),
+    });
+    const rank = auc(samples);
+    const threshold = recommendation?.threshold;
+    return {
+        name,
+        scored: samples.length,
+        positives: samples.filter(sample => sample.label).length,
+        negatives: samples.filter(sample => !sample.label).length,
+        errors: options.errors ?? 0,
+        ...(rank === undefined ? {} : { auc: rank }),
+        rows,
+        ...(recommendation === undefined ? {} : { recommended: recommendation }),
+        missed: threshold === undefined ? [] : samples.filter(sample => sample.label && sample.score < threshold),
+        flagged: threshold === undefined ? [] : samples.filter(sample => !sample.label && sample.score >= threshold),
+    };
+}
+const percent = (value) => value === undefined ? "-" : `${(value * 100).toFixed(0)}%`;
+/** Plain text, no colour: safe to write to a report file or a log. */
+export function formatCalibration(calibration) {
+    const lines = [
+        `${calibration.name}: ${calibration.scored} scored, ${calibration.positives} positives, ${calibration.negatives} negatives${calibration.errors ? `, ${calibration.errors} errors` : ""}`,
+        `AUC ${calibration.auc === undefined ? "-" : calibration.auc.toFixed(3)}`,
+        "threshold  flagged  TP  FP  FN  TN  precision  recall",
+    ];
+    for (const row of calibration.rows) {
+        lines.push(`${row.threshold.toFixed(2).padStart(9)}  ${String(row.flagged).padStart(7)}  ${String(row.tp).padStart(2)}  ${String(row.fp).padStart(2)}  ${String(row.fn).padStart(2)}  ${String(row.tn).padStart(2)}  ${percent(row.precision).padStart(9)}  ${percent(row.recall).padStart(6)}`);
+    }
+    const recommended = calibration.recommended;
+    lines.push(recommended === undefined
+        ? "recommended: none (no threshold clears the floors)"
+        : `recommended ${recommended.threshold.toFixed(2)}: precision ${percent(recommended.precision)}, recall ${percent(recommended.recall)}, flags ${percent(recommended.flagRate)}`);
+    if (calibration.missed.length)
+        lines.push(`missed positives (${calibration.missed.length}): ${calibration.missed.map(sample => sample.id ?? sample.score.toFixed(2)).join(", ").slice(0, 300)}`);
+    if (calibration.flagged.length)
+        lines.push(`flagged negatives (${calibration.flagged.length}): ${calibration.flagged.map(sample => sample.id ?? sample.score.toFixed(2)).join(", ").slice(0, 300)}`);
+    return lines.join("\n");
+}
+/**
+ * Replay labelled cases through a scorer with bounded concurrency, keeping order and capturing per-case failures.
+ * The scorer is usually one Jev question; a thrown error is recorded rather than aborting the run, so one bad case
+ * cannot destroy a long calibration. Results feed straight into `samplesOf` and `calibrate`.
+ */
+export async function replay(cases, score, options = {}) {
+    const settled = await fanOut(cases, (item, index) => score(item.data, index), {
+        ...(options.concurrency === undefined ? {} : { concurrency: options.concurrency }),
+        ...(options.signal === undefined ? {} : { signal: options.signal }),
+        ...(options.stopOn === undefined ? {} : { stopOn: options.stopOn }),
+    });
+    return settled.map((result, index) => {
+        const item = cases[index];
+        if (result.ok)
+            return { id: item.id, label: item.label, data: item.data, score: result.value, skipped: false };
+        const error = options.describeError ? options.describeError(result.error) : result.error instanceof Error ? result.error.message : "The scorer failed.";
+        return { id: item.id, label: item.label, data: item.data, error, skipped: result.skipped };
+    });
+}
+/** The scored cases of a replay, in replay order. Unscored cases are excluded and counted as errors. */
+export function samplesOf(results) {
+    const samples = [];
+    let errors = 0;
+    for (const result of results) {
+        if (result.score === undefined || !Number.isFinite(result.score)) {
+            errors++;
+            continue;
+        }
+        samples.push({ label: result.label, score: result.score, id: result.id });
+    }
+    return { samples, errors };
+}

package/node_modules/pi-typesafe/dist/client.d.ts

@@ -0,0 +1,69 @@
+import type { Fetch, Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk";
+import type { BatchEvaluation, BatchOptions } from "./batch.js";
+import type { BlockedCap, SpendCaps, UsageLedger, UsageReport } from "./usage.js";
+export interface TypeSafeOptions {
+    /** Defaults to TYPESAFE_API_KEY, then the key saved by `/typesafe login`; never returned. */
+    apiKey?: string;
+    /** Defaults to jev-latest. No model is inferred from submitted content. */
+    model?: string;
+    /** Per request. Default: 15 seconds. No automatic retries. */
+    timeoutMs?: number;
+    /** UTF-8 JSON bytes, including model/questions. Default: 64 KiB. Not a token limit. */
+    maxInputBytes?: number;
+    /** Attempts per client instance, including failed network requests. Default: 20. */
+    maxRequests?: number;
+    /** Requests per local day, counted across processes and restarts. Unlimited by default. */
+    maxRequestsPerDay?: number;
+    /** Input tokens per local day. Unlimited by default. */
+    maxInputTokensPerDay?: number;
+    /** Estimated spend per local day, in US dollars. Unlimited by default. */
+    maxUsdPerDay?: number;
+    /** Price used for the cost estimate and the USD cap. Default: DEFAULT_USD_PER_MTOK. */
+    usdPerMTok?: number;
+    /** The usage ledger; defaults to the store next to the key. Injected by tests. */
+    ledger?: UsageLedger;
+    /** Transport injection for extension authors and offline tests. */
+    fetch?: Fetch;
+}
+export interface EvaluationOptions {
+    signal?: AbortSignal;
+}
+export type Evaluation<Q extends Questions> = SystemOneResult<Q> & {
+    readonly elapsedMs: number;
+};
+/** Session counters for one client instance, plus the cost estimate they add up to. */
+export interface UsageSnapshot {
+    readonly requestsStarted: number;
+    readonly requestsSucceeded: number;
+    readonly requestsFailed: number;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly estimatedUsd: number;
+}
+/** Session counters, today's persisted counters, the caps in force, and the cap that is currently reached. */
+export interface SpendReport {
+    readonly session: UsageSnapshot;
+    readonly today: UsageReport;
+    readonly caps: SpendCaps;
+    readonly usdPerMTok: number;
+    readonly blocked?: BlockedCap;
+}
+export interface TypeSafe {
+    evaluate<Q extends Questions>(request: SystemOneRequest<Q>, options?: EvaluationOptions): Promise<Evaluation<Q>>;
+    /** Several requests with bounded concurrency; answers, usage, and model merged. Never throws. */
+    evaluateMany<Q extends Questions>(requests: readonly SystemOneRequest<Q>[], options?: BatchOptions): Promise<BatchEvaluation<Q>>;
+    /** One state, any number of questions: chunk to the per-request limit, then fan out. Never throws. */
+    evaluateAll<Q extends Questions>(request: SystemOneRequest<Q>, options?: BatchOptions & {
+        maxQuestions?: number;
+    }): Promise<BatchEvaluation<Q>>;
+    /** Model names available to the account. Verifies the key; does not count toward maxRequests. */
+    listModels(options?: EvaluationOptions): Promise<string[]>;
+    /** This client's session counters. */
+    getUsage(): UsageSnapshot;
+    /** Session counters, today's persisted totals, and the caps that stop the next request. */
+    getSpend(): SpendReport;
+}
+/** Default attempts per client instance; the extension quotes the same number in its consent copy. */
+export declare const DEFAULT_MAX_REQUESTS = 20;
+/** A bounded, server-side TypeSafe client independent of Pi's runtime. */
+export declare function createTypeSafe(options?: TypeSafeOptions): TypeSafe;