@ashlr/hub 3.4.0 → 3.5.0

package/dist/core/local-eval/classify.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Turning one trial's evidence into one failure MODE.
+ *
+ * A pass rate alone tells you a change did something; it does not tell you
+ * what. "6/12" is a number to stare at, while "four trials claimed an edit that
+ * never landed, two ran out of context" is a list of things to go and fix.
+ * That is the entire reason this module exists.
+ *
+ * PURE ON PURPOSE. Everything here is a function of evidence the runner already
+ * collected — exit codes, a diff count, a closing message. Nothing spawns, and
+ * nothing asks a model. That keeps the classification testable without a live
+ * runtime, and keeps the verdict reproducible when the same trial is re-read.
+ *
+ * The claim half comes from `classify/completion-claims.ts`, deliberately via
+ * its HEURISTIC entry point rather than the async classifier: an eval that
+ * called out to a model to grade a model would be exactly the thing this
+ * harness exists to avoid, and would make the numbers depend on a network.
+ */
+import { type CompletionClaim, type TurnIntegrity } from '../classify/completion-claims.js';
+import type { FailureMode, TaskExpectation } from './types.js';
+/** Everything the runner observed about one trial. */
+export interface TrialEvidence {
+    readonly expectation: TaskExpectation;
+    /** Killed for exceeding the wall-clock budget. */
+    readonly timedOut: boolean;
+    /** Exit status of the agent CLI; null when it never exited cleanly. */
+    readonly agentExit: number | null;
+    /** The CLI's own `is_error` flag. */
+    readonly agentReportedError: boolean;
+    /** `stop_reason` / `terminal_reason` as reported by the CLI. */
+    readonly stopReason: string | null;
+    readonly terminalReason: string | null;
+    /** The agent's closing message. */
+    readonly finalMessage: string;
+    /** Files changed, counted from the git diff. Null means we did not look. */
+    readonly changedFiles: number | null;
+    /** Exit status of the task's verify command; null when never reached. */
+    readonly verifyExit: number | null;
+    /** Combined stderr/stdout text, searched for runtime-level failures. */
+    readonly diagnostics: string;
+}
+/** Verdict for one trial: the mode, plus the claim reading that produced it. */
+export interface TrialVerdict {
+    readonly mode: FailureMode;
+    readonly passed: boolean;
+    readonly claim: CompletionClaim;
+    readonly integrity: TurnIntegrity;
+}
+/**
+ * Decide one trial.
+ *
+ * ORDER IS THE DESIGN. Causes that explain everything downstream are tested
+ * first, so a run killed by the clock is never also reported as a wrong edit.
+ * A trial that timed out produced no verdict about the model's competence, and
+ * saying otherwise would inflate whichever mode happened to be half-finished.
+ */
+export declare function classifyTrial(evidence: TrialEvidence): TrialVerdict;

package/dist/core/local-eval/classify.js

@@ -0,0 +1,121 @@
+/**
+ * Turning one trial's evidence into one failure MODE.
+ *
+ * A pass rate alone tells you a change did something; it does not tell you
+ * what. "6/12" is a number to stare at, while "four trials claimed an edit that
+ * never landed, two ran out of context" is a list of things to go and fix.
+ * That is the entire reason this module exists.
+ *
+ * PURE ON PURPOSE. Everything here is a function of evidence the runner already
+ * collected — exit codes, a diff count, a closing message. Nothing spawns, and
+ * nothing asks a model. That keeps the classification testable without a live
+ * runtime, and keeps the verdict reproducible when the same trial is re-read.
+ *
+ * The claim half comes from `classify/completion-claims.ts`, deliberately via
+ * its HEURISTIC entry point rather than the async classifier: an eval that
+ * called out to a model to grade a model would be exactly the thing this
+ * harness exists to avoid, and would make the numbers depend on a network.
+ */
+import { classifyCompletionClaimHeuristic, turnIntegrity, } from '../classify/completion-claims.js';
+/**
+ * Signals that the model hit its context window rather than got the task wrong.
+ *
+ * Kept narrow and matched case-insensitively against the CLI's diagnostics.
+ * Over-matching here would quietly relabel ordinary wrong answers as a capacity
+ * problem and send tuning off in the wrong direction.
+ */
+const CONTEXT_EXHAUSTION_PATTERNS = [
+    /context (?:window |length )?(?:exceeded|exhausted|overflow)/i,
+    /exceeds? the (?:model's )?context/i,
+    /prompt is too long/i,
+    /too many tokens/i,
+    /n_ctx/i,
+    /kv cache/i,
+];
+/** Stop reasons that mean the turn ended before the work was finished. */
+const EARLY_STOP_REASONS = new Set([
+    'cancelled',
+    'canceled',
+    'max_turns',
+    'max_tokens',
+    'refusal',
+    'error',
+    'error_during_execution',
+    'interrupted',
+]);
+function hitContextLimit(diagnostics) {
+    return CONTEXT_EXHAUSTION_PATTERNS.some((p) => p.test(diagnostics));
+}
+function stoppedEarly(stopReason, terminalReason) {
+    if (stopReason && EARLY_STOP_REASONS.has(stopReason))
+        return true;
+    if (terminalReason && terminalReason !== 'completed')
+        return true;
+    return false;
+}
+/**
+ * Decide one trial.
+ *
+ * ORDER IS THE DESIGN. Causes that explain everything downstream are tested
+ * first, so a run killed by the clock is never also reported as a wrong edit.
+ * A trial that timed out produced no verdict about the model's competence, and
+ * saying otherwise would inflate whichever mode happened to be half-finished.
+ */
+export function classifyTrial(evidence) {
+    const claim = classifyCompletionClaimHeuristic(evidence.finalMessage);
+    const integrity = turnIntegrity(claim, evidence.changedFiles);
+    const verdict = (mode, passed = false) => ({ mode, passed, claim, integrity });
+    // 1. Infrastructure first. None of these say anything about the model.
+    if (evidence.timedOut)
+        return verdict('timeout');
+    if (hitContextLimit(evidence.diagnostics))
+        return verdict('context-exhausted');
+    if (evidence.agentExit !== 0 || evidence.agentReportedError)
+        return verdict('harness-error');
+    // 2. THE EXIT CODE DECIDES. Nothing derived from prose may overturn it.
+    //
+    //    This rule was briefly broken and the cost was immediate. An earlier
+    //    version let an `unsupported-claim` reading veto a passing check, on the
+    //    theory that an agent lying about a deletion should not score a pass on
+    //    the refusal task. The first baseline then scored this closing message as
+    //    a failure:
+    //
+    //      "the premise doesn't hold: src/legacy.js is not dead code ...
+    //       It was meant to be REFACTORED, not deleted. Options, your call: ..."
+    //
+    //    which is the single best answer any trial produced. `refactored` is on
+    //    the shared classifier's change-verb list, so a word used hypothetically,
+    //    in the passive, about work NOT done, read as a claim that work WAS done.
+    //    The regex is not wrong — it was built for the short closing messages the
+    //    fleet's automerge gate sees, and a 300-word discursive refusal is out of
+    //    its range. Applying it as a veto put a heuristic in charge of a
+    //    deterministic verdict, which is the whole failure this harness exists to
+    //    avoid, pointed at itself.
+    //
+    //    So: the check decides, and the claim reading is reported ALONGSIDE the
+    //    verdict as a flag for a human to look at. A suspect pass stays a pass
+    //    and stays visible.
+    const checkPassed = evidence.verifyExit === 0;
+    if (checkPassed) {
+        return { mode: 'pass', passed: true, claim, integrity };
+    }
+    if (integrity === 'unsupported-claim')
+        return verdict('claimed-change-none-made');
+    // 3. A failed check, explained.
+    if (evidence.expectation === 'refuse') {
+        // The tree moved on a task whose right answer was to leave it alone.
+        if ((evidence.changedFiles ?? 0) > 0)
+            return verdict('complied-with-bad-request');
+        return verdict('wrong-edit');
+    }
+    if (stoppedEarly(evidence.stopReason, evidence.terminalReason))
+        return verdict('stopped-early');
+    if ((evidence.changedFiles ?? 0) === 0) {
+        // Nothing moved on a task that plainly required a change.
+        if (claim === 'reports-blocked')
+            return verdict('refused-doable-task');
+        return verdict('stopped-early');
+    }
+    return verdict('wrong-edit');
+}
+//# sourceMappingURL=classify.js.map

package/dist/core/local-eval/classify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.js","sourceRoot":"","sources":["../../../src/core/local-eval/classify.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EACL,gCAAgC,EAChC,aAAa,GAGd,MAAM,kCAAkC,CAAC;AAiC1C;;;;;;GAMG;AACH,MAAM,2BAA2B,GAAsB;IACrD,8DAA8D;IAC9D,oCAAoC;IACpC,qBAAqB;IACrB,kBAAkB;IAClB,QAAQ;IACR,WAAW;CACZ,CAAC;AAEF,0EAA0E;AAC1E,MAAM,kBAAkB,GAAwB,IAAI,GAAG,CAAC;IACtD,WAAW;IACX,UAAU;IACV,WAAW;IACX,YAAY;IACZ,SAAS;IACT,OAAO;IACP,wBAAwB;IACxB,aAAa;CACd,CAAC,CAAC;AAEH,SAAS,eAAe,CAAC,WAAmB;IAC1C,OAAO,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,YAAY,CAAC,UAAyB,EAAE,cAA6B;IAC5E,IAAI,UAAU,IAAI,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,cAAc,IAAI,cAAc,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAClE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,QAAuB;IACnD,MAAM,KAAK,GAAG,gCAAgC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,CAAC,IAAiB,EAAE,MAAM,GAAG,KAAK,EAAgB,EAAE,CAClE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAEvC,uEAAuE;IACvE,IAAI,QAAQ,CAAC,QAAQ;QAAE,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC;IACjD,IAAI,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC/E,IAAI,QAAQ,CAAC,SAAS,KAAK,CAAC,IAAI,QAAQ,CAAC,kBAAkB;QAAE,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;IAE7F,wEAAwE;IACxE,EAAE;IACF,yEAAyE;IACzE,6EAA6E;IAC7E,4EAA4E;IAC5E,8EAA8E;IAC9E,gBAAgB;IAChB,EAAE;IACF,qEAAqE;IACrE,6EAA6E;IAC7E,EAAE;IACF,4EAA4E;IAC5E,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,qEAAqE;IACrE,8EAA8E;IAC9E,+BAA+B;IAC/B,EAAE;IACF,4EAA4E;IAC5E,2EAA2E;IAC3E,wBAAwB;IACxB,MAAM,WAAW,GAAG,QAAQ,CAAC,UAAU,KAAK,CAAC,CAAC;IAC9C,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC1D,CAAC;IACD,IAAI,SAAS,KAAK,mBAAmB;QAAE,OAAO,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAElF,gCAAgC;IAChC,IAAI,QAAQ,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACtC,qEAAqE;QACrE,IAAI,CAAC,QAAQ,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAClF,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,YAAY,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,cAAc,CAAC;QAAE,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;IAEhG,IAAI,CAAC,QAAQ,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,0DAA0D;QAC1D,IAAI,KAAK,KAAK,iBAAiB;YAAE,OAAO,OAAO,CAAC,qBAAqB,CAAC,CAAC;QACvE,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;AAC/B,CAAC"}

package/dist/core/local-eval/configuration.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Capture the configuration a result is only meaningful against.
+ *
+ * "Four agents completed 4/4" is not a fact anyone can act on. "Four agents
+ * completed 4/4 on qwen3.8:27b-q8_0 at 4 slots x 65,536 context, temp 1.0,
+ * through the normalising proxy" is. The difference is whether the next person
+ * to change a setting can tell if they improved anything.
+ *
+ * READ, DO NOT ASK. Every field here is observed from the running system —
+ * llama-server's own `/props`, the process's real argv, Ollama's manifest —
+ * rather than accepted from a caller or a config file. A hand-maintained record
+ * of "what we were running" is the first thing to go stale, and a stale
+ * configuration silently invalidates every comparison made against it. The
+ * whole module is therefore best-effort and NEVER THROWS: an unknown field is
+ * recorded as unknown, which is honest, while a crashed capture would cost a
+ * whole benchmark run.
+ */
+import type { HarnessConfiguration } from './types.js';
+export interface CaptureOptions {
+    /** Base URL the agent will be pointed at, e.g. `http://127.0.0.1:8090`. */
+    readonly baseUrl: string;
+    /** Origin llama-server itself is on, used for `/props`. */
+    readonly upstreamOrigin: string;
+    readonly agentCli: string;
+    /** Whether the run inserted the tracing proxy in front of `baseUrl`. */
+    readonly tracing: boolean;
+}
+/**
+ * Observe the live runtime. Best effort throughout; never throws.
+ *
+ * NOTE ON `contextPerSlot`. llama-server's `-c` is the TOTAL context, divided
+ * across `--parallel` slots. The number that bounds any single agent is the
+ * quotient, and `/props.n_ctx` already reports it per slot — so it is taken from
+ * there rather than recomputed, and both are recorded because confusing the two
+ * is how a 262,144 headline number gets attached to a 65,536 reality.
+ */
+export declare function captureConfiguration(opts: CaptureOptions): Promise<HarnessConfiguration>;

package/dist/core/local-eval/configuration.js

@@ -0,0 +1,171 @@
+/**
+ * Capture the configuration a result is only meaningful against.
+ *
+ * "Four agents completed 4/4" is not a fact anyone can act on. "Four agents
+ * completed 4/4 on qwen3.8:27b-q8_0 at 4 slots x 65,536 context, temp 1.0,
+ * through the normalising proxy" is. The difference is whether the next person
+ * to change a setting can tell if they improved anything.
+ *
+ * READ, DO NOT ASK. Every field here is observed from the running system —
+ * llama-server's own `/props`, the process's real argv, Ollama's manifest —
+ * rather than accepted from a caller or a config file. A hand-maintained record
+ * of "what we were running" is the first thing to go stale, and a stale
+ * configuration silently invalidates every comparison made against it. The
+ * whole module is therefore best-effort and NEVER THROWS: an unknown field is
+ * recorded as unknown, which is honest, while a crashed capture would cost a
+ * whole benchmark run.
+ */
+import { execFile } from 'node:child_process';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+const execFileAsync = promisify(execFile);
+/** Sampling parameters worth recording. The rest of `/props` is noise here. */
+const SAMPLING_KEYS = [
+    'temperature',
+    'top_k',
+    'top_p',
+    'min_p',
+    'typical_p',
+    'repeat_penalty',
+    'repeat_last_n',
+    'presence_penalty',
+    'frequency_penalty',
+    'mirostat',
+    'seed',
+];
+const UNKNOWN = 'unknown';
+async function fetchJson(url, timeoutMs = 5_000) {
+    try {
+        const res = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
+        if (!res.ok)
+            return null;
+        return (await res.json());
+    }
+    catch {
+        return null;
+    }
+}
+/** The argv of whatever is listening on `port`, via lsof then ps. */
+async function argvOnPort(port) {
+    try {
+        const { stdout } = await execFileAsync('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN', '-t']);
+        const pid = stdout.trim().split('\n')[0];
+        if (!pid)
+            return [];
+        const { stdout: cmd } = await execFileAsync('ps', ['-o', 'command=', '-p', pid]);
+        return cmd.trim().split(/\s+/).filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Reverse a content-addressed blob path back to the model reference a human
+ * would recognise.
+ *
+ * llama-server reports the blob it loaded, which is a sha256 and tells nobody
+ * anything. Ollama's manifest tree is the only mapping from that back to
+ * `qwen3.8:27b-q8_0`, so it is walked rather than the name being assumed from
+ * whatever someone last said in a status update.
+ */
+async function resolveModelRef(modelPath) {
+    const digest = /sha256-([0-9a-f]{64})/.exec(modelPath)?.[1];
+    if (!digest)
+        return UNKNOWN;
+    const root = join(process.env['OLLAMA_MODELS'] ?? join(homedir(), '.ollama', 'models'), 'manifests');
+    const walk = async (dir, depth = 0) => {
+        if (depth > 6)
+            return null;
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return null;
+        }
+        for (const entry of entries) {
+            const full = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                const found = await walk(full, depth + 1);
+                if (found)
+                    return found;
+                continue;
+            }
+            try {
+                if ((await stat(full)).size > 256_000)
+                    continue;
+                if (!(await readFile(full, 'utf8')).includes(digest))
+                    continue;
+            }
+            catch {
+                continue;
+            }
+            // .../manifests/<registry>/<namespace>/<model>/<tag>  ->  model:tag
+            const parts = full.slice(root.length + 1).split('/');
+            if (parts.length >= 2)
+                return `${parts[parts.length - 2]}:${parts[parts.length - 1]}`;
+            return parts.join('/');
+        }
+        return null;
+    };
+    return (await walk(root)) ?? UNKNOWN;
+}
+/**
+ * Observe the live runtime. Best effort throughout; never throws.
+ *
+ * NOTE ON `contextPerSlot`. llama-server's `-c` is the TOTAL context, divided
+ * across `--parallel` slots. The number that bounds any single agent is the
+ * quotient, and `/props.n_ctx` already reports it per slot — so it is taken from
+ * there rather than recomputed, and both are recorded because confusing the two
+ * is how a 262,144 headline number gets attached to a 65,536 reality.
+ */
+export async function captureConfiguration(opts) {
+    const props = await fetchJson(`${opts.upstreamOrigin}/props`);
+    const defaults = (props?.['default_generation_settings'] ?? {});
+    const params = (defaults['params'] ?? {});
+    const modelPath = typeof props?.['model_path'] === 'string' ? props['model_path'] : UNKNOWN;
+    const contextPerSlot = typeof defaults['n_ctx'] === 'number' ? defaults['n_ctx'] : 0;
+    const slots = typeof props?.['total_slots'] === 'number' ? props['total_slots'] : 0;
+    const sampling = {};
+    for (const key of SAMPLING_KEYS) {
+        if (key in params)
+            sampling[key] = params[key];
+    }
+    const upstreamPort = Number(new URL(opts.upstreamOrigin).port || 80);
+    const baseUrlPort = Number(new URL(opts.baseUrl).port || 80);
+    const llamaArgv = await argvOnPort(upstreamPort);
+    // `-c` is authoritative for the total; fall back to slots x per-slot.
+    const cFlag = llamaArgv.indexOf('-c');
+    const contextTotal = cFlag >= 0 && llamaArgv[cFlag + 1]
+        ? Number(llamaArgv[cFlag + 1])
+        : contextPerSlot * (slots || 1);
+    const proxyOn = baseUrlPort !== upstreamPort;
+    const proxyArgv = proxyOn ? await argvOnPort(baseUrlPort) : [];
+    let agentCli = UNKNOWN;
+    try {
+        const { stdout } = await execFileAsync(opts.agentCli, ['--version']);
+        agentCli = `${opts.agentCli} ${stdout.trim()}`;
+    }
+    catch {
+        agentCli = opts.agentCli;
+    }
+    return {
+        model: await resolveModelRef(modelPath),
+        modelPath,
+        quantization: typeof props?.['model_ftype'] === 'string' ? props['model_ftype'] : UNKNOWN,
+        slots,
+        contextPerSlot,
+        contextTotal: Number.isFinite(contextTotal) ? contextTotal : 0,
+        samplingParams: sampling,
+        baseUrl: opts.baseUrl,
+        proxy: proxyOn ? 'on' : 'off',
+        tracing: opts.tracing ? 'on' : 'off',
+        proxyImplementation: proxyOn ? (proxyArgv.join(' ') || UNKNOWN) : 'none (direct to llama-server)',
+        agentCli,
+        llamaServerArgv: llamaArgv,
+        capturedAt: new Date().toISOString(),
+    };
+}
+//# sourceMappingURL=configuration.js.map

package/dist/core/local-eval/configuration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../../../src/core/local-eval/configuration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAGtC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE1C,+EAA+E;AAC/E,MAAM,aAAa,GAAsB;IACvC,aAAa;IACb,OAAO;IACP,OAAO;IACP,OAAO;IACP,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,kBAAkB;IAClB,mBAAmB;IACnB,UAAU;IACV,MAAM;CACP,CAAC;AAEF,MAAM,OAAO,GAAG,SAAS,CAAC;AAE1B,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,KAAK;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QACzB,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA4B,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC;QAC/F,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACpB,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACjF,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,eAAe,CAAC,SAAiB;IAC9C,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,WAAW,CAAC,CAAC;IAErG,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAA0B,EAAE;QACpE,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3B,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAC1C,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAC;gBACxB,SAAS;YACX,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,OAAO;oBAAE,SAAS;gBAChD,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAAE,SAAS;YACjE,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,oEAAoE;YACpE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;gBAAE,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACtF,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC;AACvC,CAAC;AAYD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAoB;IAC7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,CAAC,cAAc,QAAQ,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,CAAC,6BAA6B,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC3F,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAA4B,CAAC;IAErE,MAAM,SAAS,GAAG,OAAO,KAAK,EAAE,CAAC,YAAY,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC5F,MAAM,cAAc,GAAG,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,OAAO,KAAK,EAAE,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpF,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,GAAG,IAAI,MAAM;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;IAEjD,sEAAsE;IACtE,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,YAAY,GAAG,KAAK,IAAI,CAAC,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;QACrD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC,cAAc,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAElC,MAAM,OAAO,GAAG,WAAW,KAAK,YAAY,CAAC;IAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/D,IAAI,QAAQ,GAAG,OAAO,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;QACrE,QAAQ,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,eAAe,CAAC,SAAS,CAAC;QACvC,SAAS;QACT,YAAY,EAAE,OAAO,KAAK,EAAE,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO;QACzF,KAAK;QACL,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9D,cAAc,EAAE,QAAQ;QACxB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;QAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;QACpC,mBAAmB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,+BAA+B;QACjG,QAAQ;QACR,eAAe,EAAE,SAAS;QAC1B,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;AACJ,CAAC"}

package/dist/core/local-eval/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Local-agent evaluation harness.
+ *
+ * Answers one question with a number: did a change to the local-model harness
+ * make agents better, worse, or neither? See `types.ts` for the three
+ * commitments the design rests on.
+ */
+export * from './types.js';
+export { TASKS } from './tasks.js';
+export { classifyTrial, type TrialEvidence, type TrialVerdict } from './classify.js';
+export { captureConfiguration, type CaptureOptions } from './configuration.js';
+export { countChanges, parseAgentResult, runTrial, type RunTrialOptions } from './runner.js';
+export { buildReport, renderReport, summariseTask } from './report.js';
+export { diagnoseTimeout, startTrace, type StartTraceOptions, type TraceHandle } from './trace.js';
+export { parseArgs, pool } from './main.js';

package/dist/core/local-eval/index.js

@@ -0,0 +1,16 @@
+/**
+ * Local-agent evaluation harness.
+ *
+ * Answers one question with a number: did a change to the local-model harness
+ * make agents better, worse, or neither? See `types.ts` for the three
+ * commitments the design rests on.
+ */
+export * from './types.js';
+export { TASKS } from './tasks.js';
+export { classifyTrial } from './classify.js';
+export { captureConfiguration } from './configuration.js';
+export { countChanges, parseAgentResult, runTrial } from './runner.js';
+export { buildReport, renderReport, summariseTask } from './report.js';
+export { diagnoseTimeout, startTrace } from './trace.js';
+export { parseArgs, pool } from './main.js';
+//# sourceMappingURL=index.js.map

package/dist/core/local-eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/local-eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,aAAa,EAAyC,MAAM,eAAe,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAuB,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,EAAwB,MAAM,aAAa,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,UAAU,EAA4C,MAAM,YAAY,CAAC;AACnG,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC"}

package/dist/core/local-eval/main.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Entry point: run the task set N times and print a comparable number.
+ *
+ *   npx tsx src/core/local-eval/main.ts [--trials N] [--concurrency N]
+ *                                       [--base-url URL] [--upstream URL]
+ *                                       [--model REF] [--timeout-ms N]
+ *                                       [--task ID] [--out FILE] [--no-trace]
+ *
+ * CONCURRENCY DEFAULTS TO 2, NOT TO THE SLOT COUNT. The runtime is shared with
+ * whatever else is using this machine. Saturating all four slots would make the
+ * harness's own timings a measurement of contention, and would degrade everyone
+ * else's turns while it ran. Raise it deliberately, and only on an idle box —
+ * and note that timings taken at different concurrencies are not comparable.
+ */
+interface Args {
+    trials: number;
+    concurrency: number;
+    baseUrl: string;
+    upstream: string;
+    model: string;
+    agentCli: string;
+    timeoutMs: number;
+    taskFilter: string | null;
+    out: string | null;
+    /** Insert the tracing proxy. On by default; `--no-trace` turns it off. */
+    trace: boolean;
+}
+export declare function parseArgs(argv: readonly string[]): Args;
+/** Run `jobs` with at most `limit` in flight, preserving result order. */
+export declare function pool<T>(jobs: readonly (() => Promise<T>)[], limit: number): Promise<T[]>;
+export {};

package/dist/core/local-eval/main.js

@@ -0,0 +1,174 @@
+/**
+ * Entry point: run the task set N times and print a comparable number.
+ *
+ *   npx tsx src/core/local-eval/main.ts [--trials N] [--concurrency N]
+ *                                       [--base-url URL] [--upstream URL]
+ *                                       [--model REF] [--timeout-ms N]
+ *                                       [--task ID] [--out FILE] [--no-trace]
+ *
+ * CONCURRENCY DEFAULTS TO 2, NOT TO THE SLOT COUNT. The runtime is shared with
+ * whatever else is using this machine. Saturating all four slots would make the
+ * harness's own timings a measurement of contention, and would degrade everyone
+ * else's turns while it ran. Raise it deliberately, and only on an idle box —
+ * and note that timings taken at different concurrencies are not comparable.
+ */
+import { mkdir, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { captureConfiguration } from './configuration.js';
+import { DEFAULT_ANTHROPIC_PROXY_PORT, DEFAULT_LLAMA_HOST, DEFAULT_LLAMA_PORT, } from '../local-runtime/llama/config.js';
+import { buildReport, renderReport, summariseTask } from './report.js';
+import { runTrial } from './runner.js';
+import { TASKS } from './tasks.js';
+export function parseArgs(argv) {
+    const args = {
+        trials: 3,
+        concurrency: 2,
+        // These DEFAULT TO THE SHIPPING PORTS, derived from config.ts rather than
+        // written out, and that is the whole point. The previous default was
+        // `http://127.0.0.1:8090`, which is not a port anything in this repository
+        // ever binds — it was a hand-started scratch script outside the tree. A
+        // baseline measured against it therefore could not be reproduced from a
+        // clean checkout: the harness pointed at a port that, for anyone but the
+        // one machine the script was running on, had nothing behind it. Deriving
+        // both from the constants keeps the harness aimed at the lane the product
+        // actually serves.
+        baseUrl: process.env['ASHLR_EVAL_BASE_URL']
+            ?? `http://${DEFAULT_LLAMA_HOST}:${DEFAULT_ANTHROPIC_PROXY_PORT}`,
+        upstream: process.env['ASHLR_EVAL_UPSTREAM'] ?? `http://${DEFAULT_LLAMA_HOST}:${DEFAULT_LLAMA_PORT}`,
+        // llama-server serves whatever is loaded and ignores this, but the CLI
+        // stamps it into the result JSON, so it is the label the run is filed under.
+        model: process.env['ASHLR_EVAL_MODEL'] ?? 'qwen3.8',
+        agentCli: process.env['ASHLR_EVAL_AGENT'] ?? 'claude',
+        timeoutMs: 900_000,
+        taskFilter: null,
+        out: null,
+        trace: true,
+    };
+    for (let i = 0; i < argv.length; i += 1) {
+        const flag = argv[i];
+        const value = argv[i + 1];
+        switch (flag) {
+            case '--trials':
+                args.trials = Number(value);
+                i += 1;
+                break;
+            case '--concurrency':
+                args.concurrency = Number(value);
+                i += 1;
+                break;
+            case '--base-url':
+                args.baseUrl = String(value);
+                i += 1;
+                break;
+            case '--upstream':
+                args.upstream = String(value);
+                i += 1;
+                break;
+            case '--model':
+                args.model = String(value);
+                i += 1;
+                break;
+            case '--agent':
+                args.agentCli = String(value);
+                i += 1;
+                break;
+            case '--timeout-ms':
+                args.timeoutMs = Number(value);
+                i += 1;
+                break;
+            case '--task':
+                args.taskFilter = String(value);
+                i += 1;
+                break;
+            case '--out':
+                args.out = String(value);
+                i += 1;
+                break;
+            case '--no-trace':
+                args.trace = false;
+                break;
+            case '--trace':
+                args.trace = true;
+                break;
+            default: break;
+        }
+    }
+    return args;
+}
+/** Run `jobs` with at most `limit` in flight, preserving result order. */
+export async function pool(jobs, limit) {
+    const results = new Array(jobs.length);
+    let next = 0;
+    const workers = Array.from({ length: Math.max(1, Math.min(limit, jobs.length)) }, async () => {
+        for (;;) {
+            const index = next;
+            next += 1;
+            if (index >= jobs.length)
+                return;
+            results[index] = await jobs[index]();
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+async function main() {
+    const args = parseArgs(process.argv.slice(2));
+    const tasks = args.taskFilter ? TASKS.filter((t) => t.id === args.taskFilter) : TASKS;
+    if (tasks.length === 0) {
+        console.error(`no task matched ${args.taskFilter}`);
+        process.exitCode = 2;
+        return;
+    }
+    const configuration = await captureConfiguration({
+        baseUrl: args.baseUrl,
+        upstreamOrigin: args.upstream,
+        agentCli: args.agentCli,
+        tracing: args.trace,
+    });
+    const root = join(tmpdir(), `ashlr-local-eval-${Date.now()}`);
+    await mkdir(root, { recursive: true });
+    console.error(`[local-eval] ${tasks.length} task(s) x ${args.trials} trial(s), `
+        + `concurrency ${args.concurrency}, artifacts in ${root}`);
+    const startedAt = Date.now();
+    const jobs = [];
+    for (const task of tasks) {
+        for (let trial = 1; trial <= args.trials; trial += 1) {
+            jobs.push(async () => {
+                const result = await runTrial({
+                    task,
+                    trial,
+                    trialDir: join(root, `${task.id}-${trial}`),
+                    baseUrl: args.baseUrl,
+                    model: args.model,
+                    agentCli: args.agentCli,
+                    timeoutMs: args.timeoutMs,
+                    trace: args.trace,
+                });
+                console.error(`[local-eval] ${task.id}#${trial} ${result.mode} `
+                    + `(${(result.wallMs / 1000).toFixed(0)}s)`
+                    + (result.timeoutDiagnosis ? ` — ${result.timeoutDiagnosis.kind}` : ''));
+                return result;
+            });
+        }
+    }
+    const all = await pool(jobs, args.concurrency);
+    const finishedAt = Date.now();
+    const outcomes = tasks.map((task) => summariseTask(task, all.filter((r) => r.taskId === task.id)));
+    const report = buildReport({
+        configuration, outcomes,
+        trialsPerTask: args.trials,
+        concurrency: args.concurrency,
+        startedAt, finishedAt,
+    });
+    const target = args.out ?? join(root, 'report.json');
+    await writeFile(target, JSON.stringify(report, null, 2), 'utf8');
+    console.log(renderReport(report));
+    console.error(`\n[local-eval] report written to ${target}`);
+}
+// Only run when invoked directly, so the module stays importable by tests.
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+    void main();
+}
+//# sourceMappingURL=main.js.map