@otto-code/brain 0.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/bin/otto-brain +2 -0
  2. package/dist/bench/context-corpus.d.ts +58 -0
  3. package/dist/bench/context-corpus.js +210 -0
  4. package/dist/bench/corpus.d.ts +31 -0
  5. package/dist/bench/corpus.js +379 -0
  6. package/dist/bench/curated-repos.d.ts +35 -0
  7. package/dist/bench/curated-repos.js +41 -0
  8. package/dist/bench/health.d.ts +50 -0
  9. package/dist/bench/health.js +112 -0
  10. package/dist/bench/index.d.ts +103 -0
  11. package/dist/bench/index.js +209 -0
  12. package/dist/bench/mine.d.ts +32 -0
  13. package/dist/bench/mine.js +101 -0
  14. package/dist/bench/repo-task.d.ts +93 -0
  15. package/dist/bench/repo-task.js +380 -0
  16. package/dist/bench/repo.d.ts +125 -0
  17. package/dist/bench/repo.js +195 -0
  18. package/dist/bench/rescore.d.ts +41 -0
  19. package/dist/bench/rescore.js +127 -0
  20. package/dist/bench/tasks.d.ts +124 -0
  21. package/dist/bench/tasks.js +1249 -0
  22. package/dist/bench/verify.d.ts +142 -0
  23. package/dist/bench/verify.js +350 -0
  24. package/dist/cli.d.ts +13 -0
  25. package/dist/cli.js +72 -0
  26. package/dist/commands/bench.d.ts +20 -0
  27. package/dist/commands/bench.js +222 -0
  28. package/dist/commands/calibrate.d.ts +19 -0
  29. package/dist/commands/calibrate.js +75 -0
  30. package/dist/commands/catalog.d.ts +32 -0
  31. package/dist/commands/catalog.js +48 -0
  32. package/dist/commands/config.d.ts +13 -0
  33. package/dist/commands/config.js +90 -0
  34. package/dist/commands/lifecycle.d.ts +29 -0
  35. package/dist/commands/lifecycle.js +194 -0
  36. package/dist/commands/pull.d.ts +22 -0
  37. package/dist/commands/pull.js +135 -0
  38. package/dist/commands/report.d.ts +10 -0
  39. package/dist/commands/report.js +34 -0
  40. package/dist/commands/rescore.d.ts +11 -0
  41. package/dist/commands/rescore.js +48 -0
  42. package/dist/commands/runtime.d.ts +21 -0
  43. package/dist/commands/runtime.js +62 -0
  44. package/dist/commands/scan.d.ts +24 -0
  45. package/dist/commands/scan.js +55 -0
  46. package/dist/commands/search.d.ts +31 -0
  47. package/dist/commands/search.js +135 -0
  48. package/dist/commands/share.d.ts +26 -0
  49. package/dist/commands/share.js +148 -0
  50. package/dist/commands/sweep.d.ts +21 -0
  51. package/dist/commands/sweep.js +69 -0
  52. package/dist/commands/ui.d.ts +9 -0
  53. package/dist/commands/ui.js +28 -0
  54. package/dist/config/env.d.ts +10 -0
  55. package/dist/config/env.js +48 -0
  56. package/dist/config/index.d.ts +9 -0
  57. package/dist/config/index.js +9 -0
  58. package/dist/config/otto-home.d.ts +2 -0
  59. package/dist/config/otto-home.js +24 -0
  60. package/dist/config/paths.d.ts +20 -0
  61. package/dist/config/paths.js +33 -0
  62. package/dist/config/private-files.d.ts +3 -0
  63. package/dist/config/private-files.js +32 -0
  64. package/dist/config/profiles.d.ts +34 -0
  65. package/dist/config/profiles.js +116 -0
  66. package/dist/config/schema.d.ts +1035 -0
  67. package/dist/config/schema.js +191 -0
  68. package/dist/config/store.d.ts +11 -0
  69. package/dist/config/store.js +73 -0
  70. package/dist/gguf.d.ts +61 -0
  71. package/dist/gguf.js +231 -0
  72. package/dist/gpu.d.ts +12 -0
  73. package/dist/gpu.js +49 -0
  74. package/dist/index.d.ts +14 -0
  75. package/dist/index.js +13 -0
  76. package/dist/main.d.ts +2 -0
  77. package/dist/main.js +5 -0
  78. package/dist/models/dirs.d.ts +9 -0
  79. package/dist/models/dirs.js +21 -0
  80. package/dist/models/download.d.ts +31 -0
  81. package/dist/models/download.js +105 -0
  82. package/dist/models/enrich.d.ts +36 -0
  83. package/dist/models/enrich.js +68 -0
  84. package/dist/models/hf.d.ts +62 -0
  85. package/dist/models/hf.js +114 -0
  86. package/dist/models/index.d.ts +15 -0
  87. package/dist/models/index.js +49 -0
  88. package/dist/models/manage.d.ts +27 -0
  89. package/dist/models/manage.js +112 -0
  90. package/dist/models/pick.d.ts +3 -0
  91. package/dist/models/pick.js +36 -0
  92. package/dist/models/scan.d.ts +13 -0
  93. package/dist/models/scan.js +185 -0
  94. package/dist/ops/archive.d.ts +34 -0
  95. package/dist/ops/archive.js +105 -0
  96. package/dist/ops/calibrate.d.ts +49 -0
  97. package/dist/ops/calibrate.js +74 -0
  98. package/dist/ops/report.d.ts +26 -0
  99. package/dist/ops/report.js +532 -0
  100. package/dist/ops/results.d.ts +207 -0
  101. package/dist/ops/results.js +250 -0
  102. package/dist/ops/sweep.d.ts +77 -0
  103. package/dist/ops/sweep.js +126 -0
  104. package/dist/output/index.d.ts +5 -0
  105. package/dist/output/index.js +5 -0
  106. package/dist/output/render.d.ts +4 -0
  107. package/dist/output/render.js +96 -0
  108. package/dist/output/types.d.ts +52 -0
  109. package/dist/output/types.js +9 -0
  110. package/dist/output/with-output.d.ts +3 -0
  111. package/dist/output/with-output.js +42 -0
  112. package/dist/run.d.ts +5 -0
  113. package/dist/run.js +19 -0
  114. package/dist/runtime/args.d.ts +21 -0
  115. package/dist/runtime/args.js +71 -0
  116. package/dist/runtime/index.d.ts +13 -0
  117. package/dist/runtime/index.js +41 -0
  118. package/dist/runtime/lmstudio.d.ts +10 -0
  119. package/dist/runtime/lmstudio.js +88 -0
  120. package/dist/runtime/managed.d.ts +26 -0
  121. package/dist/runtime/managed.js +148 -0
  122. package/dist/service/model-selector.d.ts +36 -0
  123. package/dist/service/model-selector.js +123 -0
  124. package/dist/service/pid-lock.d.ts +18 -0
  125. package/dist/service/pid-lock.js +50 -0
  126. package/dist/service/router.d.ts +144 -0
  127. package/dist/service/router.js +746 -0
  128. package/dist/service/scheduler.d.ts +74 -0
  129. package/dist/service/scheduler.js +110 -0
  130. package/dist/service/serve.d.ts +41 -0
  131. package/dist/service/serve.js +278 -0
  132. package/dist/service/supervisor.d.ts +67 -0
  133. package/dist/service/supervisor.js +222 -0
  134. package/dist/service/tailscale.d.ts +19 -0
  135. package/dist/service/tailscale.js +62 -0
  136. package/dist/service/tls.d.ts +77 -0
  137. package/dist/service/tls.js +192 -0
  138. package/dist/sysmon.d.ts +50 -0
  139. package/dist/sysmon.js +103 -0
  140. package/dist/tui/app.d.ts +208 -0
  141. package/dist/tui/app.js +1801 -0
  142. package/dist/tui/screen.d.ts +72 -0
  143. package/dist/tui/screen.js +176 -0
  144. package/dist/types.d.ts +76 -0
  145. package/dist/types.js +7 -0
  146. package/dist/version.d.ts +2 -0
  147. package/dist/version.js +8 -0
  148. package/dist/vram.d.ts +67 -0
  149. package/dist/vram.js +132 -0
  150. package/package.json +58 -0
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Curated presets for the mined-repo (SWE-bench-style) benchmark flow.
3
+ *
4
+ * The raw `otto brain bench --repo-*` flags mine any workspace on demand; these
5
+ * presets name a few known-good, dependency-light workspaces so a repeatable
6
+ * "curated" run is one flag instead of five. Each preset still needs the caller
7
+ * to point `--repo-dir` at a working copy to mine, because the harness resets
8
+ * that copy hard (git reset --hard + git clean -fd) between tasks - it must never
9
+ * be aimed at a checkout with work you care about.
10
+ *
11
+ * Presets target this repo's own TypeScript/vitest workspaces (the miner keys off
12
+ * `.test.ts` companions), so the flow runs out of the box against a spare
13
+ * otto-code clone.
14
+ */
15
+ /** One curated mining preset. */
16
+ export interface CuratedRepo {
17
+ /** Short handle used on the command line (`--curated <name>`). */
18
+ name: string;
19
+ /** npm workspace name, e.g. `@otto-code/protocol`. */
20
+ workspace: string;
21
+ /** Workspace path relative to the repo root, e.g. `packages/protocol`. */
22
+ workspaceDir: string;
23
+ /** Git ref whose history is mined for fix commits. */
24
+ ref: string;
25
+ /** How many mined tasks to run. */
26
+ maxTasks: number;
27
+ /** One-line note shown in the preset listing. */
28
+ note: string;
29
+ }
30
+ export declare const CURATED_REPOS: CuratedRepo[];
31
+ /** Look up a curated preset by its handle. */
32
+ export declare function findCuratedRepo(name: string): CuratedRepo | null;
33
+ /** A one-per-line listing of the available presets, for help/error text. */
34
+ export declare function describeCuratedRepos(): string;
35
+ //# sourceMappingURL=curated-repos.d.ts.map
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Curated presets for the mined-repo (SWE-bench-style) benchmark flow.
3
+ *
4
+ * The raw `otto brain bench --repo-*` flags mine any workspace on demand; these
5
+ * presets name a few known-good, dependency-light workspaces so a repeatable
6
+ * "curated" run is one flag instead of five. Each preset still needs the caller
7
+ * to point `--repo-dir` at a working copy to mine, because the harness resets
8
+ * that copy hard (git reset --hard + git clean -fd) between tasks - it must never
9
+ * be aimed at a checkout with work you care about.
10
+ *
11
+ * Presets target this repo's own TypeScript/vitest workspaces (the miner keys off
12
+ * `.test.ts` companions), so the flow runs out of the box against a spare
13
+ * otto-code clone.
14
+ */
15
+ export const CURATED_REPOS = [
16
+ {
17
+ name: "protocol",
18
+ workspace: "@otto-code/protocol",
19
+ workspaceDir: "packages/protocol",
20
+ ref: "origin/main",
21
+ maxTasks: 3,
22
+ note: "schema/validation workspace — fast suite, no heavy deps",
23
+ },
24
+ {
25
+ name: "brain",
26
+ workspace: "@otto-code/brain",
27
+ workspaceDir: "packages/brain",
28
+ ref: "origin/main",
29
+ maxTasks: 3,
30
+ note: "the brain package itself — pure-logic tests (vram, gguf, bench)",
31
+ },
32
+ ];
33
+ /** Look up a curated preset by its handle. */
34
+ export function findCuratedRepo(name) {
35
+ return CURATED_REPOS.find((r) => r.name === name) ?? null;
36
+ }
37
+ /** A one-per-line listing of the available presets, for help/error text. */
38
+ export function describeCuratedRepos() {
39
+ return CURATED_REPOS.map((r) => ` ${r.name.padEnd(10)} ${r.workspace} — ${r.note}`).join("\n");
40
+ }
41
+ //# sourceMappingURL=curated-repos.js.map
@@ -0,0 +1,50 @@
1
+ /** One nvidia-smi sample. */
2
+ export interface GpuSample {
3
+ util: number | null;
4
+ temp: number | null;
5
+ power: number | null;
6
+ clock: number | null;
7
+ vramMiB: number | null;
8
+ thermal: boolean;
9
+ powerBrake: boolean;
10
+ }
11
+ /** Aggregate of a series of numeric samples. */
12
+ export interface Aggregate {
13
+ avg: number;
14
+ min: number;
15
+ max: number;
16
+ }
17
+ /** Raw samples collected during a run. */
18
+ export interface HealthSamples {
19
+ gpu: GpuSample[];
20
+ cpu: number[];
21
+ ram: number[];
22
+ }
23
+ /** Summary of system health across a run. */
24
+ export interface HealthSummary {
25
+ samples: number;
26
+ gpuUtilPct: Aggregate | null;
27
+ tempC: Aggregate | null;
28
+ powerW: Aggregate | null;
29
+ clockMHz: Aggregate | null;
30
+ vramUsedMiB: Aggregate | null;
31
+ cpuPct: Aggregate | null;
32
+ ramUsedBytes: Aggregate | null;
33
+ thermalThrottle: boolean;
34
+ powerThrottle: boolean;
35
+ }
36
+ /** A running sampler; call stop() to end it and get the summary. */
37
+ export interface HealthHandle {
38
+ stop(): HealthSummary;
39
+ }
40
+ /** Collapse raw samples into the summary stored on the result. */
41
+ export declare function summarize(samples: HealthSamples): HealthSummary;
42
+ /**
43
+ * Begin sampling; returns a handle with stop() that returns the summary.
44
+ * Sampling failures (no GPU, nvidia-smi missing) degrade to empty aggregates
45
+ * rather than throwing — health is diagnostic, never a reason to fail a bench.
46
+ */
47
+ export declare function start({ intervalMs }?: {
48
+ intervalMs?: number;
49
+ }): HealthHandle;
50
+ //# sourceMappingURL=health.d.ts.map
@@ -0,0 +1,112 @@
1
+ import os from "node:os";
2
+ import { execFile } from "node:child_process";
3
+ import { createCpuSampler } from "../sysmon.js";
4
+ /**
5
+ * System-health sampling for a benchmark run.
6
+ *
7
+ * A score is only trustworthy if the machine was in a comparable state when it
8
+ * was measured — a thermal-throttled or power-capped GPU produces a slower,
9
+ * lower run that says nothing about the model. So we sample GPU/CPU/RAM through
10
+ * the run and summarise it alongside the result, and flag when nvidia-smi itself
11
+ * reports a thermal or power slowdown (more reliable than inferring one from a
12
+ * clock dip, since the GPU also idles between tasks).
13
+ */
14
+ const FIELDS = [
15
+ "utilization.gpu",
16
+ "temperature.gpu",
17
+ "power.draw",
18
+ "clocks.sm",
19
+ "memory.used",
20
+ "clocks_throttle_reasons.sw_thermal_slowdown",
21
+ "clocks_throttle_reasons.hw_thermal_slowdown",
22
+ "clocks_throttle_reasons.hw_power_brake_slowdown",
23
+ ];
24
+ function sampleGpu() {
25
+ return new Promise((resolve) => {
26
+ execFile("nvidia-smi", [`--query-gpu=${FIELDS.join(",")}`, "--format=csv,noheader,nounits"], { timeout: 4000, windowsHide: true }, (error, stdout) => {
27
+ if (error) {
28
+ resolve(null);
29
+ return;
30
+ }
31
+ const line = String(stdout).trim().split(/\r?\n/)[0];
32
+ if (!line) {
33
+ resolve(null);
34
+ return;
35
+ }
36
+ const c = line.split(",").map((s) => s.trim());
37
+ const num = (v) => {
38
+ const n = Number(v);
39
+ return Number.isFinite(n) ? n : null;
40
+ };
41
+ const active = (v) => /active/i.test(v) && !/not\s*active/i.test(v);
42
+ resolve({
43
+ util: num(c[0]),
44
+ temp: num(c[1]),
45
+ power: num(c[2]),
46
+ clock: num(c[3]),
47
+ vramMiB: num(c[4]),
48
+ thermal: active(c[5]) || active(c[6]),
49
+ powerBrake: active(c[7]),
50
+ });
51
+ });
52
+ });
53
+ }
54
+ function agg(values) {
55
+ const nums = values.filter((v) => typeof v === "number" && Number.isFinite(v));
56
+ if (!nums.length)
57
+ return null;
58
+ const sum = nums.reduce((a, b) => a + b, 0);
59
+ return {
60
+ avg: Math.round((sum / nums.length) * 10) / 10,
61
+ min: Math.min(...nums),
62
+ max: Math.max(...nums),
63
+ };
64
+ }
65
+ /** Collapse raw samples into the summary stored on the result. */
66
+ export function summarize(samples) {
67
+ const gpu = samples.gpu || [];
68
+ return {
69
+ samples: gpu.length,
70
+ gpuUtilPct: agg(gpu.map((s) => s.util)),
71
+ tempC: agg(gpu.map((s) => s.temp)),
72
+ powerW: agg(gpu.map((s) => s.power)),
73
+ clockMHz: agg(gpu.map((s) => s.clock)),
74
+ vramUsedMiB: agg(gpu.map((s) => s.vramMiB)),
75
+ cpuPct: agg(samples.cpu || []),
76
+ ramUsedBytes: agg(samples.ram || []),
77
+ thermalThrottle: gpu.some((s) => s.thermal),
78
+ powerThrottle: gpu.some((s) => s.powerBrake),
79
+ };
80
+ }
81
+ /**
82
+ * Begin sampling; returns a handle with stop() that returns the summary.
83
+ * Sampling failures (no GPU, nvidia-smi missing) degrade to empty aggregates
84
+ * rather than throwing — health is diagnostic, never a reason to fail a bench.
85
+ */
86
+ export function start({ intervalMs = 1000 } = {}) {
87
+ const samples = { gpu: [], cpu: [], ram: [] };
88
+ const cpu = createCpuSampler();
89
+ cpu(); // establish the CPU baseline
90
+ let stopped = false;
91
+ const tick = async () => {
92
+ if (stopped)
93
+ return;
94
+ const gpu = await sampleGpu();
95
+ if (gpu)
96
+ samples.gpu.push(gpu);
97
+ const busy = cpu();
98
+ if (typeof busy === "number")
99
+ samples.cpu.push(busy * 100);
100
+ samples.ram.push(os.totalmem() - os.freemem());
101
+ };
102
+ const timer = setInterval(tick, intervalMs);
103
+ tick();
104
+ return {
105
+ stop() {
106
+ stopped = true;
107
+ clearInterval(timer);
108
+ return summarize(samples);
109
+ },
110
+ };
111
+ }
112
+ //# sourceMappingURL=health.js.map
@@ -0,0 +1,103 @@
1
+ import type { Task } from "./tasks.js";
2
+ import type { Profile } from "../config/schema.js";
3
+ /**
4
+ * Fast, locally-verifiable agentic-coding suite.
5
+ *
6
+ * This is deliberately NOT a replacement for SWE-bench or Aider Polyglot -
7
+ * those run hundreds of multi-turn exercises and take hours per model. This is
8
+ * the triage stage: a few minutes per model, enough to rank a shelf of quant
9
+ * variants and to catch the system-level failures (reasoning runaway, VRAM
10
+ * spill, throughput collapse at depth) that model-level leaderboards never
11
+ * measure because they do not know your hardware.
12
+ *
13
+ * Use it to pick finalists, then run a full public benchmark on those.
14
+ */
15
+ export declare const DEFAULT_DEPTHS: number[];
16
+ /** A progress event emitted as the suite runs. */
17
+ export interface ProgressEvent {
18
+ phase: "start" | "done" | "failed";
19
+ task?: string;
20
+ title?: string;
21
+ id?: string;
22
+ category?: string;
23
+ weight?: number;
24
+ description?: string;
25
+ score?: number;
26
+ summary?: string;
27
+ detail?: unknown;
28
+ seconds?: number;
29
+ error?: string | null;
30
+ }
31
+ /** Options for {@link runSuite}. */
32
+ export interface RunSuiteOptions {
33
+ host?: string;
34
+ port?: number;
35
+ execute?: boolean;
36
+ depths?: number[];
37
+ only?: string[] | null;
38
+ concurrency?: number;
39
+ /**
40
+ * The resident model's reasoning-token budget, forwarded to tasks that must
41
+ * size their response cap above it (the concurrency throughput task).
42
+ */
43
+ reasoningBudget?: number | null;
44
+ /** The resident model's loaded context window, for context-utilization reporting. */
45
+ contextWindow?: number | null;
46
+ timeoutMs?: number;
47
+ archiveId?: string | null;
48
+ onProgress?: (event: ProgressEvent) => void;
49
+ /**
50
+ * Task list override. Defaults to the static {@link TASKS}. Repo-backed
51
+ * SWE-bench tasks are opt-in and passed here so the static suite never
52
+ * requires a repo checkout to run.
53
+ */
54
+ tasks?: Task[] | null;
55
+ }
56
+ /** The scored result of one task within a suite run. */
57
+ export interface SuiteTaskResult {
58
+ id: string;
59
+ category: string;
60
+ weight: number;
61
+ description?: string;
62
+ score: number;
63
+ summary: string;
64
+ detail?: unknown;
65
+ seconds: number;
66
+ error: string | null;
67
+ }
68
+ /** The full scorecard produced by a suite run. */
69
+ export interface SuiteReport {
70
+ overall: number;
71
+ grade: string;
72
+ results: SuiteTaskResult[];
73
+ seconds: number;
74
+ executedCode: boolean;
75
+ pythonVersion: string | null;
76
+ archiveId: string | null;
77
+ ranAt: string;
78
+ }
79
+ /**
80
+ * Run the suite against an already-serving endpoint.
81
+ *
82
+ * @param {object} options
83
+ * @param {string} options.host endpoint host
84
+ * @param {number} options.port endpoint port
85
+ * @param {boolean} options.execute actually run generated tests
86
+ * @param {number[]} options.depths prompt depths for the scaling task
87
+ * @param {string[]} options.only task ids to run (default: all)
88
+ */
89
+ export declare function runSuite({ host, port, execute, depths, only, concurrency, reasoningBudget, contextWindow, timeoutMs, archiveId, onProgress, tasks, }?: RunSuiteOptions): Promise<SuiteReport>;
90
+ export declare function grade(score: number): string;
91
+ /** Render a scorecard for one model. */
92
+ export declare function formatReport(report: SuiteReport, { modelName, profile }?: {
93
+ modelName?: string;
94
+ profile?: Profile | null;
95
+ }): string;
96
+ /** One completed report paired with its model name, for comparison. */
97
+ export interface ComparisonEntry {
98
+ modelName: string;
99
+ report: SuiteReport;
100
+ }
101
+ /** Compare several completed reports. */
102
+ export declare function formatComparison(entries: ComparisonEntry[]): string;
103
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,209 @@
1
+ import http from "node:http";
2
+ import { TASKS } from "./tasks.js";
3
+ import { findPython } from "./verify.js";
4
+ import * as archive from "../ops/archive.js";
5
+ /**
6
+ * Fast, locally-verifiable agentic-coding suite.
7
+ *
8
+ * This is deliberately NOT a replacement for SWE-bench or Aider Polyglot -
9
+ * those run hundreds of multi-turn exercises and take hours per model. This is
10
+ * the triage stage: a few minutes per model, enough to rank a shelf of quant
11
+ * variants and to catch the system-level failures (reasoning runaway, VRAM
12
+ * spill, throughput collapse at depth) that model-level leaderboards never
13
+ * measure because they do not know your hardware.
14
+ *
15
+ * Use it to pick finalists, then run a full public benchmark on those.
16
+ */
17
+ export const DEFAULT_DEPTHS = [1000, 16000, 64000];
18
+ function chatRequest({ host, port, payload, timeoutMs }) {
19
+ return new Promise((resolve, reject) => {
20
+ const body = JSON.stringify(payload);
21
+ const req = http.request({
22
+ host,
23
+ port,
24
+ path: "/v1/chat/completions",
25
+ method: "POST",
26
+ headers: { "content-type": "application/json", "content-length": Buffer.byteLength(body) },
27
+ timeout: timeoutMs,
28
+ }, (res) => {
29
+ let text = "";
30
+ res.on("data", (c) => (text += c));
31
+ res.on("end", () => {
32
+ if (res.statusCode !== 200) {
33
+ reject(new Error(`endpoint returned ${res.statusCode}: ${text.slice(0, 300)}`));
34
+ return;
35
+ }
36
+ try {
37
+ resolve(JSON.parse(text));
38
+ }
39
+ catch (error) {
40
+ reject(error);
41
+ }
42
+ });
43
+ });
44
+ req.on("timeout", () => {
45
+ req.destroy();
46
+ reject(new Error(`request exceeded ${timeoutMs}ms`));
47
+ });
48
+ req.on("error", reject);
49
+ req.end(body);
50
+ });
51
+ }
52
+ /**
53
+ * Run the suite against an already-serving endpoint.
54
+ *
55
+ * @param {object} options
56
+ * @param {string} options.host endpoint host
57
+ * @param {number} options.port endpoint port
58
+ * @param {boolean} options.execute actually run generated tests
59
+ * @param {number[]} options.depths prompt depths for the scaling task
60
+ * @param {string[]} options.only task ids to run (default: all)
61
+ */
62
+ export async function runSuite({ host = "127.0.0.1", port = 1234, execute = true, depths = DEFAULT_DEPTHS, only = null, concurrency = 3, reasoningBudget = null, contextWindow = null, timeoutMs = 900000, archiveId = null, onProgress = () => { }, tasks = null, } = {}) {
63
+ const python = findPython();
64
+ const pool = tasks && tasks.length ? tasks : TASKS;
65
+ const selected = only && only.length ? pool.filter((t) => only.includes(t.id)) : pool;
66
+ const results = [];
67
+ const startedAll = Date.now();
68
+ for (const task of selected) {
69
+ onProgress({ phase: "start", task: task.id, title: task.category });
70
+ const started = Date.now();
71
+ // Archive every exchange for this task, so a future scorer can re-grade it
72
+ // without re-running the model.
73
+ let exchange = 0;
74
+ const chat = async (payload) => {
75
+ const response = await chatRequest({ host, port, payload, timeoutMs });
76
+ if (archiveId) {
77
+ try {
78
+ archive.put(archiveId, task.id, {
79
+ label: `exchange-${(exchange += 1)}`,
80
+ request: payload,
81
+ response,
82
+ });
83
+ }
84
+ catch {
85
+ /* archiving must never fail a benchmark run */
86
+ }
87
+ }
88
+ return response;
89
+ };
90
+ try {
91
+ const outcome = await task.run({
92
+ chat,
93
+ execute: execute && Boolean(python),
94
+ depths,
95
+ concurrency,
96
+ reasoningBudget,
97
+ contextWindow,
98
+ });
99
+ const entry = {
100
+ id: task.id,
101
+ category: task.category,
102
+ weight: task.weight,
103
+ description: task.description,
104
+ score: Math.max(0, Math.min(1, outcome.score)),
105
+ summary: outcome.summary,
106
+ detail: outcome.detail,
107
+ seconds: (Date.now() - started) / 1000,
108
+ error: null,
109
+ };
110
+ results.push(entry);
111
+ // `title` is what the progress UI renders; entry carries `category`, not
112
+ // `title`, so pass it explicitly or the label shows "undefined".
113
+ onProgress({ phase: "done", title: task.category, ...entry });
114
+ }
115
+ catch (error) {
116
+ const message = error instanceof Error ? error.message : String(error);
117
+ const entry = {
118
+ id: task.id,
119
+ category: task.category,
120
+ weight: task.weight,
121
+ score: 0,
122
+ summary: `failed: ${message}`,
123
+ seconds: (Date.now() - started) / 1000,
124
+ error: message,
125
+ };
126
+ results.push(entry);
127
+ onProgress({ phase: "failed", title: task.category, ...entry });
128
+ }
129
+ }
130
+ const totalWeight = results.reduce((sum, r) => sum + r.weight, 0);
131
+ const overall = totalWeight
132
+ ? results.reduce((sum, r) => sum + r.score * r.weight, 0) / totalWeight
133
+ : 0;
134
+ return {
135
+ overall,
136
+ grade: grade(overall),
137
+ results,
138
+ seconds: (Date.now() - startedAll) / 1000,
139
+ executedCode: execute && Boolean(python),
140
+ pythonVersion: python ? python.version : null,
141
+ archiveId,
142
+ ranAt: new Date().toISOString(),
143
+ };
144
+ }
145
+ export function grade(score) {
146
+ if (score >= 0.9)
147
+ return "excellent";
148
+ if (score >= 0.75)
149
+ return "strong";
150
+ if (score >= 0.55)
151
+ return "usable";
152
+ if (score >= 0.35)
153
+ return "weak";
154
+ return "unusable";
155
+ }
156
+ /** Render a scorecard for one model. */
157
+ export function formatReport(report, { modelName = "model", profile = null } = {}) {
158
+ const lines = [];
159
+ const bar = (score) => {
160
+ const cells = 20;
161
+ const filled = Math.round(score * cells);
162
+ return "█".repeat(filled) + "░".repeat(cells - filled);
163
+ };
164
+ lines.push("");
165
+ lines.push(` Agentic coding scorecard - ${modelName}`);
166
+ if (profile) {
167
+ lines.push(` context ${profile.contextSize.toLocaleString()} kv ${profile.cacheTypeK}/${profile.cacheTypeV} ` +
168
+ `reasoning budget ${profile.reasoningBudget}${profile.vision ? " vision on" : ""}`);
169
+ }
170
+ lines.push(` ${"-".repeat(72)}`);
171
+ for (const r of report.results) {
172
+ lines.push(` ${r.category.padEnd(18)} ${bar(r.score)} ${(r.score * 100).toFixed(0).padStart(3)}% ${r.summary}`);
173
+ }
174
+ lines.push(` ${"-".repeat(72)}`);
175
+ lines.push(` ${"OVERALL".padEnd(18)} ${bar(report.overall)} ${(report.overall * 100).toFixed(0).padStart(3)}% ${report.grade}`);
176
+ lines.push("");
177
+ lines.push(` ran in ${report.seconds.toFixed(0)}s` +
178
+ (report.executedCode
179
+ ? `, generated tests executed with ${report.pythonVersion}`
180
+ : ", code not executed"));
181
+ // Surface depth numbers, which are the most system-specific result here.
182
+ const depth = report.results.find((r) => r.id === "context-depth");
183
+ if (depth && Array.isArray(depth.detail)) {
184
+ lines.push("");
185
+ lines.push(` ${"prompt tokens".padStart(14)} ${"TTFT".padStart(8)} ${"prompt tok/s".padStart(13)} ${"gen tok/s".padStart(10)}`);
186
+ for (const p of depth.detail) {
187
+ lines.push(` ${String(p.promptTokens ?? "?").padStart(14)} ${((p.ttftSeconds ?? 0).toFixed(2) + "s").padStart(8)} ` +
188
+ `${(p.promptPerSecond ?? 0).toFixed(0).padStart(13)} ${(p.generatePerSecond ?? 0).toFixed(1).padStart(10)}`);
189
+ }
190
+ }
191
+ lines.push("");
192
+ return lines.join("\n");
193
+ }
194
+ /** Compare several completed reports. */
195
+ export function formatComparison(entries) {
196
+ const lines = [];
197
+ const categories = entries[0]?.report.results.map((r) => r.category) || [];
198
+ lines.push("");
199
+ lines.push(` ${"model".padEnd(42)}${categories.map((c) => c.slice(0, 9).padStart(10)).join("")}${"OVERALL".padStart(10)}`);
200
+ lines.push(` ${"-".repeat(42 + categories.length * 10 + 10)}`);
201
+ const ranked = [...entries].sort((a, b) => b.report.overall - a.report.overall);
202
+ for (const { modelName, report } of ranked) {
203
+ const cells = report.results.map((r) => `${(r.score * 100).toFixed(0)}%`.padStart(10)).join("");
204
+ lines.push(` ${modelName.slice(0, 41).padEnd(42)}${cells}${`${(report.overall * 100).toFixed(0)}%`.padStart(10)}`);
205
+ }
206
+ lines.push("");
207
+ return lines.join("\n");
208
+ }
209
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,32 @@
1
+ export declare function isTest(file: string): boolean;
2
+ export declare function isSource(file: string): boolean;
3
+ /** Options for {@link mine}. */
4
+ export interface MineOptions {
5
+ dir: string;
6
+ workspaceDir: string;
7
+ limit?: number;
8
+ maxFiles?: number;
9
+ since?: string | null;
10
+ ref?: string;
11
+ }
12
+ /** A benchmark task mined from a fix commit. */
13
+ export interface MinedTask {
14
+ id: string;
15
+ workspaceDir: string;
16
+ fix: string;
17
+ parent: string;
18
+ date: string;
19
+ subject: string;
20
+ testPaths: string[];
21
+ sourcePaths: string[];
22
+ looksLikeFix: boolean;
23
+ }
24
+ /**
25
+ * @param {object} options
26
+ * @param {string} options.dir repo working copy
27
+ * @param {string} options.workspaceDir e.g. "packages/protocol"
28
+ * @param {number} options.limit how many commits of history to scan
29
+ * @param {number} options.maxFiles skip sprawling commits; they make poor tasks
30
+ */
31
+ export declare function mine({ dir, workspaceDir, limit, maxFiles, since, ref, }: MineOptions): Promise<MinedTask[]>;
32
+ //# sourceMappingURL=mine.d.ts.map
@@ -0,0 +1,101 @@
1
+ import path from "node:path";
2
+ import { run } from "./repo.js";
3
+ /**
4
+ * Mine benchmark tasks out of real git history.
5
+ *
6
+ * A commit that changed both source and tests in the same workspace is a
7
+ * candidate task: check out its parent, bring in only the test changes, and the
8
+ * new tests describe a bug that the parent commit still has. The author's own
9
+ * fix is the reference solution and their tests are the oracle - which is the
10
+ * property that makes this worth more than a hand-written prompt.
11
+ *
12
+ * Nothing here reads file contents. Tasks carry SHAs and paths only, so a task
13
+ * set can be shared or version-controlled without exposing private source.
14
+ */
15
+ const TEST_PATTERN = /\.(test|spec)\.[cm]?[jt]sx?$/;
16
+ const SOURCE_PATTERN = /\.[cm]?[jt]sx?$/;
17
+ // Commits whose subject suggests a behavioural fix make the best tasks; the
18
+ // rest are still usable but ranked below these.
19
+ const FIX_HINT = /\b(fix|bug|regression|broken|crash|incorrect|wrong|fail)/i;
20
+ export function isTest(file) {
21
+ return TEST_PATTERN.test(file);
22
+ }
23
+ export function isSource(file) {
24
+ return SOURCE_PATTERN.test(file) && !isTest(file);
25
+ }
26
+ /**
27
+ * @param {object} options
28
+ * @param {string} options.dir repo working copy
29
+ * @param {string} options.workspaceDir e.g. "packages/protocol"
30
+ * @param {number} options.limit how many commits of history to scan
31
+ * @param {number} options.maxFiles skip sprawling commits; they make poor tasks
32
+ */
33
+ export async function mine({ dir, workspaceDir, limit = 2000, maxFiles = 12, since = null, ref = "origin/main", }) {
34
+ // Mine from a fixed ref, never HEAD: the harness moves HEAD around between
35
+ // tasks, and a HEAD parked on some parent commit makes every descendant
36
+ // unreachable - silently shrinking the task set instead of failing loudly.
37
+ const logArgs = [
38
+ "log",
39
+ ref,
40
+ `--max-count=${limit}`,
41
+ "--format=%H%x00%an%x00%ad%x00%s",
42
+ "--date=short",
43
+ ];
44
+ if (since)
45
+ logArgs.push(`--since=${since}`);
46
+ logArgs.push("--", workspaceDir);
47
+ const log = await run("git", logArgs, { cwd: dir, timeout: 120000 });
48
+ if (!log.ok)
49
+ throw new Error(`git log failed: ${log.stderr.trim()}`);
50
+ const commits = log.stdout
51
+ .split(/\r?\n/)
52
+ .filter(Boolean)
53
+ .map((line) => {
54
+ const [sha, author, date, subject] = line.split("\0");
55
+ return { sha, author, date, subject };
56
+ });
57
+ const candidates = [];
58
+ for (const commit of commits) {
59
+ const show = await run("git", ["show", "--name-only", "--format=", "--no-renames", commit.sha, "--", workspaceDir], { cwd: dir, timeout: 30000 });
60
+ if (!show.ok)
61
+ continue;
62
+ const files = show.stdout
63
+ .split(/\r?\n/)
64
+ .map((f) => f.trim())
65
+ .filter(Boolean);
66
+ if (!files.length || files.length > maxFiles)
67
+ continue;
68
+ const testPaths = files.filter(isTest);
69
+ const sourcePaths = files.filter(isSource);
70
+ if (!testPaths.length || !sourcePaths.length)
71
+ continue;
72
+ // A commit with no parent (or a merge) cannot define a clean "before" state.
73
+ const parents = await run("git", ["rev-list", "--parents", "-n", "1", commit.sha], {
74
+ cwd: dir,
75
+ timeout: 15000,
76
+ });
77
+ const parts = parents.stdout.trim().split(/\s+/);
78
+ if (parts.length !== 2)
79
+ continue; // 0 parents (root) or >1 (merge)
80
+ candidates.push({
81
+ id: `${path.basename(workspaceDir)}-${commit.sha.slice(0, 8)}`,
82
+ workspaceDir,
83
+ fix: commit.sha,
84
+ parent: parts[1],
85
+ date: commit.date,
86
+ subject: commit.subject,
87
+ testPaths,
88
+ sourcePaths,
89
+ looksLikeFix: FIX_HINT.test(commit.subject),
90
+ });
91
+ }
92
+ // Fix-shaped commits first, then smaller diffs - both correlate with tasks
93
+ // that have a crisp, checkable outcome.
94
+ candidates.sort((a, b) => {
95
+ if (a.looksLikeFix !== b.looksLikeFix)
96
+ return a.looksLikeFix ? -1 : 1;
97
+ return a.sourcePaths.length + a.testPaths.length - (b.sourcePaths.length + b.testPaths.length);
98
+ });
99
+ return candidates;
100
+ }
101
+ //# sourceMappingURL=mine.js.map