@otto-code/brain 0.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/bin/otto-brain +2 -0
  2. package/dist/bench/context-corpus.d.ts +58 -0
  3. package/dist/bench/context-corpus.js +210 -0
  4. package/dist/bench/corpus.d.ts +31 -0
  5. package/dist/bench/corpus.js +379 -0
  6. package/dist/bench/curated-repos.d.ts +35 -0
  7. package/dist/bench/curated-repos.js +41 -0
  8. package/dist/bench/health.d.ts +50 -0
  9. package/dist/bench/health.js +112 -0
  10. package/dist/bench/index.d.ts +103 -0
  11. package/dist/bench/index.js +209 -0
  12. package/dist/bench/mine.d.ts +32 -0
  13. package/dist/bench/mine.js +101 -0
  14. package/dist/bench/repo-task.d.ts +93 -0
  15. package/dist/bench/repo-task.js +380 -0
  16. package/dist/bench/repo.d.ts +125 -0
  17. package/dist/bench/repo.js +195 -0
  18. package/dist/bench/rescore.d.ts +41 -0
  19. package/dist/bench/rescore.js +127 -0
  20. package/dist/bench/tasks.d.ts +124 -0
  21. package/dist/bench/tasks.js +1249 -0
  22. package/dist/bench/verify.d.ts +142 -0
  23. package/dist/bench/verify.js +350 -0
  24. package/dist/cli.d.ts +13 -0
  25. package/dist/cli.js +72 -0
  26. package/dist/commands/bench.d.ts +20 -0
  27. package/dist/commands/bench.js +222 -0
  28. package/dist/commands/calibrate.d.ts +19 -0
  29. package/dist/commands/calibrate.js +75 -0
  30. package/dist/commands/catalog.d.ts +32 -0
  31. package/dist/commands/catalog.js +48 -0
  32. package/dist/commands/config.d.ts +13 -0
  33. package/dist/commands/config.js +90 -0
  34. package/dist/commands/lifecycle.d.ts +29 -0
  35. package/dist/commands/lifecycle.js +194 -0
  36. package/dist/commands/pull.d.ts +22 -0
  37. package/dist/commands/pull.js +135 -0
  38. package/dist/commands/report.d.ts +10 -0
  39. package/dist/commands/report.js +34 -0
  40. package/dist/commands/rescore.d.ts +11 -0
  41. package/dist/commands/rescore.js +48 -0
  42. package/dist/commands/runtime.d.ts +21 -0
  43. package/dist/commands/runtime.js +62 -0
  44. package/dist/commands/scan.d.ts +24 -0
  45. package/dist/commands/scan.js +55 -0
  46. package/dist/commands/search.d.ts +31 -0
  47. package/dist/commands/search.js +135 -0
  48. package/dist/commands/share.d.ts +26 -0
  49. package/dist/commands/share.js +148 -0
  50. package/dist/commands/sweep.d.ts +21 -0
  51. package/dist/commands/sweep.js +69 -0
  52. package/dist/commands/ui.d.ts +9 -0
  53. package/dist/commands/ui.js +28 -0
  54. package/dist/config/env.d.ts +10 -0
  55. package/dist/config/env.js +48 -0
  56. package/dist/config/index.d.ts +9 -0
  57. package/dist/config/index.js +9 -0
  58. package/dist/config/otto-home.d.ts +2 -0
  59. package/dist/config/otto-home.js +24 -0
  60. package/dist/config/paths.d.ts +20 -0
  61. package/dist/config/paths.js +33 -0
  62. package/dist/config/private-files.d.ts +3 -0
  63. package/dist/config/private-files.js +32 -0
  64. package/dist/config/profiles.d.ts +34 -0
  65. package/dist/config/profiles.js +116 -0
  66. package/dist/config/schema.d.ts +1035 -0
  67. package/dist/config/schema.js +191 -0
  68. package/dist/config/store.d.ts +11 -0
  69. package/dist/config/store.js +73 -0
  70. package/dist/gguf.d.ts +61 -0
  71. package/dist/gguf.js +231 -0
  72. package/dist/gpu.d.ts +12 -0
  73. package/dist/gpu.js +49 -0
  74. package/dist/index.d.ts +14 -0
  75. package/dist/index.js +13 -0
  76. package/dist/main.d.ts +2 -0
  77. package/dist/main.js +5 -0
  78. package/dist/models/dirs.d.ts +9 -0
  79. package/dist/models/dirs.js +21 -0
  80. package/dist/models/download.d.ts +31 -0
  81. package/dist/models/download.js +105 -0
  82. package/dist/models/enrich.d.ts +36 -0
  83. package/dist/models/enrich.js +68 -0
  84. package/dist/models/hf.d.ts +62 -0
  85. package/dist/models/hf.js +114 -0
  86. package/dist/models/index.d.ts +15 -0
  87. package/dist/models/index.js +49 -0
  88. package/dist/models/manage.d.ts +27 -0
  89. package/dist/models/manage.js +112 -0
  90. package/dist/models/pick.d.ts +3 -0
  91. package/dist/models/pick.js +36 -0
  92. package/dist/models/scan.d.ts +13 -0
  93. package/dist/models/scan.js +185 -0
  94. package/dist/ops/archive.d.ts +34 -0
  95. package/dist/ops/archive.js +105 -0
  96. package/dist/ops/calibrate.d.ts +49 -0
  97. package/dist/ops/calibrate.js +74 -0
  98. package/dist/ops/report.d.ts +26 -0
  99. package/dist/ops/report.js +532 -0
  100. package/dist/ops/results.d.ts +207 -0
  101. package/dist/ops/results.js +250 -0
  102. package/dist/ops/sweep.d.ts +77 -0
  103. package/dist/ops/sweep.js +126 -0
  104. package/dist/output/index.d.ts +5 -0
  105. package/dist/output/index.js +5 -0
  106. package/dist/output/render.d.ts +4 -0
  107. package/dist/output/render.js +96 -0
  108. package/dist/output/types.d.ts +52 -0
  109. package/dist/output/types.js +9 -0
  110. package/dist/output/with-output.d.ts +3 -0
  111. package/dist/output/with-output.js +42 -0
  112. package/dist/run.d.ts +5 -0
  113. package/dist/run.js +19 -0
  114. package/dist/runtime/args.d.ts +21 -0
  115. package/dist/runtime/args.js +71 -0
  116. package/dist/runtime/index.d.ts +13 -0
  117. package/dist/runtime/index.js +41 -0
  118. package/dist/runtime/lmstudio.d.ts +10 -0
  119. package/dist/runtime/lmstudio.js +88 -0
  120. package/dist/runtime/managed.d.ts +26 -0
  121. package/dist/runtime/managed.js +148 -0
  122. package/dist/service/model-selector.d.ts +36 -0
  123. package/dist/service/model-selector.js +123 -0
  124. package/dist/service/pid-lock.d.ts +18 -0
  125. package/dist/service/pid-lock.js +50 -0
  126. package/dist/service/router.d.ts +144 -0
  127. package/dist/service/router.js +746 -0
  128. package/dist/service/scheduler.d.ts +74 -0
  129. package/dist/service/scheduler.js +110 -0
  130. package/dist/service/serve.d.ts +41 -0
  131. package/dist/service/serve.js +278 -0
  132. package/dist/service/supervisor.d.ts +67 -0
  133. package/dist/service/supervisor.js +222 -0
  134. package/dist/service/tailscale.d.ts +19 -0
  135. package/dist/service/tailscale.js +62 -0
  136. package/dist/service/tls.d.ts +77 -0
  137. package/dist/service/tls.js +192 -0
  138. package/dist/sysmon.d.ts +50 -0
  139. package/dist/sysmon.js +103 -0
  140. package/dist/tui/app.d.ts +208 -0
  141. package/dist/tui/app.js +1801 -0
  142. package/dist/tui/screen.d.ts +72 -0
  143. package/dist/tui/screen.js +176 -0
  144. package/dist/types.d.ts +76 -0
  145. package/dist/types.js +7 -0
  146. package/dist/version.d.ts +2 -0
  147. package/dist/version.js +8 -0
  148. package/dist/vram.d.ts +67 -0
  149. package/dist/vram.js +132 -0
  150. package/package.json +58 -0
@@ -0,0 +1,142 @@
1
+ /**
2
+ * Objective verification of generated code.
3
+ *
4
+ * A benchmark is only worth trusting if a pass means something checkable, so
5
+ * nothing here asks a model to grade another model. Generated Python is
6
+ * written to a scratch directory and handed to the real interpreter: it either
7
+ * compiles and its tests pass, or it does not.
8
+ *
9
+ * NOTE: `runTests` executes model-generated code. It runs in a temporary
10
+ * directory under a timeout, but it is still arbitrary code execution - pass
11
+ * `execute: false` to stop at syntax checking.
12
+ */
13
+ /** A discovered Python interpreter. */
14
+ export interface PythonInfo {
15
+ exe: string;
16
+ version: string;
17
+ }
18
+ /** Outcome of running a child process. */
19
+ export interface RunResult {
20
+ ok: boolean;
21
+ code: number | string;
22
+ timedOut: boolean;
23
+ stdout: string;
24
+ stderr: string;
25
+ }
26
+ export declare function findPython(): PythonInfo | null;
27
+ /**
28
+ * Find which expected filename a snippet of text refers to.
29
+ *
30
+ * Plain `includes` is wrong here and was a real bug: "test_lru.py" contains
31
+ * "lru.py", so a naive scan attributed every test block to lru.py and silently
32
+ * overwrote it. Match longest-first and require a boundary before the name.
33
+ */
34
+ export declare function findExpectedName(source: string, expectedFiles: string[]): string | null;
35
+ /**
36
+ * Models label files in markdown, which means the label is often escaped
37
+ * (`test\_lru.py`) or decorated (`**test_lru.py**`, `### 3. test_lru.py`).
38
+ * Undo that before pattern matching or the leading segment gets lost.
39
+ */
40
+ export declare function unescapeMarkdown(text: string): string;
41
+ /** A fenced code block extracted from a model response. */
42
+ export interface CodeBlock {
43
+ language: string;
44
+ filename: string | null;
45
+ code: string;
46
+ isCode: boolean;
47
+ }
48
+ /**
49
+ * Pull fenced code blocks out of a response and work out which file each is.
50
+ *
51
+ * @param {string} text
52
+ * @param {string[]} expectedFiles names the task asked for, used to
53
+ * disambiguate unlabelled blocks
54
+ */
55
+ export declare function extractCodeBlocks(text: string, expectedFiles?: string[]): CodeBlock[];
56
+ /** True when the response elided work instead of writing it out. */
57
+ export declare function findPlaceholders(text: string): number;
58
+ /** Options for {@link verifyPython}. */
59
+ export interface VerifyPythonOptions {
60
+ expectedFiles?: string[];
61
+ execute?: boolean;
62
+ timeoutMs?: number;
63
+ }
64
+ /** Result of writing, syntax-checking, and running generated Python. */
65
+ export interface VerifyResult {
66
+ pythonAvailable: boolean;
67
+ blocks: number;
68
+ filesWritten: string[];
69
+ filesExpected: string[];
70
+ filesMissing: string[];
71
+ syntaxErrors: string[];
72
+ testFile: string | null;
73
+ nameCollisions: string[];
74
+ placeholders: number;
75
+ compiled: boolean | null;
76
+ testsRun: boolean;
77
+ testsPassed: number | null;
78
+ testsTotal: number | null;
79
+ testOutput: string | null;
80
+ }
81
+ /**
82
+ * Write the extracted files to disk, syntax-check them, and optionally run the
83
+ * test module.
84
+ */
85
+ export declare function verifyPython(text: string, { expectedFiles, execute, timeoutMs }?: VerifyPythonOptions): Promise<VerifyResult>;
86
+ /**
87
+ * Parse a `python -m unittest` transcript into passed/total counts.
88
+ *
89
+ * Pass this the unittest STDERR stream, not stdout+stderr combined: unittest
90
+ * emits its summary ("Ran N tests", then a final "OK" or "FAILED (...)" line) to
91
+ * stderr, whereas a graded solution prints to stdout. The final-status match is
92
+ * anchored to the end of the transcript so a solution that prints "OK" earlier
93
+ * cannot spoof a pass; "FAILED" is honored explicitly.
94
+ */
95
+ export declare function parseUnittest(output: string): {
96
+ total: number | null;
97
+ passed: number | null;
98
+ };
99
+ /** Outcome of running one unittest module over a set of in-memory files. */
100
+ export interface UnitRunResult {
101
+ ran: boolean;
102
+ compiled: boolean;
103
+ passed: number | null;
104
+ total: number | null;
105
+ output: string;
106
+ }
107
+ /**
108
+ * Write the given files to a scratch dir, syntax-check them, and run one
109
+ * unittest module against the real interpreter. The agentic-loop task uses this
110
+ * to grade a model's fix against a hidden test suite - objective, deterministic,
111
+ * and never a model grading a model. Returns `ran: false` when Python is absent
112
+ * or execution is disabled, so the caller can fall back to process credit.
113
+ */
114
+ export declare function runUnittestFiles(files: Record<string, string>, testModule: string, { execute, timeoutMs }?: {
115
+ execute?: boolean;
116
+ timeoutMs?: number;
117
+ }): Promise<UnitRunResult>;
118
+ /** A single tool-call the model may emit, in either Anthropic or OpenAI shape. */
119
+ export interface ToolCall {
120
+ id?: string;
121
+ name?: string;
122
+ input?: unknown;
123
+ function?: {
124
+ name?: string;
125
+ arguments?: unknown;
126
+ };
127
+ }
128
+ /** What a tool-call case expects: a name, required args, and a predicate. */
129
+ export interface ToolCallExpectation {
130
+ name: string;
131
+ requiredArgs?: string[];
132
+ check?: (args: Record<string, unknown>) => boolean | string;
133
+ }
134
+ /** Verdict from checking a tool call against an expectation. */
135
+ export interface ToolCallVerdict {
136
+ ok: boolean;
137
+ reason?: string | null;
138
+ args?: Record<string, unknown>;
139
+ }
140
+ /** Check a tool call against an expected name and argument predicate. */
141
+ export declare function verifyToolCall(toolCalls: ToolCall[] | null | undefined, expectation: ToolCallExpectation): ToolCallVerdict;
142
+ //# sourceMappingURL=verify.d.ts.map
@@ -0,0 +1,350 @@
1
+ import fs from "node:fs";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ import { execFile, execFileSync } from "node:child_process";
5
+ function which(candidates) {
6
+ for (const candidate of candidates) {
7
+ try {
8
+ const result = execFileSync(candidate, ["--version"], {
9
+ timeout: 8000,
10
+ windowsHide: true,
11
+ stdio: ["ignore", "pipe", "pipe"],
12
+ });
13
+ return { exe: candidate, version: String(result).trim() };
14
+ }
15
+ catch {
16
+ /* try the next one */
17
+ }
18
+ }
19
+ return null;
20
+ }
21
+ let cachedPython;
22
+ export function findPython() {
23
+ if (cachedPython === undefined)
24
+ cachedPython = which(["python", "py", "python3"]);
25
+ return cachedPython;
26
+ }
27
+ function run(exe, args, { cwd, timeout = 60000 }) {
28
+ return new Promise((resolve) => {
29
+ execFile(exe, args, { cwd, timeout, windowsHide: true, maxBuffer: 8 * 1024 * 1024 }, (error, stdout, stderr) => {
30
+ resolve({
31
+ ok: !error,
32
+ code: error?.code ?? 0,
33
+ timedOut: error?.killed === true || error?.signal === "SIGTERM",
34
+ stdout: String(stdout),
35
+ stderr: String(stderr),
36
+ });
37
+ });
38
+ });
39
+ }
40
+ const CODE_LANGUAGES = new Set(["python", "py", "python3", ""]);
41
+ const FILENAME_PATTERN = /([\w./-]+\.(?:py|md|txt|json|toml|cfg|ini))/;
42
+ function escapeRegExp(text) {
43
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
44
+ }
45
+ /**
46
+ * Find which expected filename a snippet of text refers to.
47
+ *
48
+ * Plain `includes` is wrong here and was a real bug: "test_lru.py" contains
49
+ * "lru.py", so a naive scan attributed every test block to lru.py and silently
50
+ * overwrote it. Match longest-first and require a boundary before the name.
51
+ */
52
+ export function findExpectedName(source, expectedFiles) {
53
+ const longestFirst = [...expectedFiles].sort((a, b) => b.length - a.length);
54
+ for (const name of longestFirst) {
55
+ const boundary = new RegExp(`(^|[^\\w.\\-])${escapeRegExp(name)}(\\W|$)`);
56
+ if (boundary.test(source))
57
+ return name;
58
+ }
59
+ return null;
60
+ }
61
+ /**
62
+ * Models label files in markdown, which means the label is often escaped
63
+ * (`test\_lru.py`) or decorated (`**test_lru.py**`, `### 3. test_lru.py`).
64
+ * Undo that before pattern matching or the leading segment gets lost.
65
+ */
66
+ export function unescapeMarkdown(text) {
67
+ return String(text).replace(/\\([_*`~[\]()#+\-.!])/g, "$1");
68
+ }
69
+ /** Guess a filename from what the code itself contains. */
70
+ function inferFromContent(code, expectedFiles) {
71
+ const isTest = /import\s+unittest|unittest\.TestCase|^\s*import\s+pytest/m.test(code);
72
+ if (isTest) {
73
+ const named = expectedFiles.find((f) => /^test_/.test(f));
74
+ if (named)
75
+ return named;
76
+ return "test_generated.py";
77
+ }
78
+ // Match a definition against an expected module name, e.g. class Metrics -> metrics.py
79
+ for (const candidate of expectedFiles) {
80
+ const stem = candidate.replace(/\.py$/, "");
81
+ if (!stem || candidate.startsWith("test_"))
82
+ continue;
83
+ const defined = new RegExp(`(?:class|def)\\s+${stem}`, "i").test(code);
84
+ if (defined)
85
+ return candidate;
86
+ }
87
+ return null;
88
+ }
89
+ /**
90
+ * Pull fenced code blocks out of a response and work out which file each is.
91
+ *
92
+ * @param {string} text
93
+ * @param {string[]} expectedFiles names the task asked for, used to
94
+ * disambiguate unlabelled blocks
95
+ */
96
+ export function extractCodeBlocks(text, expectedFiles = []) {
97
+ const clean = unescapeMarkdown(text);
98
+ const blocks = [];
99
+ const fence = /```([\w+-]*)[ \t]*([^\n]*)\n([\s\S]*?)```/g;
100
+ let match;
101
+ while ((match = fence.exec(clean)) !== null) {
102
+ const [, rawLanguage, info, code] = match;
103
+ const language = rawLanguage.toLowerCase();
104
+ // Only the 300 characters immediately before the fence, so a filename
105
+ // mentioned paragraphs earlier cannot be misattributed to this block.
106
+ const before = clean.slice(Math.max(0, match.index - 300), match.index);
107
+ const lastLines = before.split(/\n/).slice(-4).join("\n");
108
+ let filename = null;
109
+ // Strongest signal first: the fence info string, then a comment on the
110
+ // first line of the code, then the prose immediately above.
111
+ for (const source of [info, code.split("\n", 2).join("\n"), lastLines]) {
112
+ // Prefer an expected name if one appears literally.
113
+ const expected = findExpectedName(source, expectedFiles);
114
+ if (expected) {
115
+ filename = expected;
116
+ break;
117
+ }
118
+ const found = source.match(FILENAME_PATTERN);
119
+ if (found) {
120
+ filename = path.basename(found[1]);
121
+ break;
122
+ }
123
+ }
124
+ if (!filename)
125
+ filename = inferFromContent(code, expectedFiles);
126
+ blocks.push({ language, filename, code, isCode: CODE_LANGUAGES.has(language) });
127
+ }
128
+ return blocks;
129
+ }
130
+ /** True when the response elided work instead of writing it out. */
131
+ export function findPlaceholders(text) {
132
+ const patterns = [
133
+ /\.\.\.\s*(?:rest of|remaining|etc)/i,
134
+ /#\s*(?:TODO|FIXME|implement(?:ation)? (?:here|omitted)|rest of the (?:code|implementation))/i,
135
+ /\b(?:omitted for brevity|truncated for brevity|and so on)\b/i,
136
+ /<\s*(?:your code here|implementation)\s*>/i,
137
+ ];
138
+ return patterns.filter((p) => p.test(text)).length;
139
+ }
140
+ /**
141
+ * Write the extracted files to disk, syntax-check them, and optionally run the
142
+ * test module.
143
+ */
144
+ export async function verifyPython(text, { expectedFiles = [], execute = true, timeoutMs = 90000 } = {}) {
145
+ const python = findPython();
146
+ const blocks = extractCodeBlocks(text, expectedFiles);
147
+ const result = {
148
+ pythonAvailable: Boolean(python),
149
+ blocks: blocks.length,
150
+ filesWritten: [],
151
+ filesExpected: expectedFiles,
152
+ filesMissing: [],
153
+ syntaxErrors: [],
154
+ testFile: null,
155
+ nameCollisions: [],
156
+ placeholders: findPlaceholders(text),
157
+ compiled: null,
158
+ testsRun: false,
159
+ testsPassed: null,
160
+ testsTotal: null,
161
+ testOutput: null,
162
+ };
163
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "otto-brain-bench-"));
164
+ try {
165
+ // Prefer named blocks; fall back to positional naming for code blocks.
166
+ let anonymous = 0;
167
+ for (const block of blocks) {
168
+ let name = block.filename;
169
+ if (!name) {
170
+ // Never write prose (markdown, shell transcripts) into a .py file.
171
+ if (!block.isCode)
172
+ continue;
173
+ name = `block_${(anonymous += 1)}.py`;
174
+ }
175
+ // Never let a model-chosen name escape the scratch directory.
176
+ let safe = path.basename(name);
177
+ // Two blocks resolving to one name means a misattribution, not an
178
+ // intentional rewrite - keep both rather than losing one silently.
179
+ if (result.filesWritten.includes(safe)) {
180
+ const ext = path.extname(safe);
181
+ safe = `${path.basename(safe, ext)}_${result.filesWritten.length}${ext}`;
182
+ result.nameCollisions.push(name);
183
+ }
184
+ fs.writeFileSync(path.join(dir, safe), block.code, "utf8");
185
+ result.filesWritten.push(safe);
186
+ if (/^test_.*\.py$/.test(safe) || /unittest\.TestCase/.test(block.code)) {
187
+ result.testFile = safe;
188
+ }
189
+ }
190
+ result.filesMissing = expectedFiles.filter((f) => !result.filesWritten.includes(f));
191
+ if (!python)
192
+ return result;
193
+ // Syntax check every python file.
194
+ const pyFiles = result.filesWritten.filter((f) => f.endsWith(".py"));
195
+ if (pyFiles.length) {
196
+ const compileResult = await run(python.exe, ["-m", "py_compile", ...pyFiles], {
197
+ cwd: dir,
198
+ timeout: 45000,
199
+ });
200
+ result.compiled = compileResult.ok;
201
+ if (!compileResult.ok) {
202
+ for (const line of compileResult.stderr.split(/\r?\n/)) {
203
+ if (/Error|error:/.test(line))
204
+ result.syntaxErrors.push(line.trim());
205
+ }
206
+ }
207
+ }
208
+ else {
209
+ result.compiled = false;
210
+ }
211
+ // Run the tests only if everything at least parses.
212
+ const testFile = result.testFile || result.filesWritten.find((f) => /^test_.*\.py$/.test(f));
213
+ result.testFile = testFile || null;
214
+ if (execute && result.compiled && testFile) {
215
+ const testResult = await run(python.exe, ["-m", "unittest", "-v", testFile.replace(/\.py$/, "")], { cwd: dir, timeout: timeoutMs });
216
+ result.testsRun = true;
217
+ // Keep stdout+stderr for display, but the pass/fail VERDICT is parsed from
218
+ // stderr only: unittest writes its "Ran N tests" + "OK"/"FAILED" summary to
219
+ // stderr, while the model's solution prints to stdout. Concatenating them
220
+ // and scanning for "OK" anywhere let a solution that prints "OK" spoof a
221
+ // pass. parseUnittest anchors the status to the end of the transcript.
222
+ result.testOutput = `${testResult.stdout}\n${testResult.stderr}`.trim().slice(-4000);
223
+ const verdict = parseUnittest(testResult.stderr);
224
+ result.testsTotal = verdict.total;
225
+ result.testsPassed = verdict.passed;
226
+ if (testResult.timedOut)
227
+ result.testOutput = `TIMED OUT after ${timeoutMs}ms\n${result.testOutput}`;
228
+ }
229
+ return result;
230
+ }
231
+ finally {
232
+ fs.rmSync(dir, { recursive: true, force: true });
233
+ }
234
+ }
235
+ /**
236
+ * Parse a `python -m unittest` transcript into passed/total counts.
237
+ *
238
+ * Pass this the unittest STDERR stream, not stdout+stderr combined: unittest
239
+ * emits its summary ("Ran N tests", then a final "OK" or "FAILED (...)" line) to
240
+ * stderr, whereas a graded solution prints to stdout. The final-status match is
241
+ * anchored to the end of the transcript so a solution that prints "OK" earlier
242
+ * cannot spoof a pass; "FAILED" is honored explicitly.
243
+ */
244
+ export function parseUnittest(output) {
245
+ const trimmed = output.trimEnd();
246
+ const ran = trimmed.match(/Ran (\d+) tests?/);
247
+ const total = ran ? Number(ran[1]) : null;
248
+ if (total === null)
249
+ return { total: null, passed: null };
250
+ // The last line of a unittest run is "OK" (maybe with skips) or "FAILED (...)".
251
+ const status = trimmed.match(/(?:^|\n)(OK|FAILED)\b[^\n]*$/);
252
+ if (status && status[1] === "OK")
253
+ return { total, passed: total };
254
+ const failures = Number(trimmed.match(/failures=(\d+)/)?.[1] || 0);
255
+ const errors = Number(trimmed.match(/errors=(\d+)/)?.[1] || 0);
256
+ // No trailing OK and no failure counts (e.g. a crash before the summary) means
257
+ // nothing verifiably passed.
258
+ if (!status && failures === 0 && errors === 0)
259
+ return { total, passed: 0 };
260
+ return { total, passed: Math.max(0, total - failures - errors) };
261
+ }
262
+ /**
263
+ * Write the given files to a scratch dir, syntax-check them, and run one
264
+ * unittest module against the real interpreter. The agentic-loop task uses this
265
+ * to grade a model's fix against a hidden test suite - objective, deterministic,
266
+ * and never a model grading a model. Returns `ran: false` when Python is absent
267
+ * or execution is disabled, so the caller can fall back to process credit.
268
+ */
269
+ export async function runUnittestFiles(files, testModule, { execute = true, timeoutMs = 60000 } = {}) {
270
+ const result = {
271
+ ran: false,
272
+ compiled: false,
273
+ passed: null,
274
+ total: null,
275
+ output: "",
276
+ };
277
+ const python = findPython();
278
+ if (!python)
279
+ return result;
280
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "otto-brain-agentic-"));
281
+ try {
282
+ const pyFiles = [];
283
+ for (const [name, content] of Object.entries(files)) {
284
+ const safe = path.basename(name);
285
+ fs.writeFileSync(path.join(dir, safe), content, "utf8");
286
+ if (safe.endsWith(".py"))
287
+ pyFiles.push(safe);
288
+ }
289
+ const compileResult = await run(python.exe, ["-m", "py_compile", ...pyFiles], {
290
+ cwd: dir,
291
+ timeout: 30000,
292
+ });
293
+ result.compiled = compileResult.ok;
294
+ if (!compileResult.ok) {
295
+ result.output = compileResult.stderr.trim().slice(-2000);
296
+ return result;
297
+ }
298
+ if (!execute)
299
+ return result;
300
+ const testResult = await run(python.exe, ["-m", "unittest", "-v", testModule], {
301
+ cwd: dir,
302
+ timeout: timeoutMs,
303
+ });
304
+ result.ran = true;
305
+ result.output = `${testResult.stdout}\n${testResult.stderr}`.trim().slice(-2000);
306
+ // Verdict from stderr only (see parseUnittest) so model stdout can't spoof it.
307
+ const parsed = parseUnittest(testResult.stderr);
308
+ result.total = parsed.total;
309
+ result.passed = parsed.passed;
310
+ if (testResult.timedOut)
311
+ result.output = `TIMED OUT after ${timeoutMs}ms\n${result.output}`;
312
+ return result;
313
+ }
314
+ finally {
315
+ fs.rmSync(dir, { recursive: true, force: true });
316
+ }
317
+ }
318
+ /** Check a tool call against an expected name and argument predicate. */
319
+ export function verifyToolCall(toolCalls, expectation) {
320
+ if (!toolCalls || !toolCalls.length) {
321
+ return { ok: false, reason: "no tool call emitted" };
322
+ }
323
+ const call = toolCalls.find((c) => (c.function?.name || c.name) === expectation.name);
324
+ if (!call) {
325
+ const names = toolCalls.map((c) => c.function?.name || c.name).join(", ");
326
+ return { ok: false, reason: `expected ${expectation.name}, got ${names}` };
327
+ }
328
+ let args = call.function?.arguments ?? call.input ?? {};
329
+ if (typeof args === "string") {
330
+ try {
331
+ args = JSON.parse(args);
332
+ }
333
+ catch {
334
+ return { ok: false, reason: "arguments were not valid JSON" };
335
+ }
336
+ }
337
+ const record = args && typeof args === "object" ? args : {};
338
+ for (const key of expectation.requiredArgs || []) {
339
+ if (record[key] === undefined || record[key] === null || record[key] === "") {
340
+ return { ok: false, reason: `missing required argument "${key}"` };
341
+ }
342
+ }
343
+ if (expectation.check) {
344
+ const verdict = expectation.check(record);
345
+ if (verdict !== true)
346
+ return { ok: false, reason: verdict || "argument check failed" };
347
+ }
348
+ return { ok: true, args: record };
349
+ }
350
+ //# sourceMappingURL=verify.js.map
package/dist/cli.d.ts ADDED
@@ -0,0 +1,13 @@
1
+ /**
2
+ * The `otto brain` command group. Built with commander to mirror @otto-code/cli:
3
+ * a group factory (`createBrainCommand`) that adds straight into the main `otto`
4
+ * program, and a `registerBrainCommands` that mounts the same verbs on a
5
+ * standalone root program (for `bin/otto-brain` on a server without the full CLI).
6
+ * Handlers stay pure — they return typed results wrapped by withOutput.
7
+ */
8
+ import { Command } from "commander";
9
+ /** Mount every brain verb on the given command (a root program or the group). */
10
+ export declare function registerBrainCommands(program: Command): Command;
11
+ /** The `brain` command group, for adding into the main `otto` CLI. */
12
+ export declare function createBrainCommand(): Command;
13
+ //# sourceMappingURL=cli.d.ts.map
package/dist/cli.js ADDED
@@ -0,0 +1,72 @@
1
+ /**
2
+ * The `otto brain` command group. Built with commander to mirror @otto-code/cli:
3
+ * a group factory (`createBrainCommand`) that adds straight into the main `otto`
4
+ * program, and a `registerBrainCommands` that mounts the same verbs on a
5
+ * standalone root program (for `bin/otto-brain` on a server without the full CLI).
6
+ * Handlers stay pure — they return typed results wrapped by withOutput.
7
+ */
8
+ import { Command } from "commander";
9
+ import { addBenchOptions, runBenchCommand } from "./commands/bench.js";
10
+ import { addCalibrateOptions, runCalibrateCommand } from "./commands/calibrate.js";
11
+ import { addCatalogOptions, runCatalogCommand } from "./commands/catalog.js";
12
+ import { addConfigSetOptions, addConfigShowOptions, runConfigSetCommand, runConfigShowCommand, } from "./commands/config.js";
13
+ import { addShareOptions, runShareCommand } from "./commands/share.js";
14
+ import { addRestartOptions, addServeOptions, addStartOptions, addStatusOptions, addStopOptions, runRestartCommand, runServeCommand, runStartCommand, runStatusCommand, runStopCommand, } from "./commands/lifecycle.js";
15
+ import { addPullOptions, runPullCommand } from "./commands/pull.js";
16
+ import { addReportOptions, runReportCommand } from "./commands/report.js";
17
+ import { addRescoreOptions, runRescoreCommand } from "./commands/rescore.js";
18
+ import { addRuntimeInstallOptions, addRuntimeListOptions, runRuntimeInstallCommand, runRuntimeListCommand, } from "./commands/runtime.js";
19
+ import { addScanOptions, runScanCommand } from "./commands/scan.js";
20
+ import { addAddOptions, addSearchOptions, runAddCommand, runSearchCommand, } from "./commands/search.js";
21
+ import { addSweepOptions, runSweepCommand } from "./commands/sweep.js";
22
+ import { addUiOptions, runUiCommand } from "./commands/ui.js";
23
+ import { withOutput } from "./output/with-output.js";
24
+ function addGlobalOptions(cmd) {
25
+ return cmd
26
+ .option("-o, --format <format>", "output format: table, json, yaml", "table")
27
+ .option("--json", "output in JSON format (alias for --format json)")
28
+ .option("-q, --quiet", "minimal output (ids only)")
29
+ .option("--no-headers", "omit table headers")
30
+ .option("--no-color", "disable colored output");
31
+ }
32
+ /** Mount every brain verb on the given command (a root program or the group). */
33
+ export function registerBrainCommands(program) {
34
+ addGlobalOptions(program);
35
+ // Interactive: bare invocation and `ui` launch the TUI (no output wrapper).
36
+ program.action(runUiCommand);
37
+ addUiOptions(program.command("ui")).action(runUiCommand);
38
+ // Service lifecycle.
39
+ addServeOptions(program.command("serve")).action(runServeCommand);
40
+ addStartOptions(program.command("start")).action(withOutput(runStartCommand));
41
+ addStopOptions(program.command("stop")).action(withOutput(runStopCommand));
42
+ addRestartOptions(program.command("restart")).action(withOutput(runRestartCommand));
43
+ addStatusOptions(program.command("status")).action(withOutput(runStatusCommand));
44
+ // Discovery + ops.
45
+ addScanOptions(program.command("scan")).action(withOutput(runScanCommand));
46
+ addCalibrateOptions(program.command("calibrate")).action(withOutput(runCalibrateCommand));
47
+ addSweepOptions(program.command("sweep")).action(withOutput(runSweepCommand));
48
+ addCatalogOptions(program.command("catalog")).action(withOutput(runCatalogCommand));
49
+ addPullOptions(program.command("pull")).action(withOutput(runPullCommand));
50
+ addSearchOptions(program.command("search")).action(withOutput(runSearchCommand));
51
+ addAddOptions(program.command("add")).action(withOutput(runAddCommand));
52
+ // Benchmark suite (plain actions: long streaming runs with formatted reports).
53
+ addBenchOptions(program.command("bench")).action(runBenchCommand);
54
+ addRescoreOptions(program.command("rescore")).action(runRescoreCommand);
55
+ addReportOptions(program.command("report")).action(runReportCommand);
56
+ // runtime subgroup.
57
+ const runtime = program.command("runtime").description("Manage the llama.cpp runtime");
58
+ addRuntimeListOptions(runtime.command("list")).action(withOutput(runRuntimeListCommand));
59
+ addRuntimeInstallOptions(runtime.command("install")).action(withOutput(runRuntimeInstallCommand));
60
+ // config subgroup.
61
+ const config = program.command("config").description("Inspect and edit brain config");
62
+ addConfigShowOptions(config.command("show")).action(withOutput(runConfigShowCommand));
63
+ addConfigSetOptions(config.command("set")).action(withOutput(runConfigSetCommand));
64
+ addShareOptions(program.command("share")).action(withOutput(runShareCommand));
65
+ return program;
66
+ }
67
+ /** The `brain` command group, for adding into the main `otto` CLI. */
68
+ export function createBrainCommand() {
69
+ const brain = new Command("brain").description("Host local AI models (otto-brain)");
70
+ return registerBrainCommands(brain);
71
+ }
72
+ //# sourceMappingURL=cli.js.map
@@ -0,0 +1,20 @@
1
+ import type { Command } from "commander";
2
+ interface BenchOptions {
3
+ model?: string;
4
+ endpoint?: string;
5
+ only?: string;
6
+ depths?: string;
7
+ concurrency?: string;
8
+ execute?: boolean;
9
+ port?: string;
10
+ repoDir?: string;
11
+ repoWorkspace?: string;
12
+ repoWorkspaceDir?: string;
13
+ repoRef?: string;
14
+ repoMax?: string;
15
+ curated?: string;
16
+ }
17
+ export declare function addBenchOptions(cmd: Command): Command;
18
+ export declare function runBenchCommand(options: BenchOptions, _command: Command): Promise<void>;
19
+ export {};
20
+ //# sourceMappingURL=bench.d.ts.map