@otto-code/brain 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/otto-brain +2 -0
- package/dist/bench/context-corpus.d.ts +58 -0
- package/dist/bench/context-corpus.js +210 -0
- package/dist/bench/corpus.d.ts +31 -0
- package/dist/bench/corpus.js +379 -0
- package/dist/bench/curated-repos.d.ts +35 -0
- package/dist/bench/curated-repos.js +41 -0
- package/dist/bench/health.d.ts +50 -0
- package/dist/bench/health.js +112 -0
- package/dist/bench/index.d.ts +103 -0
- package/dist/bench/index.js +209 -0
- package/dist/bench/mine.d.ts +32 -0
- package/dist/bench/mine.js +101 -0
- package/dist/bench/repo-task.d.ts +93 -0
- package/dist/bench/repo-task.js +380 -0
- package/dist/bench/repo.d.ts +125 -0
- package/dist/bench/repo.js +195 -0
- package/dist/bench/rescore.d.ts +41 -0
- package/dist/bench/rescore.js +127 -0
- package/dist/bench/tasks.d.ts +124 -0
- package/dist/bench/tasks.js +1249 -0
- package/dist/bench/verify.d.ts +142 -0
- package/dist/bench/verify.js +350 -0
- package/dist/cli.d.ts +13 -0
- package/dist/cli.js +72 -0
- package/dist/commands/bench.d.ts +20 -0
- package/dist/commands/bench.js +222 -0
- package/dist/commands/calibrate.d.ts +19 -0
- package/dist/commands/calibrate.js +75 -0
- package/dist/commands/catalog.d.ts +32 -0
- package/dist/commands/catalog.js +48 -0
- package/dist/commands/config.d.ts +13 -0
- package/dist/commands/config.js +90 -0
- package/dist/commands/lifecycle.d.ts +29 -0
- package/dist/commands/lifecycle.js +194 -0
- package/dist/commands/pull.d.ts +22 -0
- package/dist/commands/pull.js +135 -0
- package/dist/commands/report.d.ts +10 -0
- package/dist/commands/report.js +34 -0
- package/dist/commands/rescore.d.ts +11 -0
- package/dist/commands/rescore.js +48 -0
- package/dist/commands/runtime.d.ts +21 -0
- package/dist/commands/runtime.js +62 -0
- package/dist/commands/scan.d.ts +24 -0
- package/dist/commands/scan.js +55 -0
- package/dist/commands/search.d.ts +31 -0
- package/dist/commands/search.js +135 -0
- package/dist/commands/share.d.ts +26 -0
- package/dist/commands/share.js +148 -0
- package/dist/commands/sweep.d.ts +21 -0
- package/dist/commands/sweep.js +69 -0
- package/dist/commands/ui.d.ts +9 -0
- package/dist/commands/ui.js +28 -0
- package/dist/config/env.d.ts +10 -0
- package/dist/config/env.js +48 -0
- package/dist/config/index.d.ts +9 -0
- package/dist/config/index.js +9 -0
- package/dist/config/otto-home.d.ts +2 -0
- package/dist/config/otto-home.js +24 -0
- package/dist/config/paths.d.ts +20 -0
- package/dist/config/paths.js +33 -0
- package/dist/config/private-files.d.ts +3 -0
- package/dist/config/private-files.js +32 -0
- package/dist/config/profiles.d.ts +34 -0
- package/dist/config/profiles.js +116 -0
- package/dist/config/schema.d.ts +1035 -0
- package/dist/config/schema.js +191 -0
- package/dist/config/store.d.ts +11 -0
- package/dist/config/store.js +73 -0
- package/dist/gguf.d.ts +61 -0
- package/dist/gguf.js +231 -0
- package/dist/gpu.d.ts +12 -0
- package/dist/gpu.js +49 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +13 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.js +5 -0
- package/dist/models/dirs.d.ts +9 -0
- package/dist/models/dirs.js +21 -0
- package/dist/models/download.d.ts +31 -0
- package/dist/models/download.js +105 -0
- package/dist/models/enrich.d.ts +36 -0
- package/dist/models/enrich.js +68 -0
- package/dist/models/hf.d.ts +62 -0
- package/dist/models/hf.js +114 -0
- package/dist/models/index.d.ts +15 -0
- package/dist/models/index.js +49 -0
- package/dist/models/manage.d.ts +27 -0
- package/dist/models/manage.js +112 -0
- package/dist/models/pick.d.ts +3 -0
- package/dist/models/pick.js +36 -0
- package/dist/models/scan.d.ts +13 -0
- package/dist/models/scan.js +185 -0
- package/dist/ops/archive.d.ts +34 -0
- package/dist/ops/archive.js +105 -0
- package/dist/ops/calibrate.d.ts +49 -0
- package/dist/ops/calibrate.js +74 -0
- package/dist/ops/report.d.ts +26 -0
- package/dist/ops/report.js +532 -0
- package/dist/ops/results.d.ts +207 -0
- package/dist/ops/results.js +250 -0
- package/dist/ops/sweep.d.ts +77 -0
- package/dist/ops/sweep.js +126 -0
- package/dist/output/index.d.ts +5 -0
- package/dist/output/index.js +5 -0
- package/dist/output/render.d.ts +4 -0
- package/dist/output/render.js +96 -0
- package/dist/output/types.d.ts +52 -0
- package/dist/output/types.js +9 -0
- package/dist/output/with-output.d.ts +3 -0
- package/dist/output/with-output.js +42 -0
- package/dist/run.d.ts +5 -0
- package/dist/run.js +19 -0
- package/dist/runtime/args.d.ts +21 -0
- package/dist/runtime/args.js +71 -0
- package/dist/runtime/index.d.ts +13 -0
- package/dist/runtime/index.js +41 -0
- package/dist/runtime/lmstudio.d.ts +10 -0
- package/dist/runtime/lmstudio.js +88 -0
- package/dist/runtime/managed.d.ts +26 -0
- package/dist/runtime/managed.js +148 -0
- package/dist/service/model-selector.d.ts +36 -0
- package/dist/service/model-selector.js +123 -0
- package/dist/service/pid-lock.d.ts +18 -0
- package/dist/service/pid-lock.js +50 -0
- package/dist/service/router.d.ts +144 -0
- package/dist/service/router.js +746 -0
- package/dist/service/scheduler.d.ts +74 -0
- package/dist/service/scheduler.js +110 -0
- package/dist/service/serve.d.ts +41 -0
- package/dist/service/serve.js +278 -0
- package/dist/service/supervisor.d.ts +67 -0
- package/dist/service/supervisor.js +222 -0
- package/dist/service/tailscale.d.ts +19 -0
- package/dist/service/tailscale.js +62 -0
- package/dist/service/tls.d.ts +77 -0
- package/dist/service/tls.js +192 -0
- package/dist/sysmon.d.ts +50 -0
- package/dist/sysmon.js +103 -0
- package/dist/tui/app.d.ts +208 -0
- package/dist/tui/app.js +1801 -0
- package/dist/tui/screen.d.ts +72 -0
- package/dist/tui/screen.js +176 -0
- package/dist/types.d.ts +76 -0
- package/dist/types.js +7 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +8 -0
- package/dist/vram.d.ts +67 -0
- package/dist/vram.js +132 -0
- package/package.json +58 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Objective verification of generated code.
|
|
3
|
+
*
|
|
4
|
+
* A benchmark is only worth trusting if a pass means something checkable, so
|
|
5
|
+
* nothing here asks a model to grade another model. Generated Python is
|
|
6
|
+
* written to a scratch directory and handed to the real interpreter: it either
|
|
7
|
+
* compiles and its tests pass, or it does not.
|
|
8
|
+
*
|
|
9
|
+
* NOTE: `runTests` executes model-generated code. It runs in a temporary
|
|
10
|
+
* directory under a timeout, but it is still arbitrary code execution - pass
|
|
11
|
+
* `execute: false` to stop at syntax checking.
|
|
12
|
+
*/
|
|
13
|
+
/** A discovered Python interpreter. */
|
|
14
|
+
export interface PythonInfo {
|
|
15
|
+
exe: string;
|
|
16
|
+
version: string;
|
|
17
|
+
}
|
|
18
|
+
/** Outcome of running a child process. */
|
|
19
|
+
export interface RunResult {
|
|
20
|
+
ok: boolean;
|
|
21
|
+
code: number | string;
|
|
22
|
+
timedOut: boolean;
|
|
23
|
+
stdout: string;
|
|
24
|
+
stderr: string;
|
|
25
|
+
}
|
|
26
|
+
export declare function findPython(): PythonInfo | null;
|
|
27
|
+
/**
|
|
28
|
+
* Find which expected filename a snippet of text refers to.
|
|
29
|
+
*
|
|
30
|
+
* Plain `includes` is wrong here and was a real bug: "test_lru.py" contains
|
|
31
|
+
* "lru.py", so a naive scan attributed every test block to lru.py and silently
|
|
32
|
+
* overwrote it. Match longest-first and require a boundary before the name.
|
|
33
|
+
*/
|
|
34
|
+
export declare function findExpectedName(source: string, expectedFiles: string[]): string | null;
|
|
35
|
+
/**
|
|
36
|
+
* Models label files in markdown, which means the label is often escaped
|
|
37
|
+
* (`test\_lru.py`) or decorated (`**test_lru.py**`, `### 3. test_lru.py`).
|
|
38
|
+
* Undo that before pattern matching or the leading segment gets lost.
|
|
39
|
+
*/
|
|
40
|
+
export declare function unescapeMarkdown(text: string): string;
|
|
41
|
+
/** A fenced code block extracted from a model response. */
|
|
42
|
+
export interface CodeBlock {
|
|
43
|
+
language: string;
|
|
44
|
+
filename: string | null;
|
|
45
|
+
code: string;
|
|
46
|
+
isCode: boolean;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Pull fenced code blocks out of a response and work out which file each is.
|
|
50
|
+
*
|
|
51
|
+
* @param {string} text
|
|
52
|
+
* @param {string[]} expectedFiles names the task asked for, used to
|
|
53
|
+
* disambiguate unlabelled blocks
|
|
54
|
+
*/
|
|
55
|
+
export declare function extractCodeBlocks(text: string, expectedFiles?: string[]): CodeBlock[];
|
|
56
|
+
/** True when the response elided work instead of writing it out. */
|
|
57
|
+
export declare function findPlaceholders(text: string): number;
|
|
58
|
+
/** Options for {@link verifyPython}. */
|
|
59
|
+
export interface VerifyPythonOptions {
|
|
60
|
+
expectedFiles?: string[];
|
|
61
|
+
execute?: boolean;
|
|
62
|
+
timeoutMs?: number;
|
|
63
|
+
}
|
|
64
|
+
/** Result of writing, syntax-checking, and running generated Python. */
|
|
65
|
+
export interface VerifyResult {
|
|
66
|
+
pythonAvailable: boolean;
|
|
67
|
+
blocks: number;
|
|
68
|
+
filesWritten: string[];
|
|
69
|
+
filesExpected: string[];
|
|
70
|
+
filesMissing: string[];
|
|
71
|
+
syntaxErrors: string[];
|
|
72
|
+
testFile: string | null;
|
|
73
|
+
nameCollisions: string[];
|
|
74
|
+
placeholders: number;
|
|
75
|
+
compiled: boolean | null;
|
|
76
|
+
testsRun: boolean;
|
|
77
|
+
testsPassed: number | null;
|
|
78
|
+
testsTotal: number | null;
|
|
79
|
+
testOutput: string | null;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Write the extracted files to disk, syntax-check them, and optionally run the
|
|
83
|
+
* test module.
|
|
84
|
+
*/
|
|
85
|
+
export declare function verifyPython(text: string, { expectedFiles, execute, timeoutMs }?: VerifyPythonOptions): Promise<VerifyResult>;
|
|
86
|
+
/**
|
|
87
|
+
* Parse a `python -m unittest` transcript into passed/total counts.
|
|
88
|
+
*
|
|
89
|
+
* Pass this the unittest STDERR stream, not stdout+stderr combined: unittest
|
|
90
|
+
* emits its summary ("Ran N tests", then a final "OK" or "FAILED (...)" line) to
|
|
91
|
+
* stderr, whereas a graded solution prints to stdout. The final-status match is
|
|
92
|
+
* anchored to the end of the transcript so a solution that prints "OK" earlier
|
|
93
|
+
* cannot spoof a pass; "FAILED" is honored explicitly.
|
|
94
|
+
*/
|
|
95
|
+
export declare function parseUnittest(output: string): {
|
|
96
|
+
total: number | null;
|
|
97
|
+
passed: number | null;
|
|
98
|
+
};
|
|
99
|
+
/** Outcome of running one unittest module over a set of in-memory files. */
|
|
100
|
+
export interface UnitRunResult {
|
|
101
|
+
ran: boolean;
|
|
102
|
+
compiled: boolean;
|
|
103
|
+
passed: number | null;
|
|
104
|
+
total: number | null;
|
|
105
|
+
output: string;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Write the given files to a scratch dir, syntax-check them, and run one
|
|
109
|
+
* unittest module against the real interpreter. The agentic-loop task uses this
|
|
110
|
+
* to grade a model's fix against a hidden test suite - objective, deterministic,
|
|
111
|
+
* and never a model grading a model. Returns `ran: false` when Python is absent
|
|
112
|
+
* or execution is disabled, so the caller can fall back to process credit.
|
|
113
|
+
*/
|
|
114
|
+
export declare function runUnittestFiles(files: Record<string, string>, testModule: string, { execute, timeoutMs }?: {
|
|
115
|
+
execute?: boolean;
|
|
116
|
+
timeoutMs?: number;
|
|
117
|
+
}): Promise<UnitRunResult>;
|
|
118
|
+
/** A single tool-call the model may emit, in either Anthropic or OpenAI shape. */
|
|
119
|
+
export interface ToolCall {
|
|
120
|
+
id?: string;
|
|
121
|
+
name?: string;
|
|
122
|
+
input?: unknown;
|
|
123
|
+
function?: {
|
|
124
|
+
name?: string;
|
|
125
|
+
arguments?: unknown;
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
/** What a tool-call case expects: a name, required args, and a predicate. */
|
|
129
|
+
export interface ToolCallExpectation {
|
|
130
|
+
name: string;
|
|
131
|
+
requiredArgs?: string[];
|
|
132
|
+
check?: (args: Record<string, unknown>) => boolean | string;
|
|
133
|
+
}
|
|
134
|
+
/** Verdict from checking a tool call against an expectation. */
|
|
135
|
+
export interface ToolCallVerdict {
|
|
136
|
+
ok: boolean;
|
|
137
|
+
reason?: string | null;
|
|
138
|
+
args?: Record<string, unknown>;
|
|
139
|
+
}
|
|
140
|
+
/** Check a tool call against an expected name and argument predicate. */
|
|
141
|
+
export declare function verifyToolCall(toolCalls: ToolCall[] | null | undefined, expectation: ToolCallExpectation): ToolCallVerdict;
|
|
142
|
+
//# sourceMappingURL=verify.d.ts.map
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { execFile, execFileSync } from "node:child_process";
|
|
5
|
+
function which(candidates) {
|
|
6
|
+
for (const candidate of candidates) {
|
|
7
|
+
try {
|
|
8
|
+
const result = execFileSync(candidate, ["--version"], {
|
|
9
|
+
timeout: 8000,
|
|
10
|
+
windowsHide: true,
|
|
11
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
12
|
+
});
|
|
13
|
+
return { exe: candidate, version: String(result).trim() };
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
/* try the next one */
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
let cachedPython;
|
|
22
|
+
export function findPython() {
|
|
23
|
+
if (cachedPython === undefined)
|
|
24
|
+
cachedPython = which(["python", "py", "python3"]);
|
|
25
|
+
return cachedPython;
|
|
26
|
+
}
|
|
27
|
+
function run(exe, args, { cwd, timeout = 60000 }) {
|
|
28
|
+
return new Promise((resolve) => {
|
|
29
|
+
execFile(exe, args, { cwd, timeout, windowsHide: true, maxBuffer: 8 * 1024 * 1024 }, (error, stdout, stderr) => {
|
|
30
|
+
resolve({
|
|
31
|
+
ok: !error,
|
|
32
|
+
code: error?.code ?? 0,
|
|
33
|
+
timedOut: error?.killed === true || error?.signal === "SIGTERM",
|
|
34
|
+
stdout: String(stdout),
|
|
35
|
+
stderr: String(stderr),
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const CODE_LANGUAGES = new Set(["python", "py", "python3", ""]);
|
|
41
|
+
const FILENAME_PATTERN = /([\w./-]+\.(?:py|md|txt|json|toml|cfg|ini))/;
|
|
42
|
+
function escapeRegExp(text) {
|
|
43
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Find which expected filename a snippet of text refers to.
|
|
47
|
+
*
|
|
48
|
+
* Plain `includes` is wrong here and was a real bug: "test_lru.py" contains
|
|
49
|
+
* "lru.py", so a naive scan attributed every test block to lru.py and silently
|
|
50
|
+
* overwrote it. Match longest-first and require a boundary before the name.
|
|
51
|
+
*/
|
|
52
|
+
export function findExpectedName(source, expectedFiles) {
|
|
53
|
+
const longestFirst = [...expectedFiles].sort((a, b) => b.length - a.length);
|
|
54
|
+
for (const name of longestFirst) {
|
|
55
|
+
const boundary = new RegExp(`(^|[^\\w.\\-])${escapeRegExp(name)}(\\W|$)`);
|
|
56
|
+
if (boundary.test(source))
|
|
57
|
+
return name;
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Models label files in markdown, which means the label is often escaped
|
|
63
|
+
* (`test\_lru.py`) or decorated (`**test_lru.py**`, `### 3. test_lru.py`).
|
|
64
|
+
* Undo that before pattern matching or the leading segment gets lost.
|
|
65
|
+
*/
|
|
66
|
+
export function unescapeMarkdown(text) {
|
|
67
|
+
return String(text).replace(/\\([_*`~[\]()#+\-.!])/g, "$1");
|
|
68
|
+
}
|
|
69
|
+
/** Guess a filename from what the code itself contains. */
|
|
70
|
+
function inferFromContent(code, expectedFiles) {
|
|
71
|
+
const isTest = /import\s+unittest|unittest\.TestCase|^\s*import\s+pytest/m.test(code);
|
|
72
|
+
if (isTest) {
|
|
73
|
+
const named = expectedFiles.find((f) => /^test_/.test(f));
|
|
74
|
+
if (named)
|
|
75
|
+
return named;
|
|
76
|
+
return "test_generated.py";
|
|
77
|
+
}
|
|
78
|
+
// Match a definition against an expected module name, e.g. class Metrics -> metrics.py
|
|
79
|
+
for (const candidate of expectedFiles) {
|
|
80
|
+
const stem = candidate.replace(/\.py$/, "");
|
|
81
|
+
if (!stem || candidate.startsWith("test_"))
|
|
82
|
+
continue;
|
|
83
|
+
const defined = new RegExp(`(?:class|def)\\s+${stem}`, "i").test(code);
|
|
84
|
+
if (defined)
|
|
85
|
+
return candidate;
|
|
86
|
+
}
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Pull fenced code blocks out of a response and work out which file each is.
|
|
91
|
+
*
|
|
92
|
+
* @param {string} text
|
|
93
|
+
* @param {string[]} expectedFiles names the task asked for, used to
|
|
94
|
+
* disambiguate unlabelled blocks
|
|
95
|
+
*/
|
|
96
|
+
export function extractCodeBlocks(text, expectedFiles = []) {
|
|
97
|
+
const clean = unescapeMarkdown(text);
|
|
98
|
+
const blocks = [];
|
|
99
|
+
const fence = /```([\w+-]*)[ \t]*([^\n]*)\n([\s\S]*?)```/g;
|
|
100
|
+
let match;
|
|
101
|
+
while ((match = fence.exec(clean)) !== null) {
|
|
102
|
+
const [, rawLanguage, info, code] = match;
|
|
103
|
+
const language = rawLanguage.toLowerCase();
|
|
104
|
+
// Only the 300 characters immediately before the fence, so a filename
|
|
105
|
+
// mentioned paragraphs earlier cannot be misattributed to this block.
|
|
106
|
+
const before = clean.slice(Math.max(0, match.index - 300), match.index);
|
|
107
|
+
const lastLines = before.split(/\n/).slice(-4).join("\n");
|
|
108
|
+
let filename = null;
|
|
109
|
+
// Strongest signal first: the fence info string, then a comment on the
|
|
110
|
+
// first line of the code, then the prose immediately above.
|
|
111
|
+
for (const source of [info, code.split("\n", 2).join("\n"), lastLines]) {
|
|
112
|
+
// Prefer an expected name if one appears literally.
|
|
113
|
+
const expected = findExpectedName(source, expectedFiles);
|
|
114
|
+
if (expected) {
|
|
115
|
+
filename = expected;
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
const found = source.match(FILENAME_PATTERN);
|
|
119
|
+
if (found) {
|
|
120
|
+
filename = path.basename(found[1]);
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (!filename)
|
|
125
|
+
filename = inferFromContent(code, expectedFiles);
|
|
126
|
+
blocks.push({ language, filename, code, isCode: CODE_LANGUAGES.has(language) });
|
|
127
|
+
}
|
|
128
|
+
return blocks;
|
|
129
|
+
}
|
|
130
|
+
/** True when the response elided work instead of writing it out. */
|
|
131
|
+
export function findPlaceholders(text) {
|
|
132
|
+
const patterns = [
|
|
133
|
+
/\.\.\.\s*(?:rest of|remaining|etc)/i,
|
|
134
|
+
/#\s*(?:TODO|FIXME|implement(?:ation)? (?:here|omitted)|rest of the (?:code|implementation))/i,
|
|
135
|
+
/\b(?:omitted for brevity|truncated for brevity|and so on)\b/i,
|
|
136
|
+
/<\s*(?:your code here|implementation)\s*>/i,
|
|
137
|
+
];
|
|
138
|
+
return patterns.filter((p) => p.test(text)).length;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Write the extracted files to disk, syntax-check them, and optionally run the
|
|
142
|
+
* test module.
|
|
143
|
+
*/
|
|
144
|
+
export async function verifyPython(text, { expectedFiles = [], execute = true, timeoutMs = 90000 } = {}) {
|
|
145
|
+
const python = findPython();
|
|
146
|
+
const blocks = extractCodeBlocks(text, expectedFiles);
|
|
147
|
+
const result = {
|
|
148
|
+
pythonAvailable: Boolean(python),
|
|
149
|
+
blocks: blocks.length,
|
|
150
|
+
filesWritten: [],
|
|
151
|
+
filesExpected: expectedFiles,
|
|
152
|
+
filesMissing: [],
|
|
153
|
+
syntaxErrors: [],
|
|
154
|
+
testFile: null,
|
|
155
|
+
nameCollisions: [],
|
|
156
|
+
placeholders: findPlaceholders(text),
|
|
157
|
+
compiled: null,
|
|
158
|
+
testsRun: false,
|
|
159
|
+
testsPassed: null,
|
|
160
|
+
testsTotal: null,
|
|
161
|
+
testOutput: null,
|
|
162
|
+
};
|
|
163
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "otto-brain-bench-"));
|
|
164
|
+
try {
|
|
165
|
+
// Prefer named blocks; fall back to positional naming for code blocks.
|
|
166
|
+
let anonymous = 0;
|
|
167
|
+
for (const block of blocks) {
|
|
168
|
+
let name = block.filename;
|
|
169
|
+
if (!name) {
|
|
170
|
+
// Never write prose (markdown, shell transcripts) into a .py file.
|
|
171
|
+
if (!block.isCode)
|
|
172
|
+
continue;
|
|
173
|
+
name = `block_${(anonymous += 1)}.py`;
|
|
174
|
+
}
|
|
175
|
+
// Never let a model-chosen name escape the scratch directory.
|
|
176
|
+
let safe = path.basename(name);
|
|
177
|
+
// Two blocks resolving to one name means a misattribution, not an
|
|
178
|
+
// intentional rewrite - keep both rather than losing one silently.
|
|
179
|
+
if (result.filesWritten.includes(safe)) {
|
|
180
|
+
const ext = path.extname(safe);
|
|
181
|
+
safe = `${path.basename(safe, ext)}_${result.filesWritten.length}${ext}`;
|
|
182
|
+
result.nameCollisions.push(name);
|
|
183
|
+
}
|
|
184
|
+
fs.writeFileSync(path.join(dir, safe), block.code, "utf8");
|
|
185
|
+
result.filesWritten.push(safe);
|
|
186
|
+
if (/^test_.*\.py$/.test(safe) || /unittest\.TestCase/.test(block.code)) {
|
|
187
|
+
result.testFile = safe;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
result.filesMissing = expectedFiles.filter((f) => !result.filesWritten.includes(f));
|
|
191
|
+
if (!python)
|
|
192
|
+
return result;
|
|
193
|
+
// Syntax check every python file.
|
|
194
|
+
const pyFiles = result.filesWritten.filter((f) => f.endsWith(".py"));
|
|
195
|
+
if (pyFiles.length) {
|
|
196
|
+
const compileResult = await run(python.exe, ["-m", "py_compile", ...pyFiles], {
|
|
197
|
+
cwd: dir,
|
|
198
|
+
timeout: 45000,
|
|
199
|
+
});
|
|
200
|
+
result.compiled = compileResult.ok;
|
|
201
|
+
if (!compileResult.ok) {
|
|
202
|
+
for (const line of compileResult.stderr.split(/\r?\n/)) {
|
|
203
|
+
if (/Error|error:/.test(line))
|
|
204
|
+
result.syntaxErrors.push(line.trim());
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
result.compiled = false;
|
|
210
|
+
}
|
|
211
|
+
// Run the tests only if everything at least parses.
|
|
212
|
+
const testFile = result.testFile || result.filesWritten.find((f) => /^test_.*\.py$/.test(f));
|
|
213
|
+
result.testFile = testFile || null;
|
|
214
|
+
if (execute && result.compiled && testFile) {
|
|
215
|
+
const testResult = await run(python.exe, ["-m", "unittest", "-v", testFile.replace(/\.py$/, "")], { cwd: dir, timeout: timeoutMs });
|
|
216
|
+
result.testsRun = true;
|
|
217
|
+
// Keep stdout+stderr for display, but the pass/fail VERDICT is parsed from
|
|
218
|
+
// stderr only: unittest writes its "Ran N tests" + "OK"/"FAILED" summary to
|
|
219
|
+
// stderr, while the model's solution prints to stdout. Concatenating them
|
|
220
|
+
// and scanning for "OK" anywhere let a solution that prints "OK" spoof a
|
|
221
|
+
// pass. parseUnittest anchors the status to the end of the transcript.
|
|
222
|
+
result.testOutput = `${testResult.stdout}\n${testResult.stderr}`.trim().slice(-4000);
|
|
223
|
+
const verdict = parseUnittest(testResult.stderr);
|
|
224
|
+
result.testsTotal = verdict.total;
|
|
225
|
+
result.testsPassed = verdict.passed;
|
|
226
|
+
if (testResult.timedOut)
|
|
227
|
+
result.testOutput = `TIMED OUT after ${timeoutMs}ms\n${result.testOutput}`;
|
|
228
|
+
}
|
|
229
|
+
return result;
|
|
230
|
+
}
|
|
231
|
+
finally {
|
|
232
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Parse a `python -m unittest` transcript into passed/total counts.
|
|
237
|
+
*
|
|
238
|
+
* Pass this the unittest STDERR stream, not stdout+stderr combined: unittest
|
|
239
|
+
* emits its summary ("Ran N tests", then a final "OK" or "FAILED (...)" line) to
|
|
240
|
+
* stderr, whereas a graded solution prints to stdout. The final-status match is
|
|
241
|
+
* anchored to the end of the transcript so a solution that prints "OK" earlier
|
|
242
|
+
* cannot spoof a pass; "FAILED" is honored explicitly.
|
|
243
|
+
*/
|
|
244
|
+
export function parseUnittest(output) {
|
|
245
|
+
const trimmed = output.trimEnd();
|
|
246
|
+
const ran = trimmed.match(/Ran (\d+) tests?/);
|
|
247
|
+
const total = ran ? Number(ran[1]) : null;
|
|
248
|
+
if (total === null)
|
|
249
|
+
return { total: null, passed: null };
|
|
250
|
+
// The last line of a unittest run is "OK" (maybe with skips) or "FAILED (...)".
|
|
251
|
+
const status = trimmed.match(/(?:^|\n)(OK|FAILED)\b[^\n]*$/);
|
|
252
|
+
if (status && status[1] === "OK")
|
|
253
|
+
return { total, passed: total };
|
|
254
|
+
const failures = Number(trimmed.match(/failures=(\d+)/)?.[1] || 0);
|
|
255
|
+
const errors = Number(trimmed.match(/errors=(\d+)/)?.[1] || 0);
|
|
256
|
+
// No trailing OK and no failure counts (e.g. a crash before the summary) means
|
|
257
|
+
// nothing verifiably passed.
|
|
258
|
+
if (!status && failures === 0 && errors === 0)
|
|
259
|
+
return { total, passed: 0 };
|
|
260
|
+
return { total, passed: Math.max(0, total - failures - errors) };
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Write the given files to a scratch dir, syntax-check them, and run one
|
|
264
|
+
* unittest module against the real interpreter. The agentic-loop task uses this
|
|
265
|
+
* to grade a model's fix against a hidden test suite - objective, deterministic,
|
|
266
|
+
* and never a model grading a model. Returns `ran: false` when Python is absent
|
|
267
|
+
* or execution is disabled, so the caller can fall back to process credit.
|
|
268
|
+
*/
|
|
269
|
+
export async function runUnittestFiles(files, testModule, { execute = true, timeoutMs = 60000 } = {}) {
|
|
270
|
+
const result = {
|
|
271
|
+
ran: false,
|
|
272
|
+
compiled: false,
|
|
273
|
+
passed: null,
|
|
274
|
+
total: null,
|
|
275
|
+
output: "",
|
|
276
|
+
};
|
|
277
|
+
const python = findPython();
|
|
278
|
+
if (!python)
|
|
279
|
+
return result;
|
|
280
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "otto-brain-agentic-"));
|
|
281
|
+
try {
|
|
282
|
+
const pyFiles = [];
|
|
283
|
+
for (const [name, content] of Object.entries(files)) {
|
|
284
|
+
const safe = path.basename(name);
|
|
285
|
+
fs.writeFileSync(path.join(dir, safe), content, "utf8");
|
|
286
|
+
if (safe.endsWith(".py"))
|
|
287
|
+
pyFiles.push(safe);
|
|
288
|
+
}
|
|
289
|
+
const compileResult = await run(python.exe, ["-m", "py_compile", ...pyFiles], {
|
|
290
|
+
cwd: dir,
|
|
291
|
+
timeout: 30000,
|
|
292
|
+
});
|
|
293
|
+
result.compiled = compileResult.ok;
|
|
294
|
+
if (!compileResult.ok) {
|
|
295
|
+
result.output = compileResult.stderr.trim().slice(-2000);
|
|
296
|
+
return result;
|
|
297
|
+
}
|
|
298
|
+
if (!execute)
|
|
299
|
+
return result;
|
|
300
|
+
const testResult = await run(python.exe, ["-m", "unittest", "-v", testModule], {
|
|
301
|
+
cwd: dir,
|
|
302
|
+
timeout: timeoutMs,
|
|
303
|
+
});
|
|
304
|
+
result.ran = true;
|
|
305
|
+
result.output = `${testResult.stdout}\n${testResult.stderr}`.trim().slice(-2000);
|
|
306
|
+
// Verdict from stderr only (see parseUnittest) so model stdout can't spoof it.
|
|
307
|
+
const parsed = parseUnittest(testResult.stderr);
|
|
308
|
+
result.total = parsed.total;
|
|
309
|
+
result.passed = parsed.passed;
|
|
310
|
+
if (testResult.timedOut)
|
|
311
|
+
result.output = `TIMED OUT after ${timeoutMs}ms\n${result.output}`;
|
|
312
|
+
return result;
|
|
313
|
+
}
|
|
314
|
+
finally {
|
|
315
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
/** Check a tool call against an expected name and argument predicate. */
|
|
319
|
+
export function verifyToolCall(toolCalls, expectation) {
|
|
320
|
+
if (!toolCalls || !toolCalls.length) {
|
|
321
|
+
return { ok: false, reason: "no tool call emitted" };
|
|
322
|
+
}
|
|
323
|
+
const call = toolCalls.find((c) => (c.function?.name || c.name) === expectation.name);
|
|
324
|
+
if (!call) {
|
|
325
|
+
const names = toolCalls.map((c) => c.function?.name || c.name).join(", ");
|
|
326
|
+
return { ok: false, reason: `expected ${expectation.name}, got ${names}` };
|
|
327
|
+
}
|
|
328
|
+
let args = call.function?.arguments ?? call.input ?? {};
|
|
329
|
+
if (typeof args === "string") {
|
|
330
|
+
try {
|
|
331
|
+
args = JSON.parse(args);
|
|
332
|
+
}
|
|
333
|
+
catch {
|
|
334
|
+
return { ok: false, reason: "arguments were not valid JSON" };
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
const record = args && typeof args === "object" ? args : {};
|
|
338
|
+
for (const key of expectation.requiredArgs || []) {
|
|
339
|
+
if (record[key] === undefined || record[key] === null || record[key] === "") {
|
|
340
|
+
return { ok: false, reason: `missing required argument "${key}"` };
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
if (expectation.check) {
|
|
344
|
+
const verdict = expectation.check(record);
|
|
345
|
+
if (verdict !== true)
|
|
346
|
+
return { ok: false, reason: verdict || "argument check failed" };
|
|
347
|
+
}
|
|
348
|
+
return { ok: true, args: record };
|
|
349
|
+
}
|
|
350
|
+
//# sourceMappingURL=verify.js.map
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `otto brain` command group. Built with commander to mirror @otto-code/cli:
|
|
3
|
+
* a group factory (`createBrainCommand`) that adds straight into the main `otto`
|
|
4
|
+
* program, and a `registerBrainCommands` that mounts the same verbs on a
|
|
5
|
+
* standalone root program (for `bin/otto-brain` on a server without the full CLI).
|
|
6
|
+
* Handlers stay pure — they return typed results wrapped by withOutput.
|
|
7
|
+
*/
|
|
8
|
+
import { Command } from "commander";
|
|
9
|
+
/** Mount every brain verb on the given command (a root program or the group). */
|
|
10
|
+
export declare function registerBrainCommands(program: Command): Command;
|
|
11
|
+
/** The `brain` command group, for adding into the main `otto` CLI. */
|
|
12
|
+
export declare function createBrainCommand(): Command;
|
|
13
|
+
//# sourceMappingURL=cli.d.ts.map
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `otto brain` command group. Built with commander to mirror @otto-code/cli:
|
|
3
|
+
* a group factory (`createBrainCommand`) that adds straight into the main `otto`
|
|
4
|
+
* program, and a `registerBrainCommands` that mounts the same verbs on a
|
|
5
|
+
* standalone root program (for `bin/otto-brain` on a server without the full CLI).
|
|
6
|
+
* Handlers stay pure — they return typed results wrapped by withOutput.
|
|
7
|
+
*/
|
|
8
|
+
import { Command } from "commander";
|
|
9
|
+
import { addBenchOptions, runBenchCommand } from "./commands/bench.js";
|
|
10
|
+
import { addCalibrateOptions, runCalibrateCommand } from "./commands/calibrate.js";
|
|
11
|
+
import { addCatalogOptions, runCatalogCommand } from "./commands/catalog.js";
|
|
12
|
+
import { addConfigSetOptions, addConfigShowOptions, runConfigSetCommand, runConfigShowCommand, } from "./commands/config.js";
|
|
13
|
+
import { addShareOptions, runShareCommand } from "./commands/share.js";
|
|
14
|
+
import { addRestartOptions, addServeOptions, addStartOptions, addStatusOptions, addStopOptions, runRestartCommand, runServeCommand, runStartCommand, runStatusCommand, runStopCommand, } from "./commands/lifecycle.js";
|
|
15
|
+
import { addPullOptions, runPullCommand } from "./commands/pull.js";
|
|
16
|
+
import { addReportOptions, runReportCommand } from "./commands/report.js";
|
|
17
|
+
import { addRescoreOptions, runRescoreCommand } from "./commands/rescore.js";
|
|
18
|
+
import { addRuntimeInstallOptions, addRuntimeListOptions, runRuntimeInstallCommand, runRuntimeListCommand, } from "./commands/runtime.js";
|
|
19
|
+
import { addScanOptions, runScanCommand } from "./commands/scan.js";
|
|
20
|
+
import { addAddOptions, addSearchOptions, runAddCommand, runSearchCommand, } from "./commands/search.js";
|
|
21
|
+
import { addSweepOptions, runSweepCommand } from "./commands/sweep.js";
|
|
22
|
+
import { addUiOptions, runUiCommand } from "./commands/ui.js";
|
|
23
|
+
import { withOutput } from "./output/with-output.js";
|
|
24
|
+
function addGlobalOptions(cmd) {
|
|
25
|
+
return cmd
|
|
26
|
+
.option("-o, --format <format>", "output format: table, json, yaml", "table")
|
|
27
|
+
.option("--json", "output in JSON format (alias for --format json)")
|
|
28
|
+
.option("-q, --quiet", "minimal output (ids only)")
|
|
29
|
+
.option("--no-headers", "omit table headers")
|
|
30
|
+
.option("--no-color", "disable colored output");
|
|
31
|
+
}
|
|
32
|
+
/** Mount every brain verb on the given command (a root program or the group). */
|
|
33
|
+
export function registerBrainCommands(program) {
|
|
34
|
+
addGlobalOptions(program);
|
|
35
|
+
// Interactive: bare invocation and `ui` launch the TUI (no output wrapper).
|
|
36
|
+
program.action(runUiCommand);
|
|
37
|
+
addUiOptions(program.command("ui")).action(runUiCommand);
|
|
38
|
+
// Service lifecycle.
|
|
39
|
+
addServeOptions(program.command("serve")).action(runServeCommand);
|
|
40
|
+
addStartOptions(program.command("start")).action(withOutput(runStartCommand));
|
|
41
|
+
addStopOptions(program.command("stop")).action(withOutput(runStopCommand));
|
|
42
|
+
addRestartOptions(program.command("restart")).action(withOutput(runRestartCommand));
|
|
43
|
+
addStatusOptions(program.command("status")).action(withOutput(runStatusCommand));
|
|
44
|
+
// Discovery + ops.
|
|
45
|
+
addScanOptions(program.command("scan")).action(withOutput(runScanCommand));
|
|
46
|
+
addCalibrateOptions(program.command("calibrate")).action(withOutput(runCalibrateCommand));
|
|
47
|
+
addSweepOptions(program.command("sweep")).action(withOutput(runSweepCommand));
|
|
48
|
+
addCatalogOptions(program.command("catalog")).action(withOutput(runCatalogCommand));
|
|
49
|
+
addPullOptions(program.command("pull")).action(withOutput(runPullCommand));
|
|
50
|
+
addSearchOptions(program.command("search")).action(withOutput(runSearchCommand));
|
|
51
|
+
addAddOptions(program.command("add")).action(withOutput(runAddCommand));
|
|
52
|
+
// Benchmark suite (plain actions: long streaming runs with formatted reports).
|
|
53
|
+
addBenchOptions(program.command("bench")).action(runBenchCommand);
|
|
54
|
+
addRescoreOptions(program.command("rescore")).action(runRescoreCommand);
|
|
55
|
+
addReportOptions(program.command("report")).action(runReportCommand);
|
|
56
|
+
// runtime subgroup.
|
|
57
|
+
const runtime = program.command("runtime").description("Manage the llama.cpp runtime");
|
|
58
|
+
addRuntimeListOptions(runtime.command("list")).action(withOutput(runRuntimeListCommand));
|
|
59
|
+
addRuntimeInstallOptions(runtime.command("install")).action(withOutput(runRuntimeInstallCommand));
|
|
60
|
+
// config subgroup.
|
|
61
|
+
const config = program.command("config").description("Inspect and edit brain config");
|
|
62
|
+
addConfigShowOptions(config.command("show")).action(withOutput(runConfigShowCommand));
|
|
63
|
+
addConfigSetOptions(config.command("set")).action(withOutput(runConfigSetCommand));
|
|
64
|
+
addShareOptions(program.command("share")).action(withOutput(runShareCommand));
|
|
65
|
+
return program;
|
|
66
|
+
}
|
|
67
|
+
/** The `brain` command group, for adding into the main `otto` CLI. */
|
|
68
|
+
export function createBrainCommand() {
|
|
69
|
+
const brain = new Command("brain").description("Host local AI models (otto-brain)");
|
|
70
|
+
return registerBrainCommands(brain);
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Command } from "commander";
|
|
2
|
+
interface BenchOptions {
|
|
3
|
+
model?: string;
|
|
4
|
+
endpoint?: string;
|
|
5
|
+
only?: string;
|
|
6
|
+
depths?: string;
|
|
7
|
+
concurrency?: string;
|
|
8
|
+
execute?: boolean;
|
|
9
|
+
port?: string;
|
|
10
|
+
repoDir?: string;
|
|
11
|
+
repoWorkspace?: string;
|
|
12
|
+
repoWorkspaceDir?: string;
|
|
13
|
+
repoRef?: string;
|
|
14
|
+
repoMax?: string;
|
|
15
|
+
curated?: string;
|
|
16
|
+
}
|
|
17
|
+
export declare function addBenchOptions(cmd: Command): Command;
|
|
18
|
+
export declare function runBenchCommand(options: BenchOptions, _command: Command): Promise<void>;
|
|
19
|
+
export {};
|
|
20
|
+
//# sourceMappingURL=bench.d.ts.map
|