@otto-code/brain 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/otto-brain +2 -0
- package/dist/bench/context-corpus.d.ts +58 -0
- package/dist/bench/context-corpus.js +210 -0
- package/dist/bench/corpus.d.ts +31 -0
- package/dist/bench/corpus.js +379 -0
- package/dist/bench/curated-repos.d.ts +35 -0
- package/dist/bench/curated-repos.js +41 -0
- package/dist/bench/health.d.ts +50 -0
- package/dist/bench/health.js +112 -0
- package/dist/bench/index.d.ts +103 -0
- package/dist/bench/index.js +209 -0
- package/dist/bench/mine.d.ts +32 -0
- package/dist/bench/mine.js +101 -0
- package/dist/bench/repo-task.d.ts +93 -0
- package/dist/bench/repo-task.js +380 -0
- package/dist/bench/repo.d.ts +125 -0
- package/dist/bench/repo.js +195 -0
- package/dist/bench/rescore.d.ts +41 -0
- package/dist/bench/rescore.js +127 -0
- package/dist/bench/tasks.d.ts +124 -0
- package/dist/bench/tasks.js +1249 -0
- package/dist/bench/verify.d.ts +142 -0
- package/dist/bench/verify.js +350 -0
- package/dist/cli.d.ts +13 -0
- package/dist/cli.js +72 -0
- package/dist/commands/bench.d.ts +20 -0
- package/dist/commands/bench.js +222 -0
- package/dist/commands/calibrate.d.ts +19 -0
- package/dist/commands/calibrate.js +75 -0
- package/dist/commands/catalog.d.ts +32 -0
- package/dist/commands/catalog.js +48 -0
- package/dist/commands/config.d.ts +13 -0
- package/dist/commands/config.js +90 -0
- package/dist/commands/lifecycle.d.ts +29 -0
- package/dist/commands/lifecycle.js +194 -0
- package/dist/commands/pull.d.ts +22 -0
- package/dist/commands/pull.js +135 -0
- package/dist/commands/report.d.ts +10 -0
- package/dist/commands/report.js +34 -0
- package/dist/commands/rescore.d.ts +11 -0
- package/dist/commands/rescore.js +48 -0
- package/dist/commands/runtime.d.ts +21 -0
- package/dist/commands/runtime.js +62 -0
- package/dist/commands/scan.d.ts +24 -0
- package/dist/commands/scan.js +55 -0
- package/dist/commands/search.d.ts +31 -0
- package/dist/commands/search.js +135 -0
- package/dist/commands/share.d.ts +26 -0
- package/dist/commands/share.js +148 -0
- package/dist/commands/sweep.d.ts +21 -0
- package/dist/commands/sweep.js +69 -0
- package/dist/commands/ui.d.ts +9 -0
- package/dist/commands/ui.js +28 -0
- package/dist/config/env.d.ts +10 -0
- package/dist/config/env.js +48 -0
- package/dist/config/index.d.ts +9 -0
- package/dist/config/index.js +9 -0
- package/dist/config/otto-home.d.ts +2 -0
- package/dist/config/otto-home.js +24 -0
- package/dist/config/paths.d.ts +20 -0
- package/dist/config/paths.js +33 -0
- package/dist/config/private-files.d.ts +3 -0
- package/dist/config/private-files.js +32 -0
- package/dist/config/profiles.d.ts +34 -0
- package/dist/config/profiles.js +116 -0
- package/dist/config/schema.d.ts +1035 -0
- package/dist/config/schema.js +191 -0
- package/dist/config/store.d.ts +11 -0
- package/dist/config/store.js +73 -0
- package/dist/gguf.d.ts +61 -0
- package/dist/gguf.js +231 -0
- package/dist/gpu.d.ts +12 -0
- package/dist/gpu.js +49 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +13 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.js +5 -0
- package/dist/models/dirs.d.ts +9 -0
- package/dist/models/dirs.js +21 -0
- package/dist/models/download.d.ts +31 -0
- package/dist/models/download.js +105 -0
- package/dist/models/enrich.d.ts +36 -0
- package/dist/models/enrich.js +68 -0
- package/dist/models/hf.d.ts +62 -0
- package/dist/models/hf.js +114 -0
- package/dist/models/index.d.ts +15 -0
- package/dist/models/index.js +49 -0
- package/dist/models/manage.d.ts +27 -0
- package/dist/models/manage.js +112 -0
- package/dist/models/pick.d.ts +3 -0
- package/dist/models/pick.js +36 -0
- package/dist/models/scan.d.ts +13 -0
- package/dist/models/scan.js +185 -0
- package/dist/ops/archive.d.ts +34 -0
- package/dist/ops/archive.js +105 -0
- package/dist/ops/calibrate.d.ts +49 -0
- package/dist/ops/calibrate.js +74 -0
- package/dist/ops/report.d.ts +26 -0
- package/dist/ops/report.js +532 -0
- package/dist/ops/results.d.ts +207 -0
- package/dist/ops/results.js +250 -0
- package/dist/ops/sweep.d.ts +77 -0
- package/dist/ops/sweep.js +126 -0
- package/dist/output/index.d.ts +5 -0
- package/dist/output/index.js +5 -0
- package/dist/output/render.d.ts +4 -0
- package/dist/output/render.js +96 -0
- package/dist/output/types.d.ts +52 -0
- package/dist/output/types.js +9 -0
- package/dist/output/with-output.d.ts +3 -0
- package/dist/output/with-output.js +42 -0
- package/dist/run.d.ts +5 -0
- package/dist/run.js +19 -0
- package/dist/runtime/args.d.ts +21 -0
- package/dist/runtime/args.js +71 -0
- package/dist/runtime/index.d.ts +13 -0
- package/dist/runtime/index.js +41 -0
- package/dist/runtime/lmstudio.d.ts +10 -0
- package/dist/runtime/lmstudio.js +88 -0
- package/dist/runtime/managed.d.ts +26 -0
- package/dist/runtime/managed.js +148 -0
- package/dist/service/model-selector.d.ts +36 -0
- package/dist/service/model-selector.js +123 -0
- package/dist/service/pid-lock.d.ts +18 -0
- package/dist/service/pid-lock.js +50 -0
- package/dist/service/router.d.ts +144 -0
- package/dist/service/router.js +746 -0
- package/dist/service/scheduler.d.ts +74 -0
- package/dist/service/scheduler.js +110 -0
- package/dist/service/serve.d.ts +41 -0
- package/dist/service/serve.js +278 -0
- package/dist/service/supervisor.d.ts +67 -0
- package/dist/service/supervisor.js +222 -0
- package/dist/service/tailscale.d.ts +19 -0
- package/dist/service/tailscale.js +62 -0
- package/dist/service/tls.d.ts +77 -0
- package/dist/service/tls.js +192 -0
- package/dist/sysmon.d.ts +50 -0
- package/dist/sysmon.js +103 -0
- package/dist/tui/app.d.ts +208 -0
- package/dist/tui/app.js +1801 -0
- package/dist/tui/screen.d.ts +72 -0
- package/dist/tui/screen.js +176 -0
- package/dist/types.d.ts +76 -0
- package/dist/types.js +7 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +8 -0
- package/dist/vram.d.ts +67 -0
- package/dist/vram.js +132 -0
- package/package.json +58 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve a model from an exact catalog id or a unique case-insensitive name
|
|
3
|
+
* fragment. An ambiguous fragment is an error that lists the matches — ported
|
|
4
|
+
* from the original CLI's pickModel, but throwing a CommandError instead of
|
|
5
|
+
* calling process.exit, so the output layer renders it.
|
|
6
|
+
*/
|
|
7
|
+
import { CommandError } from "../output/types.js";
|
|
8
|
+
export function pickModel(catalog, needle) {
|
|
9
|
+
if (!needle) {
|
|
10
|
+
throw new CommandError({
|
|
11
|
+
code: "NO_MODEL",
|
|
12
|
+
message: "specify a model with --model <name fragment>",
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
const exact = catalog.find((m) => m.id === needle);
|
|
16
|
+
if (exact)
|
|
17
|
+
return exact;
|
|
18
|
+
const lower = String(needle).toLowerCase();
|
|
19
|
+
const matches = catalog.filter((m) => m.displayName.toLowerCase().includes(lower) || m.id.toLowerCase().includes(lower));
|
|
20
|
+
if (!matches.length) {
|
|
21
|
+
throw new CommandError({
|
|
22
|
+
code: "NO_MATCH",
|
|
23
|
+
message: `no model matches "${needle}"`,
|
|
24
|
+
details: "run `otto brain scan` to list them",
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
if (matches.length > 1) {
|
|
28
|
+
throw new CommandError({
|
|
29
|
+
code: "AMBIGUOUS_MODEL",
|
|
30
|
+
message: `"${needle}" matches ${matches.length} models`,
|
|
31
|
+
details: matches.map((m) => m.displayName).join(", "),
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
return matches[0];
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=pick.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Model } from "../types.js";
|
|
2
|
+
export declare const LMSTUDIO_MODELS_DIR: string;
|
|
3
|
+
export declare function detectQuant(filename: string): string | null;
|
|
4
|
+
export declare function isProjectorFile(filename: string): boolean;
|
|
5
|
+
export interface ScanOptions {
|
|
6
|
+
modelsDir?: string;
|
|
7
|
+
withMetadata?: boolean;
|
|
8
|
+
origin?: "managed" | "lmstudio";
|
|
9
|
+
}
|
|
10
|
+
export declare function scan({ modelsDir, withMetadata, origin, }?: ScanOptions): Model[];
|
|
11
|
+
/** Human-readable size. */
|
|
12
|
+
export declare function formatBytes(bytes: number): string;
|
|
13
|
+
//# sourceMappingURL=scan.d.ts.map
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build a catalog of hostable models from disk. Weight files and their vision
|
|
3
|
+
* projectors live side by side in the same directory, so a projector found next
|
|
4
|
+
* to a model is paired with it. Ported from the original models.js; the models
|
|
5
|
+
* directory is now configurable (managed dir ∪ LM Studio) instead of hardcoded.
|
|
6
|
+
*/
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
import os from "node:os";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import * as gguf from "../gguf.js";
|
|
11
|
+
export const LMSTUDIO_MODELS_DIR = path.join(os.homedir(), ".lmstudio", "models");
|
|
12
|
+
// Quantisation labels as they appear in filenames, longest first so that
|
|
13
|
+
// Q4_K_M wins over Q4_K.
|
|
14
|
+
const QUANT_PATTERNS = [
|
|
15
|
+
"IQ1_S",
|
|
16
|
+
"IQ1_M",
|
|
17
|
+
"IQ2_XXS",
|
|
18
|
+
"IQ2_XS",
|
|
19
|
+
"IQ2_S",
|
|
20
|
+
"IQ2_M",
|
|
21
|
+
"IQ3_XXS",
|
|
22
|
+
"IQ3_XS",
|
|
23
|
+
"IQ3_S",
|
|
24
|
+
"IQ3_M",
|
|
25
|
+
"IQ4_XS",
|
|
26
|
+
"IQ4_NL",
|
|
27
|
+
"Q2_K_S",
|
|
28
|
+
"Q2_K",
|
|
29
|
+
"Q3_K_S",
|
|
30
|
+
"Q3_K_M",
|
|
31
|
+
"Q3_K_L",
|
|
32
|
+
"Q4_K_S",
|
|
33
|
+
"Q4_K_M",
|
|
34
|
+
"Q5_K_S",
|
|
35
|
+
"Q5_K_M",
|
|
36
|
+
"Q6_K",
|
|
37
|
+
"Q8_0",
|
|
38
|
+
"Q4_0",
|
|
39
|
+
"Q4_1",
|
|
40
|
+
"Q5_0",
|
|
41
|
+
"Q5_1",
|
|
42
|
+
"NVFP4",
|
|
43
|
+
"MXFP4",
|
|
44
|
+
"BF16",
|
|
45
|
+
"F16",
|
|
46
|
+
"F32",
|
|
47
|
+
];
|
|
48
|
+
const MULTIPART = /-(\d{5})-of-(\d{5})\.gguf$/i;
|
|
49
|
+
function walk(dir, out = []) {
|
|
50
|
+
let entries;
|
|
51
|
+
try {
|
|
52
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
for (const entry of entries) {
|
|
58
|
+
const full = path.join(dir, entry.name);
|
|
59
|
+
if (entry.isDirectory())
|
|
60
|
+
walk(full, out);
|
|
61
|
+
else if (entry.isFile() && entry.name.toLowerCase().endsWith(".gguf"))
|
|
62
|
+
out.push(full);
|
|
63
|
+
}
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
66
|
+
export function detectQuant(filename) {
|
|
67
|
+
const upper = filename.toUpperCase();
|
|
68
|
+
for (const pattern of QUANT_PATTERNS) {
|
|
69
|
+
if (upper.includes(pattern))
|
|
70
|
+
return pattern;
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
export function isProjectorFile(filename) {
|
|
75
|
+
return /^mmproj/i.test(path.basename(filename));
|
|
76
|
+
}
|
|
77
|
+
/** Feature flags inferred from the filename, since these are community builds. */
|
|
78
|
+
function detectFeatures(filename) {
|
|
79
|
+
const upper = path.basename(filename).toUpperCase();
|
|
80
|
+
return {
|
|
81
|
+
mtp: /\bMTP\b/.test(upper) || upper.includes("-MTP"),
|
|
82
|
+
imatrix: upper.includes("IMATRIX"),
|
|
83
|
+
distilled: upper.includes("DISTILL"),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export function scan({ modelsDir = LMSTUDIO_MODELS_DIR, withMetadata = true, origin, } = {}) {
|
|
87
|
+
const files = walk(modelsDir);
|
|
88
|
+
const projectorsByDir = new Map();
|
|
89
|
+
const weightFiles = [];
|
|
90
|
+
for (const file of files) {
|
|
91
|
+
if (isProjectorFile(file)) {
|
|
92
|
+
const dir = path.dirname(file);
|
|
93
|
+
// Prefer the largest projector if a repo ships several precisions.
|
|
94
|
+
const existing = projectorsByDir.get(dir);
|
|
95
|
+
const size = fs.statSync(file).size;
|
|
96
|
+
if (!existing || size > existing.size)
|
|
97
|
+
projectorsByDir.set(dir, { file, size });
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const parts = file.match(MULTIPART);
|
|
101
|
+
// For sharded models only the first shard is passed to llama-server.
|
|
102
|
+
if (parts && parts[1] !== "00001")
|
|
103
|
+
continue;
|
|
104
|
+
weightFiles.push(file);
|
|
105
|
+
}
|
|
106
|
+
const models = [];
|
|
107
|
+
for (const file of weightFiles) {
|
|
108
|
+
const dir = path.dirname(file);
|
|
109
|
+
const stat = fs.statSync(file);
|
|
110
|
+
let sizeBytes = stat.size;
|
|
111
|
+
// Sum shards so the VRAM estimate reflects the whole model.
|
|
112
|
+
const shard = file.match(MULTIPART);
|
|
113
|
+
if (shard) {
|
|
114
|
+
const total = Number(shard[2]);
|
|
115
|
+
const base = file.replace(MULTIPART, "");
|
|
116
|
+
for (let i = 2; i <= total; i += 1) {
|
|
117
|
+
const part = `${base}-${String(i).padStart(5, "0")}-of-${shard[2]}.gguf`;
|
|
118
|
+
try {
|
|
119
|
+
sizeBytes += fs.statSync(part).size;
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
/* missing shard: reported by validate() */
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const projector = projectorsByDir.get(dir) || null;
|
|
127
|
+
const entry = {
|
|
128
|
+
id: path.relative(modelsDir, file).replace(/\\/g, "/"),
|
|
129
|
+
displayName: path.basename(file, ".gguf"),
|
|
130
|
+
publisher: path.relative(modelsDir, dir).split(path.sep)[0] || null,
|
|
131
|
+
dir,
|
|
132
|
+
modelPath: file,
|
|
133
|
+
sizeBytes,
|
|
134
|
+
sharded: Boolean(shard),
|
|
135
|
+
quant: detectQuant(path.basename(file)),
|
|
136
|
+
features: detectFeatures(file),
|
|
137
|
+
mmprojPath: projector ? projector.file : null,
|
|
138
|
+
mmprojBytes: projector ? projector.size : 0,
|
|
139
|
+
metadata: null,
|
|
140
|
+
metadataError: null,
|
|
141
|
+
origin,
|
|
142
|
+
};
|
|
143
|
+
if (withMetadata) {
|
|
144
|
+
try {
|
|
145
|
+
entry.metadata = gguf.summarize(file);
|
|
146
|
+
}
|
|
147
|
+
catch (error) {
|
|
148
|
+
entry.metadataError = error instanceof Error ? error.message : String(error);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
models.push(entry);
|
|
152
|
+
}
|
|
153
|
+
models.sort((a, b) => a.displayName.localeCompare(b.displayName));
|
|
154
|
+
qualifyDuplicateNames(models);
|
|
155
|
+
return models;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Several publishers ship identically-named quants; qualify the duplicates so the
|
|
159
|
+
* picker never shows two indistinguishable rows.
|
|
160
|
+
*/
|
|
161
|
+
function qualifyDuplicateNames(models) {
|
|
162
|
+
const byName = new Map();
|
|
163
|
+
for (const model of models) {
|
|
164
|
+
const list = byName.get(model.displayName) || [];
|
|
165
|
+
list.push(model);
|
|
166
|
+
byName.set(model.displayName, list);
|
|
167
|
+
}
|
|
168
|
+
for (const [, group] of byName) {
|
|
169
|
+
if (group.length < 2)
|
|
170
|
+
continue;
|
|
171
|
+
for (const model of group) {
|
|
172
|
+
if (model.publisher)
|
|
173
|
+
model.displayName = `${model.displayName} (${model.publisher})`;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/** Human-readable size. */
|
|
178
|
+
export function formatBytes(bytes) {
|
|
179
|
+
if (!bytes)
|
|
180
|
+
return "0";
|
|
181
|
+
const units = ["B", "KB", "MB", "GB", "TB"];
|
|
182
|
+
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1);
|
|
183
|
+
return `${(bytes / 1024 ** i).toFixed(i >= 3 ? 2 : 0)} ${units[i]}`;
|
|
184
|
+
}
|
|
185
|
+
//# sourceMappingURL=scan.js.map
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Model } from "../types.js";
|
|
2
|
+
declare const ARCHIVE_DIR: string;
|
|
3
|
+
/** One archived request/response exchange. */
|
|
4
|
+
export interface TranscriptEntry {
|
|
5
|
+
taskId: string;
|
|
6
|
+
label: string | null;
|
|
7
|
+
at: string;
|
|
8
|
+
meta: Record<string, unknown>;
|
|
9
|
+
request: unknown;
|
|
10
|
+
response: unknown;
|
|
11
|
+
}
|
|
12
|
+
/** Arguments to `put`: one exchange to archive. */
|
|
13
|
+
export interface PutOptions {
|
|
14
|
+
label?: string;
|
|
15
|
+
request: unknown;
|
|
16
|
+
response: unknown;
|
|
17
|
+
meta?: Record<string, unknown>;
|
|
18
|
+
}
|
|
19
|
+
declare function runId(model: Model | null, timestamp?: Date): string;
|
|
20
|
+
/** Where a run's transcripts live. */
|
|
21
|
+
declare function runDir(id: string): string;
|
|
22
|
+
/**
|
|
23
|
+
* Record one exchange. `request` and `response` are stored verbatim so a future
|
|
24
|
+
* scorer sees exactly what the model saw and said.
|
|
25
|
+
*/
|
|
26
|
+
declare function put(id: string, taskId: string, { label, request, response, meta }: PutOptions): string;
|
|
27
|
+
/** Every transcript for a run, grouped by task id. */
|
|
28
|
+
declare function load(id: string): Record<string, TranscriptEntry[]>;
|
|
29
|
+
/** All archived run ids, newest first. */
|
|
30
|
+
declare function list(): string[];
|
|
31
|
+
/** Total bytes held, so the archive can be pruned knowingly. */
|
|
32
|
+
declare function size(): number;
|
|
33
|
+
export { ARCHIVE_DIR, runId, runDir, put, load, list, size };
|
|
34
|
+
//# sourceMappingURL=archive.d.ts.map
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import crypto from "node:crypto";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
/**
|
|
6
|
+
* Raw model output archive.
|
|
7
|
+
*
|
|
8
|
+
* Scoring is a separate, replayable pass over stored transcripts rather than
|
|
9
|
+
* something that only happens live. This exists because a bug in the scorer
|
|
10
|
+
* (a filename matcher that attributed every test block to the wrong file) made
|
|
11
|
+
* seven models look identical, and fixing it cost a full re-run on the GPU. The
|
|
12
|
+
* model outputs had been correct all along - only the grading was wrong.
|
|
13
|
+
*
|
|
14
|
+
* With the transcript on disk, a scorer fix re-grades history in seconds.
|
|
15
|
+
*/
|
|
16
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
17
|
+
const ROOT = path.resolve(HERE, "..", "..");
|
|
18
|
+
const ARCHIVE_DIR = path.join(ROOT, "results", "transcripts");
|
|
19
|
+
function runId(model, timestamp = new Date()) {
|
|
20
|
+
const stamp = timestamp.toISOString().replace(/[:.]/g, "-");
|
|
21
|
+
const slug = String(model?.displayName || "unknown")
|
|
22
|
+
.toLowerCase()
|
|
23
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
24
|
+
.replace(/^-|-$/g, "")
|
|
25
|
+
.slice(0, 60);
|
|
26
|
+
return `${stamp}_${slug}`;
|
|
27
|
+
}
|
|
28
|
+
/** Where a run's transcripts live. */
|
|
29
|
+
function runDir(id) {
|
|
30
|
+
return path.join(ARCHIVE_DIR, id);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Record one exchange. `request` and `response` are stored verbatim so a future
|
|
34
|
+
* scorer sees exactly what the model saw and said.
|
|
35
|
+
*/
|
|
36
|
+
function put(id, taskId, { label, request, response, meta = {} }) {
|
|
37
|
+
const dir = runDir(id);
|
|
38
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
39
|
+
const key = crypto
|
|
40
|
+
.createHash("sha1")
|
|
41
|
+
.update(`${taskId}:${label || ""}:${JSON.stringify(meta)}`)
|
|
42
|
+
.digest("hex")
|
|
43
|
+
.slice(0, 8);
|
|
44
|
+
const file = path.join(dir, `${taskId.replace(/[^\w.-]/g, "_")}.${key}.json`);
|
|
45
|
+
fs.writeFileSync(file, `${JSON.stringify({
|
|
46
|
+
taskId,
|
|
47
|
+
label: label || null,
|
|
48
|
+
at: new Date().toISOString(),
|
|
49
|
+
meta,
|
|
50
|
+
request,
|
|
51
|
+
response,
|
|
52
|
+
}, null, 2)}\n`);
|
|
53
|
+
return file;
|
|
54
|
+
}
|
|
55
|
+
/** Every transcript for a run, grouped by task id. */
|
|
56
|
+
function load(id) {
|
|
57
|
+
const dir = runDir(id);
|
|
58
|
+
if (!fs.existsSync(dir))
|
|
59
|
+
return {};
|
|
60
|
+
const byTask = {};
|
|
61
|
+
for (const name of fs.readdirSync(dir)) {
|
|
62
|
+
if (!name.endsWith(".json"))
|
|
63
|
+
continue;
|
|
64
|
+
try {
|
|
65
|
+
const entry = JSON.parse(fs.readFileSync(path.join(dir, name), "utf8"));
|
|
66
|
+
if (!byTask[entry.taskId])
|
|
67
|
+
byTask[entry.taskId] = [];
|
|
68
|
+
byTask[entry.taskId].push(entry);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
/* skip an unreadable transcript rather than failing the whole load */
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
for (const list of Object.values(byTask))
|
|
75
|
+
list.sort((a, b) => (a.at < b.at ? -1 : 1));
|
|
76
|
+
return byTask;
|
|
77
|
+
}
|
|
78
|
+
/** All archived run ids, newest first. */
|
|
79
|
+
function list() {
|
|
80
|
+
if (!fs.existsSync(ARCHIVE_DIR))
|
|
81
|
+
return [];
|
|
82
|
+
return fs
|
|
83
|
+
.readdirSync(ARCHIVE_DIR)
|
|
84
|
+
.filter((name) => fs.statSync(path.join(ARCHIVE_DIR, name)).isDirectory())
|
|
85
|
+
.sort()
|
|
86
|
+
.reverse();
|
|
87
|
+
}
|
|
88
|
+
/** Total bytes held, so the archive can be pruned knowingly. */
|
|
89
|
+
function size() {
|
|
90
|
+
let bytes = 0;
|
|
91
|
+
for (const id of list()) {
|
|
92
|
+
const dir = runDir(id);
|
|
93
|
+
for (const name of fs.readdirSync(dir)) {
|
|
94
|
+
try {
|
|
95
|
+
bytes += fs.statSync(path.join(dir, name)).size;
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
/* gone */
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return bytes;
|
|
103
|
+
}
|
|
104
|
+
export { ARCHIVE_DIR, runId, runDir, put, load, list, size };
|
|
105
|
+
//# sourceMappingURL=archive.js.map
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Model, Runtime } from "../types.js";
|
|
2
|
+
import type { Profile } from "../config/schema.js";
|
|
3
|
+
/**
|
|
4
|
+
* Measure a model's real KV-cache cost per token.
|
|
5
|
+
*
|
|
6
|
+
* The theoretical formula (layers x kv_heads x dims x bytes) overestimates
|
|
7
|
+
* badly on architectures that only keep a full cache on some layers - by ~4x
|
|
8
|
+
* on `qwen35`. So we load the model twice at two context sizes and take the
|
|
9
|
+
* slope: bytes/token = (vram_b - vram_a) / (ctx_b - ctx_a). Everything that
|
|
10
|
+
* does not scale with context (weights, projector, CUDA context, compute
|
|
11
|
+
* buffers) cancels out of the difference.
|
|
12
|
+
*/
|
|
13
|
+
export declare const DEFAULT_SAMPLES: number[];
|
|
14
|
+
/** A single context-size measurement collected during calibration. */
|
|
15
|
+
export interface CalibrationSample {
|
|
16
|
+
contextSize: number;
|
|
17
|
+
deltaBytes: number;
|
|
18
|
+
loadSeconds: number | null;
|
|
19
|
+
}
|
|
20
|
+
/** Progress event emitted while calibrating. */
|
|
21
|
+
export interface CalibrateProgress {
|
|
22
|
+
phase: "skip" | "loading" | "measured" | "failed";
|
|
23
|
+
contextSize: number;
|
|
24
|
+
reason?: string;
|
|
25
|
+
deltaBytes?: number;
|
|
26
|
+
error?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface CalibrateOptions {
|
|
29
|
+
runtime: Runtime;
|
|
30
|
+
model: Model;
|
|
31
|
+
profile: Profile;
|
|
32
|
+
samples?: number[];
|
|
33
|
+
internalPort?: number;
|
|
34
|
+
onProgress?: (event: CalibrateProgress) => void;
|
|
35
|
+
}
|
|
36
|
+
/** The measured KV-cache profile calibration produces. */
|
|
37
|
+
export interface CalibrationMeasurement {
|
|
38
|
+
kvBytesPerToken: number;
|
|
39
|
+
baseOverheadBytes: number;
|
|
40
|
+
theoreticalKvBytesPerToken: number | null;
|
|
41
|
+
theoreticalRatio: number | null;
|
|
42
|
+
samples: CalibrationSample[];
|
|
43
|
+
cacheTypeK: string;
|
|
44
|
+
cacheTypeV: string;
|
|
45
|
+
vision: boolean;
|
|
46
|
+
measuredAt: string;
|
|
47
|
+
}
|
|
48
|
+
export declare function calibrate({ runtime, model, profile, samples, internalPort, onProgress, }: CalibrateOptions): Promise<CalibrationMeasurement>;
|
|
49
|
+
//# sourceMappingURL=calibrate.d.ts.map
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { usedBytes } from "../gpu.js";
|
|
2
|
+
import * as vram from "../vram.js";
|
|
3
|
+
import { DEFAULT_INTERNAL_PORT, Supervisor } from "../service/supervisor.js";
|
|
4
|
+
/**
|
|
5
|
+
* Measure a model's real KV-cache cost per token.
|
|
6
|
+
*
|
|
7
|
+
* The theoretical formula (layers x kv_heads x dims x bytes) overestimates
|
|
8
|
+
* badly on architectures that only keep a full cache on some layers - by ~4x
|
|
9
|
+
* on `qwen35`. So we load the model twice at two context sizes and take the
|
|
10
|
+
* slope: bytes/token = (vram_b - vram_a) / (ctx_b - ctx_a). Everything that
|
|
11
|
+
* does not scale with context (weights, projector, CUDA context, compute
|
|
12
|
+
* buffers) cancels out of the difference.
|
|
13
|
+
*/
|
|
14
|
+
export const DEFAULT_SAMPLES = [8192, 65536];
|
|
15
|
+
export async function calibrate({ runtime, model, profile, samples = DEFAULT_SAMPLES, internalPort = DEFAULT_INTERNAL_PORT + 1, onProgress = () => { }, }) {
|
|
16
|
+
if (samples.length < 2)
|
|
17
|
+
throw new Error("calibration needs at least two context sizes");
|
|
18
|
+
const native = model.metadata?.contextLength || Math.max(...samples);
|
|
19
|
+
const points = [];
|
|
20
|
+
for (const contextSize of samples) {
|
|
21
|
+
if (contextSize > native) {
|
|
22
|
+
onProgress({ phase: "skip", contextSize, reason: `exceeds native context ${native}` });
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
const supervisor = new Supervisor({ runtime, internalPort });
|
|
26
|
+
onProgress({ phase: "loading", contextSize });
|
|
27
|
+
const baseline = await usedBytes();
|
|
28
|
+
try {
|
|
29
|
+
await supervisor.start(model, { ...profile, contextSize });
|
|
30
|
+
const used = supervisor.vramAtReadyBytes ?? (await usedBytes());
|
|
31
|
+
const delta = Number(used) - Number(supervisor.vramBaselineBytes ?? baseline);
|
|
32
|
+
points.push({ contextSize, deltaBytes: delta, loadSeconds: supervisor.loadSeconds });
|
|
33
|
+
onProgress({ phase: "measured", contextSize, deltaBytes: delta });
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
onProgress({
|
|
37
|
+
phase: "failed",
|
|
38
|
+
contextSize,
|
|
39
|
+
error: error instanceof Error ? error.message : String(error),
|
|
40
|
+
});
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
await supervisor.stop();
|
|
45
|
+
// Let the driver actually release the allocation before the next sample.
|
|
46
|
+
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (points.length < 2) {
|
|
50
|
+
throw new Error("calibration produced fewer than two usable samples");
|
|
51
|
+
}
|
|
52
|
+
points.sort((a, b) => a.contextSize - b.contextSize);
|
|
53
|
+
const low = points[0];
|
|
54
|
+
const high = points[points.length - 1];
|
|
55
|
+
const kvBytesPerToken = (high.deltaBytes - low.deltaBytes) / (high.contextSize - low.contextSize);
|
|
56
|
+
if (!(kvBytesPerToken > 0)) {
|
|
57
|
+
throw new Error("measured a non-positive bytes-per-token; results are unusable");
|
|
58
|
+
}
|
|
59
|
+
const fixedBytes = model.sizeBytes + (profile.vision && model.mmprojPath ? model.mmprojBytes : 0);
|
|
60
|
+
const baseOverheadBytes = Math.max(0, low.deltaBytes - fixedBytes - kvBytesPerToken * low.contextSize);
|
|
61
|
+
const theoretical = vram.theoreticalKvBytesPerToken(model.metadata, profile.cacheTypeK, profile.cacheTypeV);
|
|
62
|
+
return {
|
|
63
|
+
kvBytesPerToken,
|
|
64
|
+
baseOverheadBytes,
|
|
65
|
+
theoreticalKvBytesPerToken: theoretical,
|
|
66
|
+
theoreticalRatio: theoretical ? theoretical / kvBytesPerToken : null,
|
|
67
|
+
samples: points,
|
|
68
|
+
cacheTypeK: profile.cacheTypeK,
|
|
69
|
+
cacheTypeV: profile.cacheTypeV,
|
|
70
|
+
vision: Boolean(profile.vision && model.mmprojPath),
|
|
71
|
+
measuredAt: new Date().toISOString(),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=calibrate.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { RunRecord } from "./results.js";
|
|
2
|
+
/**
|
|
3
|
+
* Renders stored benchmark runs as a self-contained HTML report.
|
|
4
|
+
*
|
|
5
|
+
* No external assets: a strict CSP environment must be able to open the file
|
|
6
|
+
* offline, so every chart is inline SVG generated here and all CSS is local.
|
|
7
|
+
*
|
|
8
|
+
* Colour roles come from a validated categorical palette (four slots, checked
|
|
9
|
+
* for CVD separation and lightness band in both light and dark surfaces). Three
|
|
10
|
+
* light-mode slots fall below 3:1 against the light surface, so the relief rule
|
|
11
|
+
* applies: every bar carries a visible direct label and a full table view ships
|
|
12
|
+
* alongside the charts. Identity is never colour alone.
|
|
13
|
+
*/
|
|
14
|
+
declare const SERIES: {
|
|
15
|
+
light: string;
|
|
16
|
+
dark: string;
|
|
17
|
+
}[];
|
|
18
|
+
declare function build(records: RunRecord[], allRuns?: RunRecord[]): string;
|
|
19
|
+
declare const STYLE = "\n<style>\n.viz-root{\n color-scheme:light;\n --surface-1:#fcfcfb; --plane:#f9f9f7;\n --text-primary:#0b0b0b; --text-secondary:#52514e; --muted:#898781;\n --grid:#e1e0d9; --axis:#c3c2b7; --border:rgba(11,11,11,0.10);\n --series-1:#2a78d6; --series-2:#eb6834; --series-3:#1baf7a; --series-4:#eda100;\n background:var(--plane); color:var(--text-primary);\n font-family:system-ui,-apple-system,\"Segoe UI\",sans-serif;\n max-width:1000px; margin:0 auto; padding:28px 20px 56px;\n}\n@media (prefers-color-scheme:dark){\n :root:where(:not([data-theme=\"light\"])) .viz-root{\n color-scheme:dark;\n --surface-1:#1a1a19; --plane:#0d0d0d;\n --text-primary:#ffffff; --text-secondary:#c3c2b7; --muted:#898781;\n --grid:#2c2c2a; --axis:#383835; --border:rgba(255,255,255,0.10);\n --series-1:#3987e5; --series-2:#d95926; --series-3:#199e70; --series-4:#c98500;\n }\n}\n:root[data-theme=\"dark\"] .viz-root{\n color-scheme:dark;\n --surface-1:#1a1a19; --plane:#0d0d0d;\n --text-primary:#ffffff; --text-secondary:#c3c2b7; --muted:#898781;\n --grid:#2c2c2a; --axis:#383835; --border:rgba(255,255,255,0.10);\n --series-1:#3987e5; --series-2:#d95926; --series-3:#199e70; --series-4:#c98500;\n}\nh1{font-size:1.55rem;margin:0 0 4px;letter-spacing:-0.01em}\nh2{font-size:1.02rem;margin:0 0 4px}\n.sub{color:var(--text-secondary);margin:0 0 22px;font-size:.88rem}\n.cap{color:var(--text-secondary);font-size:.82rem;margin:0 0 14px;max-width:66ch;line-height:1.45}\n.tiles{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:12px;margin-bottom:20px}\n.tile{background:var(--surface-1);border:1px solid var(--border);border-radius:10px;padding:14px 16px}\n.tl{font-size:.72rem;text-transform:uppercase;letter-spacing:.05em;color:var(--muted)}\n.tv{font-size:1.5rem;font-weight:600;margin:4px 0 2px;line-height:1.15}\n.tn{font-size:.76rem;color:var(--text-secondary)}\n.card{background:var(--surface-1);border:1px solid var(--border);border-radius:12px;padding:18px 20px;margin-bottom:16px}\n.legend{display:flex;flex-wrap:wrap;gap:14px;margin-bottom:12px}\n.lg{display:inline-flex;align-items:center;gap:6px;font-size:.78rem;color:var(--text-secondary)}\n.lg i{width:11px;height:11px;border-radius:3px;display:inline-block;flex:none}\nsvg{display:block;overflow:visible}\n.grp{font-size:11.5px;font-weight:600;fill:var(--text-primary)}\n.grpmeta{font-size:10px;fill:var(--muted)}\n.grpscore{font-size:13px;font-weight:700;fill:var(--text-primary);font-variant-numeric:tabular-nums}\n.track{fill:var(--grid)}\n.val{font-size:10.5px;fill:var(--text-secondary);font-variant-numeric:tabular-nums}\n.tlabel{font-size:11px;fill:var(--text-secondary)}\n.twt{font-size:9.5px;fill:var(--muted);font-variant-numeric:tabular-nums}\n.scorebar{fill:var(--series-1)}\n.overallbar{fill:var(--series-1);stroke:var(--text-primary);stroke-width:.75}\n.olabel{font-size:11px;font-weight:600;fill:var(--text-primary)}\n.oval{font-size:11px;font-weight:700;fill:var(--text-primary);font-variant-numeric:tabular-nums}\n.grid{stroke:var(--grid);stroke-width:1}\n.axis{stroke:var(--axis);stroke-width:1}\n.tick{font-size:10px;fill:var(--muted);font-variant-numeric:tabular-nums}\n.axl{text-align:center;font-size:.74rem;color:var(--muted);margin:6px 0 0}\n.scroll{overflow-x:auto}\ntable{border-collapse:collapse;width:100%;font-size:.8rem}\nth,td{text-align:left;padding:7px 10px;border-bottom:1px solid var(--border);white-space:nowrap}\nth{font-size:.71rem;text-transform:uppercase;letter-spacing:.04em;color:var(--muted);font-weight:600}\n.num{text-align:right;font-variant-numeric:tabular-nums}\n.strong{font-weight:600}\n.muted{color:var(--muted)}\n.mono{font-family:ui-monospace,\"SFMono-Regular\",Menlo,Consolas,monospace;font-size:.78rem;background:var(--plane);border:1px solid var(--border);border-radius:4px;padding:1px 5px;white-space:nowrap}\n.wt{color:var(--muted);font-weight:400;font-size:.72em;margin-left:4px}\n.notes{margin:12px 0 0;padding-left:18px;color:var(--text-secondary);font-size:.78rem}\n.empty{color:var(--muted);font-size:.82rem}\n.v-good{color:var(--text-secondary)}\n.v-warn{color:#ec835a;font-weight:600}\n.v-bad{color:#d03b3b;font-weight:600}\ncircle:hover{r:6}\nrect[fill^=\"var\"]:hover{opacity:.82}\n</style>";
|
|
20
|
+
/** Write the report and return its path. */
|
|
21
|
+
declare function write(outFile?: string): {
|
|
22
|
+
file: string;
|
|
23
|
+
count: number;
|
|
24
|
+
};
|
|
25
|
+
export { write, build, STYLE, SERIES };
|
|
26
|
+
//# sourceMappingURL=report.d.ts.map
|