@sleepy_panda_srl/portulan 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1665 @@
1
+ #!/usr/bin/env node
2
+ // Zero-dependency local LLM runner for Portulan: loopback guard, input-byte rail, verify-gated transform with rollback, role eval, embedding dedup, its A/B record, and offline self-test.
3
+ //
4
+ // Exit codes: 0 green · 1 red check or model/verify failure · 2 could not run.
5
+
6
+ import { spawn, spawnSync } from "node:child_process";
7
+ import { createHash } from "node:crypto";
8
+ import fs from "node:fs";
9
+ import os from "node:os";
10
+ import path from "node:path";
11
+ import { performance } from "node:perf_hooks";
12
+ import process from "node:process";
13
+ import { pathToFileURL } from "node:url";
14
+
15
+ const RUNNER_FINGERPRINT = createHash("sha256").update(fs.readFileSync(new URL(import.meta.url))).digest("hex");
16
+
17
+ export const DEFAULT_BASE_URL = process.env.PORTULAN_LOCAL_LLM_URL ?? "http://127.0.0.1:1234/v1";
18
+ export const DEFAULT_MAX_INPUT_BYTES = 32 * 1024;
19
+ export const DEFAULT_RETRIES = 2;
20
+ export const FEEDBACK_CHARS = 1500;
21
+ const VERIFY_KEEP_CHARS = 64 * 1024;
22
+ export const RESTORE_SIGNALS = Object.freeze(["SIGINT", "SIGTERM", "SIGHUP"]);
23
+
24
+ // Candidates retain the historical task map; operational roles are gated by EARNED_ROUTING below.
25
+ // Evidence: evals/ab/local-worker.md; cli/local-routing.mjs checks the shipped support table.
26
+ export const ROLE_MODELS = Object.freeze({
27
+ code: process.env.PORTULAN_LOCAL_CODE_MODEL ?? "qwen/qwen3.8-27b",
28
+ prose: process.env.PORTULAN_LOCAL_PROSE_MODEL ?? "google/gemma-4-31b",
29
+ embed: process.env.PORTULAN_LOCAL_EMBED_MODEL ?? "text-embedding-nomic-embed-text-v1.5",
30
+ });
31
+ const CHAT_ROLES = new Set(["code", "prose"]);
32
+
33
+ // Embedded so a packs-only install needs no root evals/ directory. CI rederives these values.
34
+ export const EARNED_ROUTING = Object.freeze({
35
+ record: "evals/ab/local-worker.json",
36
+ sha256: "72501e02ce1284e96e19a29cfb60193552d3ac490e6e4c72391f6d68a42d01d2",
37
+ roles: Object.freeze({
38
+ code: Object.freeze({ model: "qwen/qwen3.8-27b", enabled: true }),
39
+ prose: Object.freeze({ model: "google/gemma-4-31b", enabled: false }),
40
+ }),
41
+ });
42
+
43
+ export function operationalModel(args, defaultRole) {
44
+ // An explicitly named model without a role is an operator's candidate, not earned routing.
45
+ if (args.model !== undefined && args.role === undefined) return args.model;
46
+ const role = args.role ?? defaultRole;
47
+ const earned = EARNED_ROUTING.roles[role];
48
+ if (!earned?.enabled) throw new RailError(`Local role ${role} is not supported by ${EARNED_ROUTING.record}; use the parent session, or explicitly evaluate a candidate with --model and no --role.`);
49
+ const model = args.model ?? ROLE_MODELS[role];
50
+ if (model !== earned.model) throw new RailError(`Local role ${role} was measured with ${earned.model}, not ${model}; use --model without --role for an explicit candidate.`);
51
+ if ((args.effort ?? "low") !== "low") throw new RailError(`Local role ${role} was measured at --effort low; use --model without --role for a candidate at another effort.`);
52
+ return model;
53
+ }
54
+
55
+
56
+ const LOOPBACK_HOSTS = new Set(["127.0.0.1", "localhost", "::1", "[::1]"]);
57
+
58
+ export class RailError extends Error {}
59
+ export class CouldNotRunError extends Error {}
60
+
61
+ export function assertLoopback(baseUrl, { allowRemote = false } = {}) {
62
+ let parsed;
63
+ try {
64
+ parsed = new URL(baseUrl);
65
+ } catch {
66
+ throw new CouldNotRunError(`Invalid --base-url: ${JSON.stringify(baseUrl)}`);
67
+ }
68
+ if (!allowRemote && !LOOPBACK_HOSTS.has(parsed.hostname.toLowerCase())) {
69
+ throw new RailError(
70
+ `Refused non-loopback endpoint ${parsed.origin}: local-llm requires 127.0.0.1, localhost, or [::1] unless --allow-remote is explicitly passed.`,
71
+ );
72
+ }
73
+ return parsed.href.replace(/\/+$/, "");
74
+ }
75
+
76
+ export function assertInputBudget(text, { maxInputBytes = DEFAULT_MAX_INPUT_BYTES } = {}) {
77
+ const bytes = Buffer.byteLength(text, "utf8");
78
+ if (bytes > maxInputBytes) {
79
+ throw new RailError(
80
+ `Refused prompt of ${bytes} bytes (limit: ${maxInputBytes} bytes). Outline the target file with cli/symbols.mjs and pass --lines <start>-<end> instead of sending the whole file.`,
81
+ );
82
+ }
83
+ return bytes;
84
+ }
85
+
86
+ export function stripThinking(text) {
87
+ return text.replace(/^\s*<think>[\s\S]*?<\/think>\s*/i, "").trim();
88
+ }
89
+
90
+ export function extractFencedCode(text) {
91
+ const cleaned = stripThinking(text);
92
+ const match = /```[a-zA-Z0-9_-]*\n([\s\S]*?)```/.exec(cleaned);
93
+ return match ? match[1] : cleaned;
94
+ }
95
+
96
+ const figure = (value, unit) => (value === null || value === undefined ? "?" : `${value}${unit}`);
97
+
98
+ export function formatPassReceipt({ mode, model, attempts = 1, promptTokens = null, completionTokens = null, totalMs = null, tokPerSec = null, target = "" }) {
99
+ const where = target ? ` target=${target}` : "";
100
+ return (
101
+ `OK mode=${mode} model=${model}${where} attempts=${attempts} in=${figure(promptTokens, "t")} out=${figure(completionTokens, "t")} ` +
102
+ `time=${figure(totalMs, "ms")} rate=${figure(tokPerSec, "tok/s")} spend=$0.00`
103
+ );
104
+ }
105
+
106
+ const tail = (text, chars) => (text.length > chars ? `…${text.slice(-chars)}` : text);
107
+ const sum = (a, b) => (a === null || b === null || b === undefined ? null : a + b);
108
+ const rate = (tokens, ms) => (tokens === null || !ms ? null : Number((tokens / (ms / 1000)).toFixed(1)));
109
+
110
+ export function runVerify(cmd, { onSpawn = () => {} } = {}) {
111
+ return new Promise((resolve) => {
112
+ let child;
113
+ try {
114
+ child = spawn(cmd, { shell: true, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] });
115
+ } catch (error) {
116
+ resolve({ status: null, error, stdout: "", stderr: "" });
117
+ return;
118
+ }
119
+ onSpawn(child);
120
+ let stdout = "";
121
+ let stderr = "";
122
+ child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout = (stdout + chunk).slice(-VERIFY_KEEP_CHARS)));
123
+ child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr = (stderr + chunk).slice(-VERIFY_KEEP_CHARS)));
124
+ child.on("error", (error) => resolve({ status: null, error, stdout, stderr }));
125
+ child.on("close", (status, signal) => resolve({ status, signal, stdout, stderr }));
126
+ });
127
+ }
128
+
129
+ function stopVerify(child) {
130
+ if (!child || child.exitCode !== null || child.signalCode !== null) return;
131
+ try {
132
+ process.kill(-child.pid, "SIGKILL");
133
+ } catch {
134
+ child.kill("SIGKILL");
135
+ }
136
+ }
137
+
138
+ async function localFetch(url, init, fetchImpl) {
139
+ try {
140
+ return await fetchImpl(url, { ...init, redirect: "error" });
141
+ } catch (err) {
142
+ const cause = err.cause?.message ? ` (${err.cause.message})` : "";
143
+ throw new CouldNotRunError(`Local LLM server unreachable at ${url} — ${err.message}${cause}`);
144
+ }
145
+ }
146
+
147
+ export async function callLocalChat({
148
+ baseUrl = DEFAULT_BASE_URL,
149
+ allowRemote = false,
150
+ model = ROLE_MODELS.code,
151
+ messages,
152
+ maxInputBytes = DEFAULT_MAX_INPUT_BYTES,
153
+ maxTokens = 4096,
154
+ effort = "low",
155
+ temperature = 0.1,
156
+ seed,
157
+ fetchImpl = globalThis.fetch,
158
+ }) {
159
+ const endpoint = assertLoopback(baseUrl, { allowRemote });
160
+ const serialized = messages.map((m) => m.content ?? "").join("\n");
161
+ assertInputBudget(serialized, { maxInputBytes });
162
+
163
+ const t0 = performance.now();
164
+ const res = await localFetch(
165
+ `${endpoint}/chat/completions`,
166
+ {
167
+ method: "POST",
168
+ headers: { "Content-Type": "application/json" },
169
+ body: JSON.stringify({
170
+ model,
171
+ messages,
172
+ temperature,
173
+ max_tokens: maxTokens,
174
+ reasoning_effort: effort,
175
+ chat_template_kwargs: {
176
+ enable_thinking: effort !== "none" && effort !== "low",
177
+ reasoning_effort: effort === "none" ? "low" : effort,
178
+ },
179
+ stream: false,
180
+ ...(seed === undefined ? {} : { seed }),
181
+ }),
182
+ },
183
+ fetchImpl,
184
+ );
185
+
186
+ if (!res.ok) {
187
+ const body = await res.text().catch(() => "");
188
+ throw new CouldNotRunError(`Local LLM server returned HTTP ${res.status}: ${body.slice(0, 200)}`);
189
+ }
190
+
191
+ let data;
192
+ try {
193
+ data = await res.json();
194
+ } catch (err) {
195
+ throw new CouldNotRunError(`Local LLM server returned a body that is not JSON — ${err.message}`);
196
+ }
197
+ const totalMs = Math.max(1, Math.round(performance.now() - t0));
198
+ const rawContent = data.choices?.[0]?.message?.content ?? "";
199
+ const promptTokens = data.usage?.prompt_tokens ?? null;
200
+ const completionTokens = data.usage?.completion_tokens ?? null;
201
+
202
+ return {
203
+ content: stripThinking(rawContent),
204
+ rawContent,
205
+ promptTokens,
206
+ completionTokens,
207
+ totalMs,
208
+ tokPerSec: rate(completionTokens, totalMs),
209
+ };
210
+ }
211
+
212
+ export const DEFAULT_DEDUP_THRESHOLD = 0.8;
213
+
214
+ export function parseThreshold(spec) {
215
+ const value = /^-?(\d+(\.\d*)?|\.\d+)$/.test(String(spec)) ? Number(spec) : NaN;
216
+ if (!(value >= -1 && value <= 1)) {
217
+ throw new CouldNotRunError(`--threshold takes a cosine similarity from -1 to 1, not ${JSON.stringify(spec)}`);
218
+ }
219
+ return value;
220
+ }
221
+
222
+ export function cosine(a, b) {
223
+ if (a.length !== b.length) throw new CouldNotRunError(`Cannot compare vectors of ${a.length} and ${b.length} dimensions`);
224
+ const scaleA = a.reduce((m, x) => Math.max(m, Math.abs(x)), 0);
225
+ const scaleB = b.reduce((m, x) => Math.max(m, Math.abs(x)), 0);
226
+ if (!Number.isFinite(scaleA) || !Number.isFinite(scaleB)) throw new CouldNotRunError("Cannot compare a vector that is not all finite numbers");
227
+ if (scaleA === 0 || scaleB === 0) return 0;
228
+ let dot = 0;
229
+ let aa = 0;
230
+ let bb = 0;
231
+ for (let i = 0; i < a.length; i += 1) {
232
+ const x = a[i] / scaleA;
233
+ const y = b[i] / scaleB;
234
+ dot += x * y;
235
+ aa += x * x;
236
+ bb += y * y;
237
+ }
238
+ return Math.min(1, Math.max(-1, dot / Math.sqrt(aa * bb)));
239
+ }
240
+
241
+ const byName = (x, y) => (x < y ? -1 : x > y ? 1 : 0);
242
+
243
+ export function rankDuplicatePairs(names, vectors, threshold) {
244
+ if (names.length !== vectors.length) throw new CouldNotRunError(`${vectors.length} vectors for ${names.length} files`);
245
+ const pairs = [];
246
+ for (let i = 0; i < names.length; i += 1) {
247
+ for (let j = i + 1; j < names.length; j += 1) {
248
+ const score = cosine(vectors[i], vectors[j]);
249
+ if (score >= threshold) pairs.push({ a: names[i], b: names[j], score });
250
+ }
251
+ }
252
+ return pairs.sort((x, y) => y.score - x.score || byName(x.a, y.a) || byName(x.b, y.b));
253
+ }
254
+
255
+ export function readSlot(dir, { maxInputBytes = DEFAULT_MAX_INPUT_BYTES } = {}) {
256
+ let entries;
257
+ try {
258
+ entries = fs.readdirSync(dir, { withFileTypes: true });
259
+ } catch (err) {
260
+ throw new CouldNotRunError(`Cannot read --dir ${dir}: ${err.message}`);
261
+ }
262
+ const names = entries
263
+ .filter((e) => e.isFile() && e.name.toLowerCase().endsWith(".md") && e.name.toLowerCase() !== "readme.md")
264
+ .map((e) => e.name)
265
+ .sort(byName);
266
+ if (names.length < 2) {
267
+ throw new CouldNotRunError(`--dir ${dir} holds ${names.length} Markdown file(s) besides its README; dedup needs two to compare`);
268
+ }
269
+ const each = (fn) =>
270
+ names.map((name) => {
271
+ try {
272
+ return fn(path.join(dir, name));
273
+ } catch (err) {
274
+ throw new CouldNotRunError(`Cannot read ${path.join(dir, name)}: ${err.message}`);
275
+ }
276
+ });
277
+ const refuseOver = (bytes) => {
278
+ const over = names.flatMap((name, i) => (bytes[i] > maxInputBytes ? [`${name} (${bytes[i]} bytes)`] : []));
279
+ if (over.length) {
280
+ throw new RailError(
281
+ `Refused ${over.length} file(s) over the ${maxInputBytes}-byte per-file ceiling: ${over.join(", ")}. dedup embeds whole files and never truncates one; raise --max-input-bytes only as far as the embedding model's context holds.`,
282
+ );
283
+ }
284
+ };
285
+ refuseOver(each((file) => fs.statSync(file).size));
286
+ const texts = each((file) => fs.readFileSync(file, "utf8"));
287
+ refuseOver(texts.map((text) => Buffer.byteLength(text, "utf8")));
288
+ return { names, texts };
289
+ }
290
+
291
+ function parseEmbeddings(data, count) {
292
+ const items = Array.isArray(data?.data) ? data.data : null;
293
+ if (!items || items.length !== count) {
294
+ throw new CouldNotRunError(`Local LLM server returned ${items ? items.length : "no"} embeddings for ${count} files`);
295
+ }
296
+ const vectors = new Array(count);
297
+ items.forEach((item, position) => {
298
+ const index = item?.index ?? position;
299
+ if (!Number.isInteger(index) || index < 0 || index >= count || vectors[index]) {
300
+ throw new CouldNotRunError(`Local LLM server returned an embedding with index ${JSON.stringify(item?.index)}`);
301
+ }
302
+ const vector = item.embedding;
303
+ if (!Array.isArray(vector) || vector.length === 0 || !vector.every((x) => typeof x === "number" && Number.isFinite(x))) {
304
+ throw new CouldNotRunError(`Local LLM server returned embedding ${index} that is not a list of finite numbers`);
305
+ }
306
+ vectors[index] = vector;
307
+ });
308
+ return vectors;
309
+ }
310
+
311
+ export async function runDedup({
312
+ dir,
313
+ threshold = DEFAULT_DEDUP_THRESHOLD,
314
+ model = ROLE_MODELS.embed,
315
+ baseUrl = DEFAULT_BASE_URL,
316
+ allowRemote = false,
317
+ maxInputBytes = DEFAULT_MAX_INPUT_BYTES,
318
+ fetchImpl = globalThis.fetch,
319
+ }) {
320
+ const endpoint = assertLoopback(baseUrl, { allowRemote });
321
+ const { names, texts } = readSlot(dir, { maxInputBytes });
322
+
323
+ const t0 = performance.now();
324
+ const res = await localFetch(
325
+ `${endpoint}/embeddings`,
326
+ { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model, input: texts }) },
327
+ fetchImpl,
328
+ );
329
+ if (!res.ok) {
330
+ const body = await res.text().catch(() => "");
331
+ throw new CouldNotRunError(`Local LLM server returned HTTP ${res.status}: ${body.slice(0, 200)}`);
332
+ }
333
+ let data;
334
+ try {
335
+ data = await res.json();
336
+ } catch (err) {
337
+ throw new CouldNotRunError(`Local LLM server returned a body that is not JSON — ${err.message}`);
338
+ }
339
+ const vectors = parseEmbeddings(data, names.length);
340
+ return {
341
+ pairs: rankDuplicatePairs(names, vectors, threshold),
342
+ files: names.length,
343
+ promptTokens: data.usage?.prompt_tokens ?? null,
344
+ totalMs: Math.max(1, Math.round(performance.now() - t0)),
345
+ };
346
+ }
347
+
348
+ export function formatDedupReceipt({ model, promptTokens = null, totalMs = null }) {
349
+ return formatPassReceipt({ mode: "dedup", model, promptTokens, totalMs });
350
+ }
351
+
352
+ function splitLines(text) {
353
+ const trailingNewline = text.endsWith("\n");
354
+ return { lines: (trailingNewline ? text.slice(0, -1) : text).split("\n"), trailingNewline };
355
+ }
356
+
357
+ export function parseLineRange(spec, lineCount) {
358
+ const m = /^(\d+)-(\d+)$/.exec(String(spec));
359
+ if (!m) throw new CouldNotRunError(`Invalid --lines range ${JSON.stringify(spec)} (expected start-end)`);
360
+ const start = Math.max(1, Number(m[1]));
361
+ const end = Math.min(lineCount, Number(m[2]));
362
+ if (start > end) throw new CouldNotRunError(`Invalid --lines range: start ${start} > end ${end}`);
363
+ return { start, end };
364
+ }
365
+
366
+ export async function verifyGatedTransform({
367
+ filePath,
368
+ lineRange = null,
369
+ instruction,
370
+ verifyCmd,
371
+ model = ROLE_MODELS.code,
372
+ baseUrl = DEFAULT_BASE_URL,
373
+ allowRemote = false,
374
+ maxInputBytes = DEFAULT_MAX_INPUT_BYTES,
375
+ effort = "low",
376
+ retries = DEFAULT_RETRIES,
377
+ chatFn = callLocalChat,
378
+ execFn = runVerify,
379
+ }) {
380
+ if (!verifyCmd) {
381
+ throw new CouldNotRunError("transform requires --verify <cmd>: every local edit is gated by a verify command, and kept only on its exit 0");
382
+ }
383
+ if (!fs.existsSync(filePath)) {
384
+ throw new CouldNotRunError(`Target file does not exist: ${filePath}`);
385
+ }
386
+ const originalBytes = fs.readFileSync(filePath);
387
+ const originalText = originalBytes.toString("utf8");
388
+ if (!Buffer.from(originalText, "utf8").equals(originalBytes)) {
389
+ throw new CouldNotRunError(`Refused ${filePath}: it is not valid UTF-8, so rewriting any line would change the bytes of every line`);
390
+ }
391
+ const { lines: allLines, trailingNewline } = splitLines(originalText);
392
+ const { start: startLine, end: endLine } = lineRange
393
+ ? parseLineRange(lineRange, allLines.length)
394
+ : { start: 1, end: allLines.length };
395
+ const targetSlice = allLines.slice(startLine - 1, endLine).join("\n");
396
+
397
+ let kept = false;
398
+ let verifying = null;
399
+ const restore = () => {
400
+ try {
401
+ fs.writeFileSync(filePath, originalBytes);
402
+ } catch (err) {
403
+ throw new CouldNotRunError(`Could not restore ${filePath} (${err.message}): it may still hold the model's unverified edit`);
404
+ }
405
+ };
406
+ const onSignal = (signal) => {
407
+ stopVerify(verifying);
408
+ let outcome = kept ? "the verified edit stands" : `restored ${filePath}`;
409
+ if (!kept) {
410
+ try {
411
+ restore();
412
+ } catch (err) {
413
+ outcome = err.message;
414
+ }
415
+ }
416
+ console.error(`Interrupted by ${signal}; ${outcome}.`);
417
+ process.exit(2);
418
+ };
419
+ for (const signal of RESTORE_SIGNALS) process.on(signal, onSignal);
420
+
421
+ let lastError = "";
422
+ let promptTokens = 0;
423
+ let completionTokens = 0;
424
+ let modelMs = 0;
425
+ try {
426
+ for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
427
+ const retryFeedback = lastError
428
+ ? `\n\nThe previous attempt failed verification; the end of its output:\n\`\`\`\n${tail(lastError, FEEDBACK_CHARS)}\n\`\`\`\nFix the issue and return ONLY the replacement code.`
429
+ : "";
430
+
431
+ const messages = [
432
+ {
433
+ role: "system",
434
+ content:
435
+ "You are a Portulan local code worker. Return ONLY the replacement code inside a single fenced code block (``` ... ```). " +
436
+ "Never add comments that record change history (dates, proposal/milestone numbers, PR/issue numbers, reviewer names) or narrate obvious code.",
437
+ },
438
+ {
439
+ role: "user",
440
+ content:
441
+ `Instruction: ${instruction}\n\n` +
442
+ `Target code (${path.basename(filePath)} lines ${startLine}-${endLine}):\n` +
443
+ "```\n" +
444
+ targetSlice +
445
+ "\n```" +
446
+ retryFeedback,
447
+ },
448
+ ];
449
+
450
+ const reply = await chatFn({ baseUrl, allowRemote, model, messages, maxInputBytes, effort });
451
+ promptTokens = sum(promptTokens, reply.promptTokens);
452
+ completionTokens = sum(completionTokens, reply.completionTokens);
453
+ modelMs = sum(modelMs, reply.totalMs);
454
+
455
+ const replacement = extractFencedCode(reply.content).replace(/\n$/, "");
456
+ const updatedLines = [...allLines.slice(0, startLine - 1), replacement, ...allLines.slice(endLine)];
457
+ fs.writeFileSync(filePath, updatedLines.join("\n") + (trailingNewline ? "\n" : ""), "utf8");
458
+
459
+ const check = await execFn(verifyCmd, { onSpawn: (child) => (verifying = child) });
460
+ verifying = null;
461
+ const output = [check.stdout, check.stderr].filter(Boolean).join("\n").trim();
462
+ if (check.status === 0) {
463
+ kept = true;
464
+ return {
465
+ ok: true,
466
+ attempts: attempt,
467
+ receipt: formatPassReceipt({
468
+ mode: "transform",
469
+ model,
470
+ attempts: attempt,
471
+ promptTokens,
472
+ completionTokens,
473
+ totalMs: modelMs,
474
+ tokPerSec: rate(completionTokens, modelMs),
475
+ target: path.basename(filePath),
476
+ }),
477
+ };
478
+ }
479
+ if (check.status !== 1) {
480
+ const how = check.error
481
+ ? `failed to run (${check.error.message})`
482
+ : check.signal
483
+ ? `was killed by ${check.signal}`
484
+ : `exited ${check.status}`;
485
+ throw new CouldNotRunError(
486
+ `Verify command ${how}, which is no verdict (0 green, 1 red); restored ${filePath}.${output ? `\n${tail(output, 400)}` : ""}`,
487
+ );
488
+ }
489
+ lastError = output;
490
+ }
491
+
492
+ return {
493
+ ok: false,
494
+ attempts: retries + 1,
495
+ error: `Verification remained red after ${retries + 1} attempt(s); restored ${filePath} byte-for-byte.\n${tail(lastError, FEEDBACK_CHARS)}`,
496
+ };
497
+ } finally {
498
+ try {
499
+ if (!kept) restore();
500
+ } finally {
501
+ for (const signal of RESTORE_SIGNALS) process.off(signal, onSignal);
502
+ }
503
+ }
504
+ }
505
+
506
+ export const HANDOFF_REQUIRED_TOKENS = Object.freeze([
507
+ "claude/spend-ledger",
508
+ "0038",
509
+ "cli/ledger.mjs",
510
+ "cli/advisory.mjs",
511
+ "ef91a02",
512
+ "85k",
513
+ "m13",
514
+ ]);
515
+
516
+ export async function loadPredicates(importFn = (specifier) => import(specifier)) {
517
+ try {
518
+ const [comments, doctor, goldens] = await Promise.all([
519
+ importFn("../../../../cli/comments.mjs"),
520
+ importFn("../../../../cli/doctor.mjs"),
521
+ importFn("../../../../cli/skill-goldens.mjs"),
522
+ ]);
523
+ return {
524
+ jsComments: comments.jsComments,
525
+ historyOf: comments.historyOf,
526
+ parseProvenance: doctor.parseProvenance,
527
+ fieldBlock: goldens.fieldBlock,
528
+ };
529
+ } catch (err) {
530
+ throw new CouldNotRunError(
531
+ `eval grades with Portulan's own cli/ predicates, which are not beside this pack (the repository and the npm package carry them) — ${err.message}`,
532
+ );
533
+ }
534
+ }
535
+
536
+ export function gradeRoleOutput(task, rawText, predicates) {
537
+ const cleaned = stripThinking(rawText);
538
+ if ((task === "code" || task === "codify") && !predicates) {
539
+ throw new CouldNotRunError(`Grading ${task} needs Portulan's predicates (loadPredicates)`);
540
+ }
541
+ if (task === "code") {
542
+ const code = extractFencedCode(cleaned);
543
+ const comments = predicates.jsComments(code);
544
+ const historyHits = predicates.historyOf(comments);
545
+ const exportsSummarize = /export\s+function\s+summarizeRun\b/.test(code);
546
+ const ok = Boolean(code.trim()) && historyHits.length === 0 && exportsSummarize;
547
+ return {
548
+ ok,
549
+ detail: `comments=${comments.length} history=${historyHits.length} exported=${exportsSummarize}`,
550
+ };
551
+ }
552
+ if (task === "codify") {
553
+ const normalized = cleaned.replace(/\*\*Provenance\.\*\*\s*\n?/i, "**Provenance:** ");
554
+ const prov = predicates.parseProvenance(normalized);
555
+ const fields = {
556
+ incident: predicates.fieldBlock(cleaned, "Incident") !== null,
557
+ proposedRule: predicates.fieldBlock(cleaned, "Proposed rule") !== null,
558
+ enforcement: predicates.fieldBlock(cleaned, "Enforcement") !== null,
559
+ retireWhen: predicates.fieldBlock(cleaned, "Retire when") !== null,
560
+ provenance:
561
+ prov?.present === true &&
562
+ Boolean(prov?.fields?.form) &&
563
+ (Boolean(prov?.fields?.href) || Boolean(prov?.fields?.shape)),
564
+ };
565
+ const ok = Object.values(fields).every(Boolean);
566
+ return {
567
+ ok,
568
+ detail: Object.entries(fields).map(([k, v]) => `${k}=${v}`).join(" "),
569
+ };
570
+ }
571
+ if (task === "handoff") {
572
+ const md = extractFencedCode(cleaned);
573
+ const bytes = Buffer.byteLength(md, "utf8");
574
+ const missing = HANDOFF_REQUIRED_TOKENS.filter((t) => !md.includes(t));
575
+ const ok = bytes > 0 && bytes <= 800 && missing.length === 0;
576
+ return {
577
+ ok,
578
+ detail: `bytes=${bytes}/800 missing=${missing.length}`,
579
+ };
580
+ }
581
+ throw new CouldNotRunError(`Unknown eval task: ${task}`);
582
+ }
583
+
584
+ export const EVAL_PROMPTS = Object.freeze({
585
+ code: [
586
+ {
587
+ role: "system",
588
+ content:
589
+ "Return ONLY valid JavaScript ES module code inside a single ```js fenced block. Never include comments that record change history.",
590
+ },
591
+ {
592
+ role: "user",
593
+ content:
594
+ "Refactor this module to remove all history comments and add `export function summarizeRun(records)` returning `{ requests, billableTokens, maxContext }`:\n" +
595
+ "```js\nexport function parseRange(spec) {\n const m = /^(\\d+)-(\\d+)$/.exec(String(spec).trim());\n if (!m) return null;\n const start = Number(m[1]);\n const end = Number(m[2]);\n return start >= 1 && end >= start ? { start, end } : null;\n}\n```",
596
+ },
597
+ ],
598
+ codify: [
599
+ {
600
+ role: "system",
601
+ content: "Draft a concise Portulan proposal in Markdown with bold field leads (**Incident.**, **Proposed rule.**, **Enforcement.**, **Provenance.**, **Retire when.**).",
602
+ },
603
+ {
604
+ role: "user",
605
+ content:
606
+ "Draft a Portulan proposal from these 3 recurring memory entries across `portulan`, `portulan-internal`, and `acme-web`: " +
607
+ "subagent calls inherited an 85k-token host floor (`F`) for 50-line diffs (`https://github.com/sleepy-panda-srl/portulan/pull/433`). " +
608
+ "Include `**Incident.**`, `**Proposed rule.**`, `**Enforcement.**`, `**Provenance.**` with `form=link` and `href=https://github.com/sleepy-panda-srl/portulan/pull/433`, and `**Retire when.**`.",
609
+ },
610
+ ],
611
+ handoff: [
612
+ {
613
+ role: "system",
614
+ content: "Return ONLY the Markdown handoff under 750 bytes. No code fences.",
615
+ },
616
+ {
617
+ role: "user",
618
+ content:
619
+ "Compress this session into a Portulan handoff (`<= 800` bytes) with `## Did`, `## Open`, and `## Next`, preserving every token in " +
620
+ JSON.stringify(HANDOFF_REQUIRED_TOKENS) +
621
+ ":\nBranch `claude/spend-ledger` implemented Proposal `0038` items 2 and 5 in `cli/ledger.mjs` and `cli/advisory.mjs` at commit `ef91a02`. " +
622
+ "Verified `85k` fresh context arithmetic. Open: wire `m13` persona A/B routing next.",
623
+ },
624
+ ],
625
+ });
626
+
627
+ export async function runRoleEval({
628
+ model = ROLE_MODELS.code,
629
+ tasks = ["code", "codify", "handoff"],
630
+ baseUrl = DEFAULT_BASE_URL,
631
+ allowRemote = false,
632
+ effort = "low",
633
+ chatFn = callLocalChat,
634
+ predicates,
635
+ } = {}) {
636
+ for (const task of tasks) {
637
+ if (!Object.hasOwn(EVAL_PROMPTS, task)) throw new CouldNotRunError(`Unknown eval task: ${task}`);
638
+ }
639
+ const grading = predicates ?? (await loadPredicates());
640
+
641
+ const results = [];
642
+ for (const task of tasks) {
643
+ const reply = await chatFn({ baseUrl, allowRemote, model, effort, messages: EVAL_PROMPTS[task] });
644
+ const verdict = gradeRoleOutput(task, reply.content, grading);
645
+ results.push({
646
+ task,
647
+ model,
648
+ ok: verdict.ok,
649
+ detail: verdict.detail,
650
+ totalMs: reply.totalMs,
651
+ tokPerSec: reply.tokPerSec,
652
+ });
653
+ }
654
+ return results;
655
+ }
656
+
657
+ // ---------------------------------------------------------------- the A/B record
658
+
659
+ export const AB_RECORD = "evals/ab/local-worker.json";
660
+ export const AB_REGISTER = "evals/ab/local-worker.md";
661
+ export const AB_K = 5;
662
+ export const AB_K_RULED = "2026-08-31";
663
+ export const AB_TASKS = Object.freeze(["code", "codify", "handoff"]);
664
+ export const AB_ARMS = Object.freeze(["l", "h"]);
665
+ export const TASK_ROLES = Object.freeze({ code: "code", codify: "prose", handoff: "prose" });
666
+ export const HOST_TURN_TIMEOUT_MS = 10 * 60 * 1000;
667
+ export const HOST_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
668
+ export const PREDICATE_SOURCES = Object.freeze(["cli/comments.mjs", "cli/doctor.mjs", "cli/skill-goldens.mjs"]);
669
+ export const JOURNAL_GIT_PATH = "portulan/local-worker-journal";
670
+ const JOURNAL_CONDITIONS = "conditions.json";
671
+
672
+ export const AB_LIMITATIONS = Object.freeze([
673
+ "**What may not be concluded from the figures above.**",
674
+ "",
675
+ "- **`k = 5` per cell supports a recorded rate and nothing else**, the limit `evals/ab/baseline.md`",
676
+ " carries and `evals/ab/corpus.md` argues: no significance, no interval, and no claim that a",
677
+ " difference between two cells is a difference between the models.",
678
+ "- **A pass is the screen's predicate and nothing more.** `gradeRoleOutput()` asks what `eval` asks:",
679
+ " no history comment and the export for `code`, five fields and a provenance for `codify`, 800 bytes",
680
+ " and every required token for `handoff`. It does not ask whether an output is correct or good.",
681
+ "- **The verdicts are the screen's as it ships, not only as it was.** `eval --verify` regrades every",
682
+ " completed turn's output with the predicates of the commit it runs on, and any verdict that moves is",
683
+ " red, so these figures hold under the screen that ships. The fingerprints above name the screen the",
684
+ " capture graded under, as recorded; a changed predicate file whose verdicts all regrade the same is",
685
+ " no finding.",
686
+ "- **The arms differ in more than the model.** Arm L runs with no host floor at `--effort low` and",
687
+ " receives a system and a user message; arm H is the host agent under the operator's own",
688
+ " environment, credentials and configuration, with whatever it loads in an empty directory, and",
689
+ " receives the two as one text.",
690
+ " That difference is the routing question, not a confound this record removes.",
691
+ "- **Arm H's model is the operator's declaration**, which the runner records and does not observe.",
692
+ "- **`did-not-complete` is a fact about a turn**, a host exit other than `0`, a timeout, or a host",
693
+ " ended by a signal the capture did not send, and is folded into neither count.",
694
+ "- **Which routing these figures support is not stated here.** It is stated where the routing ships,",
695
+ " the pack's README and `ROLE_MODELS`, which cite this record.",
696
+ ]);
697
+
698
+ export function turnSeed(seed, task, run) {
699
+ return Number.parseInt(createHash("sha256").update(`${seed}\0${task}\0${run}`).digest("hex").slice(0, 8), 16);
700
+ }
701
+
702
+ export function screenFingerprint(readFn = (rel) => fs.readFileSync(new URL(`../../../../${rel}`, import.meta.url))) {
703
+ const hash = createHash("sha256");
704
+ for (const rel of PREDICATE_SOURCES) hash.update(`${rel}\0`).update(readFn(rel)).update("\0");
705
+ return hash.update(gradeRoleOutput.toString()).update("\0").update(JSON.stringify(HANDOFF_REQUIRED_TOKENS)).digest("hex");
706
+ }
707
+
708
+ export function promptsFingerprint() {
709
+ return createHash("sha256").update(JSON.stringify(EVAL_PROMPTS)).digest("hex");
710
+ }
711
+
712
+ export function currentScreen() {
713
+ return { predicates: screenFingerprint(), prompts: promptsFingerprint() };
714
+ }
715
+
716
+ export function hostPrompt(task) {
717
+ return EVAL_PROMPTS[task].map((m) => m.content).join("\n\n");
718
+ }
719
+
720
+ export function parseHostCommand(spec) {
721
+ const argv = String(spec ?? "").trim().split(/\s+/).filter(Boolean);
722
+ if (argv.length === 0) throw new CouldNotRunError("--host-cmd names no command");
723
+ return argv;
724
+ }
725
+
726
+ export function machineDescription() {
727
+ const cpu = os.cpus()[0]?.model?.trim() || "an unnamed CPU";
728
+ const gib = Math.round(os.totalmem() / 2 ** 30);
729
+ return `${cpu}, ${gib} GiB, ${os.platform()} ${os.arch()}, node ${process.version}`;
730
+ }
731
+
732
+ export function runHostTurn({ command, prompt, timeoutMs = HOST_TURN_TIMEOUT_MS, maxOutputBytes = HOST_MAX_OUTPUT_BYTES, onSpawn = () => {} }) {
733
+ const cwd = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "portulan-local-llm-host-"));
734
+ const started = performance.now();
735
+ const unspawnable = (error) =>
736
+ new CouldNotRunError(`\`${command[0]}\` could not be spawned — ${error.code ?? error.message}. That is a fact about this machine, not a turn of arm H`);
737
+ return new Promise((resolve, reject) => {
738
+ let child;
739
+ try {
740
+ child = spawn(command[0], [...command.slice(1), prompt], { cwd, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] });
741
+ } catch (error) {
742
+ reject(unspawnable(error));
743
+ return;
744
+ }
745
+ onSpawn(child, cwd);
746
+ const chunks = [];
747
+ let bytes = 0;
748
+ let overflowed = false;
749
+ let timedOut = false;
750
+ const timer = setTimeout(() => {
751
+ timedOut = true;
752
+ stopVerify(child);
753
+ }, timeoutMs);
754
+ child.stdout.on("data", (chunk) => {
755
+ bytes += chunk.length;
756
+ if (bytes > maxOutputBytes) {
757
+ overflowed = true;
758
+ stopVerify(child);
759
+ } else chunks.push(chunk);
760
+ });
761
+ child.stderr.resume();
762
+ child.on("error", (error) => {
763
+ clearTimeout(timer);
764
+ reject(unspawnable(error));
765
+ });
766
+ child.on("close", (status) => {
767
+ clearTimeout(timer);
768
+ if (overflowed) {
769
+ reject(new CouldNotRunError(`arm H wrote more than ${maxOutputBytes} bytes, which no screen task asks for; the turn is not journalled, and runs again on resume`));
770
+ return;
771
+ }
772
+ const ms = Math.round(performance.now() - started);
773
+ resolve({ completed: !timedOut && status === 0, exit: timedOut ? null : status, timedOut, ms, output: Buffer.concat(chunks).toString("utf8") });
774
+ });
775
+ }).finally(() => fs.rmSync(cwd, { recursive: true, force: true }));
776
+ }
777
+
778
+ export function abConditions({ repoRoot, hostCommand, spawnFn = spawnSync }) {
779
+ const git = (...argv) => spawnFn("git", argv, { cwd: repoRoot, encoding: "utf8" });
780
+ const head = git("rev-parse", "HEAD");
781
+ if (head.error || head.status !== 0) throw new CouldNotRunError(`${repoRoot} names no commit (\`git rev-parse HEAD\` failed), and a record must name the one it was captured at`);
782
+ const status = git("status", "--porcelain");
783
+ if (status.error || status.status !== 0) throw new CouldNotRunError(`\`git status\` failed in ${repoRoot}, so the record could not say whether its tree was clean`);
784
+ const journal = git("rev-parse", "--git-path", JOURNAL_GIT_PATH);
785
+ if (journal.error || journal.status !== 0) throw new CouldNotRunError(`\`git rev-parse --git-path\` failed in ${repoRoot}, so the capture has nowhere to journal its turns`);
786
+ const version = spawnFn(hostCommand[0], [...hostCommand.slice(1), "--version"], { cwd: os.tmpdir(), encoding: "utf8", timeout: 60_000 });
787
+ const line = (version.stdout ?? "").trim().split("\n")[0]?.trim() ?? "";
788
+ if (version.error || version.status !== 0 || line === "") {
789
+ throw new CouldNotRunError(`\`${hostCommand.join(" ")} --version\` did not name a version, and a record that cannot name its host is a figure with no conditions`);
790
+ }
791
+ return {
792
+ journal: path.resolve(repoRoot, journal.stdout.trim()),
793
+ captured: new Date().toISOString(),
794
+ commit: head.stdout.trim(),
795
+ clean: status.stdout.trim() === "",
796
+ machine: machineDescription(),
797
+ hostVersion: line,
798
+ };
799
+ }
800
+
801
+ export function journalFile(dir, id) {
802
+ return path.join(dir, `${id.task}__${id.arm}__${id.run}.json`);
803
+ }
804
+
805
+ function writeAtomic(file, value) {
806
+ fs.writeFileSync(`${file}.partial`, `${JSON.stringify(value, null, 2)}\n`);
807
+ fs.renameSync(`${file}.partial`, file);
808
+ }
809
+
810
+ function readJournalEntry(file) {
811
+ try {
812
+ return JSON.parse(fs.readFileSync(file, "utf8"));
813
+ } catch (cause) {
814
+ if (cause.code === "ENOENT") return null;
815
+ throw new CouldNotRunError(`${file} could not be read — ${cause.code ?? cause.message}. A journal that cannot be read is not an absent one`);
816
+ }
817
+ }
818
+
819
+ export function openJournal(dir, header) {
820
+ fs.mkdirSync(dir, { recursive: true });
821
+ const file = path.join(dir, JOURNAL_CONDITIONS);
822
+ const existing = readJournalEntry(file);
823
+ if (existing === null) {
824
+ writeAtomic(file, header);
825
+ return header;
826
+ }
827
+ const differing = [...new Set([...Object.keys(header), ...Object.keys(existing)])].filter(
828
+ (key) => key !== "captured" && JSON.stringify(existing[key]) !== JSON.stringify(header[key]),
829
+ );
830
+ if (differing.length > 0) {
831
+ throw new CouldNotRunError(
832
+ `${dir} journals turns taken under other conditions (${differing.join(", ")}). Its arm-H turns were paid for: inspect it, and remove it yourself to capture afresh`,
833
+ );
834
+ }
835
+ return existing;
836
+ }
837
+
838
+ export async function captureAb({
839
+ seed,
840
+ k = AB_K,
841
+ hostCommand,
842
+ hostModel,
843
+ models = { code: ROLE_MODELS.code, prose: ROLE_MODELS.prose },
844
+ baseUrl = DEFAULT_BASE_URL,
845
+ conditions,
846
+ screen = currentScreen(),
847
+ journal = null,
848
+ chatFn = callLocalChat,
849
+ hostFn = runHostTurn,
850
+ predicates,
851
+ onTurn = () => {},
852
+ }) {
853
+ const endpoint = assertLoopback(baseUrl);
854
+ const grading = predicates ?? (await loadPredicates());
855
+ const grade = (task, output) => {
856
+ const verdict = gradeRoleOutput(task, output, grading);
857
+ return { ok: verdict.ok, detail: verdict.detail };
858
+ };
859
+ const header = {
860
+ k,
861
+ seed,
862
+ commit: conditions.commit,
863
+ clean: conditions.clean,
864
+ runner: RUNNER_FINGERPRINT,
865
+ machine: conditions.machine,
866
+ endpoint,
867
+ models,
868
+ hostCommand,
869
+ hostVersion: conditions.hostVersion,
870
+ hostModel,
871
+ screen,
872
+ captured: conditions.captured,
873
+ };
874
+ const opened = journal === null ? header : openJournal(journal, header);
875
+
876
+ const turns = [];
877
+ let running = null;
878
+ const onSignal = (signal) => {
879
+ if (running !== null) {
880
+ stopVerify(running.child);
881
+ fs.rmSync(running.cwd, { recursive: true, force: true });
882
+ }
883
+ console.error(`Interrupted by ${signal}; ${journal === null ? `${turns.length} turn(s) lost, with no journal` : `${turns.length} turn(s) kept in ${journal}, and the same command resumes from them`}.`);
884
+ process.exit(2);
885
+ };
886
+ const take = async (id, run) => {
887
+ const file = journal === null ? null : journalFile(journal, id);
888
+ const kept = file === null ? null : readJournalEntry(file);
889
+ if (kept !== null) {
890
+ if (kept.task !== id.task || kept.arm !== id.arm || kept.run !== id.run) throw new CouldNotRunError(`${file} holds another turn than its name says`);
891
+ turns.push(kept);
892
+ onTurn(kept, true);
893
+ return;
894
+ }
895
+ const turn = await run();
896
+ if (file !== null) writeAtomic(file, turn);
897
+ turns.push(turn);
898
+ onTurn(turn, false);
899
+ };
900
+ for (const signal of RESTORE_SIGNALS) process.on(signal, onSignal);
901
+ try {
902
+ for (const task of AB_TASKS) {
903
+ for (let run = 0; run < k; run += 1) {
904
+ await take({ task, arm: "l", run }, async () => {
905
+ const model = models[TASK_ROLES[task]];
906
+ const turnSeedValue = turnSeed(seed, task, run);
907
+ const reply = await chatFn({ baseUrl: endpoint, model, effort: "low", messages: EVAL_PROMPTS[task], seed: turnSeedValue });
908
+ return { task, arm: "l", run, seed: turnSeedValue, model, completed: true, exit: null, timedOut: false, ms: reply.totalMs, ...grade(task, reply.content), output: reply.content };
909
+ });
910
+ }
911
+ }
912
+ for (const task of AB_TASKS) {
913
+ for (let run = 0; run < k; run += 1) {
914
+ await take({ task, arm: "h", run }, async () => {
915
+ const res = await hostFn({
916
+ command: hostCommand,
917
+ prompt: hostPrompt(task),
918
+ timeoutMs: HOST_TURN_TIMEOUT_MS,
919
+ onSpawn: (child, cwd) => (running = { child, cwd }),
920
+ });
921
+ running = null;
922
+ const verdict = res.completed ? grade(task, res.output) : { ok: null, detail: null };
923
+ return { task, arm: "h", run, seed: null, model: hostModel, completed: res.completed, exit: res.exit, timedOut: res.timedOut, ms: res.ms, ...verdict, output: res.output };
924
+ });
925
+ }
926
+ }
927
+ } finally {
928
+ for (const signal of RESTORE_SIGNALS) process.off(signal, onSignal);
929
+ }
930
+ const order = (t) => AB_TASKS.indexOf(t.task) * 2 * k + AB_ARMS.indexOf(t.arm) * k + t.run;
931
+ turns.sort((a, b) => order(a) - order(b));
932
+ const record = {
933
+ portulan: { abLocalWorker: "1" },
934
+ captured: opened.captured,
935
+ source: { commit: conditions.commit, clean: conditions.clean, runner: opened.runner },
936
+ machine: conditions.machine,
937
+ k,
938
+ seed,
939
+ rulings: { k: AB_K_RULED },
940
+ screen: { ...screen },
941
+ arms: {
942
+ l: { endpoint, effort: "low", models: { ...models }, routing: { ...TASK_ROLES } },
943
+ h: { command: [...hostCommand], version: conditions.hostVersion, model: hostModel, timeoutMs: HOST_TURN_TIMEOUT_MS },
944
+ },
945
+ turns,
946
+ };
947
+ record.cells = foldAb(turns, k);
948
+ return record;
949
+ }
950
+
951
+ export function foldAb(turns, k = AB_K) {
952
+ const cells = [];
953
+ for (const task of AB_TASKS) {
954
+ for (const arm of AB_ARMS) {
955
+ const mine = turns.filter((t) => t?.task === task && t?.arm === arm);
956
+ if (mine.length !== k) throw new RailError(`\`${task}\`/${arm} holds ${mine.length} turn(s) and k is ${k} — a rate over an incomplete cell is a rate about something else`);
957
+ const named = new Set(mine.map((t) => t.model));
958
+ if (named.size !== 1) throw new RailError(`\`${task}\`/${arm} names ${named.size} models — a cell is one model's`);
959
+ const cell = {
960
+ task,
961
+ arm,
962
+ model: mine[0].model,
963
+ passed: mine.filter((t) => t.completed === true && t.ok === true).length,
964
+ failed: mine.filter((t) => t.completed === true && t.ok === false).length,
965
+ didNotComplete: mine.filter((t) => t.completed === false).length,
966
+ };
967
+ if (cell.passed + cell.failed + cell.didNotComplete !== k) {
968
+ throw new RailError(`\`${task}\`/${arm} accounts for ${cell.passed + cell.failed + cell.didNotComplete} of ${k} turns — the three states must be total`);
969
+ }
970
+ cells.push(cell);
971
+ }
972
+ }
973
+ return cells;
974
+ }
975
+
976
+ const isNonEmpty = (v) => typeof v === "string" && v.trim() !== "";
977
+
978
+ export function verifyAbShape(rec) {
979
+ const red = [];
980
+ if (rec?.portulan?.abLocalWorker !== "1") red.push("the record does not declare `portulan.abLocalWorker: \"1\"` — this is not a local-worker A/B capture");
981
+ if (!isNonEmpty(rec?.captured)) red.push("the record carries no `captured` time");
982
+ if (!isNonEmpty(rec?.source?.commit) || typeof rec?.source?.clean !== "boolean") red.push("the record's `source` names no commit, or does not say whether its tree was clean");
983
+ if (!isNonEmpty(rec?.machine)) red.push("the record names no machine");
984
+ if (rec?.k !== AB_K) red.push(`the record's k is ${JSON.stringify(rec?.k)} where the maintainer ruled ${AB_K}`);
985
+ if (!isNonEmpty(rec?.seed)) red.push("the record carries no seed, so no arm-L seed in it can be recomputed");
986
+ if (!isNonEmpty(rec?.rulings?.k)) red.push("the record carries no ruling for its k");
987
+ const sha = (v) => typeof v === "string" && /^[0-9a-f]{64}$/.test(v);
988
+ if (!sha(rec?.source?.runner)) red.push("the record carries no sha256 fingerprint of the loaded runner source");
989
+ if (!sha(rec?.screen?.predicates) || !sha(rec?.screen?.prompts)) red.push("the record carries no sha256 fingerprint of the predicates and the prompts it was graded under");
990
+ if (rec?.fixture !== undefined && !isNonEmpty(rec.fixture)) red.push("the record's `fixture` is present and is not a non-empty string");
991
+ const l = rec?.arms?.l;
992
+ const h = rec?.arms?.h;
993
+ if (!isNonEmpty(l?.endpoint) || l?.effort !== "low" || !isNonEmpty(l?.models?.code) || !isNonEmpty(l?.models?.prose)) {
994
+ red.push("arm L does not record a loopback endpoint, `--effort low`, and a model for each chat role");
995
+ }
996
+ if (JSON.stringify(l?.routing) !== JSON.stringify(TASK_ROLES)) red.push("arm L's recorded routing is not the task-to-role map this runner routes by");
997
+ if (!Array.isArray(h?.command) || h.command.length === 0 || !h.command.every(isNonEmpty)) red.push("arm H records no host command");
998
+ if (!isNonEmpty(h?.version)) red.push("arm H names no host version");
999
+ if (!isNonEmpty(h?.model)) red.push("arm H names no model");
1000
+ if (!Number.isInteger(h?.timeoutMs) || h.timeoutMs <= 0) red.push("arm H records no per-turn timeout");
1001
+ if (!Array.isArray(rec?.turns) || rec.turns.length === 0) red.push("the record carries no turns");
1002
+ if (!Array.isArray(rec?.cells) || rec.cells.length === 0) red.push("the record publishes no cells");
1003
+ if (Array.isArray(rec?.turns)) {
1004
+ for (const t of rec.turns) {
1005
+ const id = `(${t?.task}, ${t?.arm}, run ${t?.run})`;
1006
+ if (!AB_TASKS.includes(t?.task) || !AB_ARMS.includes(t?.arm) || !Number.isInteger(t?.run)) red.push(`${id} is not a turn of this matrix`);
1007
+ if (typeof t?.completed !== "boolean" || typeof t?.timedOut !== "boolean") red.push(`${id} carries no boolean \`completed\` or \`timedOut\``);
1008
+ if (t?.completed === true && (typeof t?.ok !== "boolean" || typeof t?.detail !== "string")) red.push(`${id} completed and carries no verdict`);
1009
+ if (t?.completed === false && (t?.ok !== null || t?.detail !== null)) red.push(`${id} did not complete and carries a verdict anyway`);
1010
+ if (typeof t?.output !== "string") red.push(`${id} carries no output, so nobody can regrade it`);
1011
+ if (!isNonEmpty(t?.model)) red.push(`${id} names no model`);
1012
+ if (t?.exit !== null && !Number.isInteger(t?.exit)) red.push(`${id} carries an \`exit\` that is neither \`null\` nor an integer`);
1013
+ if (!Number.isInteger(t?.ms) || t.ms < 0) red.push(`${id} carries no whole-millisecond time`);
1014
+ }
1015
+ const shapes = new Set(rec.turns.map((t) => Object.keys(t ?? {}).sort().join(",")));
1016
+ if (shapes.size > 1) red.push(`the record's turns do not all carry the same fields (${shapes.size} shapes) — a row missing what its neighbours record is an edit, not a capture`);
1017
+ }
1018
+ if (red.length > 0) return red;
1019
+ let rendered;
1020
+ try {
1021
+ rendered = renderAbRegister(rec);
1022
+ } catch (cause) {
1023
+ return [`the register cannot be rendered from this record — ${cause.message}`];
1024
+ }
1025
+ for (const hole of ["undefined", "NaN"]) {
1026
+ if (rendered.includes(hole)) red.push(`the register rendered from this record contains \`${hole}\` — a field the renderer reads is missing`);
1027
+ }
1028
+ return red;
1029
+ }
1030
+
1031
+ export function verifyAb(rec, { predicates, screen }) {
1032
+ const red = verifyAbShape(rec);
1033
+ if (red.length > 0) return red;
1034
+ if (rec.screen.prompts !== screen.prompts) red.push("the eval prompts moved since capture, so the record measures a screen that no longer ships");
1035
+ const moved = rec.screen.predicates !== screen.predicates;
1036
+ for (const t of rec.turns.filter((turn) => turn.completed)) {
1037
+ const now = gradeRoleOutput(t.task, t.output, predicates);
1038
+ if (now.ok !== t.ok || now.detail !== t.detail) {
1039
+ red.push(
1040
+ `(${t.task}, ${t.arm}, run ${t.run}) records ${t.ok ? "pass" : "fail"} \`${t.detail}\` where its output regrades ${now.ok ? "pass" : "fail"} \`${now.detail}\` — ` +
1041
+ (moved ? "the predicates moved since capture, and the figures must hold under the screen that ships" : "a recorded verdict does not match its output"),
1042
+ );
1043
+ }
1044
+ }
1045
+ try {
1046
+ assertLoopback(rec.arms.l.endpoint);
1047
+ } catch (cause) {
1048
+ red.push(`arm L's endpoint is not loopback, so it is not the local model — ${cause.message}`);
1049
+ }
1050
+ const seen = new Set(rec.turns.map((t) => `${t.task}\0${t.arm}\0${t.run}`));
1051
+ if (rec.turns.length !== AB_TASKS.length * AB_ARMS.length * rec.k) red.push(`the record holds ${rec.turns.length} turn(s) where k=${rec.k} needs ${AB_TASKS.length * AB_ARMS.length * rec.k}`);
1052
+ for (const task of AB_TASKS) {
1053
+ for (const arm of AB_ARMS) {
1054
+ for (let run = 0; run < rec.k; run += 1) {
1055
+ if (!seen.has(`${task}\0${arm}\0${run}`)) red.push(`no turn for (${task}, ${arm}, run ${run}) — the matrix is not total over k`);
1056
+ }
1057
+ }
1058
+ }
1059
+ for (const t of rec.turns) {
1060
+ const id = `(${t.task}, ${t.arm}, run ${t.run})`;
1061
+ if (t.arm === "l") {
1062
+ const want = turnSeed(rec.seed, t.task, t.run);
1063
+ if (t.seed !== want) red.push(`${id} records seed ${t.seed} where the record's seed derives ${want}`);
1064
+ if (t.model !== rec.arms.l.models[TASK_ROLES[t.task]]) red.push(`${id} names \`${t.model}\`, not the model arm L routes \`${t.task}\` to`);
1065
+ if (t.completed !== true || t.exit !== null || t.timedOut !== false) red.push(`${id} is an arm-L turn recorded as a process — a local call either answers or aborts the capture`);
1066
+ } else {
1067
+ if (t.seed !== null) red.push(`${id} records a seed, and arm H's command takes none`);
1068
+ if (t.model !== rec.arms.h.model) red.push(`${id} names \`${t.model}\`, not arm H's declared model`);
1069
+ if (t.completed !== (t.exit === 0 && t.timedOut === false)) red.push(`${id} records \`completed\` against its own exit and timeout`);
1070
+ }
1071
+ }
1072
+ try {
1073
+ if (JSON.stringify(foldAb(rec.turns, rec.k)) !== JSON.stringify(rec.cells)) red.push("the published cells do not match a fresh fold of the turns — a figure has drifted from its own data");
1074
+ } catch (cause) {
1075
+ red.push(`the turns do not fold: ${cause.message}`);
1076
+ }
1077
+ return red;
1078
+ }
1079
+
1080
+ export function renderAbRegister(rec) {
1081
+ const k = rec.k;
1082
+ const cell = (task, arm) => {
1083
+ const found = rec.cells.find((c) => c.task === task && c.arm === arm);
1084
+ if (found === undefined) throw new Error(`the record publishes no cell for \`${task}\`/${arm}`);
1085
+ return found;
1086
+ };
1087
+ const lines = [];
1088
+ lines.push("# The local-worker A/B record — what a run of the arms measured");
1089
+ lines.push("");
1090
+ lines.push(`> Rendered from \`${AB_RECORD}\` by \`local-llm.mjs eval --write\`. Do not edit by hand: it is`);
1091
+ lines.push("> regenerated from that file and byte-compared, so a hand-edit survives exactly until the next run.");
1092
+ lines.push(">");
1093
+ lines.push("> **The record is the unreproducible half.** It is derived from **events**, model turns, which do");
1094
+ lines.push("> not repeat. So the turns are captured once, and the rail holds this document to that capture");
1095
+ lines.push("> rather than to the world.");
1096
+ if (rec.fixture !== undefined) {
1097
+ lines.push("");
1098
+ lines.push(`> **SYNTHETIC FIXTURE: not a record of any run.** ${rec.fixture}`);
1099
+ }
1100
+ lines.push("");
1101
+ lines.push("## The conditions");
1102
+ lines.push("");
1103
+ lines.push(`- **Taken:** ${rec.captured}`);
1104
+ lines.push(`- **Commit:** \`${rec.source.commit}\`${rec.source.clean ? "" : " — **a dirty tree**, which is a fact about the runner and the predicates that graded it"}`);
1105
+ lines.push(`- **Machine:** ${rec.machine}`);
1106
+ lines.push(`- **k:** ${k} per cell, the k the maintainer ruled for \`evals/ab/baseline.md\` on ${rec.rulings.k}`);
1107
+ lines.push(`- **Seed:** \`${rec.seed}\` — each arm-L turn's sampling seed derives from it, so a reader can recompute them; arm H's command takes none`);
1108
+ lines.push(`- **Arm L:** the local model as routed, through \`local-llm.mjs\` at \`--effort ${rec.arms.l.effort}\` with no host floor, against \`${rec.arms.l.endpoint}\``);
1109
+ for (const task of AB_TASKS) lines.push(` - \`${task}\` → role \`${rec.arms.l.routing[task]}\` → \`${rec.arms.l.models[rec.arms.l.routing[task]]}\``);
1110
+ lines.push(`- **Arm H:** \`${rec.arms.h.command.join(" ")} <prompt>\`, host \`${rec.arms.h.version}\` as \`${rec.arms.h.command.join(" ")} --version\` reports it, in an empty working directory, with a per-turn timeout of ${Math.round(rec.arms.h.timeoutMs / 1000)}s`);
1111
+ lines.push(`- **Arm H's model:** \`${rec.arms.h.model}\`, as the operator declared it — the runner does not observe it`);
1112
+ lines.push("- **Prompt:** `EVAL_PROMPTS`' own, verbatim. Arm L receives its system and user messages; arm H the two joined by a blank line, as its command's last argument");
1113
+ lines.push("- **Graded by:** `gradeRoleOutput()`, the predicates `eval` screens with, the same for both arms");
1114
+ lines.push(`- **Runner source:** sha256 \`${rec.source.runner}\``);
1115
+ lines.push(`- **Screen fingerprints:** predicates \`${rec.screen.predicates}\`, the sha256 over ${PREDICATE_SOURCES.map((p) => `\`${p}\``).join(", ")}, \`gradeRoleOutput()\` and its required tokens; prompts \`${rec.screen.prompts}\``);
1116
+ lines.push("");
1117
+ lines.push("## The figures");
1118
+ lines.push("");
1119
+ lines.push("| Task | Arm | Model | passed | failed | did-not-complete |");
1120
+ lines.push("|---|---|---|---|---|---|");
1121
+ for (const c of rec.cells) lines.push(`| \`${c.task}\` | ${c.arm.toUpperCase()} | \`${c.model}\` | **${c.passed}**/${k} | ${c.failed} | ${c.didNotComplete} |`);
1122
+ lines.push("");
1123
+ const total = (arm) => AB_TASKS.reduce((n, task) => n + cell(task, arm).passed, 0);
1124
+ const totalL = total("l");
1125
+ const totalH = total("h");
1126
+ const denominator = k * AB_TASKS.length;
1127
+ const verdict = totalL === totalH ? "a tie" : `a difference of ${totalL - totalH > 0 ? "+" : ""}${totalL - totalH}`;
1128
+ lines.push(`**Arm L ${totalL}/${denominator}, arm H ${totalH}/${denominator} — ${verdict}, recorded as measured.**`);
1129
+ lines.push("");
1130
+ lines.push(`That total is a sum of ${AB_TASKS.length} counts of ${k}, and NOT a rate over ${denominator} independent`);
1131
+ lines.push("trials. It carries no significance, no interval, and no claim that a difference between the arms is");
1132
+ lines.push("a difference between the models. The per-task rows above are the measurement.");
1133
+ lines.push("");
1134
+ lines.push("### Arm H's absolute rate, beside every contrast");
1135
+ lines.push("");
1136
+ lines.push("| Task | arm L | arm H | difference |");
1137
+ lines.push("|---|---|---|---|");
1138
+ for (const task of AB_TASKS) {
1139
+ const diff = cell(task, "l").passed - cell(task, "h").passed;
1140
+ lines.push(`| \`${task}\` | ${cell(task, "l").passed}/${k} | ${cell(task, "h").passed}/${k} | ${diff >= 0 ? "+" : ""}${diff} |`);
1141
+ }
1142
+ lines.push("");
1143
+ lines.push(`A difference here is a difference between two counts of ${k}. It is not a measurement of an`);
1144
+ lines.push("effect, and the block below is not a formality.");
1145
+ lines.push("");
1146
+ lines.push("## Every turn, so the figures above can be checked against them");
1147
+ lines.push("");
1148
+ lines.push("| Task | Arm | run | seed | verdict | exit | ms | detail |");
1149
+ lines.push("|---|---|---|---|---|---|---|---|");
1150
+ for (const t of rec.turns) {
1151
+ const state = t.completed ? `\`${t.ok ? "pass" : "fail"}\`` : t.timedOut ? "**did-not-complete** (timed out)" : "**did-not-complete**";
1152
+ lines.push(`| \`${t.task}\` | ${t.arm.toUpperCase()} | ${t.run} | ${t.seed ?? "—"} | ${state} | ${t.exit ?? "—"} | ${t.ms} | ${t.detail === null ? "—" : `\`${t.detail}\``} |`);
1153
+ }
1154
+ lines.push("");
1155
+ lines.push(...AB_LIMITATIONS);
1156
+ return `${lines.join("\n")}\n`;
1157
+ }
1158
+
1159
+ function readAbRecord(repoRoot) {
1160
+ const file = path.join(repoRoot, AB_RECORD);
1161
+ let text;
1162
+ try {
1163
+ text = fs.readFileSync(file, "utf8");
1164
+ } catch (cause) {
1165
+ if (cause.code === "ENOENT") {
1166
+ throw new CouldNotRunError(`${AB_RECORD} does not exist — no A/B record has been captured, which is not a verdict about one`);
1167
+ }
1168
+ throw new CouldNotRunError(`${AB_RECORD} could not be read — ${cause.code ?? cause.message}`);
1169
+ }
1170
+ try {
1171
+ return JSON.parse(text);
1172
+ } catch (cause) {
1173
+ throw new RailError(`${AB_RECORD} is not JSON — ${cause.message}`);
1174
+ }
1175
+ }
1176
+
1177
+ const findings = (red) => `${red.length} finding(s):\n - ${red.join("\n - ")}`;
1178
+
1179
+ async function regrading() {
1180
+ const predicates = await loadPredicates();
1181
+ return { predicates, screen: currentScreen() };
1182
+ }
1183
+
1184
+ function refuseFixture(rec, allowFixture) {
1185
+ if (rec?.fixture !== undefined && !allowFixture) {
1186
+ throw new RailError(`${AB_RECORD} is a synthetic fixture, and a synthetic fixture is not a capture: no run stands behind its figures`);
1187
+ }
1188
+ }
1189
+
1190
+ export async function verifyAbRecord(repoRoot, { allowFixture = false } = {}) {
1191
+ const rec = readAbRecord(repoRoot);
1192
+ refuseFixture(rec, allowFixture);
1193
+ const shape = verifyAbShape(rec);
1194
+ if (shape.length > 0) throw new RailError(findings(shape));
1195
+ const grading = await regrading();
1196
+ const red = verifyAb(rec, grading);
1197
+ const rendered = renderAbRegister(rec);
1198
+ const registerPath = path.join(repoRoot, AB_REGISTER);
1199
+ const onDisk = fs.existsSync(registerPath) ? fs.readFileSync(registerPath, "utf8") : null;
1200
+ if (onDisk === null) red.push(`${AB_REGISTER} is missing — render it with \`local-llm.mjs eval --write\``);
1201
+ else if (onDisk !== rendered) red.push(`${AB_REGISTER} does not match a fresh render of ${AB_RECORD} — a published figure has drifted from its own data`);
1202
+ if (!rendered.includes(AB_LIMITATIONS[0])) red.push("the rendered register carries no limitation block");
1203
+ if (red.length > 0) throw new RailError(findings(red));
1204
+ const moved = rec.screen.predicates === grading.screen.predicates ? "" : "; the predicate files changed since capture, and every verdict regrades the same";
1205
+ return `${rec.turns.length} turn(s) over ${rec.cells.length} cell(s) at k=${rec.k}, regraded and folded consistently, and ${AB_REGISTER} matches byte for byte${moved}`;
1206
+ }
1207
+
1208
+ export async function writeAbRegister(repoRoot, { allowFixture = false } = {}) {
1209
+ const rec = readAbRecord(repoRoot);
1210
+ refuseFixture(rec, allowFixture);
1211
+ const shape = verifyAbShape(rec);
1212
+ if (shape.length > 0) throw new RailError(`${findings(shape)}\n${AB_REGISTER} was NOT written`);
1213
+ const red = verifyAb(rec, await regrading());
1214
+ if (red.length > 0) throw new RailError(`${findings(red)}\n${AB_REGISTER} was NOT written`);
1215
+ fs.writeFileSync(path.join(repoRoot, AB_REGISTER), renderAbRegister(rec));
1216
+ return `wrote ${AB_REGISTER} from ${AB_RECORD}`;
1217
+ }
1218
+
1219
+ export function publishAb(repoRoot, rec, grading) {
1220
+ fs.mkdirSync(path.join(repoRoot, path.dirname(AB_RECORD)), { recursive: true });
1221
+ try {
1222
+ fs.writeFileSync(path.join(repoRoot, AB_RECORD), `${JSON.stringify(rec, null, 2)}\n`, { flag: "wx" });
1223
+ } catch (cause) {
1224
+ if (cause.code === "EEXIST") throw new CouldNotRunError(`${AB_RECORD} exists at publication: its bytes and register were preserved; the current journal remains for recovery`);
1225
+ throw new CouldNotRunError(`${AB_RECORD} could not be published — ${cause.code ?? cause.message}; the current journal remains for recovery`);
1226
+ }
1227
+ const red = verifyAb(rec, grading);
1228
+ if (red.length > 0) return { ok: false, message: `wrote ${AB_RECORD}, which is kept because its turns do not repeat; ${findings(red)}\n${AB_REGISTER} was NOT written` };
1229
+ fs.writeFileSync(path.join(repoRoot, AB_REGISTER), renderAbRegister(rec));
1230
+ return { ok: true, message: `wrote ${AB_RECORD} and ${AB_REGISTER}. k=${rec.k} per cell supports a recorded rate and nothing else` };
1231
+ }
1232
+
1233
+ export async function runOfflineVerify() {
1234
+ let caughtRemote = false;
1235
+ try {
1236
+ assertLoopback("https://api.openai.com/v1");
1237
+ } catch (e) {
1238
+ caughtRemote = e instanceof RailError;
1239
+ }
1240
+ if (!caughtRemote) return { status: 1, message: "FAIL: non-loopback endpoint was not refused by assertLoopback" };
1241
+
1242
+ let caughtBudget = false;
1243
+ try {
1244
+ assertInputBudget("x".repeat(1024), { maxInputBytes: 512 });
1245
+ } catch (e) {
1246
+ caughtBudget = e instanceof RailError;
1247
+ }
1248
+ if (!caughtBudget) return { status: 1, message: "FAIL: oversized input was not refused by assertInputBudget" };
1249
+
1250
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "portulan-local-llm-"));
1251
+ const sampleFile = path.join(tmpDir, "sample.mjs");
1252
+ const originalBytes = "export const x = 1;\n";
1253
+ fs.writeFileSync(sampleFile, originalBytes, "utf8");
1254
+ const editing = async () => ({ content: "```js\nexport const x = 2;\n```", promptTokens: 10, completionTokens: 5, totalMs: 50, tokPerSec: 40 });
1255
+ try {
1256
+ const res = await verifyGatedTransform({
1257
+ filePath: sampleFile,
1258
+ instruction: "change x to 2",
1259
+ verifyCmd: "exit 1",
1260
+ retries: 1,
1261
+ chatFn: editing,
1262
+ execFn: () => ({ status: 1, stdout: "assertion failed: x !== 1", stderr: "" }),
1263
+ });
1264
+ if (res.ok || fs.readFileSync(sampleFile, "utf8") !== originalBytes) {
1265
+ return { status: 1, message: "FAIL: verifyGatedTransform did not restore original bytes after red verify" };
1266
+ }
1267
+
1268
+ let unjudged = null;
1269
+ try {
1270
+ await verifyGatedTransform({
1271
+ filePath: sampleFile,
1272
+ instruction: "change x to 2",
1273
+ verifyCmd: "missing-verify-command",
1274
+ chatFn: editing,
1275
+ execFn: () => ({ status: 127, stdout: "", stderr: "missing-verify-command: not found" }),
1276
+ });
1277
+ } catch (e) {
1278
+ unjudged = e;
1279
+ }
1280
+ if (!(unjudged instanceof CouldNotRunError) || fs.readFileSync(sampleFile, "utf8") !== originalBytes) {
1281
+ return { status: 1, message: "FAIL: a verify exit other than 0 or 1 was not treated as could-not-run with the file restored" };
1282
+ }
1283
+
1284
+ const green = await verifyGatedTransform({
1285
+ filePath: sampleFile,
1286
+ instruction: "change x to 2",
1287
+ verifyCmd: "exit 0",
1288
+ chatFn: editing,
1289
+ execFn: () => ({ status: 0, stdout: "", stderr: "" }),
1290
+ });
1291
+ if (!green.ok || fs.readFileSync(sampleFile, "utf8") !== "export const x = 2;\n" || green.receipt.includes("\n")) {
1292
+ return { status: 1, message: "FAIL: a green verify did not keep the edit with a single-line receipt" };
1293
+ }
1294
+
1295
+ fs.writeFileSync(path.join(tmpDir, "a.md"), "short\n");
1296
+ fs.writeFileSync(path.join(tmpDir, "b.md"), "x".repeat(1024));
1297
+ let requested = 0;
1298
+ const recording = async () => {
1299
+ requested += 1;
1300
+ throw new Error("dedup reached the network");
1301
+ };
1302
+ const refusals = [];
1303
+ for (const options of [{ baseUrl: "https://api.openai.com/v1" }, { baseUrl: "http://127.0.0.1:1/v1", maxInputBytes: 512 }]) {
1304
+ try {
1305
+ await runDedup({ dir: tmpDir, fetchImpl: recording, ...options });
1306
+ } catch (e) {
1307
+ refusals.push(e instanceof RailError);
1308
+ }
1309
+ }
1310
+ if (requested !== 0 || refusals.length !== 2 || !refusals.every(Boolean)) {
1311
+ return { status: 1, message: "FAIL: dedup did not refuse a non-loopback endpoint and an over-ceiling file before any request" };
1312
+ }
1313
+ } finally {
1314
+ fs.rmSync(tmpDir, { recursive: true, force: true });
1315
+ }
1316
+
1317
+ return { status: 0, message: "OK tools/local-llm:runner-rails (6/6 offline rails held)" };
1318
+ }
1319
+
1320
+ const BOOLEAN_FLAGS = Object.freeze({
1321
+ "--verify-offline": "verifyOffline",
1322
+ "--allow-remote": "allowRemote",
1323
+ "--ab": "ab",
1324
+ "--write": "write",
1325
+ "--help": "help",
1326
+ });
1327
+ const VALUE_FLAGS = Object.freeze({
1328
+ "--base-url": "baseUrl",
1329
+ "--model": "model",
1330
+ "--role": "role",
1331
+ "--effort": "effort",
1332
+ "--task": "task",
1333
+ "--file": "file",
1334
+ "--lines": "lines",
1335
+ "--verify": "verify",
1336
+ "--retries": "retries",
1337
+ "--max-input-bytes": "maxInputBytes",
1338
+ "--prompt": "prompt",
1339
+ "--k": "k",
1340
+ "--seed": "seed",
1341
+ "--host-cmd": "hostCmd",
1342
+ "--host-model": "hostModel",
1343
+ "--repo-root": "repoRoot",
1344
+ "--dir": "dir",
1345
+ "--threshold": "threshold",
1346
+ });
1347
+ const COUNT_FLAGS = Object.freeze({ retries: ["--retries", 0], maxInputBytes: ["--max-input-bytes", 1], k: ["--k", 1] });
1348
+ const FLAG_OF = Object.freeze({
1349
+ ...Object.fromEntries([...Object.entries(BOOLEAN_FLAGS), ...Object.entries(VALUE_FLAGS)].map(([flag, key]) => [key, flag])),
1350
+ verifyRecord: "--verify",
1351
+ });
1352
+ const EVAL_ONLY = Object.freeze(["ab", "write", "verifyRecord", "k", "seed", "hostCmd", "hostModel", "repoRoot"]);
1353
+ const AB_ONLY = Object.freeze(["k", "seed", "hostCmd", "hostModel", "repoRoot"]);
1354
+ const AB_TAKES = Object.freeze(new Set(["_", "ab", "write", "k", "seed", "hostCmd", "hostModel", "repoRoot", "baseUrl", "effort"]));
1355
+ const RECORD_TAKES = Object.freeze(new Set(["_", "verifyRecord", "write", "repoRoot"]));
1356
+ const AB_REFUSALS = Object.freeze({
1357
+ model: "arm L is each task's model as routed",
1358
+ role: "arm L is each task's model as routed",
1359
+ task: "the record is over every task",
1360
+ allowRemote: "arm L is the local model, and a remote endpoint is not that arm",
1361
+ });
1362
+
1363
+ function refuseOutside(args, takes, mode) {
1364
+ for (const key of Object.keys(args)) {
1365
+ if (takes.has(key)) continue;
1366
+ const why = AB_REFUSALS[key] ? `: ${AB_REFUSALS[key]}` : "";
1367
+ throw new CouldNotRunError(`${FLAG_OF[key] ?? key} does not apply to ${mode}${why}`);
1368
+ }
1369
+ }
1370
+
1371
+ export const USAGE = `local-llm.mjs — a zero-dependency runner for a loopback OpenAI-compatible model.
1372
+
1373
+ local-llm.mjs status
1374
+ local-llm.mjs chat|summarize [--role code|prose | --model <id>] [--file <path> [--lines <a>-<b>]] --prompt <text>
1375
+ local-llm.mjs transform --file <path> --verify <cmd> --prompt <instruction> [--lines <a>-<b>] [--retries <n>]
1376
+ local-llm.mjs eval [--role code|prose | --model <id>] [--task code|codify|handoff]
1377
+ local-llm.mjs eval --ab --k ${AB_K} --seed <s> --host-cmd "<command>" --host-model <id> --write [--repo-root <dir>]
1378
+ local-llm.mjs eval --verify [--repo-root <dir>]
1379
+ local-llm.mjs eval --write [--repo-root <dir>]
1380
+ local-llm.mjs --verify-offline
1381
+
1382
+ eval the screen: one sample per task and no baseline arm. A model that fails it is not
1383
+ routed; passing it earns nothing.
1384
+ eval --ab capture the A/B record routing is earned by, k=${AB_K} per (task, arm) over code, codify and
1385
+ handoff, both arms graded by the screen's predicates. Arm L is each task's model as
1386
+ routed, at --effort low, on loopback, at $0.00, and runs first. Arm H is the host agent:
1387
+ ${AB_K * AB_TASKS.length} PAID calls whose prompts LEAVE THIS MACHINE through the command --host-cmd names,
1388
+ under the operator's own environment and credentials, crossing the loopback rail this
1389
+ runner otherwise holds. The command is split on whitespace, so no argument of it can
1390
+ hold a space; it is given the prompt as its last argument, and its version is read by
1391
+ running it with --version appended. Without --host-cmd, --ab refuses with exit 1 and
1392
+ contacts nothing; a command that bypasses permission prompts is refused. --host-model is
1393
+ recorded as declared, never observed. Every turn is journalled under the repository's
1394
+ git directory (${JOURNAL_GIT_PATH}) before the next starts, so an interrupted capture
1395
+ (SIGINT, SIGTERM, SIGHUP: exit 2) resumes from it when run again under the same
1396
+ conditions, and refuses a journal of others. Refuses where ${AB_RECORD} exists: a
1397
+ record is captured once. Creates ${AB_RECORD} exclusively; a late existing record is
1398
+ preserved with the journal (exit2). Removes the journal after writing a new record,
1399
+ and writes ${AB_REGISTER} only where the capture passes --verify.
1400
+ eval --verify no model: the record's shape, its matrix total over k, every arm-L seed, every completed
1401
+ turn REGRADED by the predicates as they ship, the cells re-folded from the turns, and
1402
+ ${AB_REGISTER} byte-compared with a fresh render. Exit 2 where no record exists or cli/
1403
+ is not beside packs/ to regrade with, 1 where anything disagrees.
1404
+ eval --write re-render ${AB_REGISTER} from the committed record, refusing one --verify would red.
1405
+
1406
+ Routing evidence: ${AB_REGISTER}, k=5. Code is enabled (L5/H5); shared prose is disabled
1407
+ (codify L0/H3, handoff L5/H5). Operational defaults and --role require the measured model at low
1408
+ effort. --model without --role is an explicit candidate; eval and eval --ab remain screens/captures
1409
+ and never enable a role automatically.
1410
+
1411
+ Every chat request is refused off loopback unless --allow-remote is passed, and is refused over
1412
+ --max-input-bytes (default ${DEFAULT_MAX_INPUT_BYTES}). --effort defaults to low.
1413
+
1414
+ Exit codes: 0 green · 1 red check or model/verify failure · 2 could not run.`;
1415
+
1416
+ export function parseArgs(argv) {
1417
+ const args = { _: [] };
1418
+ for (let i = 0; i < argv.length; i += 1) {
1419
+ const tok = argv[i];
1420
+ if (Object.hasOwn(BOOLEAN_FLAGS, tok)) {
1421
+ args[BOOLEAN_FLAGS[tok]] = true;
1422
+ } else if (tok === "--verify" && args._[0] === "eval") {
1423
+ args.verifyRecord = true;
1424
+ } else if (Object.hasOwn(VALUE_FLAGS, tok)) {
1425
+ if (i + 1 >= argv.length) throw new CouldNotRunError(`${tok} needs a value`);
1426
+ args[VALUE_FLAGS[tok]] = argv[++i];
1427
+ } else if (tok.startsWith("--")) {
1428
+ throw new CouldNotRunError(`Unknown option ${tok}`);
1429
+ } else {
1430
+ args._.push(tok);
1431
+ }
1432
+ }
1433
+ for (const [key, [flag, min]] of Object.entries(COUNT_FLAGS)) {
1434
+ if (args[key] === undefined) continue;
1435
+ if (!/^\d+$/.test(args[key]) || Number(args[key]) < min) {
1436
+ throw new CouldNotRunError(`${flag} takes a whole number of at least ${min}, not ${JSON.stringify(args[key])}`);
1437
+ }
1438
+ args[key] = Number(args[key]);
1439
+ }
1440
+ if (args.threshold !== undefined) args.threshold = parseThreshold(args.threshold);
1441
+ if (args.role !== undefined && !CHAT_ROLES.has(args.role)) {
1442
+ throw new CouldNotRunError(`--role is one of ${[...CHAT_ROLES].join(", ")}, not ${JSON.stringify(args.role)}`);
1443
+ }
1444
+ return args;
1445
+ }
1446
+
1447
+ export function buildPrompt(args) {
1448
+ let input = args.prompt ?? args._.slice(1).join(" ");
1449
+ if (args.file) {
1450
+ const text = fs.readFileSync(args.file, "utf8");
1451
+ let span = text;
1452
+ if (args.lines) {
1453
+ const { lines } = splitLines(text);
1454
+ const { start, end } = parseLineRange(args.lines, lines.length);
1455
+ span = lines.slice(start - 1, end).join("\n");
1456
+ }
1457
+ input = `${input}\n\n\`\`\`\n${span}\n\`\`\``;
1458
+ } else if (args.lines) {
1459
+ throw new CouldNotRunError("--lines selects a span of --file, and no --file was given");
1460
+ }
1461
+ return input;
1462
+ }
1463
+
1464
+ async function loadHostRail(importFn = (specifier) => import(specifier)) {
1465
+ try {
1466
+ return (await importFn("../../../../cli/ab-run.mjs")).dissolvesTheTreatment;
1467
+ } catch (err) {
1468
+ throw new CouldNotRunError(`eval --ab checks the host command with cli/ab-run.mjs, which is not beside this pack — ${err.message}`);
1469
+ }
1470
+ }
1471
+
1472
+ async function abMode(args, { fetchImpl }) {
1473
+ const repoRoot = path.resolve(args.repoRoot ?? ".");
1474
+ if (args.ab && args.verifyRecord) throw new CouldNotRunError("--ab and --verify are two modes — pick one");
1475
+ if (args.verifyRecord) {
1476
+ refuseOutside(args, RECORD_TAKES, "eval --verify");
1477
+ if (args.write) throw new CouldNotRunError("--verify writes nothing; --write alone re-renders the register");
1478
+ console.log(`local-llm: ${await verifyAbRecord(repoRoot)}`);
1479
+ return 0;
1480
+ }
1481
+ if (!args.ab) {
1482
+ refuseOutside(args, RECORD_TAKES, "eval --write");
1483
+ console.log(`local-llm: ${await writeAbRegister(repoRoot)}`);
1484
+ return 0;
1485
+ }
1486
+ refuseOutside(args, AB_TAKES, "eval --ab");
1487
+ if (args.hostCmd === undefined) {
1488
+ throw new RailError(
1489
+ `Refused eval --ab: arm H is a PAID run of ${AB_K * AB_TASKS.length} host calls whose prompts leave this machine, which crosses the loopback rail this runner holds. ` +
1490
+ 'Name the host command with --host-cmd "<command>" to declare that crossing; nothing was contacted.',
1491
+ );
1492
+ }
1493
+ if (!args.write) throw new CouldNotRunError(`--ab captures turns that do not repeat and keeps them: pass --write, which writes ${AB_RECORD}`);
1494
+ if (fs.existsSync(path.join(repoRoot, AB_RECORD))) {
1495
+ throw new CouldNotRunError(`${AB_RECORD} exists, and a record is captured once: removing it to capture another is the operator's deliberate act, not this command's`);
1496
+ }
1497
+ const k = args.k ?? AB_K;
1498
+ if (k !== AB_K) throw new CouldNotRunError(`--k ${k}: the maintainer ruled ${AB_K} for evals/ab/baseline.md, and a record at another k is another experiment`);
1499
+ if (args.seed === undefined || args.seed.trim() === "") throw new CouldNotRunError("--ab needs --seed <s>, which every arm-L sampling seed derives from");
1500
+ if (args.hostModel === undefined || args.hostModel.trim() === "") throw new CouldNotRunError("--ab needs --host-model <id>: the record names both arms' models");
1501
+ if ((args.effort ?? "low") !== "low") throw new CouldNotRunError("--ab runs arm L at --effort low, the effort the routing ships with");
1502
+ const hostCommand = parseHostCommand(args.hostCmd);
1503
+ const bypassing = (await loadHostRail())(hostCommand);
1504
+ if (bypassing.length > 0) throw new CouldNotRunError(`the host command carries ${bypassing.join(" and ")}, which lets arm H act without a permission prompt — refused, as cli/ab-run.mjs refuses it`);
1505
+ const baseUrl = assertLoopback(args.baseUrl ?? DEFAULT_BASE_URL);
1506
+ const grading = await regrading();
1507
+ const conditions = abConditions({ repoRoot, hostCommand });
1508
+ console.error(`local-llm: arm H sends ${AB_K * AB_TASKS.length} prompts off this machine through \`${hostCommand.join(" ")}\`, billed to its account, after arm L's ${AB_K * AB_TASKS.length} on ${baseUrl}; turns are journalled in ${conditions.journal}`);
1509
+ const rec = await captureAb({
1510
+ seed: args.seed,
1511
+ k,
1512
+ hostCommand,
1513
+ hostModel: args.hostModel,
1514
+ baseUrl,
1515
+ conditions,
1516
+ screen: grading.screen,
1517
+ journal: conditions.journal,
1518
+ predicates: grading.predicates,
1519
+ chatFn: (options) => callLocalChat({ ...options, fetchImpl }),
1520
+ onTurn: (t, reused) =>
1521
+ console.error(`local-llm: ${t.arm.toUpperCase()} ${t.task} run ${t.run} ${t.completed ? (t.ok ? "PASS" : "FAIL") : "DID-NOT-COMPLETE"} ${t.ms}ms${reused ? " · from the journal, not re-run" : ""}`),
1522
+ });
1523
+ const published = publishAb(repoRoot, rec, grading);
1524
+ fs.rmSync(conditions.journal, { recursive: true, force: true });
1525
+ if (published.ok) console.log(`local-llm: ${published.message}`);
1526
+ else console.error(`local-llm: ${published.message}`);
1527
+ return published.ok ? 0 : 1;
1528
+ }
1529
+
1530
+ export async function main(argv = process.argv.slice(2), { fetchImpl = globalThis.fetch } = {}) {
1531
+ try {
1532
+ const args = parseArgs(argv);
1533
+ if (args.help) {
1534
+ console.log(USAGE);
1535
+ return 0;
1536
+ }
1537
+ if (args.verifyOffline) {
1538
+ const res = await runOfflineVerify();
1539
+ if (res.status === 0) console.log(res.message);
1540
+ else console.error(res.message);
1541
+ return res.status;
1542
+ }
1543
+
1544
+ const cmd = args._[0] ?? "status";
1545
+ if (cmd !== "eval") {
1546
+ for (const key of EVAL_ONLY) {
1547
+ if (args[key] !== undefined) throw new CouldNotRunError(`${FLAG_OF[key]} belongs to eval, not ${cmd}`);
1548
+ }
1549
+ }
1550
+ if (cmd === "eval" && args._.length > 1) throw new CouldNotRunError(`eval takes no positional argument, not ${JSON.stringify(args._[1])}`);
1551
+ const baseUrl = args.baseUrl ?? DEFAULT_BASE_URL;
1552
+ const model = args.model ?? ROLE_MODELS[args.role ?? "code"];
1553
+
1554
+ if (cmd === "status") {
1555
+ const endpoint = assertLoopback(baseUrl, { allowRemote: args.allowRemote });
1556
+ const res = await localFetch(`${endpoint}/models`, { method: "GET" }, fetchImpl);
1557
+ if (!res.ok) throw new CouldNotRunError(`HTTP ${res.status} from ${endpoint}/models`);
1558
+ const data = await res.json();
1559
+ const ids = (data.data ?? []).map((m) => m.id);
1560
+ console.log(`OK endpoint=${endpoint} models=${ids.length} [${ids.join(", ")}]`);
1561
+ return 0;
1562
+ }
1563
+
1564
+ if (cmd === "chat" || cmd === "summarize") {
1565
+ const chosen = operationalModel(args, cmd === "summarize" ? "prose" : "code");
1566
+ const reply = await callLocalChat({
1567
+ baseUrl,
1568
+ allowRemote: args.allowRemote,
1569
+ model: chosen,
1570
+ effort: args.effort ?? "low",
1571
+ maxInputBytes: args.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES,
1572
+ messages: [{ role: "user", content: buildPrompt(args) }],
1573
+ fetchImpl,
1574
+ });
1575
+ console.log(reply.content);
1576
+ console.error(formatPassReceipt({ mode: cmd, model: chosen, ...reply }));
1577
+ return 0;
1578
+ }
1579
+
1580
+ if (cmd === "transform") {
1581
+ if (!args.file || !args.prompt || !args.verify) {
1582
+ throw new CouldNotRunError(
1583
+ "Usage: local-llm.mjs transform --file <path> --verify <cmd> --prompt <instruction> [--lines <start>-<end>]",
1584
+ );
1585
+ }
1586
+ const res = await verifyGatedTransform({
1587
+ filePath: args.file,
1588
+ lineRange: args.lines ?? null,
1589
+ instruction: args.prompt,
1590
+ verifyCmd: args.verify,
1591
+ model: operationalModel(args, "code"),
1592
+ baseUrl,
1593
+ allowRemote: args.allowRemote,
1594
+ maxInputBytes: args.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES,
1595
+ effort: args.effort ?? "low",
1596
+ retries: args.retries ?? DEFAULT_RETRIES,
1597
+ chatFn: (options) => callLocalChat({ ...options, fetchImpl }),
1598
+ });
1599
+ if (res.ok) {
1600
+ console.log(res.receipt);
1601
+ return 0;
1602
+ }
1603
+ console.error(res.error);
1604
+ return 1;
1605
+ }
1606
+
1607
+ if (cmd === "eval" && (args.ab || args.verifyRecord || args.write)) return await abMode(args, { fetchImpl });
1608
+ if (cmd === "dedup") {
1609
+ if (!args.dir) {
1610
+ throw new CouldNotRunError(
1611
+ "Usage: local-llm.mjs dedup --dir <slot-dir> [--threshold 0.80] [--model <embedding model>] [--max-input-bytes <per file>]",
1612
+ );
1613
+ }
1614
+ if (args.role) throw new CouldNotRunError(`--role picks a chat model; dedup embeds with ${ROLE_MODELS.embed} unless --model names another`);
1615
+ const chosen = args.model ?? ROLE_MODELS.embed;
1616
+ const threshold = args.threshold ?? DEFAULT_DEDUP_THRESHOLD;
1617
+ const res = await runDedup({
1618
+ dir: args.dir,
1619
+ threshold,
1620
+ model: chosen,
1621
+ baseUrl,
1622
+ allowRemote: args.allowRemote,
1623
+ maxInputBytes: args.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES,
1624
+ fetchImpl,
1625
+ });
1626
+ for (const p of res.pairs) console.log(`${p.score.toFixed(3)} ${p.a} ${p.b}`);
1627
+ console.error(`dedup files=${res.files} pairs=${res.pairs.length} threshold=${threshold}`);
1628
+ console.error(formatDedupReceipt({ model: chosen, promptTokens: res.promptTokens, totalMs: res.totalMs }));
1629
+ return 0;
1630
+ }
1631
+
1632
+ if (cmd === "eval") {
1633
+ for (const key of AB_ONLY) {
1634
+ if (args[key] !== undefined) throw new CouldNotRunError(`${FLAG_OF[key]} belongs to eval --ab, --verify or --write; the screen takes one sample and runs no host`);
1635
+ }
1636
+ const results = await runRoleEval({
1637
+ model,
1638
+ tasks: args.task ? [args.task] : ["code", "codify", "handoff"],
1639
+ baseUrl,
1640
+ allowRemote: args.allowRemote,
1641
+ effort: args.effort ?? "low",
1642
+ chatFn: (options) => callLocalChat({ ...options, fetchImpl }),
1643
+ });
1644
+ for (const r of results) {
1645
+ console.log(`${r.ok ? "PASS" : "FAIL"} task=${r.task} model=${r.model} time=${r.totalMs}ms rate=${figure(r.tokPerSec, "tok/s")} ${r.detail}`);
1646
+ }
1647
+ return results.every((r) => r.ok) ? 0 : 1;
1648
+ }
1649
+
1650
+ throw new CouldNotRunError(`Unknown command: ${cmd}`);
1651
+ } catch (err) {
1652
+ console.error(err.message);
1653
+ return err instanceof RailError ? 1 : 2;
1654
+ }
1655
+ }
1656
+
1657
+ if (process.argv[1] && import.meta.url === pathToFileURL(fs.realpathSync(process.argv[1])).href) {
1658
+ main().then(
1659
+ (code) => process.exit(code),
1660
+ (err) => {
1661
+ console.error(err?.message ?? err);
1662
+ process.exit(2);
1663
+ },
1664
+ );
1665
+ }