@seanhogg/builderforce-memory 2026.6.35 → 2026.6.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/limbic/LimbicSession.d.ts +31 -1
- package/dist/limbic/LimbicSession.d.ts.map +1 -1
- package/dist/limbic/LimbicSession.js +47 -3
- package/dist/limbic/LimbicSession.js.map +1 -1
- package/dist/workflow/code-eval.d.ts +54 -0
- package/dist/workflow/code-eval.d.ts.map +1 -0
- package/dist/workflow/code-eval.js +135 -0
- package/dist/workflow/code-eval.js.map +1 -0
- package/dist/workflow/index.d.ts +3 -2
- package/dist/workflow/index.d.ts.map +1 -1
- package/dist/workflow/index.js +3 -2
- package/dist/workflow/index.js.map +1 -1
- package/dist/workflow/steps.d.ts +20 -0
- package/dist/workflow/steps.d.ts.map +1 -1
- package/dist/workflow/steps.js +321 -7
- package/dist/workflow/steps.js.map +1 -1
- package/dist/workflow/templates.d.ts +15 -0
- package/dist/workflow/templates.d.ts.map +1 -1
- package/dist/workflow/templates.js +56 -1
- package/dist/workflow/templates.js.map +1 -1
- package/package.json +7 -4
- package/src/index.ts +6 -0
- package/src/limbic/LimbicSession.ts +61 -1
- package/src/workflow/code-eval.ts +143 -0
- package/src/workflow/index.ts +9 -1
- package/src/workflow/steps.ts +367 -6
- package/src/workflow/templates.ts +59 -1
package/src/workflow/steps.ts
CHANGED
|
@@ -16,13 +16,19 @@ import {
|
|
|
16
16
|
BPETokenizer,
|
|
17
17
|
EvermindModelPackage,
|
|
18
18
|
exportEvermind,
|
|
19
|
+
importEvermind,
|
|
19
20
|
benchmarkText,
|
|
20
21
|
type ExportFormat,
|
|
21
22
|
type BenchmarkReport,
|
|
23
|
+
type ImportOptions,
|
|
22
24
|
} from "@seanhogg/builderforce-memory-engine";
|
|
23
25
|
import { EvermindCognition } from "../cognition/index.js";
|
|
24
26
|
import { hybridRetrieve, chunkText } from "../retrieval/index.js";
|
|
25
27
|
import { MemoryStore } from "../memory/MemoryStore.js";
|
|
28
|
+
import { OpenAIBridge } from "../bridges/OpenAIBridge.js";
|
|
29
|
+
import { AnthropicBridge } from "../bridges/AnthropicBridge.js";
|
|
30
|
+
import type { TransformerBridge } from "../bridges/TransformerBridge.js";
|
|
31
|
+
import { analyzeCode, runJsCases, type CodeCase } from "./code-eval.js";
|
|
26
32
|
import type { StackStep } from "../diagnostics/stack-diagnostic.js";
|
|
27
33
|
import type { StepFactory, StepTypeInfo, WorkflowStepConfig } from "./types.js";
|
|
28
34
|
|
|
@@ -47,20 +53,185 @@ function pBool(cfg: WorkflowStepConfig, key: string, def: boolean): boolean {
|
|
|
47
53
|
const v = cfg.params?.[key];
|
|
48
54
|
return typeof v === "boolean" ? v : def;
|
|
49
55
|
}
|
|
56
|
+
/** Read a string[] param (non-strings dropped). */
|
|
57
|
+
function pStrArr(cfg: WorkflowStepConfig, key: string): string[] {
|
|
58
|
+
const v = cfg.params?.[key];
|
|
59
|
+
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Default teacher instruction for code distillation — code only, no prose/fences. */
|
|
63
|
+
const CODE_TEACHER_SYSTEM =
|
|
64
|
+
"You are an expert programmer. Respond with correct, idiomatic code that fulfils the request. " +
|
|
65
|
+
"Output code only — no explanation, no markdown fences.";
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* A (prompt → teacher completion) training exemplar produced by distillation.
|
|
69
|
+
* `cases` (when present) execution-verify the completion; `teacher` records
|
|
70
|
+
* which panel member produced the surviving answer (attribution/telemetry).
|
|
71
|
+
*/
|
|
72
|
+
interface DistillPair {
|
|
73
|
+
prompt: string;
|
|
74
|
+
completion: string;
|
|
75
|
+
cases?: CodeCase[];
|
|
76
|
+
teacher?: string;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** One coding task to distil: a prompt plus the tests its answer must pass. */
|
|
80
|
+
interface DistillTask {
|
|
81
|
+
prompt: string;
|
|
82
|
+
cases?: CodeCase[];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** A single teacher's answer to a prompt, before selection. */
|
|
86
|
+
interface Candidate {
|
|
87
|
+
teacher: string;
|
|
88
|
+
completion: string;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A configured teacher in the panel (its display label + bridge). */
|
|
92
|
+
interface Teacher {
|
|
93
|
+
label: string;
|
|
94
|
+
bridge: TransformerBridge;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** One teacher's config — a panel entry, or the legacy single-teacher shape. */
|
|
98
|
+
interface TeacherSpec {
|
|
99
|
+
provider?: string;
|
|
100
|
+
apiKey?: string;
|
|
101
|
+
model?: string;
|
|
102
|
+
baseUrl?: string;
|
|
103
|
+
systemPrompt?: string;
|
|
104
|
+
maxTokens?: number;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Strip a wrapping ```lang … ``` markdown fence some teachers add despite the
|
|
109
|
+
* "code only, no fences" instruction — otherwise the fence fails every code
|
|
110
|
+
* gate and the exemplar is wrongly dropped.
|
|
111
|
+
*/
|
|
112
|
+
export function stripCodeFences(text: string): string {
|
|
113
|
+
const t = text.trim();
|
|
114
|
+
const m = /^```[\w-]*\r?\n([\s\S]*?)\r?\n```$/.exec(t);
|
|
115
|
+
return m ? m[1]!.trim() : t;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Read `cases` off a param blob, dropping malformed entries. */
|
|
119
|
+
function normalizeCases(raw: unknown): CodeCase[] {
|
|
120
|
+
return Array.isArray(raw)
|
|
121
|
+
? raw.filter((c): c is CodeCase => typeof (c as { call?: unknown })?.call === "string")
|
|
122
|
+
: [];
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Best-of-N selection across a teacher panel. When the task carries execution
|
|
127
|
+
* `cases`, keep only fully-passing candidates and return the first — teachers
|
|
128
|
+
* are listed strongest-first (e.g. Opus), so the strongest correct answer wins
|
|
129
|
+
* and a wrong teacher answer is dropped rather than trained on. With no cases,
|
|
130
|
+
* take the first non-empty candidate (teacher-order priority).
|
|
131
|
+
*/
|
|
132
|
+
export function selectBestCandidate(candidates: Candidate[], cases: CodeCase[]): Candidate | null {
|
|
133
|
+
if (candidates.length === 0) return null;
|
|
134
|
+
if (cases.length === 0) return candidates[0]!;
|
|
135
|
+
for (const c of candidates) {
|
|
136
|
+
if (runJsCases(c.completion, cases).passRate === 1) return c;
|
|
137
|
+
}
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Build one teacher bridge from its spec. Any external LLM can teach Evermind:
|
|
143
|
+
* • provider "openai" (default) — the BuilderForce gateway, OpenRouter, OpenAI,
|
|
144
|
+
* or any OpenAI-compatible server (Ollama / vLLM / LM Studio), by `baseUrl`.
|
|
145
|
+
* • provider "anthropic" — the Anthropic Messages API directly (Opus etc).
|
|
146
|
+
* Returns null when the entry lacks the credentials it needs.
|
|
147
|
+
*/
|
|
148
|
+
function buildTeacher(spec: TeacherSpec, defSystem: string, defMaxTokens: number): Teacher | null {
|
|
149
|
+
const provider = (spec.provider ?? "openai").toLowerCase();
|
|
150
|
+
const apiKey = spec.apiKey ?? "";
|
|
151
|
+
const model = spec.model ?? "";
|
|
152
|
+
const systemPrompt = spec.systemPrompt ?? defSystem;
|
|
153
|
+
const maxTokens = spec.maxTokens ?? defMaxTokens;
|
|
154
|
+
if (provider === "anthropic") {
|
|
155
|
+
if (!apiKey) return null;
|
|
156
|
+
return { label: `anthropic:${model || "default"}`, bridge: new AnthropicBridge({ apiKey, systemPrompt, maxTokens, ...(model ? { model } : {}) }) };
|
|
157
|
+
}
|
|
158
|
+
// openai-compatible (gateway / OpenRouter / OpenAI / local) — needs a baseUrl.
|
|
159
|
+
const baseUrl = spec.baseUrl ?? "";
|
|
160
|
+
if (!baseUrl) return null;
|
|
161
|
+
return { label: `${provider}:${model || "default"}`, bridge: new OpenAIBridge({ apiKey, baseUrl, systemPrompt, maxTokens, ...(model ? { model } : {}) }) };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Build the teacher panel from step params. `teachers: TeacherSpec[]` runs
|
|
166
|
+
* several models as a best-of-N panel (list the strongest FIRST — e.g. Opus —
|
|
167
|
+
* so it wins ties); the legacy single-teacher shape (top-level
|
|
168
|
+
* provider/baseUrl/apiKey/model) is honoured as a one-entry panel. Entries
|
|
169
|
+
* without credentials are skipped, so an offline `pairs`-only run yields [].
|
|
170
|
+
*/
|
|
171
|
+
function makeTeachers(cfg: WorkflowStepConfig): Teacher[] {
|
|
172
|
+
const defSystem = pStr(cfg, "systemPrompt", CODE_TEACHER_SYSTEM);
|
|
173
|
+
const defMaxTokens = pNum(cfg, "maxTokens", 512);
|
|
174
|
+
const raw = cfg.params?.teachers;
|
|
175
|
+
const specs: TeacherSpec[] =
|
|
176
|
+
Array.isArray(raw) && raw.length > 0
|
|
177
|
+
? (raw as TeacherSpec[])
|
|
178
|
+
: [{ provider: pStr(cfg, "provider", "openai"), apiKey: pStr(cfg, "apiKey", ""), model: pStr(cfg, "model", ""), baseUrl: pStr(cfg, "baseUrl", "") }];
|
|
179
|
+
const out: Teacher[] = [];
|
|
180
|
+
for (const s of specs) {
|
|
181
|
+
const t = buildTeacher(s ?? {}, defSystem, defMaxTokens);
|
|
182
|
+
if (t) out.push(t);
|
|
183
|
+
}
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Read the coding tasks to distil: rich `tasks[{prompt,cases}]` or plain `prompts[]`. */
|
|
188
|
+
function readTasks(cfg: WorkflowStepConfig): DistillTask[] {
|
|
189
|
+
const raw = cfg.params?.tasks;
|
|
190
|
+
if (Array.isArray(raw)) {
|
|
191
|
+
const out: DistillTask[] = [];
|
|
192
|
+
for (const t of raw) {
|
|
193
|
+
const prompt = (t as DistillTask)?.prompt;
|
|
194
|
+
if (typeof prompt !== "string") continue;
|
|
195
|
+
const cases = normalizeCases((t as DistillTask).cases);
|
|
196
|
+
out.push(cases.length ? { prompt, cases } : { prompt });
|
|
197
|
+
}
|
|
198
|
+
return out;
|
|
199
|
+
}
|
|
200
|
+
return pStrArr(cfg, "prompts").map((prompt) => ({ prompt }));
|
|
201
|
+
}
|
|
50
202
|
|
|
51
203
|
/** Byte length of an emitted export file (text or binary). */
|
|
52
204
|
function fileBytes(data: Uint8Array | string): number {
|
|
53
205
|
return typeof data === "string" ? new TextEncoder().encode(data).length : data.length;
|
|
54
206
|
}
|
|
55
207
|
|
|
208
|
+
/** Coerce a workflow-param weight blob (bytes, ArrayBuffer, number[], or base64) to Uint8Array. */
|
|
209
|
+
function toBytes(value: unknown, base64?: string): Uint8Array | null {
|
|
210
|
+
if (value instanceof Uint8Array) return value;
|
|
211
|
+
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
212
|
+
if (Array.isArray(value) && value.every((n) => typeof n === "number")) return Uint8Array.from(value as number[]);
|
|
213
|
+
if (typeof base64 === "string" && base64) {
|
|
214
|
+
if (typeof atob !== "function") throw new Error("import-model: base64 decoding needs atob (browser/Worker/Node 16+)");
|
|
215
|
+
const bin = atob(base64);
|
|
216
|
+
const out = new Uint8Array(bin.length);
|
|
217
|
+
for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
|
|
218
|
+
return out;
|
|
219
|
+
}
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
|
|
56
223
|
function mkStep(cfg: WorkflowStepConfig, layer: string, defaultLabel: string, run: StackStep["run"]): StackStep {
|
|
57
224
|
return { id: cfg.id, layer, label: cfg.label ?? defaultLabel, run };
|
|
58
225
|
}
|
|
59
226
|
|
|
60
|
-
/**
|
|
227
|
+
/**
|
|
228
|
+
* Encode a corpus into next-token training sequences. Split on sentence
|
|
229
|
+
* boundaries (prose) OR blank lines (code blocks / distilled prompt→code pairs),
|
|
230
|
+
* so both natural-language and code corpora yield multiple trainable sequences.
|
|
231
|
+
*/
|
|
61
232
|
function corpusToSequences(corpus: string, tok: BPETokenizer): number[][] {
|
|
62
233
|
return corpus
|
|
63
|
-
.split(/(?<=\.)\s
|
|
234
|
+
.split(/(?<=\.)\s+|\n\s*\n/)
|
|
64
235
|
.map((s) => tok.encode(s.trim()))
|
|
65
236
|
.filter((ids) => ids.length >= 2);
|
|
66
237
|
}
|
|
@@ -175,16 +346,38 @@ export const BUILTIN_STEPS: Record<string, Registered> = {
|
|
|
175
346
|
|
|
176
347
|
// ── LLM-creation pipeline (BUILD) — produces a custom .evermind model ────────
|
|
177
348
|
"train-tokenizer": {
|
|
178
|
-
info: {
|
|
349
|
+
info: {
|
|
350
|
+
type: "train-tokenizer",
|
|
351
|
+
layer: "BUILD",
|
|
352
|
+
label: "Train tokenizer",
|
|
353
|
+
description:
|
|
354
|
+
"Build a BPE vocab — train fresh from your corpus, OR import merges from a proven code tokenizer (params: corpus, numMerges, hfTokenizer, hfTokenizerUrl)",
|
|
355
|
+
},
|
|
179
356
|
factory: (cfg) =>
|
|
180
357
|
mkStep(cfg, "BUILD", "Train tokenizer (BPE)", async (ctx) => {
|
|
181
|
-
const corpus = pStr(cfg, "corpus", KNOWLEDGE.repeat(4));
|
|
182
|
-
const numMerges = pNum(cfg, "numMerges", 100);
|
|
183
358
|
const tok = new BPETokenizer();
|
|
359
|
+
// Two paths, both first-class (this is an IDE choice):
|
|
360
|
+
// • import-merges: seed from a Hugging Face tokenizer.json (code vocab on day one)
|
|
361
|
+
// • train-fresh: learn a vocab from the corpus
|
|
362
|
+
const hfTokenizer = cfg.params?.hfTokenizer as Record<string, unknown> | undefined;
|
|
363
|
+
const hfTokenizerUrl = pStr(cfg, "hfTokenizerUrl", "");
|
|
364
|
+
if (hfTokenizer && typeof hfTokenizer === "object") {
|
|
365
|
+
tok.loadHuggingFace(hfTokenizer);
|
|
366
|
+
ctx.bag.tokenizer = tok;
|
|
367
|
+
return `imported vocab ${tok.vocabSize} (Hugging Face merges)`;
|
|
368
|
+
}
|
|
369
|
+
if (hfTokenizerUrl) {
|
|
370
|
+
await tok.loadHuggingFaceUrl(hfTokenizerUrl);
|
|
371
|
+
ctx.bag.tokenizer = tok;
|
|
372
|
+
return `imported vocab ${tok.vocabSize} from ${hfTokenizerUrl}`;
|
|
373
|
+
}
|
|
374
|
+
// train-fresh — default to the distilled corpus from a prior step when present.
|
|
375
|
+
const corpus = pStr(cfg, "corpus", (ctx.bag.corpus as string) ?? KNOWLEDGE.repeat(4));
|
|
376
|
+
const numMerges = pNum(cfg, "numMerges", 100);
|
|
184
377
|
tok.train(corpus, { numMerges });
|
|
185
378
|
ctx.bag.tokenizer = tok;
|
|
186
379
|
ctx.bag.corpus = corpus;
|
|
187
|
-
return `vocab ${tok.vocabSize} (${numMerges} merges)`;
|
|
380
|
+
return `trained vocab ${tok.vocabSize} (${numMerges} merges)`;
|
|
188
381
|
}),
|
|
189
382
|
},
|
|
190
383
|
"train-model": {
|
|
@@ -415,4 +608,172 @@ export const BUILTIN_STEPS: Record<string, Registered> = {
|
|
|
415
608
|
return `exported ${format}: ${result.files.length} file(s), ${total} bytes, ${result.paramCount} params`;
|
|
416
609
|
}),
|
|
417
610
|
},
|
|
611
|
+
|
|
612
|
+
"import-model": {
|
|
613
|
+
info: {
|
|
614
|
+
type: "import-model",
|
|
615
|
+
layer: "BUILD",
|
|
616
|
+
label: "Import model (warm-start)",
|
|
617
|
+
description:
|
|
618
|
+
"Warm-start an EvermindLM from a .safetensors checkpoint instead of random init — round-trips our own exports, or maps a foreign SSM checkpoint via renameMap (params: safetensors, safetensorsBase64, topK, renameMap)",
|
|
619
|
+
},
|
|
620
|
+
factory: (cfg) =>
|
|
621
|
+
mkStep(cfg, "BUILD", "Import model (warm-start)", async (ctx) => {
|
|
622
|
+
const bytes = toBytes(cfg.params?.safetensors, pStr(cfg, "safetensorsBase64", ""));
|
|
623
|
+
if (!bytes) {
|
|
624
|
+
throw new Error("import-model: provide 'safetensors' (bytes/ArrayBuffer/number[]) or 'safetensorsBase64'");
|
|
625
|
+
}
|
|
626
|
+
const opts: ImportOptions = {};
|
|
627
|
+
const topK = pNum(cfg, "topK", 0);
|
|
628
|
+
if (topK > 0) opts.topK = topK;
|
|
629
|
+
// renameMap is the JSON-friendly form of ImportOptions.rename — used to
|
|
630
|
+
// translate a foreign SSM checkpoint's tensor names to our canonical names.
|
|
631
|
+
const renameMap = cfg.params?.renameMap as Record<string, string> | undefined;
|
|
632
|
+
if (renameMap && typeof renameMap === "object") {
|
|
633
|
+
opts.rename = (name) => (name in renameMap ? renameMap[name]! : name);
|
|
634
|
+
}
|
|
635
|
+
const model = importEvermind(bytes, opts);
|
|
636
|
+
ctx.bag.model = model;
|
|
637
|
+
const params = model.parameters().reduce((n, p) => n + p.data.length, 0);
|
|
638
|
+
const c = model.config;
|
|
639
|
+
return `imported ${params} params (dModel ${c.dModel}, ${c.numLayers}L, ${c.numExperts}E) from ${bytes.length} bytes`;
|
|
640
|
+
}),
|
|
641
|
+
},
|
|
642
|
+
|
|
643
|
+
// ── Teaching Evermind to code — multi-teacher distillation + exec gates ───────
|
|
644
|
+
"distill-corpus": {
|
|
645
|
+
info: {
|
|
646
|
+
type: "distill-corpus",
|
|
647
|
+
layer: "BUILD",
|
|
648
|
+
label: "Distill from teacher panel",
|
|
649
|
+
description:
|
|
650
|
+
"Build a training corpus from a panel of teacher LLMs (Opus + any other model — the gateway, OpenRouter, OpenAI, a local server, Anthropic). Per task, queries every teacher, keeps only completions that PASS the task's execution cases (best-of-N, strongest teacher first), tags each with its teacher, and ACCUMULATES so stacked steps add up. Feeds the corpus to train-model (params: teachers[{provider,baseUrl,apiKey,model}], tasks[{prompt,cases}], prompts, provider, baseUrl, apiKey, model, systemPrompt, maxTokens, temperature, pairs)",
|
|
651
|
+
},
|
|
652
|
+
factory: (cfg) =>
|
|
653
|
+
mkStep(cfg, "BUILD", "Distill corpus from teacher panel", async (ctx) => {
|
|
654
|
+
// Accumulate across steps so multiple teacher panels stack instead of
|
|
655
|
+
// overwriting a prior distill step's exemplars.
|
|
656
|
+
const pairs: DistillPair[] = Array.isArray(ctx.bag.distillPairs) ? (ctx.bag.distillPairs as DistillPair[]) : [];
|
|
657
|
+
const startCount = pairs.length;
|
|
658
|
+
let offlineKept = 0;
|
|
659
|
+
let verified = 0;
|
|
660
|
+
let dropped = 0;
|
|
661
|
+
|
|
662
|
+
// Offline exemplars (pre-captured) — a single trusted candidate each,
|
|
663
|
+
// execution-verified when the pair carries `cases`.
|
|
664
|
+
const offline = cfg.params?.pairs;
|
|
665
|
+
if (Array.isArray(offline)) {
|
|
666
|
+
for (const p of offline) {
|
|
667
|
+
const prompt = (p as DistillPair)?.prompt;
|
|
668
|
+
const completion = (p as DistillPair)?.completion;
|
|
669
|
+
if (typeof prompt !== "string" || typeof completion !== "string") continue;
|
|
670
|
+
const cases = normalizeCases((p as DistillPair).cases);
|
|
671
|
+
const chosen = selectBestCandidate([{ teacher: "offline", completion: stripCodeFences(completion) }], cases);
|
|
672
|
+
if (!chosen) { dropped++; continue; }
|
|
673
|
+
const pair: DistillPair = { prompt, completion: chosen.completion, teacher: "offline" };
|
|
674
|
+
if (cases.length) pair.cases = cases;
|
|
675
|
+
pairs.push(pair);
|
|
676
|
+
offlineKept++;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// Live teacher panel — Opus and/or any other models, best-of-N + exec-filtered.
|
|
681
|
+
const teachers = makeTeachers(cfg);
|
|
682
|
+
const tasks = readTasks(cfg);
|
|
683
|
+
if (teachers.length > 0 && tasks.length > 0) {
|
|
684
|
+
const temperature = pNum(cfg, "temperature", 0.2);
|
|
685
|
+
// Dedupe identical (teacher, prompt) calls within this run so a repeated
|
|
686
|
+
// prompt is not billed twice.
|
|
687
|
+
const memo = new Map<string, string>();
|
|
688
|
+
for (const task of tasks) {
|
|
689
|
+
const candidates: Candidate[] = [];
|
|
690
|
+
for (const t of teachers) {
|
|
691
|
+
const key = `${t.label}${task.prompt}`;
|
|
692
|
+
let completion = memo.get(key);
|
|
693
|
+
if (completion === undefined) {
|
|
694
|
+
completion = stripCodeFences(await t.bridge.generate(task.prompt, { temperature }));
|
|
695
|
+
memo.set(key, completion);
|
|
696
|
+
}
|
|
697
|
+
if (completion.trim()) candidates.push({ teacher: t.label, completion });
|
|
698
|
+
}
|
|
699
|
+
const chosen = selectBestCandidate(candidates, task.cases ?? []);
|
|
700
|
+
if (!chosen) { dropped++; continue; }
|
|
701
|
+
const pair: DistillPair = { prompt: task.prompt, completion: chosen.completion, teacher: chosen.teacher };
|
|
702
|
+
if (task.cases?.length) pair.cases = task.cases;
|
|
703
|
+
pairs.push(pair);
|
|
704
|
+
verified++;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
if (pairs.length === 0) {
|
|
709
|
+
throw new Error(
|
|
710
|
+
"distill-corpus produced no exemplars — set teachers (provider + apiKey + prompts/tasks; baseUrl for openai-compatible) or supply offline 'pairs'",
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
const corpus = pairs.map((p) => `${p.prompt}\n${p.completion}`).join("\n\n");
|
|
715
|
+
ctx.bag.corpus = corpus;
|
|
716
|
+
ctx.bag.distillPairs = pairs;
|
|
717
|
+
|
|
718
|
+
const added = pairs.length - startCount;
|
|
719
|
+
const parts: string[] = [];
|
|
720
|
+
if (offlineKept) parts.push(`${offlineKept} offline`);
|
|
721
|
+
if (verified) parts.push(`${verified} verified via ${teachers.length} teacher(s): ${teachers.map((t) => t.label).join(", ")}`);
|
|
722
|
+
if (dropped) parts.push(`${dropped} dropped`);
|
|
723
|
+
return `distilled ${added} exemplar(s) (${pairs.length} total, ${corpus.length} chars)${parts.length ? ` — ${parts.join(", ")}` : ""}`;
|
|
724
|
+
}),
|
|
725
|
+
},
|
|
726
|
+
"code-parse-check": {
|
|
727
|
+
info: {
|
|
728
|
+
type: "code-parse-check",
|
|
729
|
+
layer: "BUILD",
|
|
730
|
+
label: "Code parse check",
|
|
731
|
+
description:
|
|
732
|
+
"Validate generated output is structurally valid code — balanced delimiters/strings, optional JS parse (params: code, language, minScore)",
|
|
733
|
+
},
|
|
734
|
+
factory: (cfg) =>
|
|
735
|
+
mkStep(cfg, "BUILD", "Code structure / parse gate", async (ctx) => {
|
|
736
|
+
const code = pStr(cfg, "code", (ctx.bag.sample as string) ?? "");
|
|
737
|
+
if (!code.trim()) throw new Error("no code to check — run 'evaluate'/'generate-check' first or pass a 'code' param");
|
|
738
|
+
const language = pStr(cfg, "language", "js");
|
|
739
|
+
const analysis = analyzeCode(code, language);
|
|
740
|
+
ctx.bag.codeParse = analysis;
|
|
741
|
+
const minScore = pNum(cfg, "minScore", 0); // only enforced when configured
|
|
742
|
+
if (minScore > 0 && analysis.score < minScore) {
|
|
743
|
+
throw new Error(
|
|
744
|
+
`code structure score ${analysis.score.toFixed(2)} < min ${minScore} (unbalanced delimiters/strings — output is not valid ${language})`,
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
return `structure ${(analysis.score * 100) | 0}%${language === "js" ? `, js-parse ${analysis.jsParse ? "ok" : "fail"}` : ""}`;
|
|
748
|
+
}),
|
|
749
|
+
},
|
|
750
|
+
"code-eval": {
|
|
751
|
+
info: {
|
|
752
|
+
type: "code-eval",
|
|
753
|
+
layer: "BUILD",
|
|
754
|
+
label: "Code eval (reward)",
|
|
755
|
+
description:
|
|
756
|
+
"Execution-grounded reward: run the generated JS against test cases and score the pass-rate (params: code, cases, minPassRate)",
|
|
757
|
+
},
|
|
758
|
+
factory: (cfg) =>
|
|
759
|
+
mkStep(cfg, "BUILD", "Code execution reward", async (ctx) => {
|
|
760
|
+
const code = pStr(cfg, "code", (ctx.bag.sample as string) ?? "");
|
|
761
|
+
if (!code.trim()) throw new Error("no code to evaluate — run 'evaluate'/'generate-check' first or pass a 'code' param");
|
|
762
|
+
const rawCases = cfg.params?.cases;
|
|
763
|
+
const cases = Array.isArray(rawCases)
|
|
764
|
+
? rawCases.filter((c): c is { call: string; expect: unknown } => typeof (c as { call?: unknown })?.call === "string")
|
|
765
|
+
: [];
|
|
766
|
+
const result = runJsCases(code, cases);
|
|
767
|
+
ctx.bag.codeEval = result;
|
|
768
|
+
const minPassRate = pNum(cfg, "minPassRate", 0); // only enforced when configured
|
|
769
|
+
if (minPassRate > 0 && result.passRate < minPassRate) {
|
|
770
|
+
throw new Error(
|
|
771
|
+
`code-eval pass-rate ${(result.passRate * 100) | 0}% < min ${(minPassRate * 100) | 0}%${result.error ? ` (${result.error})` : ""}`,
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
return result.total > 0
|
|
775
|
+
? `${result.passed}/${result.total} cases passed (${(result.passRate * 100) | 0}%)`
|
|
776
|
+
: "no cases — execution reward skipped";
|
|
777
|
+
}),
|
|
778
|
+
},
|
|
418
779
|
};
|
|
@@ -69,7 +69,65 @@ export const TRAIN_LLM: WorkflowConfig = {
|
|
|
69
69
|
],
|
|
70
70
|
};
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
/**
|
|
73
|
+
* Out-of-the-box (prompt → code) exemplars so "Teach Evermind to Code" runs with
|
|
74
|
+
* no network. Each carries execution `cases` so the same best-of-N / exec-filter
|
|
75
|
+
* path a live run uses is exercised offline (an exemplar whose code fails its
|
|
76
|
+
* cases is dropped, never trained on).
|
|
77
|
+
*
|
|
78
|
+
* A real run instead sets a teacher PANEL on the `distill-corpus` step —
|
|
79
|
+
* `teachers: [{provider:"anthropic", model:"claude-opus-4-8", apiKey},
|
|
80
|
+
* {provider:"openai", baseUrl:<gateway>, model:<coder>, apiKey}]`
|
|
81
|
+
* (strongest first) — plus `tasks: [{prompt, cases}]`. Per task every teacher is
|
|
82
|
+
* queried, only completions that PASS the cases survive, and the strongest
|
|
83
|
+
* correct answer is kept: Opus is the authority, cheaper models contribute only
|
|
84
|
+
* where they are verifiably right.
|
|
85
|
+
*/
|
|
86
|
+
const CODE_DISTILL_PAIRS = [
|
|
87
|
+
{ prompt: "Write a JS function add(a, b) that returns their sum.", completion: "function add(a, b) {\n return a + b;\n}", cases: [{ call: "add(2, 3)", expect: 5 }] },
|
|
88
|
+
{ prompt: "Write a JS function isEven(n) that returns true when n is even.", completion: "function isEven(n) {\n return n % 2 === 0;\n}", cases: [{ call: "isEven(4)", expect: true }, { call: "isEven(7)", expect: false }] },
|
|
89
|
+
{ prompt: "Write a JS function max(a, b) returning the larger value.", completion: "function max(a, b) {\n return a > b ? a : b;\n}", cases: [{ call: "max(2, 9)", expect: 9 }] },
|
|
90
|
+
{ prompt: "Write a JS function reverse(s) that reverses a string.", completion: "function reverse(s) {\n return s.split('').reverse().join('');\n}", cases: [{ call: "reverse('abc')", expect: "cba" }] },
|
|
91
|
+
{ prompt: "Write a JS function clamp(x, lo, hi) bounding x to [lo, hi].", completion: "function clamp(x, lo, hi) {\n return Math.min(hi, Math.max(lo, x));\n}", cases: [{ call: "clamp(12, 0, 10)", expect: 10 }] },
|
|
92
|
+
{ prompt: "Write a JS function sum(xs) that adds an array of numbers.", completion: "function sum(xs) {\n return xs.reduce((a, b) => a + b, 0);\n}", cases: [{ call: "sum([1, 2, 3])", expect: 6 }] },
|
|
93
|
+
];
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Teach Evermind to Code — the foundation pipeline for the coding skill.
|
|
97
|
+
*
|
|
98
|
+
* Distil a PANEL of teacher coders — Opus as the authority plus any cheaper
|
|
99
|
+
* models (gateway/OpenRouter/local) — into an execution-verified (prompt → code)
|
|
100
|
+
* corpus (best-of-N: only completions that pass their tests survive), learn a
|
|
101
|
+
* code tokenizer (train fresh by default; set `hfTokenizerUrl` on the tokenizer
|
|
102
|
+
* step to import merges from a proven code tokenizer instead), train the model,
|
|
103
|
+
* and gate it with the standard convergence/benchmark checks PLUS code-specific
|
|
104
|
+
* ones: structural/parse validity and an execution-grounded test reward.
|
|
105
|
+
*
|
|
106
|
+
* The code gates ship report-only (no thresholds) so the template completes and
|
|
107
|
+
* surfaces metrics; a real run sets `minScore` / `minPassRate` to enforce them.
|
|
108
|
+
*/
|
|
109
|
+
export const TEACH_CODE: WorkflowConfig = {
|
|
110
|
+
id: "teach-code",
|
|
111
|
+
name: "Teach Evermind to Code",
|
|
112
|
+
description:
|
|
113
|
+
"Distil a teacher coder into a (prompt → code) corpus, train an EvermindLM on it, and validate with code-aware gates: structural parse + an execution test reward. Tokenizer supports both train-fresh and import-merges.",
|
|
114
|
+
steps: [
|
|
115
|
+
{ id: "distill", type: "distill-corpus", params: { pairs: CODE_DISTILL_PAIRS } },
|
|
116
|
+
{ id: "tok", type: "train-tokenizer", params: { numMerges: 200 } },
|
|
117
|
+
{ id: "data", type: "dataset-quality", params: { minSequences: 3, minWords: 20 } },
|
|
118
|
+
{ id: "model", type: "train-model", params: { epochs: 60, dModel: 24, numLayers: 2, hiddenDim: 32 } },
|
|
119
|
+
{ id: "converge", type: "convergence" },
|
|
120
|
+
{ id: "eval", type: "evaluate", params: { prompt: "function" } },
|
|
121
|
+
{ id: "gen", type: "generate-check", params: { prompt: "function" } },
|
|
122
|
+
{ id: "parse", type: "code-parse-check", params: { language: "js" } },
|
|
123
|
+
{ id: "reward", type: "code-eval", params: { cases: [{ call: "add(2, 3)", expect: 5 }] } },
|
|
124
|
+
{ id: "bench", type: "benchmark" },
|
|
125
|
+
{ id: "pkg", type: "roundtrip", params: { name: "evermind-coder" } },
|
|
126
|
+
{ id: "export", type: "export", params: { format: "huggingface", name: "evermind-coder", version: "1.0.0" } },
|
|
127
|
+
],
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
export const WORKFLOW_TEMPLATES: WorkflowConfig[] = [AGENTIC_SEVEN_LAYER, TRAIN_LLM, TEACH_CODE];
|
|
73
131
|
|
|
74
132
|
export function getTemplate(id: string): WorkflowConfig | undefined {
|
|
75
133
|
return WORKFLOW_TEMPLATES.find((t) => t.id === id);
|