@matchbox-ai/train 0.1.0

package/dist/project-BP331m20.js

@@ -0,0 +1,125 @@
+import { n as pipelineSchema } from "./define-pipeline-Colvwt7u.js";
+import { z } from "zod";
+import { readArtifact, tensorPredictor } from "@matchbox-ai/core/internal";
+import { access, readFile, readdir, stat } from "node:fs/promises";
+import { basename, dirname, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { createParser } from "@matchbox-ai/core/runtime";
+//#region src/project/config.ts
+const configSchema = z.strictObject({
+	formatVersion: z.literal(1).default(1),
+	task: z.string().default("./parser.ts"),
+	train: z.string().default("./data/train.jsonl"),
+	validation: z.string().default("./evals/validation.jsonl"),
+	eval: z.string().default("./evals/test.jsonl"),
+	output: z.string().endsWith(".matchbox"),
+	minAccuracy: z.number().min(0).max(1).default(.95),
+	maxBytes: z.number().positive().default(64e3),
+	sequence: z.strictObject({
+		recipe: z.string(),
+		decoder: z.string()
+	}).optional(),
+	baseline: z.string().optional(),
+	challenges: z.string().optional()
+});
+async function loadConfig(path) {
+	const directory = (await stat(path)).isDirectory();
+	const root = directory ? resolve(path) : dirname(resolve(path));
+	const exists = (file) => access(resolve(root, file)).then(() => true, () => false);
+	const configPath = directory ? resolve(root, "matchbox.config.ts") : resolve(path);
+	const authored = await access(configPath).then(async () => (await import(pathToFileURL(configPath).href)).default, () => ({}));
+	let defaults = {};
+	if (await exists("pipeline.ts")) {
+		const pipeline = pipelineSchema.parse((await import(pathToFileURL(resolve(root, "pipeline.ts")).href)).default);
+		defaults = { ...pipeline.acceptance };
+		if (pipeline.prediction.kind === "token-classifier") defaults.sequence = {
+			recipe: pipeline.prediction.recipe,
+			decoder: pipeline.prediction.decode
+		};
+	} else if (!authored.sequence) throw new Error(`Missing ${resolve(root, "pipeline.ts")}. Author an explicit pipeline before training.`);
+	const config = configSchema.parse({
+		output: resolve(root, "../../.matchbox", basename(root), "model.matchbox"),
+		...defaults,
+		...authored
+	});
+	if (!config.baseline && await exists("evals/baseline.ts")) config.baseline = "./evals/baseline.ts";
+	if (!config.challenges && await exists("evals/challenges.json")) config.challenges = "./evals/challenges.json";
+	return {
+		config,
+		root
+	};
+}
+//#endregion
+//#region src/project/discover.ts
+const exists = (path) => access(path).then(() => true, () => false);
+async function select(root) {
+	if (await exists(resolve(root, "parser.ts"))) return root;
+	if (await exists(resolve(root, "matchbox.config.ts"))) return resolve(root, "matchbox.config.ts");
+	const directory = resolve(root, "matchbox");
+	const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
+	const tasks = [];
+	for (const entry of entries) if (entry.isDirectory() && await exists(resolve(directory, entry.name, "parser.ts"))) tasks.push(entry.name);
+	tasks.sort();
+	if (tasks.length > 1) throw new Error(`Choose a task: ${tasks.join(", ")}. For example, matchbox-ai train ${tasks[0]}.`);
+	return tasks.length ? resolve(directory, tasks[0]) : void 0;
+}
+async function discover(target, cwd = process.cwd()) {
+	if (target && await exists(resolve(cwd, target))) {
+		const path = resolve(cwd, target);
+		if ((await stat(path)).isFile()) return path;
+		const selected = await select(path);
+		if (selected) return selected;
+		throw new Error(`No Matchbox task found in ${path}.`);
+	}
+	let root = resolve(cwd);
+	for (;;) {
+		if (target) {
+			const task = resolve(root, "matchbox", target);
+			if (await exists(resolve(task, "parser.ts"))) return task;
+		} else {
+			const selected = await select(root);
+			if (selected) return selected;
+		}
+		const parent = dirname(root);
+		if (parent === root) throw new Error(target ? `Task not found: ${target}.` : "No Matchbox project found. Run matchbox-ai init <name>.");
+		root = parent;
+	}
+}
+//#endregion
+//#region src/project/artifact.ts
+async function loadArtifact(configPath) {
+	const { config, root } = await loadConfig(configPath);
+	const output = resolve(root, config.output);
+	const text = await readFile(output, "utf8").catch(() => {
+		throw new Error(`No readable model at ${output}. Run matchbox-ai train first.`);
+	});
+	const artifact = readArtifact(JSON.parse(text));
+	const task = (await import(pathToFileURL(resolve(dirname(output), artifact.taskModule)).href)).default;
+	const decode = artifact.decoderModule === null ? void 0 : (await import(pathToFileURL(resolve(dirname(output), artifact.decoderModule)).href)).default;
+	return {
+		artifact,
+		task,
+		decode,
+		parser: createParser(artifact, task, decode),
+		inspect: async (input) => {
+			const predictor = await tensorPredictor(artifact);
+			try {
+				if (artifact.kind === "record-parser") return predictor.record(input);
+				const tokens = predictor.sequence(input);
+				return {
+					tokens,
+					candidate: decode(tokens, input)
+				};
+			} finally {
+				predictor.dispose();
+			}
+		},
+		output,
+		config,
+		root
+	};
+}
+//#endregion
+export { discover as n, loadConfig as r, loadArtifact as t };
+
+//# sourceMappingURL=project-BP331m20.js.map

package/dist/project-BP331m20.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"project-BP331m20.js","names":[],"sources":["../src/project/config.ts","../src/project/discover.ts","../src/project/artifact.ts"],"sourcesContent":["import { access, stat } from \"node:fs/promises\";\nimport { basename, dirname, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { z } from \"zod\";\nimport { pipelineSchema } from \"../pipeline/index.js\";\nexport const configSchema = z.strictObject({\n  formatVersion: z.literal(1).default(1),\n  task: z.string().default(\"./parser.ts\"),\n  train: z.string().default(\"./data/train.jsonl\"),\n  validation: z.string().default(\"./evals/validation.jsonl\"),\n  eval: z.string().default(\"./evals/test.jsonl\"),\n  output: z.string().endsWith(\".matchbox\"),\n  minAccuracy: z.number().min(0).max(1).default(0.95),\n  maxBytes: z.number().positive().default(64000),\n  sequence: z.strictObject({ recipe: z.string(), decoder: z.string() }).optional(),\n  baseline: z.string().optional(),\n  challenges: z.string().optional(),\n});\nexport async function loadConfig(path: string) {\n  const directory = (await stat(path)).isDirectory();\n  const root = directory ? resolve(path) : dirname(resolve(path));\n  const exists = (file: string) =>\n    access(resolve(root, file)).then(\n      () => true,\n      () => false,\n    );\n  const configPath = directory ? resolve(root, \"matchbox.config.ts\") : resolve(path);\n  const authored = await access(configPath).then(\n    async () => (await import(pathToFileURL(configPath).href)).default,\n    () => ({}),\n  );\n  let defaults: Record<string, unknown> = {};\n  if (await exists(\"pipeline.ts\")) {\n    const pipeline = pipelineSchema.parse(\n      (await import(pathToFileURL(resolve(root, \"pipeline.ts\")).href)).default,\n    );\n    defaults = { ...pipeline.acceptance };\n    if (pipeline.prediction.kind === \"token-classifier\")\n      defaults.sequence = {\n        recipe: pipeline.prediction.recipe,\n        decoder: pipeline.prediction.decode,\n      };\n  } else if (!authored.sequence)\n    throw new Error(\n      `Missing ${resolve(root, \"pipeline.ts\")}. Author an explicit pipeline before training.`,\n    );\n  const config = configSchema.parse({\n    output: resolve(root, \"../../.matchbox\", basename(root), \"model.matchbox\"),\n    ...defaults,\n    ...authored,\n  });\n  if (!config.baseline && (await exists(\"evals/baseline.ts\")))\n    config.baseline = \"./evals/baseline.ts\";\n  if (!config.challenges && (await exists(\"evals/challenges.json\")))\n    config.challenges = \"./evals/challenges.json\";\n  return { config, root };\n}\n","import { access, readdir, stat } from \"node:fs/promises\";\nimport { dirname, resolve } from \"node:path\";\nconst exists = (path: string) =>\n  access(path).then(\n    () => true,\n    () => false,\n  );\nasync function select(root: string): Promise<string | undefined> {\n  if (await exists(resolve(root, \"parser.ts\"))) return root;\n  if (await exists(resolve(root, \"matchbox.config.ts\"))) return resolve(root, \"matchbox.config.ts\");\n  const directory = resolve(root, \"matchbox\");\n  const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);\n  const tasks: string[] = [];\n  for (const entry of entries)\n    if (entry.isDirectory() && (await exists(resolve(directory, entry.name, \"parser.ts\"))))\n      tasks.push(entry.name);\n  tasks.sort();\n  if (tasks.length > 1)\n    throw new Error(\n      `Choose a task: ${tasks.join(\", \")}. For example, matchbox-ai train ${tasks[0]}.`,\n    );\n  return tasks.length ? resolve(directory, tasks[0]!) : undefined;\n}\nexport async function discover(target?: string, cwd = process.cwd()): Promise<string> {\n  if (target && (await exists(resolve(cwd, target)))) {\n    const path = resolve(cwd, target);\n    if ((await stat(path)).isFile()) return path;\n    const selected = await select(path);\n    if (selected) return selected;\n    throw new Error(`No Matchbox task found in ${path}.`);\n  }\n  let root = resolve(cwd);\n  for (;;) {\n    if (target) {\n      const task = resolve(root, \"matchbox\", target);\n      if (await exists(resolve(task, \"parser.ts\"))) return task;\n    } else {\n      const selected = await select(root);\n      if (selected) return selected;\n    }\n    const parent = dirname(root);\n    if (parent === root)\n      throw new Error(\n        target\n          ? `Task not found: ${target}.`\n          : \"No Matchbox project found. Run matchbox-ai init <name>.\",\n      );\n    root = parent;\n  }\n}\n","import { readFile } from \"node:fs/promises\";\nimport { dirname, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { createParser } from \"@matchbox-ai/core/runtime\";\nimport { readArtifact, tensorPredictor } from \"@matchbox-ai/core/internal\";\nimport type { SequenceDecoder } from \"@matchbox-ai/core/runtime\";\nimport type { ParserDefinition } from \"@matchbox-ai/core\";\nimport type { z } from \"zod\";\nimport { loadConfig } from \"./config.js\";\nexport async function loadArtifact(configPath: string) {\n  const { config, root } = await loadConfig(configPath);\n  const output = resolve(root, config.output);\n  const text = await readFile(output, \"utf8\").catch(() => {\n    throw new Error(`No readable model at ${output}. Run matchbox-ai train first.`);\n  });\n  const artifact = readArtifact(JSON.parse(text));\n  const task: ParserDefinition<z.ZodType> = (\n    await import(pathToFileURL(resolve(dirname(output), artifact.taskModule)).href)\n  ).default;\n  const decode: SequenceDecoder | undefined =\n    artifact.decoderModule === null\n      ? undefined\n      : (await import(pathToFileURL(resolve(dirname(output), artifact.decoderModule)).href))\n          .default;\n  return {\n    artifact,\n    task,\n    decode,\n    parser: createParser(artifact, task, decode),\n    inspect: async (input: string) => {\n      const predictor = await tensorPredictor(artifact);\n      try {\n        if (artifact.kind === \"record-parser\") return predictor.record(input);\n        const tokens = predictor.sequence(input);\n        return { tokens, candidate: decode!(tokens, input) };\n      } finally {\n        predictor.dispose();\n      }\n    },\n    output,\n    config,\n    root,\n  };\n}\n"],"mappings":";;;;;;;;AAKA,MAAa,eAAe,EAAE,aAAa;CACzC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;CACrC,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,aAAa;CACtC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,oBAAoB;CAC9C,YAAY,EAAE,OAAO,CAAC,CAAC,QAAQ,0BAA0B;CACzD,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,oBAAoB;CAC7C,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,WAAW;CACvC,aAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAI;CAClD,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,IAAK;CAC7C,UAAU,EAAE,aAAa;EAAE,QAAQ,EAAE,OAAO;EAAG,SAAS,EAAE,OAAO;CAAE,CAAC,CAAC,CAAC,SAAS;CAC/E,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS;CAC9B,YAAY,EAAE,OAAO,CAAC,CAAC,SAAS;AAClC,CAAC;AACD,eAAsB,WAAW,MAAc;CAC7C,MAAM,aAAa,MAAM,KAAK,IAAI,EAAA,CAAG,YAAY;CACjD,MAAM,OAAO,YAAY,QAAQ,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC;CAC9D,MAAM,UAAU,SACd,OAAO,QAAQ,MAAM,IAAI,CAAC,CAAC,CAAC,WACpB,YACA,KACR;CACF,MAAM,aAAa,YAAY,QAAQ,MAAM,oBAAoB,IAAI,QAAQ,IAAI;CACjF,MAAM,WAAW,MAAM,OAAO,UAAU,CAAC,CAAC,KACxC,aAAa,MAAM,OAAO,cAAc,UAAU,CAAC,CAAC,MAAA,CAAO,gBACpD,CAAC,EACV;CACA,IAAI,WAAoC,CAAC;CACzC,IAAI,MAAM,OAAO,aAAa,GAAG;EAC/B,MAAM,WAAW,eAAe,OAC7B,MAAM,OAAO,cAAc,QAAQ,MAAM,aAAa,CAAC,CAAC,CAAC,MAAA,CAAO,OACnE;EACA,WAAW,EAAE,GAAG,SAAS,WAAW;EACpC,IAAI,SAAS,WAAW,SAAS,oBAC/B,SAAS,WAAW;GAClB,QAAQ,SAAS,WAAW;GAC5B,SAAS,SAAS,WAAW;EAC/B;CACJ,OAAO,IAAI,CAAC,SAAS,UACnB,MAAM,IAAI,MACR,WAAW,QAAQ,MAAM,aAAa,EAAE,+CAC1C;CACF,MAAM,SAAS,aAAa,MAAM;EAChC,QAAQ,QAAQ,MAAM,mBAAmB,SAAS,IAAI,GAAG,gBAAgB;EACzE,GAAG;EACH,GAAG;CACL,CAAC;CACD,IAAI,CAAC,OAAO,YAAa,MAAM,OAAO,mBAAmB,GACvD,OAAO,WAAW;CACpB,IAAI,CAAC,OAAO,cAAe,MAAM,OAAO,uBAAuB,GAC7D,OAAO,aAAa;CACtB,OAAO;EAAE;EAAQ;CAAK;AACxB;;;ACtDA,MAAM,UAAU,SACd,OAAO,IAAI,CAAC,CAAC,WACL,YACA,KACR;AACF,eAAe,OAAO,MAA2C;CAC/D,IAAI,MAAM,OAAO,QAAQ,MAAM,WAAW,CAAC,GAAG,OAAO;CACrD,IAAI,MAAM,OAAO,QAAQ,MAAM,oBAAoB,CAAC,GAAG,OAAO,QAAQ,MAAM,oBAAoB;CAChG,MAAM,YAAY,QAAQ,MAAM,UAAU;CAC1C,MAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChF,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,YAAY,KAAM,MAAM,OAAO,QAAQ,WAAW,MAAM,MAAM,WAAW,CAAC,GAClF,MAAM,KAAK,MAAM,IAAI;CACzB,MAAM,KAAK;CACX,IAAI,MAAM,SAAS,GACjB,MAAM,IAAI,MACR,kBAAkB,MAAM,KAAK,IAAI,EAAE,mCAAmC,MAAM,GAAG,EACjF;CACF,OAAO,MAAM,SAAS,QAAQ,WAAW,MAAM,EAAG,IAAI,KAAA;AACxD;AACA,eAAsB,SAAS,QAAiB,MAAM,QAAQ,IAAI,GAAoB;CACpF,IAAI,UAAW,MAAM,OAAO,QAAQ,KAAK,MAAM,CAAC,GAAI;EAClD,MAAM,OAAO,QAAQ,KAAK,MAAM;EAChC,KAAK,MAAM,KAAK,IAAI,EAAA,CAAG,OAAO,GAAG,OAAO;EACxC,MAAM,WAAW,MAAM,OAAO,IAAI;EAClC,IAAI,UAAU,OAAO;EACrB,MAAM,IAAI,MAAM,6BAA6B,KAAK,EAAE;CACtD;CACA,IAAI,OAAO,QAAQ,GAAG;CACtB,SAAS;EACP,IAAI,QAAQ;GACV,MAAM,OAAO,QAAQ,MAAM,YAAY,MAAM;GAC7C,IAAI,MAAM,OAAO,QAAQ,MAAM,WAAW,CAAC,GAAG,OAAO;EACvD,OAAO;GACL,MAAM,WAAW,MAAM,OAAO,IAAI;GAClC,IAAI,UAAU,OAAO;EACvB;EACA,MAAM,SAAS,QAAQ,IAAI;EAC3B,IAAI,WAAW,MACb,MAAM,IAAI,MACR,SACI,mBAAmB,OAAO,KAC1B,yDACN;EACF,OAAO;CACT;AACF;;;ACxCA,eAAsB,aAAa,YAAoB;CACrD,MAAM,EAAE,QAAQ,SAAS,MAAM,WAAW,UAAU;CACpD,MAAM,SAAS,QAAQ,MAAM,OAAO,MAAM;CAC1C,MAAM,OAAO,MAAM,SAAS,QAAQ,MAAM,CAAC,CAAC,YAAY;EACtD,MAAM,IAAI,MAAM,wBAAwB,OAAO,+BAA+B;CAChF,CAAC;CACD,MAAM,WAAW,aAAa,KAAK,MAAM,IAAI,CAAC;CAC9C,MAAM,QACJ,MAAM,OAAO,cAAc,QAAQ,QAAQ,MAAM,GAAG,SAAS,UAAU,CAAC,CAAC,CAAC,MAAA,CAC1E;CACF,MAAM,SACJ,SAAS,kBAAkB,OACvB,KAAA,KACC,MAAM,OAAO,cAAc,QAAQ,QAAQ,MAAM,GAAG,SAAS,aAAa,CAAC,CAAC,CAAC,MAAA,CAC3E;CACT,OAAO;EACL;EACA;EACA;EACA,QAAQ,aAAa,UAAU,MAAM,MAAM;EAC3C,SAAS,OAAO,UAAkB;GAChC,MAAM,YAAY,MAAM,gBAAgB,QAAQ;GAChD,IAAI;IACF,IAAI,SAAS,SAAS,iBAAiB,OAAO,UAAU,OAAO,KAAK;IACpE,MAAM,SAAS,UAAU,SAAS,KAAK;IACvC,OAAO;KAAE;KAAQ,WAAW,OAAQ,QAAQ,KAAK;IAAE;GACrD,UAAU;IACR,UAAU,QAAQ;GACpB;EACF;EACA;EACA;EACA;CACF;AACF"}

package/dist/record-BTZ5LWHF.js

@@ -0,0 +1,205 @@
+import { t as evaluateSequence } from "./evaluate-sequence-ZCwyomzQ.js";
+import { t as packageModel } from "./package-model-CPj0qnQs.js";
+import { readRecordArtifact, recordFeatures, recordTokens, tensorPredictor } from "@matchbox-ai/core/internal";
+import { dirname, relative } from "node:path";
+import { createParser } from "@matchbox-ai/core/runtime";
+import { createHash } from "node:crypto";
+import * as tf from "@tensorflow/tfjs-node";
+//#region src/models/record/verify-export.ts
+async function verifyExport(model, artifact, probes) {
+	const native = probes.map((input) => tf.tidy(() => {
+		const logits = model.predict(tf.tensor2d([recordFeatures(input, artifact.vocabulary)]));
+		let offset = 0;
+		return artifact.fields.map((field) => {
+			const probabilities = Array.from(tf.softmax(tf.slice(logits, [0, offset], [1, field.values.length])).dataSync());
+			offset += field.values.length;
+			return probabilities;
+		});
+	}));
+	const predictor = await tensorPredictor(artifact);
+	let maxConfidenceError = 0;
+	let labelDisagreements = 0;
+	try {
+		probes.forEach((input, probeIndex) => {
+			predictor.record(input).fields.forEach((prediction, index) => {
+				const scores = native[probeIndex][index];
+				const maximum = Math.max(...scores);
+				maxConfidenceError = Math.max(maxConfidenceError, Math.abs(maximum - prediction.confidence));
+				if (prediction.value !== artifact.fields[index].values[scores.indexOf(maximum)]) labelDisagreements++;
+			});
+		});
+	} finally {
+		predictor.dispose();
+		await tf.setBackend("tensorflow");
+	}
+	if (labelDisagreements || maxConfidenceError > 1e-5) throw new Error("Serialized TensorFlow model disagrees with native predictions.");
+	return {
+		examples: probes.length,
+		labelDisagreements,
+		maxConfidenceError
+	};
+}
+//#endregion
+//#region src/models/record/fit.ts
+async function fitRecord(examples, metadata, probes, progress) {
+	const schema = metadata.taskMetadata.output;
+	if (schema.type !== "object" || !schema.properties || schema.additionalProperties !== false) throw new Error("The default trainer currently supports strict flat objects with primitive field values. Use an explicit sequence pipeline for other shapes.");
+	const fields = Object.keys(schema.properties).map((name) => {
+		return {
+			name,
+			values: [...new Map(examples.map((row) => {
+				const value = row.output[name];
+				if (value !== null && ![
+					"number",
+					"string",
+					"boolean"
+				].includes(typeof value)) throw new Error(`output.${name}: the default trainer requires primitive field values.`);
+				return [JSON.stringify(value), value];
+			})).entries()].sort(([a], [b]) => a.localeCompare(b)).map(([, value]) => value)
+		};
+	});
+	const vocabulary = [...new Set(examples.flatMap((row) => recordTokens(row.input)))].sort();
+	const outputs = fields.reduce((sum, field) => sum + field.values.length, 0);
+	if (fields.length > 32 || fields.some((field) => field.values.length > 256) || vocabulary.length > 1e4) throw new Error("Default trainer capacity exceeded. Use a custom pipeline for larger output domains.");
+	await tf.setBackend("tensorflow");
+	await tf.ready();
+	const model = tf.sequential({ layers: [tf.layers.dense({
+		inputShape: [vocabulary.length],
+		units: 32,
+		activation: "tanh",
+		kernelInitializer: tf.initializers.glorotUniform({ seed: 42 })
+	}), tf.layers.dense({
+		units: outputs,
+		kernelInitializer: tf.initializers.glorotUniform({ seed: 43 })
+	})] });
+	const optimizer = tf.train.adam(.02);
+	model.compile({
+		optimizer,
+		loss: (gold, logits) => tf.tidy(() => {
+			let offset = 0;
+			const losses = fields.map((field) => {
+				const count = field.values.length;
+				const target = tf.slice(gold, [0, offset], [-1, count]);
+				const scores = tf.slice(logits, [0, offset], [-1, count]);
+				offset += count;
+				return tf.neg(tf.sum(tf.mul(target, tf.logSoftmax(scores)), 1));
+			});
+			return tf.mean(tf.addN(losses));
+		})
+	});
+	const x = tf.tensor2d(examples.map((row) => recordFeatures(row.input, vocabulary)));
+	const y = tf.tensor2d(examples.map((row) => fields.flatMap((field) => field.values.map((value) => value === row.output[field.name] ? 1 : 0))));
+	function exported(precision) {
+		const weights = model.getWeights().map((tensor, index) => {
+			const values = Array.from(tensor.dataSync());
+			const scale = precision === "int8" ? values.reduce((max, value) => Math.max(max, Math.abs(value)), 0) / 127 || 1 : 1;
+			return {
+				name: model.weights[index].originalName,
+				shape: tensor.shape,
+				values: precision === "int8" ? values.map((value) => Math.round(value / scale)) : values,
+				scale
+			};
+		});
+		return readRecordArtifact({
+			formatVersion: 2,
+			kind: "record-parser",
+			architecture: "bag-of-words-mlp",
+			...metadata,
+			modelTopology: JSON.parse(model.toJSON()),
+			decoderModule: null,
+			fields,
+			vocabulary,
+			threshold: .75,
+			precision,
+			weights
+		});
+	}
+	const untrained = exported("float32");
+	const history = [];
+	try {
+		await model.fit(x, y, {
+			epochs: 100,
+			batchSize: 128,
+			shuffle: false,
+			verbose: 0,
+			callbacks: { onEpochEnd(epoch, logs) {
+				history.push(Number(logs?.loss));
+				progress?.(epoch + 1, Number(logs?.loss));
+			} }
+		});
+		const float = exported("float32");
+		return {
+			float,
+			quantized: exported("int8"),
+			untrained,
+			history,
+			parity: await verifyExport(model, float, probes)
+		};
+	} finally {
+		x.dispose();
+		y.dispose();
+		model.dispose();
+		optimizer.dispose();
+	}
+}
+//#endregion
+//#region src/models/record/run.ts
+async function runRecord(project, progress) {
+	const start = performance.now();
+	const taskModule = relative(dirname(project.output), project.taskPath).replaceAll("\\", "/");
+	const fit = await fitRecord(project.train, {
+		taskModule: taskModule.startsWith(".") ? taskModule : `./${taskModule}`,
+		taskMetadata: project.task.toJSON()
+	}, project.validation.map((row) => row.input), progress);
+	const evaluate = async (artifact, rows = project.evaluation) => {
+		const parser = createParser(artifact, project.task);
+		try {
+			return await evaluateSequence(parser, rows, (value) => project.task.validateOutput(value).success);
+		} finally {
+			parser.dispose();
+		}
+	};
+	const validation = await evaluate(fit.quantized, project.validation);
+	const bytes = Buffer.byteLength(JSON.stringify(fit.quantized));
+	if (validation.exactAccuracy < project.config.minAccuracy || bytes > project.config.maxBytes) throw new Error(`Record model failed validation/size requirements (${validation.exactAccuracy}, ${bytes} bytes). ${JSON.stringify(validation.failures.slice(0, 10))}`);
+	const report = {
+		formatVersion: 1,
+		architecture: fit.quantized.architecture,
+		backend: "TensorFlow native CPU",
+		seed: 42,
+		artifactSha256: createHash("sha256").update(JSON.stringify(fit.quantized)).digest("hex"),
+		bytes,
+		parameters: fit.quantized.weights.reduce((sum, weight) => sum + weight.values.length, 0),
+		datasetSha256: project.sources.map((source) => ({
+			source: source.source,
+			sha256: createHash("sha256").update(source.text).digest("hex")
+		})),
+		examples: {
+			train: project.train.length,
+			validation: project.validation.length,
+			eval: project.evaluation.length
+		},
+		loss: fit.history,
+		exportParity: fit.parity,
+		validation,
+		quantized: await evaluate(fit.quantized),
+		float: await evaluate(fit.float),
+		untrained: await evaluate(fit.untrained),
+		untrainedUngated: await evaluate({
+			...fit.untrained,
+			threshold: 0
+		}),
+		baseline: project.baseline ? await evaluateSequence(project.baseline, project.evaluation, (value) => project.task.validateOutput(value).success) : null,
+		trainingMs: performance.now() - start,
+		notes: "Each field classifies values present in training data. Vocabulary and value domains are fitted only on training. There are no number dictionaries, aliases, normalization rules, or span annotations. Unknown tokens abstain. Bag-of-words ignores order. Confidence is uncalibrated. This model cannot emit unseen numeric values."
+	};
+	await packageModel(project.output, fit.quantized, report);
+	return {
+		report,
+		output: project.output
+	};
+}
+//#endregion
+export { runRecord };
+
+//# sourceMappingURL=record-BTZ5LWHF.js.map

package/dist/record-BTZ5LWHF.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"record-BTZ5LWHF.js","names":[],"sources":["../src/models/record/verify-export.ts","../src/models/record/fit.ts","../src/models/record/run.ts"],"sourcesContent":["import * as tf from \"@tensorflow/tfjs-node\";\nimport { recordFeatures, tensorPredictor } from \"@matchbox-ai/core/internal\";\nimport type { RecordArtifact } from \"@matchbox-ai/core/internal\";\n\nexport async function verifyExport(\n  model: tf.LayersModel,\n  artifact: RecordArtifact,\n  probes: readonly string[],\n) {\n  // Snapshot predictions on the native backend before loading the browser model.\n  const native = probes.map((input) =>\n    tf.tidy(() => {\n      const logits = model.predict(\n        tf.tensor2d([recordFeatures(input, artifact.vocabulary)]),\n      ) as tf.Tensor;\n      let offset = 0;\n      return artifact.fields.map((field) => {\n        const probabilities = Array.from(\n          tf.softmax(tf.slice(logits, [0, offset], [1, field.values.length])).dataSync(),\n        );\n        offset += field.values.length;\n        return probabilities;\n      });\n    }),\n  );\n  const predictor = await tensorPredictor(artifact);\n  let maxConfidenceError = 0;\n  let labelDisagreements = 0;\n  try {\n    probes.forEach((input, probeIndex) => {\n      predictor.record(input).fields.forEach((prediction, index) => {\n        const scores = native[probeIndex]![index]!;\n        const maximum = Math.max(...scores);\n        maxConfidenceError = Math.max(\n          maxConfidenceError,\n          Math.abs(maximum - prediction.confidence),\n        );\n        if (prediction.value !== artifact.fields[index]!.values[scores.indexOf(maximum)])\n          labelDisagreements++;\n      });\n    });\n  } finally {\n    predictor.dispose();\n    await tf.setBackend(\"tensorflow\");\n  }\n  if (labelDisagreements || maxConfidenceError > 1e-5)\n    throw new Error(\"Serialized TensorFlow model disagrees with native predictions.\");\n  return { examples: probes.length, labelDisagreements, maxConfidenceError };\n}\n","import { verifyExport } from \"./verify-export.js\";\nimport * as tf from \"@tensorflow/tfjs-node\";\nimport { recordFeatures, recordTokens, readRecordArtifact } from \"@matchbox-ai/core/internal\";\nimport type { RecordArtifact } from \"@matchbox-ai/core/internal\";\nimport type { DatasetExample, ParserMetadata } from \"@matchbox-ai/core\";\nexport async function fitRecord(\n  examples: readonly DatasetExample<unknown>[],\n  metadata: { taskModule: string; taskMetadata: ParserMetadata },\n  probes: readonly string[],\n  progress?: (epoch: number, loss: number) => void,\n) {\n  const schema = metadata.taskMetadata.output;\n  if (schema.type !== \"object\" || !schema.properties || schema.additionalProperties !== false)\n    throw new Error(\n      \"The default trainer currently supports strict flat objects with primitive field values. Use an explicit sequence pipeline for other shapes.\",\n    );\n  const fields = Object.keys(schema.properties).map((name) => {\n    const values = [\n      ...new Map(\n        examples.map((row) => {\n          const value = (row.output as Record<string, unknown>)[name];\n          if (value !== null && ![\"number\", \"string\", \"boolean\"].includes(typeof value))\n            throw new Error(`output.${name}: the default trainer requires primitive field values.`);\n          return [JSON.stringify(value), value as string | number | boolean | null] as const;\n        }),\n      ).entries(),\n    ]\n      .sort(([a], [b]) => a.localeCompare(b))\n      .map(([, value]) => value);\n    return { name, values };\n  });\n  const vocabulary = [...new Set(examples.flatMap((row) => recordTokens(row.input)))].sort();\n  const outputs = fields.reduce((sum, field) => sum + field.values.length, 0);\n  if (\n    fields.length > 32 ||\n    fields.some((field) => field.values.length > 256) ||\n    vocabulary.length > 10000\n  )\n    throw new Error(\n      \"Default trainer capacity exceeded. Use a custom pipeline for larger output domains.\",\n    );\n  await tf.setBackend(\"tensorflow\");\n  await tf.ready();\n  const model = tf.sequential({\n    layers: [\n      tf.layers.dense({\n        inputShape: [vocabulary.length],\n        units: 32,\n        activation: \"tanh\",\n        kernelInitializer: tf.initializers.glorotUniform({ seed: 42 }),\n      }),\n      tf.layers.dense({\n        units: outputs,\n        kernelInitializer: tf.initializers.glorotUniform({ seed: 43 }),\n      }),\n    ],\n  });\n  const optimizer = tf.train.adam(0.02);\n  model.compile({\n    optimizer,\n    loss: (gold, logits) =>\n      tf.tidy(() => {\n        let offset = 0;\n        const losses = fields.map((field) => {\n          const count = field.values.length;\n          const target = tf.slice(gold, [0, offset], [-1, count]);\n          const scores = tf.slice(logits, [0, offset], [-1, count]);\n          offset += count;\n          return tf.neg(tf.sum(tf.mul(target, tf.logSoftmax(scores)), 1));\n        });\n        return tf.mean(tf.addN(losses));\n      }),\n  });\n  const x = tf.tensor2d(examples.map((row) => recordFeatures(row.input, vocabulary)));\n  const y = tf.tensor2d(\n    examples.map((row) =>\n      fields.flatMap((field) =>\n        field.values.map((value) =>\n          value === (row.output as Record<string, unknown>)[field.name] ? 1 : 0,\n        ),\n      ),\n    ),\n  );\n  function exported(precision: \"int8\" | \"float32\"): RecordArtifact {\n    const weights = model.getWeights().map((tensor, index) => {\n      const values = Array.from(tensor.dataSync());\n      const scale =\n        precision === \"int8\"\n          ? values.reduce((max, value) => Math.max(max, Math.abs(value)), 0) / 127 || 1\n          : 1;\n      return {\n        name: model.weights[index]!.originalName,\n        shape: tensor.shape,\n        values: precision === \"int8\" ? values.map((value) => Math.round(value / scale)) : values,\n        scale,\n      };\n    }) as RecordArtifact[\"weights\"];\n    return readRecordArtifact({\n      formatVersion: 2,\n      kind: \"record-parser\",\n      architecture: \"bag-of-words-mlp\",\n      ...metadata,\n      modelTopology: JSON.parse(model.toJSON() as string),\n      decoderModule: null,\n      fields,\n      vocabulary,\n      threshold: 0.75,\n      precision,\n      weights,\n    });\n  }\n  const untrained = exported(\"float32\");\n  const history: number[] = [];\n  try {\n    await model.fit(x, y, {\n      epochs: 100,\n      batchSize: 128,\n      shuffle: false,\n      verbose: 0,\n      callbacks: {\n        onEpochEnd(epoch, logs) {\n          history.push(Number(logs?.loss));\n          progress?.(epoch + 1, Number(logs?.loss));\n        },\n      },\n    });\n    const float = exported(\"float32\"),\n      quantized = exported(\"int8\");\n    const parity = await verifyExport(model, float, probes);\n    return {\n      float,\n      quantized,\n      untrained,\n      history,\n      parity,\n    };\n  } finally {\n    x.dispose();\n    y.dispose();\n    model.dispose();\n    optimizer.dispose();\n  }\n}\n","import { createHash } from \"node:crypto\";\nimport { dirname, relative } from \"node:path\";\nimport { createParser } from \"@matchbox-ai/core/runtime\";\nimport type { RecordArtifact } from \"@matchbox-ai/core/internal\";\nimport type { loadProject } from \"../../load-project.js\";\nimport { packageModel } from \"../../packaging/package-model.js\";\nimport { evaluateSequence } from \"../../evaluation/evaluate-sequence.js\";\nimport { fitRecord } from \"./fit.js\";\nexport async function runRecord(\n  project: Awaited<ReturnType<typeof loadProject>>,\n  progress?: (epoch: number, loss: number) => void,\n) {\n  const start = performance.now();\n  const taskModule = relative(dirname(project.output), project.taskPath).replaceAll(\"\\\\\", \"/\");\n  const fit = await fitRecord(\n    project.train,\n    {\n      taskModule: taskModule.startsWith(\".\") ? taskModule : `./${taskModule}`,\n      taskMetadata: project.task.toJSON(),\n    },\n    project.validation.map((row) => row.input),\n    progress,\n  );\n  const evaluate = async (artifact: RecordArtifact, rows = project.evaluation) => {\n    const parser = createParser(artifact, project.task);\n    try {\n      return await evaluateSequence(\n        parser,\n        rows,\n        (value) => project.task.validateOutput(value).success,\n      );\n    } finally {\n      parser.dispose();\n    }\n  };\n  const validation = await evaluate(fit.quantized, project.validation);\n  const bytes = Buffer.byteLength(JSON.stringify(fit.quantized));\n  if (validation.exactAccuracy < project.config.minAccuracy || bytes > project.config.maxBytes)\n    throw new Error(\n      `Record model failed validation/size requirements (${validation.exactAccuracy}, ${bytes} bytes). ${JSON.stringify(validation.failures.slice(0, 10))}`,\n    );\n  const report = {\n    formatVersion: 1,\n    architecture: fit.quantized.architecture,\n    backend: \"TensorFlow native CPU\",\n    seed: 42,\n    artifactSha256: createHash(\"sha256\").update(JSON.stringify(fit.quantized)).digest(\"hex\"),\n    bytes,\n    parameters: fit.quantized.weights.reduce((sum, weight) => sum + weight.values.length, 0),\n    datasetSha256: project.sources.map((source) => ({\n      source: source.source,\n      sha256: createHash(\"sha256\").update(source.text).digest(\"hex\"),\n    })),\n    examples: {\n      train: project.train.length,\n      validation: project.validation.length,\n      eval: project.evaluation.length,\n    },\n    loss: fit.history,\n    exportParity: fit.parity,\n    validation,\n    quantized: await evaluate(fit.quantized),\n    float: await evaluate(fit.float),\n    untrained: await evaluate(fit.untrained),\n    untrainedUngated: await evaluate({ ...fit.untrained, threshold: 0 }),\n    baseline: project.baseline\n      ? await evaluateSequence(\n          project.baseline,\n          project.evaluation,\n          (value) => project.task.validateOutput(value).success,\n        )\n      : null,\n    trainingMs: performance.now() - start,\n    notes:\n      \"Each field classifies values present in training data. Vocabulary and value domains are fitted only on training. There are no number dictionaries, aliases, normalization rules, or span annotations. Unknown tokens abstain. Bag-of-words ignores order. Confidence is uncalibrated. This model cannot emit unseen numeric values.\",\n  };\n  await packageModel(project.output, fit.quantized, report);\n  return { report, output: project.output };\n}\n"],"mappings":";;;;;;;;AAIA,eAAsB,aACpB,OACA,UACA,QACA;CAEA,MAAM,SAAS,OAAO,KAAK,UACzB,GAAG,WAAW;EACZ,MAAM,SAAS,MAAM,QACnB,GAAG,SAAS,CAAC,eAAe,OAAO,SAAS,UAAU,CAAC,CAAC,CAC1D;EACA,IAAI,SAAS;EACb,OAAO,SAAS,OAAO,KAAK,UAAU;GACpC,MAAM,gBAAgB,MAAM,KAC1B,GAAG,QAAQ,GAAG,MAAM,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG,MAAM,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAC/E;GACA,UAAU,MAAM,OAAO;GACvB,OAAO;EACT,CAAC;CACH,CAAC,CACH;CACA,MAAM,YAAY,MAAM,gBAAgB,QAAQ;CAChD,IAAI,qBAAqB;CACzB,IAAI,qBAAqB;CACzB,IAAI;EACF,OAAO,SAAS,OAAO,eAAe;GACpC,UAAU,OAAO,KAAK,CAAC,CAAC,OAAO,SAAS,YAAY,UAAU;IAC5D,MAAM,SAAS,OAAO,WAAW,CAAE;IACnC,MAAM,UAAU,KAAK,IAAI,GAAG,MAAM;IAClC,qBAAqB,KAAK,IACxB,oBACA,KAAK,IAAI,UAAU,WAAW,UAAU,CAC1C;IACA,IAAI,WAAW,UAAU,SAAS,OAAO,MAAM,CAAE,OAAO,OAAO,QAAQ,OAAO,IAC5E;GACJ,CAAC;EACH,CAAC;CACH,UAAU;EACR,UAAU,QAAQ;EAClB,MAAM,GAAG,WAAW,YAAY;CAClC;CACA,IAAI,sBAAsB,qBAAqB,MAC7C,MAAM,IAAI,MAAM,gEAAgE;CAClF,OAAO;EAAE,UAAU,OAAO;EAAQ;EAAoB;CAAmB;AAC3E;;;AC3CA,eAAsB,UACpB,UACA,UACA,QACA,UACA;CACA,MAAM,SAAS,SAAS,aAAa;CACrC,IAAI,OAAO,SAAS,YAAY,CAAC,OAAO,cAAc,OAAO,yBAAyB,OACpF,MAAM,IAAI,MACR,6IACF;CACF,MAAM,SAAS,OAAO,KAAK,OAAO,UAAU,CAAC,CAAC,KAAK,SAAS;EAa1D,OAAO;GAAE;GAAM,QAZA,CACb,GAAG,IAAI,IACL,SAAS,KAAK,QAAQ;IACpB,MAAM,QAAS,IAAI,OAAmC;IACtD,IAAI,UAAU,QAAQ,CAAC;KAAC;KAAU;KAAU;IAAS,CAAC,CAAC,SAAS,OAAO,KAAK,GAC1E,MAAM,IAAI,MAAM,UAAU,KAAK,uDAAuD;IACxF,OAAO,CAAC,KAAK,UAAU,KAAK,GAAG,KAAyC;GAC1E,CAAC,CACH,CAAC,CAAC,QAAQ,CACZ,CAAC,CACE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CACtC,KAAK,GAAG,WAAW,KACF;EAAE;CACxB,CAAC;CACD,MAAM,aAAa,CAAC,GAAG,IAAI,IAAI,SAAS,SAAS,QAAQ,aAAa,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;CACzF,MAAM,UAAU,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,QAAQ,CAAC;CAC1E,IACE,OAAO,SAAS,MAChB,OAAO,MAAM,UAAU,MAAM,OAAO,SAAS,GAAG,KAChD,WAAW,SAAS,KAEpB,MAAM,IAAI,MACR,qFACF;CACF,MAAM,GAAG,WAAW,YAAY;CAChC,MAAM,GAAG,MAAM;CACf,MAAM,QAAQ,GAAG,WAAW,EAC1B,QAAQ,CACN,GAAG,OAAO,MAAM;EACd,YAAY,CAAC,WAAW,MAAM;EAC9B,OAAO;EACP,YAAY;EACZ,mBAAmB,GAAG,aAAa,cAAc,EAAE,MAAM,GAAG,CAAC;CAC/D,CAAC,GACD,GAAG,OAAO,MAAM;EACd,OAAO;EACP,mBAAmB,GAAG,aAAa,cAAc,EAAE,MAAM,GAAG,CAAC;CAC/D,CAAC,CACH,EACF,CAAC;CACD,MAAM,YAAY,GAAG,MAAM,KAAK,GAAI;CACpC,MAAM,QAAQ;EACZ;EACA,OAAO,MAAM,WACX,GAAG,WAAW;GACZ,IAAI,SAAS;GACb,MAAM,SAAS,OAAO,KAAK,UAAU;IACnC,MAAM,QAAQ,MAAM,OAAO;IAC3B,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC;IACtD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC;IACxD,UAAU;IACV,OAAO,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,QAAQ,GAAG,WAAW,MAAM,CAAC,GAAG,CAAC,CAAC;GAChE,CAAC;GACD,OAAO,GAAG,KAAK,GAAG,KAAK,MAAM,CAAC;EAChC,CAAC;CACL,CAAC;CACD,MAAM,IAAI,GAAG,SAAS,SAAS,KAAK,QAAQ,eAAe,IAAI,OAAO,UAAU,CAAC,CAAC;CAClF,MAAM,IAAI,GAAG,SACX,SAAS,KAAK,QACZ,OAAO,SAAS,UACd,MAAM,OAAO,KAAK,UAChB,UAAW,IAAI,OAAmC,MAAM,QAAQ,IAAI,CACtE,CACF,CACF,CACF;CACA,SAAS,SAAS,WAA+C;EAC/D,MAAM,UAAU,MAAM,WAAW,CAAC,CAAC,KAAK,QAAQ,UAAU;GACxD,MAAM,SAAS,MAAM,KAAK,OAAO,SAAS,CAAC;GAC3C,MAAM,QACJ,cAAc,SACV,OAAO,QAAQ,KAAK,UAAU,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,OAAO,IAC1E;GACN,OAAO;IACL,MAAM,MAAM,QAAQ,MAAM,CAAE;IAC5B,OAAO,OAAO;IACd,QAAQ,cAAc,SAAS,OAAO,KAAK,UAAU,KAAK,MAAM,QAAQ,KAAK,CAAC,IAAI;IAClF;GACF;EACF,CAAC;EACD,OAAO,mBAAmB;GACxB,eAAe;GACf,MAAM;GACN,cAAc;GACd,GAAG;GACH,eAAe,KAAK,MAAM,MAAM,OAAO,CAAW;GAClD,eAAe;GACf;GACA;GACA,WAAW;GACX;GACA;EACF,CAAC;CACH;CACA,MAAM,YAAY,SAAS,SAAS;CACpC,MAAM,UAAoB,CAAC;CAC3B,IAAI;EACF,MAAM,MAAM,IAAI,GAAG,GAAG;GACpB,QAAQ;GACR,WAAW;GACX,SAAS;GACT,SAAS;GACT,WAAW,EACT,WAAW,OAAO,MAAM;IACtB,QAAQ,KAAK,OAAO,MAAM,IAAI,CAAC;IAC/B,WAAW,QAAQ,GAAG,OAAO,MAAM,IAAI,CAAC;GAC1C,EACF;EACF,CAAC;EACD,MAAM,QAAQ,SAAS,SAAS;EAGhC,OAAO;GACL;GACA,WAJY,SAAS,MAIb;GACR;GACA;GACA,QAAA,MANmB,aAAa,OAAO,OAAO,MAAM;EAOtD;CACF,UAAU;EACR,EAAE,QAAQ;EACV,EAAE,QAAQ;EACV,MAAM,QAAQ;EACd,UAAU,QAAQ;CACpB;AACF;;;ACtIA,eAAsB,UACpB,SACA,UACA;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,aAAa,SAAS,QAAQ,QAAQ,MAAM,GAAG,QAAQ,QAAQ,CAAC,CAAC,WAAW,MAAM,GAAG;CAC3F,MAAM,MAAM,MAAM,UAChB,QAAQ,OACR;EACE,YAAY,WAAW,WAAW,GAAG,IAAI,aAAa,KAAK;EAC3D,cAAc,QAAQ,KAAK,OAAO;CACpC,GACA,QAAQ,WAAW,KAAK,QAAQ,IAAI,KAAK,GACzC,QACF;CACA,MAAM,WAAW,OAAO,UAA0B,OAAO,QAAQ,eAAe;EAC9E,MAAM,SAAS,aAAa,UAAU,QAAQ,IAAI;EAClD,IAAI;GACF,OAAO,MAAM,iBACX,QACA,OACC,UAAU,QAAQ,KAAK,eAAe,KAAK,CAAC,CAAC,OAChD;EACF,UAAU;GACR,OAAO,QAAQ;EACjB;CACF;CACA,MAAM,aAAa,MAAM,SAAS,IAAI,WAAW,QAAQ,UAAU;CACnE,MAAM,QAAQ,OAAO,WAAW,KAAK,UAAU,IAAI,SAAS,CAAC;CAC7D,IAAI,WAAW,gBAAgB,QAAQ,OAAO,eAAe,QAAQ,QAAQ,OAAO,UAClF,MAAM,IAAI,MACR,qDAAqD,WAAW,cAAc,IAAI,MAAM,WAAW,KAAK,UAAU,WAAW,SAAS,MAAM,GAAG,EAAE,CAAC,GACpJ;CACF,MAAM,SAAS;EACb,eAAe;EACf,cAAc,IAAI,UAAU;EAC5B,SAAS;EACT,MAAM;EACN,gBAAgB,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC,OAAO,KAAK;EACvF;EACA,YAAY,IAAI,UAAU,QAAQ,QAAQ,KAAK,WAAW,MAAM,OAAO,OAAO,QAAQ,CAAC;EACvF,eAAe,QAAQ,QAAQ,KAAK,YAAY;GAC9C,QAAQ,OAAO;GACf,QAAQ,WAAW,QAAQ,CAAC,CAAC,OAAO,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK;EAC/D,EAAE;EACF,UAAU;GACR,OAAO,QAAQ,MAAM;GACrB,YAAY,QAAQ,WAAW;GAC/B,MAAM,QAAQ,WAAW;EAC3B;EACA,MAAM,IAAI;EACV,cAAc,IAAI;EAClB;EACA,WAAW,MAAM,SAAS,IAAI,SAAS;EACvC,OAAO,MAAM,SAAS,IAAI,KAAK;EAC/B,WAAW,MAAM,SAAS,IAAI,SAAS;EACvC,kBAAkB,MAAM,SAAS;GAAE,GAAG,IAAI;GAAW,WAAW;EAAE,CAAC;EACnE,UAAU,QAAQ,WACd,MAAM,iBACJ,QAAQ,UACR,QAAQ,aACP,UAAU,QAAQ,KAAK,eAAe,KAAK,CAAC,CAAC,OAChD,IACA;EACJ,YAAY,YAAY,IAAI,IAAI;EAChC,OACE;CACJ;CACA,MAAM,aAAa,QAAQ,QAAQ,IAAI,WAAW,MAAM;CACxD,OAAO;EAAE;EAAQ,QAAQ,QAAQ;CAAO;AAC1C"}

package/dist/run-Cwa7vD_T.js

@@ -0,0 +1,72 @@
+import { r as loadConfig } from "./project-BP331m20.js";
+import "zod";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { parseDatasets } from "@matchbox-ai/core";
+//#region src/load-project.ts
+async function loadProject(path) {
+	const { config, root } = await loadConfig(path);
+	const taskPath = resolve(root, config.task);
+	const task = (await import(pathToFileURL(taskPath).href)).default;
+	const baseline = config.baseline ? (await import(pathToFileURL(resolve(root, config.baseline)).href)).default : null;
+	const sources = await Promise.all([
+		config.train,
+		config.validation,
+		config.eval
+	].map(async (source) => ({
+		source,
+		text: await readFile(resolve(root, source), "utf8")
+	})));
+	const training = parseDatasets(task, {
+		formatVersion: 1,
+		train: sources[0],
+		eval: sources[1]
+	});
+	const evaluation = parseDatasets(task, {
+		formatVersion: 1,
+		train: sources[0],
+		eval: sources[2]
+	});
+	for (const result of [training, evaluation]) if (!result.success) throw new Error(result.issues.map((issue) => `${issue.source}:${issue.line} ${issue.path.join(".")}: ${issue.message}`).join("\n"));
+	if (!training.success || !evaluation.success) throw new Error("Invalid datasets.");
+	const groups = [
+		training.data.train,
+		training.data.eval,
+		evaluation.data.eval
+	];
+	const seen = /* @__PURE__ */ new Map();
+	groups.forEach((group, split) => group.forEach((row) => {
+		const key = row.input.trim().toLowerCase();
+		if (seen.has(key) && seen.get(key) !== split) throw new Error(`Input overlaps dataset splits: ${row.input}`);
+		seen.set(key, split);
+	}));
+	return {
+		root,
+		config,
+		task,
+		baseline,
+		taskPath,
+		output: resolve(root, config.output),
+		sources,
+		train: training.data.train,
+		validation: training.data.eval,
+		evaluation: evaluation.data.eval
+	};
+}
+//#endregion
+//#region src/run.ts
+async function run(command, path, progress) {
+	const project = await loadProject(path);
+	if (project.config.sequence) {
+		const { runSequence } = await import("./sequence-C_WfVGO1.js");
+		return runSequence(command, project, progress);
+	}
+	if (command !== "train") throw new Error("Use the eval CLI command to evaluate a saved artifact.");
+	const { runRecord } = await import("./record-BTZ5LWHF.js");
+	return runRecord(project, progress);
+}
+//#endregion
+export { run };
+
+//# sourceMappingURL=run-Cwa7vD_T.js.map

package/dist/run-Cwa7vD_T.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-Cwa7vD_T.js","names":[],"sources":["../src/load-project.ts","../src/run.ts"],"sourcesContent":["import { loadConfig } from \"./project/index.js\";\nimport { readFile } from \"node:fs/promises\";\nimport { resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { z } from \"zod\";\nimport { parseDatasets } from \"@matchbox-ai/core\";\nimport type { ParserDefinition } from \"@matchbox-ai/core\";\nimport type { MatchboxParser } from \"@matchbox-ai/core/runtime\";\nexport async function loadProject(path: string) {\n  const { config, root } = await loadConfig(path);\n  const taskPath = resolve(root, config.task);\n  const task: ParserDefinition<z.ZodType> = (await import(pathToFileURL(taskPath).href)).default;\n  const baseline: MatchboxParser<unknown> | null = config.baseline\n    ? (await import(pathToFileURL(resolve(root, config.baseline)).href)).default\n    : null;\n  const sources = await Promise.all(\n    [config.train, config.validation, config.eval].map(async (source) => ({\n      source,\n      text: await readFile(resolve(root, source), \"utf8\"),\n    })),\n  );\n  const training = parseDatasets(task, { formatVersion: 1, train: sources[0]!, eval: sources[1]! });\n  const evaluation = parseDatasets(task, {\n    formatVersion: 1,\n    train: sources[0]!,\n    eval: sources[2]!,\n  });\n  for (const result of [training, evaluation])\n    if (!result.success)\n      throw new Error(\n        result.issues\n          .map((issue) => `${issue.source}:${issue.line} ${issue.path.join(\".\")}: ${issue.message}`)\n          .join(\"\\n\"),\n      );\n  if (!training.success || !evaluation.success) throw new Error(\"Invalid datasets.\");\n  const groups = [training.data.train, training.data.eval, evaluation.data.eval];\n  const seen = new Map<string, number>();\n  groups.forEach((group, split) =>\n    group.forEach((row) => {\n      const key = row.input.trim().toLowerCase();\n      if (seen.has(key) && seen.get(key) !== split)\n        throw new Error(`Input overlaps dataset splits: ${row.input}`);\n      seen.set(key, split);\n    }),\n  );\n  return {\n    root,\n    config,\n    task,\n    baseline,\n    taskPath,\n    output: resolve(root, config.output),\n    sources,\n    train: training.data.train,\n    validation: training.data.eval,\n    evaluation: evaluation.data.eval,\n  };\n}\n","import { loadProject } from \"./load-project.js\";\nexport async function run(\n  command: \"train\" | \"eval\",\n  path: string,\n  progress?: (epoch: number, loss: number) => void,\n) {\n  const project = await loadProject(path);\n  if (project.config.sequence) {\n    const { runSequence } = await import(\"./models/sequence/index.js\");\n    return runSequence(command, project, progress);\n  }\n  if (command !== \"train\")\n    throw new Error(\"Use the eval CLI command to evaluate a saved artifact.\");\n  const { runRecord } = await import(\"./models/record/index.js\");\n  return runRecord(project, progress);\n}\n"],"mappings":";;;;;;;AAQA,eAAsB,YAAY,MAAc;CAC9C,MAAM,EAAE,QAAQ,SAAS,MAAM,WAAW,IAAI;CAC9C,MAAM,WAAW,QAAQ,MAAM,OAAO,IAAI;CAC1C,MAAM,QAAqC,MAAM,OAAO,cAAc,QAAQ,CAAC,CAAC,MAAA,CAAO;CACvF,MAAM,WAA2C,OAAO,YACnD,MAAM,OAAO,cAAc,QAAQ,MAAM,OAAO,QAAQ,CAAC,CAAC,CAAC,MAAA,CAAO,UACnE;CACJ,MAAM,UAAU,MAAM,QAAQ,IAC5B;EAAC,OAAO;EAAO,OAAO;EAAY,OAAO;CAAI,CAAC,CAAC,IAAI,OAAO,YAAY;EACpE;EACA,MAAM,MAAM,SAAS,QAAQ,MAAM,MAAM,GAAG,MAAM;CACpD,EAAE,CACJ;CACA,MAAM,WAAW,cAAc,MAAM;EAAE,eAAe;EAAG,OAAO,QAAQ;EAAK,MAAM,QAAQ;CAAI,CAAC;CAChG,MAAM,aAAa,cAAc,MAAM;EACrC,eAAe;EACf,OAAO,QAAQ;EACf,MAAM,QAAQ;CAChB,CAAC;CACD,KAAK,MAAM,UAAU,CAAC,UAAU,UAAU,GACxC,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,MACR,OAAO,OACJ,KAAK,UAAU,GAAG,MAAM,OAAO,GAAG,MAAM,KAAK,GAAG,MAAM,KAAK,KAAK,GAAG,EAAE,IAAI,MAAM,SAAS,CAAC,CACzF,KAAK,IAAI,CACd;CACJ,IAAI,CAAC,SAAS,WAAW,CAAC,WAAW,SAAS,MAAM,IAAI,MAAM,mBAAmB;CACjF,MAAM,SAAS;EAAC,SAAS,KAAK;EAAO,SAAS,KAAK;EAAM,WAAW,KAAK;CAAI;CAC7E,MAAM,uBAAO,IAAI,IAAoB;CACrC,OAAO,SAAS,OAAO,UACrB,MAAM,SAAS,QAAQ;EACrB,MAAM,MAAM,IAAI,MAAM,KAAK,CAAC,CAAC,YAAY;EACzC,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,MAAM,OACrC,MAAM,IAAI,MAAM,kCAAkC,IAAI,OAAO;EAC/D,KAAK,IAAI,KAAK,KAAK;CACrB,CAAC,CACH;CACA,OAAO;EACL;EACA;EACA;EACA;EACA;EACA,QAAQ,QAAQ,MAAM,OAAO,MAAM;EACnC;EACA,OAAO,SAAS,KAAK;EACrB,YAAY,SAAS,KAAK;EAC1B,YAAY,WAAW,KAAK;CAC9B;AACF;;;ACxDA,eAAsB,IACpB,SACA,MACA,UACA;CACA,MAAM,UAAU,MAAM,YAAY,IAAI;CACtC,IAAI,QAAQ,OAAO,UAAU;EAC3B,MAAM,EAAE,gBAAgB,MAAM,OAAO;EACrC,OAAO,YAAY,SAAS,SAAS,QAAQ;CAC/C;CACA,IAAI,YAAY,SACd,MAAM,IAAI,MAAM,wDAAwD;CAC1E,MAAM,EAAE,cAAc,MAAM,OAAO;CACnC,OAAO,UAAU,SAAS,QAAQ;AACpC"}