@matchbox-ai/train 0.1.0

package/dist/run.d.ts

@@ -0,0 +1,273 @@
+export declare function run(command: "train" | "eval", path: string, progress?: (epoch: number, loss: number) => void): Promise<{
+    evaluation: {
+        examples: number;
+        invalidOutputRate: number;
+        exactAccuracy: number;
+        accepted: number;
+        acceptedAccuracy: number | null;
+        abstentionRate: number;
+        correctAbstentions: number;
+        failures: {
+            input: string;
+            expected: unknown;
+            actual: unknown;
+        }[];
+    };
+    report?: never;
+    output?: never;
+} | {
+    evaluation?: never;
+    report: {
+        formatVersion: number;
+        architecture: "embedding-window-mlp";
+        backend: string;
+        seed: number;
+        artifactSha256: string;
+        bytes: number;
+        parameters: number;
+        datasetSha256: {
+            source: string;
+            sha256: string;
+        }[];
+        examples: {
+            train: number;
+            validation: number;
+            eval: number;
+        };
+        shuffledLabelsUngated: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        } | null;
+        supervisedTokens: number;
+        loss: number[];
+        exportParity: {
+            examples: number;
+            maxConfidenceError: number;
+            labelDisagreements: number;
+        };
+        validation: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        untrainedUngated: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        untrained: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        float: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        quantized: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        challenges: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        } | null;
+        baseline: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        } | null;
+        trainingMs: number;
+        notes: string;
+    };
+    output: string;
+} | {
+    report: {
+        formatVersion: number;
+        architecture: "bag-of-words-mlp";
+        backend: string;
+        seed: number;
+        artifactSha256: string;
+        bytes: number;
+        parameters: number;
+        datasetSha256: {
+            source: string;
+            sha256: string;
+        }[];
+        examples: {
+            train: number;
+            validation: number;
+            eval: number;
+        };
+        loss: number[];
+        exportParity: {
+            examples: number;
+            labelDisagreements: number;
+            maxConfidenceError: number;
+        };
+        validation: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        quantized: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        float: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        untrained: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        untrainedUngated: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        };
+        baseline: {
+            examples: number;
+            invalidOutputRate: number;
+            exactAccuracy: number;
+            accepted: number;
+            acceptedAccuracy: number | null;
+            abstentionRate: number;
+            correctAbstentions: number;
+            failures: {
+                input: string;
+                expected: unknown;
+                actual: unknown;
+            }[];
+        } | null;
+        trainingMs: number;
+        notes: string;
+    };
+    output: string;
+}>;
+//# sourceMappingURL=run.d.ts.map

package/dist/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AACA,wBAAsB,GAAG,CACvB,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAWjD"}

package/dist/sequence-C_WfVGO1.js

@@ -0,0 +1,251 @@
+import { n as sameOutput, t as evaluateSequence } from "./evaluate-sequence-ZCwyomzQ.js";
+import { t as packageModel } from "./package-model-CPj0qnQs.js";
+import { z } from "zod";
+import { readSequenceArtifact, tensorPredictor, tokenize, windows } from "@matchbox-ai/core/internal";
+import { readFile } from "node:fs/promises";
+import { dirname, relative, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { createParser } from "@matchbox-ai/core/runtime";
+import { createHash } from "node:crypto";
+import * as tf from "@tensorflow/tfjs-node";
+//#region src/models/sequence/create-network.ts
+function createNetwork(vocabularySize, labelCount) {
+	return tf.sequential({ layers: [
+		tf.layers.embedding({
+			inputDim: vocabularySize + 2,
+			outputDim: 8,
+			inputLength: 3,
+			embeddingsInitializer: tf.initializers.randomUniform({
+				minval: -.1,
+				maxval: .1,
+				seed: 42
+			})
+		}),
+		tf.layers.flatten(),
+		tf.layers.dense({
+			units: 16,
+			activation: "tanh",
+			kernelInitializer: tf.initializers.glorotUniform({ seed: 43 })
+		}),
+		tf.layers.dense({
+			units: labelCount,
+			activation: "softmax",
+			kernelInitializer: tf.initializers.glorotUniform({ seed: 44 })
+		})
+	] });
+}
+//#endregion
+//#region src/models/sequence/fit-sequence.ts
+async function fitSequence(examples, recipe, metadata, probes = [], shuffleLabels = false, progress) {
+	await tf.setBackend("tensorflow");
+	await tf.ready();
+	const vocabulary = [...new Set(examples.flatMap((row) => tokenize(row.input, recipe.tokenizer).map((token) => token.key)))].sort();
+	const radius = 1;
+	const inputs = [];
+	const labels = [];
+	for (const example of examples) {
+		const tokens = tokenize(example.input, recipe.tokenizer);
+		const annotations = recipe.annotate(example, tokens);
+		if (annotations.length !== tokens.length) throw new Error(`Annotation length mismatch: ${example.input}`);
+		windows(tokens, vocabulary, radius).forEach((window, position) => {
+			const label = annotations[position];
+			if (label === null) return;
+			const id = recipe.labels.indexOf(label);
+			if (id < 0) throw new Error(`Unknown annotation ${label}: ${example.input}`);
+			inputs.push(window);
+			labels.push(id);
+		});
+	}
+	if (recipe.labels.some((_, id) => !labels.includes(id))) throw new Error("Every label needs supervised training examples.");
+	if (shuffleLabels) {
+		let state = 173;
+		for (let index = labels.length - 1; index > 0; index--) {
+			state = Math.imul(state, 1664525) + 1013904223 >>> 0;
+			const other = state % (index + 1);
+			[labels[index], labels[other]] = [labels[other], labels[index]];
+		}
+	}
+	const model = createNetwork(vocabulary.length, recipe.labels.length);
+	const optimizer = tf.train.adam(.02);
+	model.compile({
+		optimizer,
+		loss: "categoricalCrossentropy"
+	});
+	const x = tf.tensor2d(inputs, [inputs.length, 3], "int32");
+	const y = tf.tidy(() => tf.oneHot(tf.tensor1d(labels, "int32"), recipe.labels.length));
+	function exportModel(precision) {
+		const weights = model.getWeights().map((tensor, index) => {
+			const values = Array.from(tensor.dataSync());
+			const scale = precision === "int8" ? Math.max(...values.map(Math.abs)) / 127 || 1 : 1;
+			return {
+				name: model.weights[index].originalName,
+				shape: tensor.shape,
+				scale,
+				values: precision === "int8" ? values.map((value) => Math.round(value / scale)) : values
+			};
+		});
+		return {
+			formatVersion: 2,
+			kind: "sequence-parser",
+			architecture: "embedding-window-mlp",
+			...metadata,
+			modelTopology: JSON.parse(model.toJSON()),
+			tokenizer: recipe.tokenizer,
+			readout: recipe.readout,
+			vocabulary,
+			labels: [...recipe.labels],
+			radius,
+			threshold: .75,
+			precision,
+			weights
+		};
+	}
+	const untrained = exportModel("float32");
+	const history = [];
+	try {
+		await model.fit(x, y, {
+			epochs: 55,
+			batchSize: 128,
+			shuffle: false,
+			verbose: 0,
+			callbacks: { onEpochEnd(epoch, logs) {
+				history.push(Number(logs?.loss));
+				progress?.(epoch + 1, Number(logs?.loss));
+			} }
+		});
+		const float = exportModel("float32");
+		const quantized = exportModel("int8");
+		let maxConfidenceError = 0;
+		let labelDisagreements = 0;
+		const checkedInputs = [...examples.slice(0, 16).map((row) => row.input), ...probes];
+		const nativeProbabilities = checkedInputs.map((inputText) => tf.tidy(() => {
+			const tokens = tokenize(inputText, recipe.tokenizer);
+			const input = tf.tensor2d(windows(tokens, vocabulary, radius), [tokens.length, 3], "int32");
+			return model.predict(input).arraySync();
+		}));
+		const portable = await tensorPredictor(float);
+		try {
+			checkedInputs.forEach((inputText, probeIndex) => {
+				portable.sequence(inputText).forEach((token, index) => {
+					const scores = nativeProbabilities[probeIndex][index];
+					const maximum = Math.max(...scores);
+					maxConfidenceError = Math.max(maxConfidenceError, Math.abs(maximum - token.confidence));
+					if (recipe.labels[scores.indexOf(maximum)] !== token.label) labelDisagreements++;
+				});
+			});
+		} finally {
+			portable.dispose();
+			await tf.setBackend("tensorflow");
+		}
+		if (labelDisagreements || maxConfidenceError > 1e-5) throw new Error("Exported runtime disagrees with TensorFlow.js.");
+		return {
+			untrained,
+			float,
+			quantized,
+			history,
+			parity: {
+				examples: checkedInputs.length,
+				maxConfidenceError,
+				labelDisagreements
+			},
+			supervisedTokens: inputs.length
+		};
+	} finally {
+		x.dispose();
+		y.dispose();
+		model.dispose();
+		optimizer.dispose();
+	}
+}
+//#endregion
+//#region src/models/sequence/run-sequence.ts
+const hash = (value) => createHash("sha256").update(value).digest("hex");
+async function runSequence(command, project, progress) {
+	const { task, config } = project;
+	const root = project.root;
+	const sequence = config.sequence;
+	if (!sequence) throw new Error("A sequence pipeline needs both recipe and decoder modules.");
+	const savedArtifact = command === "eval" ? readSequenceArtifact(JSON.parse(await readFile(project.output, "utf8"))) : null;
+	const decoderPath = savedArtifact ? resolve(dirname(project.output), savedArtifact.decoderModule) : resolve(root, sequence.decoder);
+	const decode = (await import(pathToFileURL(decoderPath).href)).default;
+	const evaluateSequence$1 = (parser, examples) => evaluateSequence(parser, examples, (value) => task.validateOutput(value).success).finally(() => {
+		if ("dispose" in parser && typeof parser.dispose === "function") parser.dispose();
+	});
+	const parser = (artifact) => createParser(artifact, task, decode);
+	if (savedArtifact) return { evaluation: await evaluateSequence$1(parser(savedArtifact), project.evaluation) };
+	const recipe = (await import(pathToFileURL(resolve(root, sequence.recipe)).href)).default;
+	const modulePath = (path) => {
+		const value = relative(dirname(project.output), path).replaceAll("\\", "/");
+		return value.startsWith(".") ? value : `./${value}`;
+	};
+	for (const example of project.train) {
+		const tokens = tokenize(example.input, recipe.tokenizer);
+		const labels = recipe.annotate(example, tokens);
+		const value = decode(tokens.map((token, index) => ({
+			...token,
+			label: labels[index] ?? "O",
+			confidence: 1
+		})), example.input);
+		if (!sameOutput(value, example.output)) throw new Error(`Training annotations do not decode to the supplied output: ${example.input}`);
+	}
+	const started = performance.now();
+	const fit = await fitSequence(project.train, recipe, {
+		taskMetadata: task.toJSON(),
+		taskModule: modulePath(project.taskPath),
+		decoderModule: modulePath(resolve(root, sequence.decoder))
+	}, project.validation.map((row) => row.input), false, progress);
+	const validation = await evaluateSequence$1(parser(fit.quantized), project.validation);
+	const bytes = Buffer.byteLength(JSON.stringify(fit.quantized));
+	if (validation.exactAccuracy < config.minAccuracy || bytes > config.maxBytes) throw new Error(`Sequence model failed validation/size requirements (${validation.exactAccuracy}, ${bytes} bytes). ${JSON.stringify(validation.failures.slice(0, 10))}`);
+	const shuffledControl = recipe.readout === "last" ? await fitSequence(project.train, recipe, fit.quantized, [], true) : null;
+	const challenges = config.challenges ? z.array(z.strictObject({
+		input: z.string(),
+		output: z.null()
+	})).min(1).parse(JSON.parse(await readFile(resolve(root, config.challenges), "utf8"))) : null;
+	const report = {
+		formatVersion: 1,
+		architecture: fit.quantized.architecture,
+		backend: "TensorFlow native CPU",
+		seed: 42,
+		artifactSha256: hash(JSON.stringify(fit.quantized)),
+		bytes,
+		parameters: fit.quantized.weights.reduce((sum, weight) => sum + weight.values.length, 0),
+		datasetSha256: project.sources.map((source) => ({
+			source: source.source,
+			sha256: hash(source.text)
+		})),
+		examples: {
+			train: project.train.length,
+			validation: project.validation.length,
+			eval: project.evaluation.length
+		},
+		shuffledLabelsUngated: shuffledControl ? await evaluateSequence$1(parser({
+			...shuffledControl.quantized,
+			threshold: 0
+		}), project.evaluation) : null,
+		supervisedTokens: fit.supervisedTokens,
+		loss: fit.history,
+		exportParity: fit.parity,
+		validation,
+		untrainedUngated: await evaluateSequence$1(parser({
+			...fit.untrained,
+			threshold: 0
+		}), project.evaluation),
+		untrained: await evaluateSequence$1(parser(fit.untrained), project.evaluation),
+		float: await evaluateSequence$1(parser(fit.float), project.evaluation),
+		quantized: await evaluateSequence$1(parser(fit.quantized), project.evaluation),
+		challenges: challenges ? await evaluateSequence$1(parser(fit.quantized), challenges) : null,
+		baseline: project.baseline ? await evaluateSequence$1(project.baseline, project.evaluation) : null,
+		trainingMs: performance.now() - started,
+		notes: "Validation gates export. Eval labels do not influence selection. Scores are uncalibrated. Unknown tokens abstain. JSON int8 arrays are portable but not a packed binary format."
+	};
+	await packageModel(project.output, fit.quantized, report);
+	return {
+		report,
+		output: project.output
+	};
+}
+//#endregion
+export { runSequence };
+
+//# sourceMappingURL=sequence-C_WfVGO1.js.map

package/dist/sequence-C_WfVGO1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sequence-C_WfVGO1.js","names":["evaluateSequence","evaluate"],"sources":["../src/models/sequence/create-network.ts","../src/models/sequence/fit-sequence.ts","../src/models/sequence/run-sequence.ts"],"sourcesContent":["import * as tf from \"@tensorflow/tfjs-node\";\nexport function createNetwork(vocabularySize: number, labelCount: number) {\n  return tf.sequential({\n    layers: [\n      tf.layers.embedding({\n        inputDim: vocabularySize + 2,\n        outputDim: 8,\n        inputLength: 3,\n        embeddingsInitializer: tf.initializers.randomUniform({\n          minval: -0.1,\n          maxval: 0.1,\n          seed: 42,\n        }),\n      }),\n      tf.layers.flatten(),\n      tf.layers.dense({\n        units: 16,\n        activation: \"tanh\",\n        kernelInitializer: tf.initializers.glorotUniform({ seed: 43 }),\n      }),\n      tf.layers.dense({\n        units: labelCount,\n        activation: \"softmax\",\n        kernelInitializer: tf.initializers.glorotUniform({ seed: 44 }),\n      }),\n    ],\n  });\n}\n","import { createNetwork } from \"./create-network.js\";\nimport * as tf from \"@tensorflow/tfjs-node\";\nimport { tensorPredictor, tokenize, windows } from \"@matchbox-ai/core/internal\";\nimport type { SequenceArtifact } from \"@matchbox-ai/core/internal\";\nimport type { DatasetExample } from \"@matchbox-ai/core\";\nimport type { SequenceRecipe } from \"./types.js\";\nexport async function fitSequence(\n  examples: readonly DatasetExample<unknown>[],\n  recipe: SequenceRecipe,\n  metadata: Pick<SequenceArtifact, \"taskModule\" | \"taskMetadata\" | \"decoderModule\">,\n  probes: readonly string[] = [],\n  shuffleLabels = false,\n  progress?: (epoch: number, loss: number) => void,\n) {\n  await tf.setBackend(\"tensorflow\");\n  await tf.ready();\n  const vocabulary = [\n    ...new Set(\n      examples.flatMap((row) => tokenize(row.input, recipe.tokenizer).map((token) => token.key)),\n    ),\n  ].sort();\n  const radius = 1;\n  const inputs: number[][] = [];\n  const labels: number[] = [];\n  for (const example of examples) {\n    const tokens = tokenize(example.input, recipe.tokenizer);\n    const annotations = recipe.annotate(example, tokens);\n    if (annotations.length !== tokens.length)\n      throw new Error(`Annotation length mismatch: ${example.input}`);\n    windows(tokens, vocabulary, radius).forEach((window, position) => {\n      const label = annotations[position];\n      if (label === null) return;\n      const id = recipe.labels.indexOf(label!);\n      if (id < 0) throw new Error(`Unknown annotation ${label}: ${example.input}`);\n      inputs.push(window);\n      labels.push(id);\n    });\n  }\n  if (recipe.labels.some((_, id) => !labels.includes(id)))\n    throw new Error(\"Every label needs supervised training examples.\");\n  if (shuffleLabels) {\n    let state = 173;\n    for (let index = labels.length - 1; index > 0; index--) {\n      state = (Math.imul(state, 1664525) + 1013904223) >>> 0;\n      const other = state % (index + 1);\n      [labels[index], labels[other]] = [labels[other]!, labels[index]!];\n    }\n  }\n  const model = createNetwork(vocabulary.length, recipe.labels.length);\n  const optimizer = tf.train.adam(0.02);\n  model.compile({ optimizer, loss: \"categoricalCrossentropy\" });\n  const x = tf.tensor2d(inputs, [inputs.length, 3], \"int32\");\n  const y = tf.tidy(() => tf.oneHot(tf.tensor1d(labels, \"int32\"), recipe.labels.length));\n  function exportModel(precision: \"float32\" | \"int8\"): SequenceArtifact {\n    const weights = model.getWeights().map((tensor, index) => {\n      const values = Array.from(tensor.dataSync());\n      const scale = precision === \"int8\" ? Math.max(...values.map(Math.abs)) / 127 || 1 : 1;\n      return {\n        name: model.weights[index]!.originalName,\n        shape: tensor.shape,\n        scale,\n        values: precision === \"int8\" ? values.map((value) => Math.round(value / scale)) : values,\n      };\n    }) as SequenceArtifact[\"weights\"];\n    return {\n      formatVersion: 2,\n      kind: \"sequence-parser\",\n      architecture: \"embedding-window-mlp\",\n      ...metadata,\n      modelTopology: JSON.parse(model.toJSON() as string),\n      tokenizer: recipe.tokenizer,\n      readout: recipe.readout,\n      vocabulary,\n      labels: [...recipe.labels],\n      radius,\n      threshold: 0.75,\n      precision,\n      weights,\n    };\n  }\n  const untrained = exportModel(\"float32\");\n  const history: number[] = [];\n  try {\n    await model.fit(x, y, {\n      epochs: 55,\n      batchSize: 128,\n      shuffle: false,\n      verbose: 0,\n      callbacks: {\n        onEpochEnd(epoch, logs) {\n          history.push(Number(logs?.loss));\n          progress?.(epoch + 1, Number(logs?.loss));\n        },\n      },\n    });\n    const float = exportModel(\"float32\");\n    const quantized = exportModel(\"int8\");\n    let maxConfidenceError = 0;\n    let labelDisagreements = 0;\n    const checkedInputs = [...examples.slice(0, 16).map((row) => row.input), ...probes];\n    const nativeProbabilities = checkedInputs.map((inputText) =>\n      tf.tidy(() => {\n        const tokens = tokenize(inputText, recipe.tokenizer);\n        const input = tf.tensor2d(windows(tokens, vocabulary, radius), [tokens.length, 3], \"int32\");\n        return (model.predict(input) as tf.Tensor).arraySync() as number[][];\n      }),\n    );\n    const portable = await tensorPredictor(float);\n    try {\n      checkedInputs.forEach((inputText, probeIndex) => {\n        portable.sequence(inputText).forEach((token, index) => {\n          const scores = nativeProbabilities[probeIndex]![index]!;\n          const maximum = Math.max(...scores);\n          maxConfidenceError = Math.max(maxConfidenceError, Math.abs(maximum - token.confidence));\n          if (recipe.labels[scores.indexOf(maximum)] !== token.label) labelDisagreements++;\n        });\n      });\n    } finally {\n      portable.dispose();\n      await tf.setBackend(\"tensorflow\");\n    }\n    if (labelDisagreements || maxConfidenceError > 1e-5)\n      throw new Error(\"Exported runtime disagrees with TensorFlow.js.\");\n    return {\n      untrained,\n      float,\n      quantized,\n      history,\n      parity: { examples: checkedInputs.length, maxConfidenceError, labelDisagreements },\n      supervisedTokens: inputs.length,\n    };\n  } finally {\n    x.dispose();\n    y.dispose();\n    model.dispose();\n    optimizer.dispose();\n  }\n}\n","import { sameOutput } from \"../../evaluation/same-output.js\";\nimport { z } from \"zod\";\nimport { createHash } from \"node:crypto\";\nimport { readFile } from \"node:fs/promises\";\nimport { dirname, relative, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { createParser } from \"@matchbox-ai/core/runtime\";\nimport { readSequenceArtifact, tokenize } from \"@matchbox-ai/core/internal\";\nimport type { DatasetExample } from \"@matchbox-ai/core\";\nimport type { MatchboxParser, SequenceDecoder } from \"@matchbox-ai/core/runtime\";\nimport type { loadProject } from \"../../load-project.js\";\nimport { packageModel } from \"../../packaging/package-model.js\";\nimport { fitSequence } from \"./fit-sequence.js\";\nimport { evaluateSequence as evaluate } from \"../../evaluation/evaluate-sequence.js\";\nimport type { SequenceRecipe } from \"./types.js\";\nconst hash = (value: string) => createHash(\"sha256\").update(value).digest(\"hex\");\nexport async function runSequence(\n  command: \"train\" | \"eval\",\n  project: Awaited<ReturnType<typeof loadProject>>,\n  progress?: (epoch: number, loss: number) => void,\n) {\n  const { task, config } = project;\n  const root = project.root;\n  const sequence = config.sequence;\n  if (!sequence) throw new Error(\"A sequence pipeline needs both recipe and decoder modules.\");\n  const savedArtifact =\n    command === \"eval\"\n      ? readSequenceArtifact(JSON.parse(await readFile(project.output, \"utf8\")))\n      : null;\n  const decoderPath = savedArtifact\n    ? resolve(dirname(project.output), savedArtifact.decoderModule)\n    : resolve(root, sequence.decoder);\n  const decode: SequenceDecoder = (await import(pathToFileURL(decoderPath).href)).default;\n  const evaluateSequence = (\n    parser: MatchboxParser<unknown>,\n    examples: readonly DatasetExample<unknown>[],\n  ) =>\n    evaluate(parser, examples, (value) => task.validateOutput(value).success).finally(() => {\n      if (\"dispose\" in parser && typeof parser.dispose === \"function\") parser.dispose();\n    });\n  const parser = (artifact: unknown) => createParser(artifact, task, decode);\n  if (savedArtifact) {\n    return { evaluation: await evaluateSequence(parser(savedArtifact), project.evaluation) };\n  }\n  const recipe: SequenceRecipe = (await import(pathToFileURL(resolve(root, sequence.recipe)).href))\n    .default;\n  const modulePath = (path: string) => {\n    const value = relative(dirname(project.output), path).replaceAll(\"\\\\\", \"/\");\n    return value.startsWith(\".\") ? value : `./${value}`;\n  };\n  for (const example of project.train) {\n    const tokens = tokenize(example.input, recipe.tokenizer);\n    const labels = recipe.annotate(example, tokens);\n    const value = decode(\n      tokens.map((token, index) => ({ ...token, label: labels[index] ?? \"O\", confidence: 1 })),\n      example.input,\n    );\n    if (!sameOutput(value, example.output))\n      throw new Error(\n        `Training annotations do not decode to the supplied output: ${example.input}`,\n      );\n  }\n  const started = performance.now();\n  const fit = await fitSequence(\n    project.train,\n    recipe,\n    {\n      taskMetadata: task.toJSON(),\n      taskModule: modulePath(project.taskPath),\n      decoderModule: modulePath(resolve(root, sequence.decoder)),\n    },\n    project.validation.map((row) => row.input),\n    false,\n    progress,\n  );\n  const validation = await evaluateSequence(parser(fit.quantized), project.validation);\n  const bytes = Buffer.byteLength(JSON.stringify(fit.quantized));\n  if (validation.exactAccuracy < config.minAccuracy || bytes > config.maxBytes)\n    throw new Error(\n      `Sequence model failed validation/size requirements (${validation.exactAccuracy}, ${bytes} bytes). ${JSON.stringify(validation.failures.slice(0, 10))}`,\n    );\n  const shuffledControl =\n    recipe.readout === \"last\"\n      ? await fitSequence(project.train, recipe, fit.quantized, [], true)\n      : null;\n  const challenges = config.challenges\n    ? z\n        .array(z.strictObject({ input: z.string(), output: z.null() }))\n        .min(1)\n        .parse(JSON.parse(await readFile(resolve(root, config.challenges), \"utf8\")))\n    : null;\n  const report = {\n    formatVersion: 1,\n    architecture: fit.quantized.architecture,\n    backend: \"TensorFlow native CPU\",\n    seed: 42,\n    artifactSha256: hash(JSON.stringify(fit.quantized)),\n    bytes,\n    parameters: fit.quantized.weights.reduce((sum, weight) => sum + weight.values.length, 0),\n    datasetSha256: project.sources.map((source) => ({\n      source: source.source,\n      sha256: hash(source.text),\n    })),\n    examples: {\n      train: project.train.length,\n      validation: project.validation.length,\n      eval: project.evaluation.length,\n    },\n    shuffledLabelsUngated: shuffledControl\n      ? await evaluateSequence(\n          parser({ ...shuffledControl.quantized, threshold: 0 }),\n          project.evaluation,\n        )\n      : null,\n    supervisedTokens: fit.supervisedTokens,\n    loss: fit.history,\n    exportParity: fit.parity,\n    validation,\n    untrainedUngated: await evaluateSequence(\n      parser({ ...fit.untrained, threshold: 0 }),\n      project.evaluation,\n    ),\n    untrained: await evaluateSequence(parser(fit.untrained), project.evaluation),\n    float: await evaluateSequence(parser(fit.float), project.evaluation),\n    quantized: await evaluateSequence(parser(fit.quantized), project.evaluation),\n    challenges: challenges ? await evaluateSequence(parser(fit.quantized), challenges) : null,\n    baseline: project.baseline\n      ? await evaluateSequence(project.baseline, project.evaluation)\n      : null,\n    trainingMs: performance.now() - started,\n    notes:\n      \"Validation gates export. Eval labels do not influence selection. Scores are uncalibrated. Unknown tokens abstain. JSON int8 arrays are portable but not a packed binary format.\",\n  };\n  await packageModel(project.output, fit.quantized, report);\n  return { report, output: project.output };\n}\n"],"mappings":";;;;;;;;;;;AACA,SAAgB,cAAc,gBAAwB,YAAoB;CACxE,OAAO,GAAG,WAAW,EACnB,QAAQ;EACN,GAAG,OAAO,UAAU;GAClB,UAAU,iBAAiB;GAC3B,WAAW;GACX,aAAa;GACb,uBAAuB,GAAG,aAAa,cAAc;IACnD,QAAQ;IACR,QAAQ;IACR,MAAM;GACR,CAAC;EACH,CAAC;EACD,GAAG,OAAO,QAAQ;EAClB,GAAG,OAAO,MAAM;GACd,OAAO;GACP,YAAY;GACZ,mBAAmB,GAAG,aAAa,cAAc,EAAE,MAAM,GAAG,CAAC;EAC/D,CAAC;EACD,GAAG,OAAO,MAAM;GACd,OAAO;GACP,YAAY;GACZ,mBAAmB,GAAG,aAAa,cAAc,EAAE,MAAM,GAAG,CAAC;EAC/D,CAAC;CACH,EACF,CAAC;AACH;;;ACrBA,eAAsB,YACpB,UACA,QACA,UACA,SAA4B,CAAC,GAC7B,gBAAgB,OAChB,UACA;CACA,MAAM,GAAG,WAAW,YAAY;CAChC,MAAM,GAAG,MAAM;CACf,MAAM,aAAa,CACjB,GAAG,IAAI,IACL,SAAS,SAAS,QAAQ,SAAS,IAAI,OAAO,OAAO,SAAS,CAAC,CAAC,KAAK,UAAU,MAAM,GAAG,CAAC,CAC3F,CACF,CAAC,CAAC,KAAK;CACP,MAAM,SAAS;CACf,MAAM,SAAqB,CAAC;CAC5B,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,WAAW,UAAU;EAC9B,MAAM,SAAS,SAAS,QAAQ,OAAO,OAAO,SAAS;EACvD,MAAM,cAAc,OAAO,SAAS,SAAS,MAAM;EACnD,IAAI,YAAY,WAAW,OAAO,QAChC,MAAM,IAAI,MAAM,+BAA+B,QAAQ,OAAO;EAChE,QAAQ,QAAQ,YAAY,MAAM,CAAC,CAAC,SAAS,QAAQ,aAAa;GAChE,MAAM,QAAQ,YAAY;GAC1B,IAAI,UAAU,MAAM;GACpB,MAAM,KAAK,OAAO,OAAO,QAAQ,KAAM;GACvC,IAAI,KAAK,GAAG,MAAM,IAAI,MAAM,sBAAsB,MAAM,IAAI,QAAQ,OAAO;GAC3E,OAAO,KAAK,MAAM;GAClB,OAAO,KAAK,EAAE;EAChB,CAAC;CACH;CACA,IAAI,OAAO,OAAO,MAAM,GAAG,OAAO,CAAC,OAAO,SAAS,EAAE,CAAC,GACpD,MAAM,IAAI,MAAM,iDAAiD;CACnE,IAAI,eAAe;EACjB,IAAI,QAAQ;EACZ,KAAK,IAAI,QAAQ,OAAO,SAAS,GAAG,QAAQ,GAAG,SAAS;GACtD,QAAS,KAAK,KAAK,OAAO,OAAO,IAAI,eAAgB;GACrD,MAAM,QAAQ,SAAS,QAAQ;GAC/B,CAAC,OAAO,QAAQ,OAAO,UAAU,CAAC,OAAO,QAAS,OAAO,MAAO;EAClE;CACF;CACA,MAAM,QAAQ,cAAc,WAAW,QAAQ,OAAO,OAAO,MAAM;CACnE,MAAM,YAAY,GAAG,MAAM,KAAK,GAAI;CACpC,MAAM,QAAQ;EAAE;EAAW,MAAM;CAA0B,CAAC;CAC5D,MAAM,IAAI,GAAG,SAAS,QAAQ,CAAC,OAAO,QAAQ,CAAC,GAAG,OAAO;CACzD,MAAM,IAAI,GAAG,WAAW,GAAG,OAAO,GAAG,SAAS,QAAQ,OAAO,GAAG,OAAO,OAAO,MAAM,CAAC;CACrF,SAAS,YAAY,WAAiD;EACpE,MAAM,UAAU,MAAM,WAAW,CAAC,CAAC,KAAK,QAAQ,UAAU;GACxD,MAAM,SAAS,MAAM,KAAK,OAAO,SAAS,CAAC;GAC3C,MAAM,QAAQ,cAAc,SAAS,KAAK,IAAI,GAAG,OAAO,IAAI,KAAK,GAAG,CAAC,IAAI,OAAO,IAAI;GACpF,OAAO;IACL,MAAM,MAAM,QAAQ,MAAM,CAAE;IAC5B,OAAO,OAAO;IACd;IACA,QAAQ,cAAc,SAAS,OAAO,KAAK,UAAU,KAAK,MAAM,QAAQ,KAAK,CAAC,IAAI;GACpF;EACF,CAAC;EACD,OAAO;GACL,eAAe;GACf,MAAM;GACN,cAAc;GACd,GAAG;GACH,eAAe,KAAK,MAAM,MAAM,OAAO,CAAW;GAClD,WAAW,OAAO;GAClB,SAAS,OAAO;GAChB;GACA,QAAQ,CAAC,GAAG,OAAO,MAAM;GACzB;GACA,WAAW;GACX;GACA;EACF;CACF;CACA,MAAM,YAAY,YAAY,SAAS;CACvC,MAAM,UAAoB,CAAC;CAC3B,IAAI;EACF,MAAM,MAAM,IAAI,GAAG,GAAG;GACpB,QAAQ;GACR,WAAW;GACX,SAAS;GACT,SAAS;GACT,WAAW,EACT,WAAW,OAAO,MAAM;IACtB,QAAQ,KAAK,OAAO,MAAM,IAAI,CAAC;IAC/B,WAAW,QAAQ,GAAG,OAAO,MAAM,IAAI,CAAC;GAC1C,EACF;EACF,CAAC;EACD,MAAM,QAAQ,YAAY,SAAS;EACnC,MAAM,YAAY,YAAY,MAAM;EACpC,IAAI,qBAAqB;EACzB,IAAI,qBAAqB;EACzB,MAAM,gBAAgB,CAAC,GAAG,SAAS,MAAM,GAAG,EAAE,CAAC,CAAC,KAAK,QAAQ,IAAI,KAAK,GAAG,GAAG,MAAM;EAClF,MAAM,sBAAsB,cAAc,KAAK,cAC7C,GAAG,WAAW;GACZ,MAAM,SAAS,SAAS,WAAW,OAAO,SAAS;GACnD,MAAM,QAAQ,GAAG,SAAS,QAAQ,QAAQ,YAAY,MAAM,GAAG,CAAC,OAAO,QAAQ,CAAC,GAAG,OAAO;GAC1F,OAAQ,MAAM,QAAQ,KAAK,CAAC,CAAe,UAAU;EACvD,CAAC,CACH;EACA,MAAM,WAAW,MAAM,gBAAgB,KAAK;EAC5C,IAAI;GACF,cAAc,SAAS,WAAW,eAAe;IAC/C,SAAS,SAAS,SAAS,CAAC,CAAC,SAAS,OAAO,UAAU;KACrD,MAAM,SAAS,oBAAoB,WAAW,CAAE;KAChD,MAAM,UAAU,KAAK,IAAI,GAAG,MAAM;KAClC,qBAAqB,KAAK,IAAI,oBAAoB,KAAK,IAAI,UAAU,MAAM,UAAU,CAAC;KACtF,IAAI,OAAO,OAAO,OAAO,QAAQ,OAAO,OAAO,MAAM,OAAO;IAC9D,CAAC;GACH,CAAC;EACH,UAAU;GACR,SAAS,QAAQ;GACjB,MAAM,GAAG,WAAW,YAAY;EAClC;EACA,IAAI,sBAAsB,qBAAqB,MAC7C,MAAM,IAAI,MAAM,gDAAgD;EAClE,OAAO;GACL;GACA;GACA;GACA;GACA,QAAQ;IAAE,UAAU,cAAc;IAAQ;IAAoB;GAAmB;GACjF,kBAAkB,OAAO;EAC3B;CACF,UAAU;EACR,EAAE,QAAQ;EACV,EAAE,QAAQ;EACV,MAAM,QAAQ;EACd,UAAU,QAAQ;CACpB;AACF;;;AC1HA,MAAM,QAAQ,UAAkB,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK;AAC/E,eAAsB,YACpB,SACA,SACA,UACA;CACA,MAAM,EAAE,MAAM,WAAW;CACzB,MAAM,OAAO,QAAQ;CACrB,MAAM,WAAW,OAAO;CACxB,IAAI,CAAC,UAAU,MAAM,IAAI,MAAM,4DAA4D;CAC3F,MAAM,gBACJ,YAAY,SACR,qBAAqB,KAAK,MAAM,MAAM,SAAS,QAAQ,QAAQ,MAAM,CAAC,CAAC,IACvE;CACN,MAAM,cAAc,gBAChB,QAAQ,QAAQ,QAAQ,MAAM,GAAG,cAAc,aAAa,IAC5D,QAAQ,MAAM,SAAS,OAAO;CAClC,MAAM,UAA2B,MAAM,OAAO,cAAc,WAAW,CAAC,CAAC,MAAA,CAAO;CAChF,MAAMA,sBACJ,QACA,aAEAC,iBAAS,QAAQ,WAAW,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,cAAc;EACtF,IAAI,aAAa,UAAU,OAAO,OAAO,YAAY,YAAY,OAAO,QAAQ;CAClF,CAAC;CACH,MAAM,UAAU,aAAsB,aAAa,UAAU,MAAM,MAAM;CACzE,IAAI,eACF,OAAO,EAAE,YAAY,MAAMD,mBAAiB,OAAO,aAAa,GAAG,QAAQ,UAAU,EAAE;CAEzF,MAAM,UAA0B,MAAM,OAAO,cAAc,QAAQ,MAAM,SAAS,MAAM,CAAC,CAAC,CAAC,MAAA,CACxF;CACH,MAAM,cAAc,SAAiB;EACnC,MAAM,QAAQ,SAAS,QAAQ,QAAQ,MAAM,GAAG,IAAI,CAAC,CAAC,WAAW,MAAM,GAAG;EAC1E,OAAO,MAAM,WAAW,GAAG,IAAI,QAAQ,KAAK;CAC9C;CACA,KAAK,MAAM,WAAW,QAAQ,OAAO;EACnC,MAAM,SAAS,SAAS,QAAQ,OAAO,OAAO,SAAS;EACvD,MAAM,SAAS,OAAO,SAAS,SAAS,MAAM;EAC9C,MAAM,QAAQ,OACZ,OAAO,KAAK,OAAO,WAAW;GAAE,GAAG;GAAO,OAAO,OAAO,UAAU;GAAK,YAAY;EAAE,EAAE,GACvF,QAAQ,KACV;EACA,IAAI,CAAC,WAAW,OAAO,QAAQ,MAAM,GACnC,MAAM,IAAI,MACR,8DAA8D,QAAQ,OACxE;CACJ;CACA,MAAM,UAAU,YAAY,IAAI;CAChC,MAAM,MAAM,MAAM,YAChB,QAAQ,OACR,QACA;EACE,cAAc,KAAK,OAAO;EAC1B,YAAY,WAAW,QAAQ,QAAQ;EACvC,eAAe,WAAW,QAAQ,MAAM,SAAS,OAAO,CAAC;CAC3D,GACA,QAAQ,WAAW,KAAK,QAAQ,IAAI,KAAK,GACzC,OACA,QACF;CACA,MAAM,aAAa,MAAMA,mBAAiB,OAAO,IAAI,SAAS,GAAG,QAAQ,UAAU;CACnF,MAAM,QAAQ,OAAO,WAAW,KAAK,UAAU,IAAI,SAAS,CAAC;CAC7D,IAAI,WAAW,gBAAgB,OAAO,eAAe,QAAQ,OAAO,UAClE,MAAM,IAAI,MACR,uDAAuD,WAAW,cAAc,IAAI,MAAM,WAAW,KAAK,UAAU,WAAW,SAAS,MAAM,GAAG,EAAE,CAAC,GACtJ;CACF,MAAM,kBACJ,OAAO,YAAY,SACf,MAAM,YAAY,QAAQ,OAAO,QAAQ,IAAI,WAAW,CAAC,GAAG,IAAI,IAChE;CACN,MAAM,aAAa,OAAO,aACtB,EACG,MAAM,EAAE,aAAa;EAAE,OAAO,EAAE,OAAO;EAAG,QAAQ,EAAE,KAAK;CAAE,CAAC,CAAC,CAAC,CAC9D,IAAI,CAAC,CAAC,CACN,MAAM,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,OAAO,UAAU,GAAG,MAAM,CAAC,CAAC,IAC7E;CACJ,MAAM,SAAS;EACb,eAAe;EACf,cAAc,IAAI,UAAU;EAC5B,SAAS;EACT,MAAM;EACN,gBAAgB,KAAK,KAAK,UAAU,IAAI,SAAS,CAAC;EAClD;EACA,YAAY,IAAI,UAAU,QAAQ,QAAQ,KAAK,WAAW,MAAM,OAAO,OAAO,QAAQ,CAAC;EACvF,eAAe,QAAQ,QAAQ,KAAK,YAAY;GAC9C,QAAQ,OAAO;GACf,QAAQ,KAAK,OAAO,IAAI;EAC1B,EAAE;EACF,UAAU;GACR,OAAO,QAAQ,MAAM;GACrB,YAAY,QAAQ,WAAW;GAC/B,MAAM,QAAQ,WAAW;EAC3B;EACA,uBAAuB,kBACnB,MAAMA,mBACJ,OAAO;GAAE,GAAG,gBAAgB;GAAW,WAAW;EAAE,CAAC,GACrD,QAAQ,UACV,IACA;EACJ,kBAAkB,IAAI;EACtB,MAAM,IAAI;EACV,cAAc,IAAI;EAClB;EACA,kBAAkB,MAAMA,mBACtB,OAAO;GAAE,GAAG,IAAI;GAAW,WAAW;EAAE,CAAC,GACzC,QAAQ,UACV;EACA,WAAW,MAAMA,mBAAiB,OAAO,IAAI,SAAS,GAAG,QAAQ,UAAU;EAC3E,OAAO,MAAMA,mBAAiB,OAAO,IAAI,KAAK,GAAG,QAAQ,UAAU;EACnE,WAAW,MAAMA,mBAAiB,OAAO,IAAI,SAAS,GAAG,QAAQ,UAAU;EAC3E,YAAY,aAAa,MAAMA,mBAAiB,OAAO,IAAI,SAAS,GAAG,UAAU,IAAI;EACrF,UAAU,QAAQ,WACd,MAAMA,mBAAiB,QAAQ,UAAU,QAAQ,UAAU,IAC3D;EACJ,YAAY,YAAY,IAAI,IAAI;EAChC,OACE;CACJ;CACA,MAAM,aAAa,QAAQ,QAAQ,IAAI,WAAW,MAAM;CACxD,OAAO;EAAE;EAAQ,QAAQ,QAAQ;CAAO;AAC1C"}