@matchbox-ai/train 0.2.2 → 0.3.0

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @matchbox-ai/train
 
+## 0.3.0
+
+### Minor Changes
+
+- 6158277: Add explicit case preservation to token recipes and configurable context radius to token classifiers. Native Burn training and WASM inference use the same serialized encoding and window shape. Existing recipes retain lowercase keys and a radius of one, and the runtime continues to read version 3 sequence artifacts.
+
+  Store training windows in packed integer arrays to limit preparation memory for larger corpora and wider contexts.
+
+### Patch Changes
+
+- Updated dependencies [6158277]
+  - @matchbox-ai/core@0.3.0
+
 ## 0.2.2
 
 ### Patch Changes

package/dist/{define-pipeline-D6I-jAE_.js → define-pipeline-DEkjmOte.js}

@@ -4,6 +4,7 @@ const pipelineSchema = z.strictObject({
 	prediction: z.discriminatedUnion("kind", [z.strictObject({ kind: z.literal("field-classifier") }), z.strictObject({
 		kind: z.literal("token-classifier"),
 		recipe: z.string().min(1).default("./recipe"),
+		contextRadius: z.number().int().min(1).max(16).optional(),
 		decode: z.string().min(1).default("./decode")
 	})]),
 	acceptance: z.strictObject({
@@ -17,4 +18,4 @@ function definePipeline(pipeline) {
 //#endregion
 export { pipelineSchema as n, definePipeline as t };
 
-//# sourceMappingURL=define-pipeline-D6I-jAE_.js.map
+//# sourceMappingURL=define-pipeline-DEkjmOte.js.map

package/dist/define-pipeline-DEkjmOte.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-pipeline-DEkjmOte.js","names":[],"sources":["../src/pipeline/define-pipeline.ts"],"sourcesContent":["import { z } from \"zod\";\nexport const pipelineSchema = z.strictObject({\n  prediction: z.discriminatedUnion(\"kind\", [\n    z.strictObject({ kind: z.literal(\"field-classifier\") }),\n    z.strictObject({\n      kind: z.literal(\"token-classifier\"),\n      recipe: z.string().min(1).default(\"./recipe\"),\n      contextRadius: z.number().int().min(1).max(16).optional(),\n      decode: z.string().min(1).default(\"./decode\"),\n    }),\n  ]),\n  acceptance: z\n    .strictObject({\n      minAccuracy: z.number().min(0).max(1).optional(),\n      maxBytes: z.number().positive().optional(),\n    })\n    .optional(),\n});\nexport type Pipeline = z.infer<typeof pipelineSchema>;\nexport function definePipeline(pipeline: Pipeline): Pipeline {\n  return pipelineSchema.parse(pipeline);\n}\n"],"mappings":";;AACA,MAAa,iBAAiB,EAAE,aAAa;CAC3C,YAAY,EAAE,mBAAmB,QAAQ,CACvC,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,kBAAkB,EAAE,CAAC,GACtD,EAAE,aAAa;EACb,MAAM,EAAE,QAAQ,kBAAkB;EAClC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,UAAU;EAC5C,eAAe,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS;EACxD,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,UAAU;CAC9C,CAAC,CACH,CAAC;CACD,YAAY,EACT,aAAa;EACZ,aAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;EAC/C,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC3C,CAAC,CAAC,CACD,SAAS;AACd,CAAC;AAED,SAAgB,eAAe,UAA8B;CAC3D,OAAO,eAAe,MAAM,QAAQ;AACtC"}

package/dist/index.js

@@ -1,4 +1,4 @@
-import { t as definePipeline } from "./define-pipeline-D6I-jAE_.js";
+import { t as definePipeline } from "./define-pipeline-DEkjmOte.js";
 import { t as evaluateSequence } from "./evaluate-sequence-ZCwyomzQ.js";
 import { tokenize } from "@matchbox-ai/core/internal";
 //#region src/models/presets.ts
@@ -20,7 +20,7 @@ function tokenClassifier(options = {}) {
 /** Runs the same validation, selection, and packaging pipeline as the CLI. */
 async function train(target, options = {}) {
 	const { discover } = await import("./project/index.js");
-	const { run } = await import("./run-Y4hS_j62.js");
+	const { run } = await import("./run-MgK9OCQv.js");
 	return run("train", await discover(target), options.onProgress);
 }
 //#endregion

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../src/models/presets.ts","../src/train.ts"],"sourcesContent":["/** Independent field classification over training values. Numeric outputs are finite classes. */\nexport function fieldClassifier() {\n  return { kind: \"field-classifier\" } as const;\n}\n/** Application-owned span supervision and deterministic decoding. Paths are relative to the task directory. */\nexport function tokenClassifier(options: { recipe?: string; decode?: string } = {}) {\n  return { kind: \"token-classifier\", recipe: \"./recipe\", decode: \"./decode\", ...options } as const;\n}\n","/** Runs the same validation, selection, and packaging pipeline as the CLI. */\nexport async function train(\n  target: string,\n  options: { onProgress?: (epoch: number, loss: number) => void } = {},\n) {\n  const { discover } = await import(\"./project/index.js\");\n  const { run } = await import(\"./run.js\");\n  return run(\"train\", await discover(target), options.onProgress);\n}\n"],"mappings":";;;;;AACA,SAAgB,kBAAkB;CAChC,OAAO,EAAE,MAAM,mBAAmB;AACpC;;AAEA,SAAgB,gBAAgB,UAAgD,CAAC,GAAG;CAClF,OAAO;EAAE,MAAM;EAAoB,QAAQ;EAAY,QAAQ;EAAY,GAAG;CAAQ;AACxF;;;;ACNA,eAAsB,MACpB,QACA,UAAkE,CAAC,GACnE;CACA,MAAM,EAAE,aAAa,MAAM,OAAO;CAClC,MAAM,EAAE,QAAQ,MAAM,OAAO;CAC7B,OAAO,IAAI,SAAS,MAAM,SAAS,MAAM,GAAG,QAAQ,UAAU;AAChE"}
+{"version":3,"file":"index.js","names":[],"sources":["../src/models/presets.ts","../src/train.ts"],"sourcesContent":["/** Independent field classification over training values. Numeric outputs are finite classes. */\nexport function fieldClassifier() {\n  return { kind: \"field-classifier\" } as const;\n}\n/** Application-owned span supervision and deterministic decoding. Paths are relative to the task directory. */\nexport function tokenClassifier(\n  options: { recipe?: string; decode?: string; contextRadius?: number } = {},\n) {\n  return { kind: \"token-classifier\", recipe: \"./recipe\", decode: \"./decode\", ...options } as const;\n}\n","/** Runs the same validation, selection, and packaging pipeline as the CLI. */\nexport async function train(\n  target: string,\n  options: { onProgress?: (epoch: number, loss: number) => void } = {},\n) {\n  const { discover } = await import(\"./project/index.js\");\n  const { run } = await import(\"./run.js\");\n  return run(\"train\", await discover(target), options.onProgress);\n}\n"],"mappings":";;;;;AACA,SAAgB,kBAAkB;CAChC,OAAO,EAAE,MAAM,mBAAmB;AACpC;;AAEA,SAAgB,gBACd,UAAwE,CAAC,GACzE;CACA,OAAO;EAAE,MAAM;EAAoB,QAAQ;EAAY,QAAQ;EAAY,GAAG;CAAQ;AACxF;;;;ACRA,eAAsB,MACpB,QACA,UAAkE,CAAC,GACnE;CACA,MAAM,EAAE,aAAa,MAAM,OAAO;CAClC,MAAM,EAAE,QAAQ,MAAM,OAAO;CAC7B,OAAO,IAAI,SAAS,MAAM,SAAS,MAAM,GAAG,QAAQ,UAAU;AAChE"}

package/dist/load-project.d.ts

@@ -13,6 +13,7 @@ export declare function loadProject(path: string): Promise<{
         sequence?: {
             recipe: string;
             decoder: string;
+            contextRadius?: number | undefined;
         } | undefined;
         challenges?: string | undefined;
     };

package/dist/load-project.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"load-project.d.ts","sourceRoot":"","sources":["../src/load-project.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,wBAAsB,WAAW,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;GAkD7C"}
+{"version":3,"file":"load-project.d.ts","sourceRoot":"","sources":["../src/load-project.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,wBAAsB,WAAW,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkD7C"}

package/dist/models/presets.d.ts

@@ -6,7 +6,9 @@ export declare function fieldClassifier(): {
 export declare function tokenClassifier(options?: {
     recipe?: string;
     decode?: string;
+    contextRadius?: number;
 }): {
+    readonly contextRadius?: number;
     readonly kind: "token-classifier";
     recipe: string;
     decode: string;

package/dist/models/presets.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"presets.d.ts","sourceRoot":"","sources":["../../src/models/presets.ts"],"names":[],"mappings":"AAAA,iGAAiG;AACjG,wBAAgB,eAAe;aACpB,IAAI,EAAE,kBAAkB;EAClC;AACD,+GAA+G;AAC/G,wBAAgB,eAAe,CAAC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO;mBACjE,kBAAkB;;;EAClC"}
+{"version":3,"file":"presets.d.ts","sourceRoot":"","sources":["../../src/models/presets.ts"],"names":[],"mappings":"AAAA,iGAAiG;AACjG,wBAAgB,eAAe;aACpB,IAAI,EAAE,kBAAkB;EAClC;AACD,+GAA+G;AAC/G,wBAAgB,eAAe,CAC7B,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAO;6BAAb,MAAM;mBAEpD,kBAAkB;;;EAClC"}

package/dist/models/sequence/fit-sequence.d.ts

@@ -1,9 +1,8 @@
 import type { SequenceArtifact } from "@matchbox-ai/core/internal";
 import type { DatasetExample } from "@matchbox-ai/core";
 import type { SequenceRecipe } from "./types.js";
-export declare function fitSequence(examples: readonly DatasetExample<unknown>[], recipe: SequenceRecipe, metadata: Pick<SequenceArtifact, "taskModule" | "taskMetadata" | "decoderModule">, probes?: readonly string[], progress?: (epoch: number, loss: number) => void): Promise<{
+export declare function fitSequence(examples: readonly DatasetExample<unknown>[], recipe: SequenceRecipe, metadata: Pick<SequenceArtifact, "taskModule" | "taskMetadata" | "decoderModule">, probes?: readonly string[], progress?: (epoch: number, loss: number) => void, contextRadius?: number): Promise<{
     model: {
-        formatVersion: 3;
         engine: "burn-0.21";
         kind: "sequence-parser";
         architecture: "embedding-window-mlp";
@@ -14,11 +13,31 @@ export declare function fitSequence(examples: readonly DatasetExample<unknown>[]
         tokenizer: "characters" | "words";
         vocabulary: string[];
         labels: string[];
+        unknownTokens: "abstain" | "predict";
+        threshold: number;
+        precision: "float32";
+        weights: string;
+        formatVersion: 3;
         radius: 1;
+        casing?: "lowercase" | undefined;
+    } | {
+        engine: "burn-0.21";
+        kind: "sequence-parser";
+        architecture: "embedding-window-mlp";
+        taskModule: string;
+        decoderModule: string;
+        taskMetadata: unknown;
+        readout: "all" | "last";
+        tokenizer: "characters" | "words";
+        vocabulary: string[];
+        labels: string[];
         unknownTokens: "abstain" | "predict";
         threshold: number;
         precision: "float32";
         weights: string;
+        formatVersion: 4;
+        radius: number;
+        casing: "lowercase" | "preserve";
     };
     parameters: number;
     history: number[];

package/dist/models/sequence/fit-sequence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fit-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/fit-sequence.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,SAAS,cAAc,CAAC,OAAO,CAAC,EAAE,EAC5C,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,YAAY,GAAG,cAAc,GAAG,eAAe,CAAC,EACjF,MAAM,GAAE,SAAS,MAAM,EAAO,EAC9B,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDjD"}
+{"version":3,"file":"fit-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/fit-sequence.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,SAAS,cAAc,CAAC,OAAO,CAAC,EAAE,EAC5C,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,YAAY,GAAG,cAAc,GAAG,eAAe,CAAC,EACjF,MAAM,GAAE,SAAS,MAAM,EAAO,EAC9B,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,EAChD,aAAa,SAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqElB"}

package/dist/models/sequence/prepare-supervision.d.ts

@@ -1,10 +1,10 @@
 import type { DatasetExample } from "@matchbox-ai/core";
 import type { SequenceRecipe } from "./types.js";
-export declare function prepareSupervision(examples: readonly DatasetExample<unknown>[], recipe: SequenceRecipe): {
+export declare function prepareSupervision(examples: readonly DatasetExample<unknown>[], recipe: SequenceRecipe, radius?: number): {
     vocabulary: string[];
     radius: number;
     dropout: number;
-    inputs: number[][];
-    labels: number[];
+    inputs: Int32Array<ArrayBuffer>;
+    labels: Int32Array<ArrayBuffer>;
 };
 //# sourceMappingURL=prepare-supervision.d.ts.map

package/dist/models/sequence/prepare-supervision.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prepare-supervision.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/prepare-supervision.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,SAAS,cAAc,CAAC,OAAO,CAAC,EAAE,EAC5C,MAAM,EAAE,cAAc;;;;;;EAqDvB"}
+{"version":3,"file":"prepare-supervision.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/prepare-supervision.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,SAAS,cAAc,CAAC,OAAO,CAAC,EAAE,EAC5C,MAAM,EAAE,cAAc,EACtB,MAAM,SAAI;;;;;;EAkFX"}

package/dist/models/sequence/run-sequence.d.ts

@@ -21,6 +21,11 @@ export declare function runSequence(command: "train" | "eval", project: Awaited<
     report: {
         formatVersion: number;
         architecture: "embedding-window-mlp";
+        encoding: {
+            tokenizer: "characters" | "words";
+            casing: "lowercase" | "preserve" | undefined;
+        };
+        contextRadius: number;
         backend: string;
         seed: number;
         artifactSha256: string;

package/dist/models/sequence/run-sequence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/run-sequence.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAMzD,wBAAsB,WAAW,CAC/B,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,EAChD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAmG5C,KAAK;YACL,UAAU;YACV,UAAU;YACV,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsBT"}
+{"version":3,"file":"run-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/run-sequence.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAMzD,wBAAsB,WAAW,CAC/B,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,EAChD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;YA0FlC,SAAS;YAAuB,MAAM;;;;;;;;;;;;;YAYhD,KAAK;YACL,UAAU;YACV,UAAU;YACV,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsBT"}

package/dist/models/sequence/types.d.ts

@@ -7,6 +7,8 @@ export interface SequenceRecipe {
     /** Train the unknown-token embedding by masking this fraction of training token IDs. */
     tokenDropout?: number;
     tokenizer: "characters" | "words";
+    /** Preserve token casing or lowercase keys. Defaults to lowercase for existing recipes. */
+    casing?: "lowercase" | "preserve";
     readout: "all" | "last";
     labels: readonly string[];
     annotate(example: DatasetExample<unknown>, tokens: readonly Token[]): readonly (string | null)[];

package/dist/models/sequence/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,+FAA+F;AAC/F,MAAM,WAAW,cAAc;IAC7B,iFAAiF;IACjF,UAAU,CAAC,EAAE,SAAS,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,YAAY,GAAG,OAAO,CAAC;IAClC,OAAO,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;CAClG"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,+FAA+F;AAC/F,MAAM,WAAW,cAAc;IAC7B,iFAAiF;IACjF,UAAU,CAAC,EAAE,SAAS,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,YAAY,GAAG,OAAO,CAAC;IAClC,2FAA2F;IAC3F,MAAM,CAAC,EAAE,WAAW,GAAG,UAAU,CAAC;IAClC,OAAO,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;CAClG"}

package/dist/native/fit.d.ts

@@ -6,10 +6,12 @@ interface NativeResult {
 export declare function predict(config: {
     vocabularySize: number;
     labelCount: number;
-}, weights: Uint8Array, inputs: number[][]): number[];
+    contextRadius?: number;
+}, weights: Uint8Array, inputs: number[][] | Int32Array): number[];
 export declare function fit(config: {
     vocabularySize: number;
     labelCount: number;
-}, inputs: number[][], labels: number[], progress?: (epoch: number, loss: number) => void): Promise<NativeResult>;
+    contextRadius?: number;
+}, inputs: number[][] | Int32Array, labels: number[] | Int32Array, progress?: (epoch: number, loss: number) => void): Promise<NativeResult>;
 export {};
 //# sourceMappingURL=fit.d.ts.map

package/dist/native/fit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fit.d.ts","sourceRoot":"","sources":["../../src/native/fit.ts"],"names":[],"mappings":"AACA,UAAU,YAAY;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,wBAAgB,OAAO,CACrB,MAAM,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EACtD,OAAO,EAAE,UAAU,EACnB,MAAM,EAAE,MAAM,EAAE,EAAE,GACjB,MAAM,EAAE,CASV;AACD,wBAAgB,GAAG,CACjB,MAAM,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EACtD,MAAM,EAAE,MAAM,EAAE,EAAE,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GAC/C,OAAO,CAAC,YAAY,CAAC,CAmBvB"}
+{"version":3,"file":"fit.d.ts","sourceRoot":"","sources":["../../src/native/fit.ts"],"names":[],"mappings":"AACA,UAAU,YAAY;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,wBAAgB,OAAO,CACrB,MAAM,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,EAC9E,OAAO,EAAE,UAAU,EACnB,MAAM,EAAE,MAAM,EAAE,EAAE,GAAG,UAAU,GAC9B,MAAM,EAAE,CASV;AACD,wBAAgB,GAAG,CACjB,MAAM,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,EAC9E,MAAM,EAAE,MAAM,EAAE,EAAE,GAAG,UAAU,EAC/B,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,EAC7B,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GAC/C,OAAO,CAAC,YAAY,CAAC,CAmBvB"}

package/dist/pipeline/define-pipeline.d.ts

@@ -5,6 +5,7 @@ export declare const pipelineSchema: z.ZodObject<{
     }, z.core.$strict>, z.ZodObject<{
         kind: z.ZodLiteral<"token-classifier">;
         recipe: z.ZodDefault<z.ZodString>;
+        contextRadius: z.ZodOptional<z.ZodNumber>;
         decode: z.ZodDefault<z.ZodString>;
     }, z.core.$strict>], "kind">;
     acceptance: z.ZodOptional<z.ZodObject<{

package/dist/pipeline/define-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"define-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/define-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,eAAO,MAAM,cAAc;;;;;;;;;;;;kBAezB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAE3D"}
+{"version":3,"file":"define-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/define-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,eAAO,MAAM,cAAc;;;;;;;;;;;;;kBAgBzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAE3D"}

package/dist/project/artifact.d.ts

@@ -19,7 +19,6 @@ export declare function loadArtifact(configPath: string): Promise<{
         precision: "float32";
         weights: string;
     } | {
-        formatVersion: 3;
         engine: "burn-0.21";
         kind: "sequence-parser";
         architecture: "embedding-window-mlp";
@@ -30,11 +29,31 @@ export declare function loadArtifact(configPath: string): Promise<{
         tokenizer: "characters" | "words";
         vocabulary: string[];
         labels: string[];
+        unknownTokens: "abstain" | "predict";
+        threshold: number;
+        precision: "float32";
+        weights: string;
+        formatVersion: 3;
         radius: 1;
+        casing?: "lowercase" | undefined;
+    } | {
+        engine: "burn-0.21";
+        kind: "sequence-parser";
+        architecture: "embedding-window-mlp";
+        taskModule: string;
+        decoderModule: string;
+        taskMetadata: unknown;
+        readout: "all" | "last";
+        tokenizer: "characters" | "words";
+        vocabulary: string[];
+        labels: string[];
         unknownTokens: "abstain" | "predict";
         threshold: number;
         precision: "float32";
         weights: string;
+        formatVersion: 4;
+        radius: number;
+        casing: "lowercase" | "preserve";
     };
     task: ParserDefinition<z.ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>>;
     decode: SequenceDecoder | undefined;
@@ -75,6 +94,7 @@ export declare function loadArtifact(configPath: string): Promise<{
         sequence?: {
             recipe: string;
             decoder: string;
+            contextRadius?: number | undefined;
         } | undefined;
         challenges?: string | undefined;
     };

package/dist/project/artifact.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"artifact.d.ts","sourceRoot":"","sources":["../../src/project/artifact.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAoB1B,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgBhC"}
+{"version":3,"file":"artifact.d.ts","sourceRoot":"","sources":["../../src/project/artifact.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAoB1B,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgBhC"}

package/dist/project/config.d.ts

@@ -10,6 +10,7 @@ export declare const configSchema: z.ZodObject<{
     sequence: z.ZodOptional<z.ZodObject<{
         recipe: z.ZodString;
         decoder: z.ZodString;
+        contextRadius: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strict>>;
     challenges: z.ZodOptional<z.ZodString>;
 }, z.core.$strict>;
@@ -25,6 +26,7 @@ export declare function loadConfig(path: string): Promise<{
         sequence?: {
             recipe: string;
             decoder: string;
+            contextRadius?: number | undefined;
         } | undefined;
         challenges?: string | undefined;
     };

package/dist/project/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/project/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,YAAY;;;;;;;;;;;;;kBAUvB,CAAC;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;GA2C5C"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/project/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,YAAY;;;;;;;;;;;;;;kBAgBvB,CAAC;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;GA4C5C"}

package/dist/project/index.js

@@ -1,2 +1,2 @@
-import { i as loadConfig, n as discover, r as listTasks, t as loadArtifact } from "../project-BR13wXHr.js";
+import { i as loadConfig, n as discover, r as listTasks, t as loadArtifact } from "../project-CgNwV9Dj.js";
 export { discover, listTasks, loadArtifact, loadConfig };

package/dist/{project-BR13wXHr.js → project-CgNwV9Dj.js}

@@ -1,4 +1,4 @@
-import { n as pipelineSchema } from "./define-pipeline-D6I-jAE_.js";
+import { n as pipelineSchema } from "./define-pipeline-DEkjmOte.js";
 import { z } from "zod";
 import { readArtifact, tensorPredictor } from "@matchbox-ai/core/internal";
 import { access, readFile, readdir, stat } from "node:fs/promises";
@@ -39,7 +39,8 @@ const configSchema = z.strictObject({
 	maxBytes: z.number().positive().default(64e3),
 	sequence: z.strictObject({
 		recipe: z.string(),
-		decoder: z.string()
+		decoder: z.string(),
+		contextRadius: z.number().int().min(1).max(16).optional()
 	}).optional(),
 	challenges: z.string().optional()
 });
@@ -56,7 +57,8 @@ async function loadConfig(path) {
 		defaults = { ...pipeline.acceptance };
 		if (pipeline.prediction.kind === "token-classifier") defaults.sequence = {
 			recipe: pipeline.prediction.recipe,
-			decoder: pipeline.prediction.decode
+			decoder: pipeline.prediction.decode,
+			contextRadius: pipeline.prediction.contextRadius
 		};
 	} else if (!authored.sequence) throw new Error(`Missing pipeline.ts or pipeline/pipeline.ts in ${root}. Author an explicit pipeline before training.`);
 	const config = configSchema.parse({
@@ -155,4 +157,4 @@ async function loadArtifact(configPath) {
 //#endregion
 export { loadConfig as i, discover as n, listTasks as r, loadArtifact as t };
 
-//# sourceMappingURL=project-BR13wXHr.js.map
+//# sourceMappingURL=project-CgNwV9Dj.js.map

package/dist/project-CgNwV9Dj.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"project-CgNwV9Dj.js","names":[],"sources":["../src/project/entry.ts","../src/project/config.ts","../src/project/discover.ts","../src/project/artifact.ts"],"sourcesContent":["import { stat } from \"node:fs/promises\";\nimport { basename, extname, resolve } from \"node:path\";\nasync function isFile(path: string): Promise<boolean> {\n  try {\n    return (await stat(path)).isFile();\n  } catch (error) {\n    if ([\"ENOENT\", \"ENOTDIR\"].includes((error as NodeJS.ErrnoException).code ?? \"\")) {\n      return false;\n    }\n    throw error;\n  }\n}\n/** Named task modules have one flat or directory-backed TypeScript entry point. */\nexport async function findEntry(root: string, name: string): Promise<string | undefined> {\n  const base = resolve(root, name);\n  const candidates = [`${base}.ts`, resolve(base, `${basename(base)}.ts`)];\n  const found = (await Promise.all(candidates.map(isFile)))\n    .map((exists, index) => (exists ? candidates[index] : undefined))\n    .filter((path): path is string => path !== undefined);\n  if (found.length > 1) {\n    throw new Error(`Conflicting task entry points: ${found.join(\" and \")}. Keep only one.`);\n  }\n  return found[0];\n}\n/** Explicit filenames stay exact; extensionless references use the named entry convention. */\nexport async function resolveModule(root: string, reference: string): Promise<string> {\n  if (extname(reference)) {\n    return resolve(root, reference);\n  }\n  // Missing training modules are reported when imported, so artifact evaluation needs no recipe.\n  return (await findEntry(root, reference)) ?? resolve(root, `${reference}.ts`);\n}\n","import { access, stat } from \"node:fs/promises\";\nimport { basename, dirname, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { z } from \"zod\";\nimport { findEntry, resolveModule } from \"./entry.js\";\nimport { pipelineSchema } from \"../pipeline/index.js\";\nexport const configSchema = z.strictObject({\n  task: z.string().default(\"./parser\"),\n  train: z.string().default(\"./data/train.jsonl\"),\n  validation: z.string().default(\"./evals/validation.jsonl\"),\n  eval: z.string().default(\"./evals/test.jsonl\"),\n  output: z.string().endsWith(\".matchbox\"),\n  minAccuracy: z.number().min(0).max(1).default(0.95),\n  maxBytes: z.number().positive().default(64000),\n  sequence: z\n    .strictObject({\n      recipe: z.string(),\n      decoder: z.string(),\n      contextRadius: z.number().int().min(1).max(16).optional(),\n    })\n    .optional(),\n  challenges: z.string().optional(),\n});\nexport async function loadConfig(path: string) {\n  const directory = (await stat(path)).isDirectory();\n  const root = directory ? resolve(path) : dirname(resolve(path));\n  const exists = (file: string) =>\n    access(resolve(root, file)).then(\n      () => true,\n      () => false,\n    );\n  const configPath = directory ? resolve(root, \"matchbox.config.ts\") : resolve(path);\n  const authored = await access(configPath).then(\n    async () => (await import(pathToFileURL(configPath).href)).default,\n    () => ({}),\n  );\n  let defaults: Record<string, unknown> = {};\n  const pipelinePath = await findEntry(root, \"pipeline\");\n  if (pipelinePath) {\n    const pipeline = pipelineSchema.parse((await import(pathToFileURL(pipelinePath).href)).default);\n    defaults = { ...pipeline.acceptance };\n    if (pipeline.prediction.kind === \"token-classifier\") {\n      defaults.sequence = {\n        recipe: pipeline.prediction.recipe,\n        decoder: pipeline.prediction.decode,\n        contextRadius: pipeline.prediction.contextRadius,\n      };\n    }\n  } else if (!authored.sequence) {\n    throw new Error(\n      `Missing pipeline.ts or pipeline/pipeline.ts in ${root}. Author an explicit pipeline before training.`,\n    );\n  }\n  const config = configSchema.parse({\n    output: resolve(root, \"../../.matchbox\", basename(root), \"model.matchbox\"),\n    ...defaults,\n    ...authored,\n  });\n  if (!config.challenges && (await exists(\"evals/challenges.json\"))) {\n    config.challenges = \"./evals/challenges.json\";\n  }\n  config.task = await resolveModule(root, config.task);\n  if (config.sequence) {\n    config.sequence.recipe = await resolveModule(root, config.sequence.recipe);\n    config.sequence.decoder = await resolveModule(root, config.sequence.decoder);\n  }\n  return { config, root, pipelinePath };\n}\n","import { access, readdir, stat } from \"node:fs/promises\";\nimport { findEntry } from \"./entry.js\";\nimport { basename, dirname, resolve } from \"node:path\";\nconst exists = (path: string) =>\n  access(path).then(\n    () => true,\n    () => false,\n  );\nasync function select(root: string): Promise<string | undefined> {\n  // A parser/parser.ts module directory is inside its task, not a second task.\n  if (\n    basename(root) === \"parser\" &&\n    (await findEntry(dirname(root), \"pipeline\")) &&\n    (await findEntry(dirname(root), \"parser\")) === resolve(root, \"parser.ts\")\n  ) {\n    return dirname(root);\n  }\n  if (await findEntry(root, \"parser\")) {\n    return root;\n  }\n  if (await exists(resolve(root, \"matchbox.config.ts\"))) {\n    return resolve(root, \"matchbox.config.ts\");\n  }\n  const tasks = await listTasks(root);\n  const directory = resolve(root, \"matchbox\");\n  if (tasks.length > 1) {\n    throw new Error(\n      `Choose a task: ${tasks.join(\", \")}. For example, matchbox-ai train ${tasks[0]}.`,\n    );\n  }\n  return tasks.length ? resolve(directory, tasks[0]!) : undefined;\n}\nexport async function discover(target?: string, cwd = process.cwd()): Promise<string> {\n  if (target && (await exists(resolve(cwd, target)))) {\n    const path = resolve(cwd, target);\n    if ((await stat(path)).isFile()) {\n      return path;\n    }\n    const selected = await select(path);\n    if (selected) {\n      return selected;\n    }\n    throw new Error(`No Matchbox task found in ${path}.`);\n  }\n  let root = resolve(cwd);\n  for (;;) {\n    if (target) {\n      const task = resolve(root, \"matchbox\", target);\n      if (await findEntry(task, \"parser\")) {\n        return task;\n      }\n    } else {\n      const selected = await select(root);\n      if (selected) {\n        return selected;\n      }\n    }\n    const parent = dirname(root);\n    if (parent === root) {\n      throw new Error(\n        target\n          ? `Task not found: ${target}.`\n          : \"No Matchbox project found. Run matchbox-ai init <name>.\",\n      );\n    }\n    root = parent;\n  }\n}\n\n/** Lists conventional tasks immediately beneath an application’s matchbox directory. */\nexport async function listTasks(root: string): Promise<string[]> {\n  const directory = resolve(root, \"matchbox\");\n  const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);\n  const tasks: string[] = [];\n  for (const entry of entries) {\n    if (entry.isDirectory() && (await findEntry(resolve(directory, entry.name), \"parser\"))) {\n      tasks.push(entry.name);\n    }\n  }\n  return tasks.sort();\n}\n","import { readFile } from \"node:fs/promises\";\nimport { dirname, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { createParser } from \"@matchbox-ai/core/runtime\";\nimport { readArtifact, tensorPredictor } from \"@matchbox-ai/core/internal\";\nimport type { SequenceDecoder } from \"@matchbox-ai/core/runtime\";\nimport type { ParserDefinition } from \"@matchbox-ai/core\";\nimport type { z } from \"zod\";\nimport { loadConfig } from \"./config.js\";\nexport async function loadArtifact(configPath: string) {\n  const { config, root } = await loadConfig(configPath);\n  const output = resolve(root, config.output);\n  const text = await readFile(output, \"utf8\").catch(() => {\n    throw new Error(`No readable model at ${output}. Run matchbox-ai train first.`);\n  });\n  const artifact = readArtifact(JSON.parse(text));\n  const task: ParserDefinition<z.ZodType> = (\n    await import(pathToFileURL(resolve(dirname(output), artifact.taskModule)).href)\n  ).default;\n  const decode: SequenceDecoder | undefined =\n    artifact.decoderModule === null\n      ? undefined\n      : (await import(pathToFileURL(resolve(dirname(output), artifact.decoderModule)).href))\n          .default;\n  return {\n    artifact,\n    task,\n    decode,\n    parser: createParser(artifact, task, decode),\n    inspect: async (input: string) => {\n      const predictor = await tensorPredictor(artifact);\n      try {\n        if (artifact.kind === \"record-parser\") {\n          return predictor.record(input);\n        }\n        const tokens = predictor.sequence(input);\n        return { tokens, candidate: decode!(tokens, input) };\n      } finally {\n        predictor.dispose();\n      }\n    },\n    output,\n    config,\n    root,\n  };\n}\n"],"mappings":";;;;;;;;AAEA,eAAe,OAAO,MAAgC;CACpD,IAAI;EACF,QAAQ,MAAM,KAAK,IAAI,EAAA,CAAG,OAAO;CACnC,SAAS,OAAO;EACd,IAAI,CAAC,UAAU,SAAS,CAAC,CAAC,SAAU,MAAgC,QAAQ,EAAE,GAC5E,OAAO;EAET,MAAM;CACR;AACF;;AAEA,eAAsB,UAAU,MAAc,MAA2C;CACvF,MAAM,OAAO,QAAQ,MAAM,IAAI;CAC/B,MAAM,aAAa,CAAC,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG,SAAS,IAAI,EAAE,IAAI,CAAC;CACvE,MAAM,SAAS,MAAM,QAAQ,IAAI,WAAW,IAAI,MAAM,CAAC,EAAA,CACpD,KAAK,QAAQ,UAAW,SAAS,WAAW,SAAS,KAAA,CAAU,CAAC,CAChE,QAAQ,SAAyB,SAAS,KAAA,CAAS;CACtD,IAAI,MAAM,SAAS,GACjB,MAAM,IAAI,MAAM,kCAAkC,MAAM,KAAK,OAAO,EAAE,iBAAiB;CAEzF,OAAO,MAAM;AACf;;AAEA,eAAsB,cAAc,MAAc,WAAoC;CACpF,IAAI,QAAQ,SAAS,GACnB,OAAO,QAAQ,MAAM,SAAS;CAGhC,OAAQ,MAAM,UAAU,MAAM,SAAS,KAAM,QAAQ,MAAM,GAAG,UAAU,IAAI;AAC9E;;;ACzBA,MAAa,eAAe,EAAE,aAAa;CACzC,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,UAAU;CACnC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,oBAAoB;CAC9C,YAAY,EAAE,OAAO,CAAC,CAAC,QAAQ,0BAA0B;CACzD,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,oBAAoB;CAC7C,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,WAAW;CACvC,aAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAI;CAClD,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,IAAK;CAC7C,UAAU,EACP,aAAa;EACZ,QAAQ,EAAE,OAAO;EACjB,SAAS,EAAE,OAAO;EAClB,eAAe,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS;CAC1D,CAAC,CAAC,CACD,SAAS;CACZ,YAAY,EAAE,OAAO,CAAC,CAAC,SAAS;AAClC,CAAC;AACD,eAAsB,WAAW,MAAc;CAC7C,MAAM,aAAa,MAAM,KAAK,IAAI,EAAA,CAAG,YAAY;CACjD,MAAM,OAAO,YAAY,QAAQ,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC;CAC9D,MAAM,UAAU,SACd,OAAO,QAAQ,MAAM,IAAI,CAAC,CAAC,CAAC,WACpB,YACA,KACR;CACF,MAAM,aAAa,YAAY,QAAQ,MAAM,oBAAoB,IAAI,QAAQ,IAAI;CACjF,MAAM,WAAW,MAAM,OAAO,UAAU,CAAC,CAAC,KACxC,aAAa,MAAM,OAAO,cAAc,UAAU,CAAC,CAAC,MAAA,CAAO,gBACpD,CAAC,EACV;CACA,IAAI,WAAoC,CAAC;CACzC,MAAM,eAAe,MAAM,UAAU,MAAM,UAAU;CACrD,IAAI,cAAc;EAChB,MAAM,WAAW,eAAe,OAAO,MAAM,OAAO,cAAc,YAAY,CAAC,CAAC,MAAA,CAAO,OAAO;EAC9F,WAAW,EAAE,GAAG,SAAS,WAAW;EACpC,IAAI,SAAS,WAAW,SAAS,oBAC/B,SAAS,WAAW;GAClB,QAAQ,SAAS,WAAW;GAC5B,SAAS,SAAS,WAAW;GAC7B,eAAe,SAAS,WAAW;EACrC;CAEJ,OAAO,IAAI,CAAC,SAAS,UACnB,MAAM,IAAI,MACR,kDAAkD,KAAK,+CACzD;CAEF,MAAM,SAAS,aAAa,MAAM;EAChC,QAAQ,QAAQ,MAAM,mBAAmB,SAAS,IAAI,GAAG,gBAAgB;EACzE,GAAG;EACH,GAAG;CACL,CAAC;CACD,IAAI,CAAC,OAAO,cAAe,MAAM,OAAO,uBAAuB,GAC7D,OAAO,aAAa;CAEtB,OAAO,OAAO,MAAM,cAAc,MAAM,OAAO,IAAI;CACnD,IAAI,OAAO,UAAU;EACnB,OAAO,SAAS,SAAS,MAAM,cAAc,MAAM,OAAO,SAAS,MAAM;EACzE,OAAO,SAAS,UAAU,MAAM,cAAc,MAAM,OAAO,SAAS,OAAO;CAC7E;CACA,OAAO;EAAE;EAAQ;EAAM;CAAa;AACtC;;;AChEA,MAAM,UAAU,SACd,OAAO,IAAI,CAAC,CAAC,WACL,YACA,KACR;AACF,eAAe,OAAO,MAA2C;CAE/D,IACE,SAAS,IAAI,MAAM,YAClB,MAAM,UAAU,QAAQ,IAAI,GAAG,UAAU,KACzC,MAAM,UAAU,QAAQ,IAAI,GAAG,QAAQ,MAAO,QAAQ,MAAM,WAAW,GAExE,OAAO,QAAQ,IAAI;CAErB,IAAI,MAAM,UAAU,MAAM,QAAQ,GAChC,OAAO;CAET,IAAI,MAAM,OAAO,QAAQ,MAAM,oBAAoB,CAAC,GAClD,OAAO,QAAQ,MAAM,oBAAoB;CAE3C,MAAM,QAAQ,MAAM,UAAU,IAAI;CAClC,MAAM,YAAY,QAAQ,MAAM,UAAU;CAC1C,IAAI,MAAM,SAAS,GACjB,MAAM,IAAI,MACR,kBAAkB,MAAM,KAAK,IAAI,EAAE,mCAAmC,MAAM,GAAG,EACjF;CAEF,OAAO,MAAM,SAAS,QAAQ,WAAW,MAAM,EAAG,IAAI,KAAA;AACxD;AACA,eAAsB,SAAS,QAAiB,MAAM,QAAQ,IAAI,GAAoB;CACpF,IAAI,UAAW,MAAM,OAAO,QAAQ,KAAK,MAAM,CAAC,GAAI;EAClD,MAAM,OAAO,QAAQ,KAAK,MAAM;EAChC,KAAK,MAAM,KAAK,IAAI,EAAA,CAAG,OAAO,GAC5B,OAAO;EAET,MAAM,WAAW,MAAM,OAAO,IAAI;EAClC,IAAI,UACF,OAAO;EAET,MAAM,IAAI,MAAM,6BAA6B,KAAK,EAAE;CACtD;CACA,IAAI,OAAO,QAAQ,GAAG;CACtB,SAAS;EACP,IAAI,QAAQ;GACV,MAAM,OAAO,QAAQ,MAAM,YAAY,MAAM;GAC7C,IAAI,MAAM,UAAU,MAAM,QAAQ,GAChC,OAAO;EAEX,OAAO;GACL,MAAM,WAAW,MAAM,OAAO,IAAI;GAClC,IAAI,UACF,OAAO;EAEX;EACA,MAAM,SAAS,QAAQ,IAAI;EAC3B,IAAI,WAAW,MACb,MAAM,IAAI,MACR,SACI,mBAAmB,OAAO,KAC1B,yDACN;EAEF,OAAO;CACT;AACF;;AAGA,eAAsB,UAAU,MAAiC;CAC/D,MAAM,YAAY,QAAQ,MAAM,UAAU;CAC1C,MAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChF,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,YAAY,KAAM,MAAM,UAAU,QAAQ,WAAW,MAAM,IAAI,GAAG,QAAQ,GAClF,MAAM,KAAK,MAAM,IAAI;CAGzB,OAAO,MAAM,KAAK;AACpB;;;ACvEA,eAAsB,aAAa,YAAoB;CACrD,MAAM,EAAE,QAAQ,SAAS,MAAM,WAAW,UAAU;CACpD,MAAM,SAAS,QAAQ,MAAM,OAAO,MAAM;CAC1C,MAAM,OAAO,MAAM,SAAS,QAAQ,MAAM,CAAC,CAAC,YAAY;EACtD,MAAM,IAAI,MAAM,wBAAwB,OAAO,+BAA+B;CAChF,CAAC;CACD,MAAM,WAAW,aAAa,KAAK,MAAM,IAAI,CAAC;CAC9C,MAAM,QACJ,MAAM,OAAO,cAAc,QAAQ,QAAQ,MAAM,GAAG,SAAS,UAAU,CAAC,CAAC,CAAC,MAAA,CAC1E;CACF,MAAM,SACJ,SAAS,kBAAkB,OACvB,KAAA,KACC,MAAM,OAAO,cAAc,QAAQ,QAAQ,MAAM,GAAG,SAAS,aAAa,CAAC,CAAC,CAAC,MAAA,CAC3E;CACT,OAAO;EACL;EACA;EACA;EACA,QAAQ,aAAa,UAAU,MAAM,MAAM;EAC3C,SAAS,OAAO,UAAkB;GAChC,MAAM,YAAY,MAAM,gBAAgB,QAAQ;GAChD,IAAI;IACF,IAAI,SAAS,SAAS,iBACpB,OAAO,UAAU,OAAO,KAAK;IAE/B,MAAM,SAAS,UAAU,SAAS,KAAK;IACvC,OAAO;KAAE;KAAQ,WAAW,OAAQ,QAAQ,KAAK;IAAE;GACrD,UAAU;IACR,UAAU,QAAQ;GACpB;EACF;EACA;EACA;EACA;CACF;AACF"}

package/dist/{run-Y4hS_j62.js → run-MgK9OCQv.js}

@@ -1,4 +1,4 @@
-import { i as loadConfig } from "./project-BR13wXHr.js";
+import { i as loadConfig } from "./project-CgNwV9Dj.js";
 import "zod";
 import { readFile } from "node:fs/promises";
 import { resolve } from "node:path";
@@ -57,7 +57,7 @@ async function loadProject(path) {
 async function run(command, path, progress) {
 	const project = await loadProject(path);
 	if (project.config.sequence) {
-		const { runSequence } = await import("./sequence-CZLL5c0A.js");
+		const { runSequence } = await import("./sequence-NevJFVwZ.js");
 		return runSequence(command, project, progress);
 	}
 	if (command !== "train") throw new Error("Use the eval CLI command to evaluate a saved artifact.");
@@ -67,4 +67,4 @@ async function run(command, path, progress) {
 //#endregion
 export { run };
 
-//# sourceMappingURL=run-Y4hS_j62.js.map
+//# sourceMappingURL=run-MgK9OCQv.js.map

package/dist/{run-Y4hS_j62.js.map → run-MgK9OCQv.js.map}

@@ -1 +1 @@
-{"version":3,"file":"run-Y4hS_j62.js","names":[],"sources":["../src/load-project.ts","../src/run.ts"],"sourcesContent":["import { loadConfig } from \"./project/index.js\";\nimport { readFile } from \"node:fs/promises\";\nimport { resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { z } from \"zod\";\nimport { parseDatasets } from \"@matchbox-ai/core\";\nimport type { ParserDefinition } from \"@matchbox-ai/core\";\nexport async function loadProject(path: string) {\n  const { config, root } = await loadConfig(path);\n  const taskPath = resolve(root, config.task);\n  const task: ParserDefinition<z.ZodType> = (await import(pathToFileURL(taskPath).href)).default;\n  const sources = await Promise.all(\n    [config.train, config.validation, config.eval].map(async (source) => ({\n      source,\n      text: await readFile(resolve(root, source), \"utf8\"),\n    })),\n  );\n  const training = parseDatasets(task, { formatVersion: 1, train: sources[0]!, eval: sources[1]! });\n  const evaluation = parseDatasets(task, {\n    formatVersion: 1,\n    train: sources[0]!,\n    eval: sources[2]!,\n  });\n  for (const result of [training, evaluation]) {\n    if (!result.success) {\n      throw new Error(\n        result.issues\n          .map((issue) => `${issue.source}:${issue.line} ${issue.path.join(\".\")}: ${issue.message}`)\n          .join(\"\\n\"),\n      );\n    }\n  }\n  if (!training.success || !evaluation.success) {\n    throw new Error(\"Invalid datasets.\");\n  }\n  const groups = [training.data.train, training.data.eval, evaluation.data.eval];\n  const seen = new Map<string, number>();\n  groups.forEach((group, split) =>\n    group.forEach((row) => {\n      const key = row.input.trim().toLowerCase();\n      if (seen.has(key) && seen.get(key) !== split) {\n        throw new Error(`Input overlaps dataset splits: ${row.input}`);\n      }\n      seen.set(key, split);\n    }),\n  );\n  return {\n    root,\n    config,\n    task,\n    taskPath,\n    output: resolve(root, config.output),\n    sources,\n    train: training.data.train,\n    validation: training.data.eval,\n    evaluation: evaluation.data.eval,\n  };\n}\n","import { loadProject } from \"./load-project.js\";\nexport async function run(\n  command: \"train\" | \"eval\",\n  path: string,\n  progress?: (epoch: number, loss: number) => void,\n) {\n  const project = await loadProject(path);\n  if (project.config.sequence) {\n    const { runSequence } = await import(\"./models/sequence/index.js\");\n    return runSequence(command, project, progress);\n  }\n  if (command !== \"train\") {\n    throw new Error(\"Use the eval CLI command to evaluate a saved artifact.\");\n  }\n  const { runRecord } = await import(\"./models/record/index.js\");\n  return runRecord(project, progress);\n}\n"],"mappings":";;;;;;;AAOA,eAAsB,YAAY,MAAc;CAC9C,MAAM,EAAE,QAAQ,SAAS,MAAM,WAAW,IAAI;CAC9C,MAAM,WAAW,QAAQ,MAAM,OAAO,IAAI;CAC1C,MAAM,QAAqC,MAAM,OAAO,cAAc,QAAQ,CAAC,CAAC,MAAA,CAAO;CACvF,MAAM,UAAU,MAAM,QAAQ,IAC5B;EAAC,OAAO;EAAO,OAAO;EAAY,OAAO;CAAI,CAAC,CAAC,IAAI,OAAO,YAAY;EACpE;EACA,MAAM,MAAM,SAAS,QAAQ,MAAM,MAAM,GAAG,MAAM;CACpD,EAAE,CACJ;CACA,MAAM,WAAW,cAAc,MAAM;EAAE,eAAe;EAAG,OAAO,QAAQ;EAAK,MAAM,QAAQ;CAAI,CAAC;CAChG,MAAM,aAAa,cAAc,MAAM;EACrC,eAAe;EACf,OAAO,QAAQ;EACf,MAAM,QAAQ;CAChB,CAAC;CACD,KAAK,MAAM,UAAU,CAAC,UAAU,UAAU,GACxC,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,MACR,OAAO,OACJ,KAAK,UAAU,GAAG,MAAM,OAAO,GAAG,MAAM,KAAK,GAAG,MAAM,KAAK,KAAK,GAAG,EAAE,IAAI,MAAM,SAAS,CAAC,CACzF,KAAK,IAAI,CACd;CAGJ,IAAI,CAAC,SAAS,WAAW,CAAC,WAAW,SACnC,MAAM,IAAI,MAAM,mBAAmB;CAErC,MAAM,SAAS;EAAC,SAAS,KAAK;EAAO,SAAS,KAAK;EAAM,WAAW,KAAK;CAAI;CAC7E,MAAM,uBAAO,IAAI,IAAoB;CACrC,OAAO,SAAS,OAAO,UACrB,MAAM,SAAS,QAAQ;EACrB,MAAM,MAAM,IAAI,MAAM,KAAK,CAAC,CAAC,YAAY;EACzC,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,MAAM,OACrC,MAAM,IAAI,MAAM,kCAAkC,IAAI,OAAO;EAE/D,KAAK,IAAI,KAAK,KAAK;CACrB,CAAC,CACH;CACA,OAAO;EACL;EACA;EACA;EACA;EACA,QAAQ,QAAQ,MAAM,OAAO,MAAM;EACnC;EACA,OAAO,SAAS,KAAK;EACrB,YAAY,SAAS,KAAK;EAC1B,YAAY,WAAW,KAAK;CAC9B;AACF;;;ACxDA,eAAsB,IACpB,SACA,MACA,UACA;CACA,MAAM,UAAU,MAAM,YAAY,IAAI;CACtC,IAAI,QAAQ,OAAO,UAAU;EAC3B,MAAM,EAAE,gBAAgB,MAAM,OAAO;EACrC,OAAO,YAAY,SAAS,SAAS,QAAQ;CAC/C;CACA,IAAI,YAAY,SACd,MAAM,IAAI,MAAM,wDAAwD;CAE1E,MAAM,EAAE,cAAc,MAAM,OAAO;CACnC,OAAO,UAAU,SAAS,QAAQ;AACpC"}
+{"version":3,"file":"run-MgK9OCQv.js","names":[],"sources":["../src/load-project.ts","../src/run.ts"],"sourcesContent":["import { loadConfig } from \"./project/index.js\";\nimport { readFile } from \"node:fs/promises\";\nimport { resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { z } from \"zod\";\nimport { parseDatasets } from \"@matchbox-ai/core\";\nimport type { ParserDefinition } from \"@matchbox-ai/core\";\nexport async function loadProject(path: string) {\n  const { config, root } = await loadConfig(path);\n  const taskPath = resolve(root, config.task);\n  const task: ParserDefinition<z.ZodType> = (await import(pathToFileURL(taskPath).href)).default;\n  const sources = await Promise.all(\n    [config.train, config.validation, config.eval].map(async (source) => ({\n      source,\n      text: await readFile(resolve(root, source), \"utf8\"),\n    })),\n  );\n  const training = parseDatasets(task, { formatVersion: 1, train: sources[0]!, eval: sources[1]! });\n  const evaluation = parseDatasets(task, {\n    formatVersion: 1,\n    train: sources[0]!,\n    eval: sources[2]!,\n  });\n  for (const result of [training, evaluation]) {\n    if (!result.success) {\n      throw new Error(\n        result.issues\n          .map((issue) => `${issue.source}:${issue.line} ${issue.path.join(\".\")}: ${issue.message}`)\n          .join(\"\\n\"),\n      );\n    }\n  }\n  if (!training.success || !evaluation.success) {\n    throw new Error(\"Invalid datasets.\");\n  }\n  const groups = [training.data.train, training.data.eval, evaluation.data.eval];\n  const seen = new Map<string, number>();\n  groups.forEach((group, split) =>\n    group.forEach((row) => {\n      const key = row.input.trim().toLowerCase();\n      if (seen.has(key) && seen.get(key) !== split) {\n        throw new Error(`Input overlaps dataset splits: ${row.input}`);\n      }\n      seen.set(key, split);\n    }),\n  );\n  return {\n    root,\n    config,\n    task,\n    taskPath,\n    output: resolve(root, config.output),\n    sources,\n    train: training.data.train,\n    validation: training.data.eval,\n    evaluation: evaluation.data.eval,\n  };\n}\n","import { loadProject } from \"./load-project.js\";\nexport async function run(\n  command: \"train\" | \"eval\",\n  path: string,\n  progress?: (epoch: number, loss: number) => void,\n) {\n  const project = await loadProject(path);\n  if (project.config.sequence) {\n    const { runSequence } = await import(\"./models/sequence/index.js\");\n    return runSequence(command, project, progress);\n  }\n  if (command !== \"train\") {\n    throw new Error(\"Use the eval CLI command to evaluate a saved artifact.\");\n  }\n  const { runRecord } = await import(\"./models/record/index.js\");\n  return runRecord(project, progress);\n}\n"],"mappings":";;;;;;;AAOA,eAAsB,YAAY,MAAc;CAC9C,MAAM,EAAE,QAAQ,SAAS,MAAM,WAAW,IAAI;CAC9C,MAAM,WAAW,QAAQ,MAAM,OAAO,IAAI;CAC1C,MAAM,QAAqC,MAAM,OAAO,cAAc,QAAQ,CAAC,CAAC,MAAA,CAAO;CACvF,MAAM,UAAU,MAAM,QAAQ,IAC5B;EAAC,OAAO;EAAO,OAAO;EAAY,OAAO;CAAI,CAAC,CAAC,IAAI,OAAO,YAAY;EACpE;EACA,MAAM,MAAM,SAAS,QAAQ,MAAM,MAAM,GAAG,MAAM;CACpD,EAAE,CACJ;CACA,MAAM,WAAW,cAAc,MAAM;EAAE,eAAe;EAAG,OAAO,QAAQ;EAAK,MAAM,QAAQ;CAAI,CAAC;CAChG,MAAM,aAAa,cAAc,MAAM;EACrC,eAAe;EACf,OAAO,QAAQ;EACf,MAAM,QAAQ;CAChB,CAAC;CACD,KAAK,MAAM,UAAU,CAAC,UAAU,UAAU,GACxC,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,MACR,OAAO,OACJ,KAAK,UAAU,GAAG,MAAM,OAAO,GAAG,MAAM,KAAK,GAAG,MAAM,KAAK,KAAK,GAAG,EAAE,IAAI,MAAM,SAAS,CAAC,CACzF,KAAK,IAAI,CACd;CAGJ,IAAI,CAAC,SAAS,WAAW,CAAC,WAAW,SACnC,MAAM,IAAI,MAAM,mBAAmB;CAErC,MAAM,SAAS;EAAC,SAAS,KAAK;EAAO,SAAS,KAAK;EAAM,WAAW,KAAK;CAAI;CAC7E,MAAM,uBAAO,IAAI,IAAoB;CACrC,OAAO,SAAS,OAAO,UACrB,MAAM,SAAS,QAAQ;EACrB,MAAM,MAAM,IAAI,MAAM,KAAK,CAAC,CAAC,YAAY;EACzC,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,MAAM,OACrC,MAAM,IAAI,MAAM,kCAAkC,IAAI,OAAO;EAE/D,KAAK,IAAI,KAAK,KAAK;CACrB,CAAC,CACH;CACA,OAAO;EACL;EACA;EACA;EACA;EACA,QAAQ,QAAQ,MAAM,OAAO,MAAM;EACnC;EACA,OAAO,SAAS,KAAK;EACrB,YAAY,SAAS,KAAK;EAC1B,YAAY,WAAW,KAAK;CAC9B;AACF;;;ACxDA,eAAsB,IACpB,SACA,MACA,UACA;CACA,MAAM,UAAU,MAAM,YAAY,IAAI;CACtC,IAAI,QAAQ,OAAO,UAAU;EAC3B,MAAM,EAAE,gBAAgB,MAAM,OAAO;EACrC,OAAO,YAAY,SAAS,SAAS,QAAQ;CAC/C;CACA,IAAI,YAAY,SACd,MAAM,IAAI,MAAM,wDAAwD;CAE1E,MAAM,EAAE,cAAc,MAAM,OAAO;CACnC,OAAO,UAAU,SAAS,QAAQ;AACpC"}

package/dist/run.d.ts

@@ -20,6 +20,11 @@ export declare function run(command: "train" | "eval", path: string, progress?:
     report: {
         formatVersion: number;
         architecture: "embedding-window-mlp";
+        encoding: {
+            tokenizer: "characters" | "words";
+            casing: "lowercase" | "preserve" | undefined;
+        };
+        contextRadius: number;
         backend: string;
         seed: number;
         artifactSha256: string;

package/dist/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AACA,wBAAsB,GAAG,CACvB,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAYjD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AACA,wBAAsB,GAAG,CACvB,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAYjD"}