@matchbox-ai/train 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +1 -1
- package/THIRD-PARTY-NOTICES.md +1 -1
- package/dist/define-pipeline-D6I-jAE_.js +20 -0
- package/dist/define-pipeline-D6I-jAE_.js.map +1 -0
- package/dist/encoders/index.d.ts +0 -1
- package/dist/encoders/index.d.ts.map +1 -1
- package/dist/evaluate-sequence-ZCwyomzQ.js.map +1 -1
- package/dist/evaluation/evaluate-sequence.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -11
- package/dist/index.js.map +1 -1
- package/dist/load-project.d.ts +0 -4
- package/dist/load-project.d.ts.map +1 -1
- package/dist/models/presets.d.ts +6 -6
- package/dist/models/presets.d.ts.map +1 -1
- package/dist/models/record/fit.d.ts +6 -99
- package/dist/models/record/fit.d.ts.map +1 -1
- package/dist/models/record/run.d.ts +1 -57
- package/dist/models/record/run.d.ts.map +1 -1
- package/dist/models/record/verify-export.d.ts +1 -2
- package/dist/models/record/verify-export.d.ts.map +1 -1
- package/dist/models/sequence/fit-sequence.d.ts +9 -116
- package/dist/models/sequence/fit-sequence.d.ts.map +1 -1
- package/dist/models/sequence/prepare-supervision.d.ts +10 -0
- package/dist/models/sequence/prepare-supervision.d.ts.map +1 -0
- package/dist/models/sequence/run-sequence.d.ts +3 -71
- package/dist/models/sequence/run-sequence.d.ts.map +1 -1
- package/dist/models/sequence/types.d.ts +4 -0
- package/dist/models/sequence/types.d.ts.map +1 -1
- package/dist/native/fit.d.ts +15 -0
- package/dist/native/fit.d.ts.map +1 -0
- package/dist/native/index.d.ts +3 -0
- package/dist/native/index.d.ts.map +1 -0
- package/dist/native/record.d.ts +12 -0
- package/dist/native/record.d.ts.map +1 -0
- package/dist/package-model-CPj0qnQs.js.map +1 -1
- package/dist/packaging/package-model.d.ts.map +1 -1
- package/dist/pipeline/define-pipeline.d.ts +2 -5
- package/dist/pipeline/define-pipeline.d.ts.map +1 -1
- package/dist/project/artifact.d.ts +21 -67
- package/dist/project/artifact.d.ts.map +1 -1
- package/dist/project/config.d.ts +1 -4
- package/dist/project/config.d.ts.map +1 -1
- package/dist/project/discover.d.ts +2 -0
- package/dist/project/discover.d.ts.map +1 -1
- package/dist/project/entry.d.ts +5 -0
- package/dist/project/entry.d.ts.map +1 -0
- package/dist/project/index.d.ts +1 -1
- package/dist/project/index.d.ts.map +1 -1
- package/dist/project/index.js +2 -2
- package/dist/{project-BP331m20.js → project-BR13wXHr.js} +51 -18
- package/dist/project-BR13wXHr.js.map +1 -0
- package/dist/{record-BTZ5LWHF.js → record-MCYVi9O6.js} +60 -114
- package/dist/record-MCYVi9O6.js.map +1 -0
- package/dist/{run-Cwa7vD_T.js → run-PB_jiKnw.js} +4 -6
- package/dist/run-PB_jiKnw.js.map +1 -0
- package/dist/run.d.ts +4 -128
- package/dist/run.d.ts.map +1 -1
- package/dist/sequence-DGoGU7kG.js +219 -0
- package/dist/sequence-DGoGU7kG.js.map +1 -0
- package/dist/train.d.ts +4 -128
- package/dist/train.d.ts.map +1 -1
- package/dist/types.d.ts +0 -2
- package/dist/types.d.ts.map +1 -1
- package/docs/README.md +12 -7
- package/docs/agent-skill.md +8 -4
- package/docs/cli.md +85 -28
- package/docs/contributing.md +43 -0
- package/docs/dataset-format.md +4 -2
- package/docs/evaluation.md +66 -5
- package/docs/example-evaluation.md +47 -0
- package/docs/example-results.json +280 -0
- package/docs/examples/money.md +19 -0
- package/docs/examples/time.md +43 -0
- package/docs/getting-started.md +48 -54
- package/docs/native-packages.md +31 -0
- package/docs/neural-training.md +1 -1
- package/docs/parser-api.md +16 -16
- package/docs/pipelines.md +5 -6
- package/docs/primitives/README.md +12 -11
- package/docs/project-structure.md +17 -3
- package/docs/react.md +49 -7
- package/docs/reference/README.md +25 -0
- package/docs/reference/configuration.md +43 -0
- package/docs/reference/evaluation.md +21 -0
- package/docs/reference/pipeline.md +43 -0
- package/docs/reference/runtime.md +56 -0
- package/docs/reference/supervision.md +87 -0
- package/docs/reference/training.md +26 -0
- package/docs/reference/vite.md +20 -0
- package/docs/releases.md +6 -16
- package/docs/repository-design.md +2 -2
- package/docs/runtime-backends.md +8 -11
- package/docs/training.md +73 -0
- package/docs/website-deployment.md +11 -0
- package/native.cjs +1 -0
- package/package.json +10 -5
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/darwin-x64/node.napi.node +0 -0
- package/prebuilds/linux-arm64/node.napi.glibc.node +0 -0
- package/prebuilds/linux-arm64/node.napi.musl.node +0 -0
- package/prebuilds/linux-x64/node.napi.glibc.node +0 -0
- package/prebuilds/linux-x64/node.napi.musl.node +0 -0
- package/prebuilds/win32-arm64/node.napi.node +0 -0
- package/prebuilds/win32-x64/node.napi.node +0 -0
- package/dist/define-pipeline-Colvwt7u.js +0 -32
- package/dist/define-pipeline-Colvwt7u.js.map +0 -1
- package/dist/encoders/word-tokens.d.ts +0 -5
- package/dist/encoders/word-tokens.d.ts.map +0 -1
- package/dist/models/sequence/create-network.d.ts +0 -3
- package/dist/models/sequence/create-network.d.ts.map +0 -1
- package/dist/project-BP331m20.js.map +0 -1
- package/dist/record-BTZ5LWHF.js.map +0 -1
- package/dist/run-Cwa7vD_T.js.map +0 -1
- package/dist/sequence-C_WfVGO1.js +0 -251
- package/dist/sequence-C_WfVGO1.js.map +0 -1
- package/docs/filter-research.md +0 -63
|
@@ -32,23 +32,11 @@ export declare function runSequence(command: "train" | "eval", project: Awaited<
|
|
|
32
32
|
}[];
|
|
33
33
|
examples: {
|
|
34
34
|
train: number;
|
|
35
|
+
rejections: number;
|
|
35
36
|
validation: number;
|
|
36
37
|
eval: number;
|
|
37
38
|
};
|
|
38
|
-
|
|
39
|
-
examples: number;
|
|
40
|
-
invalidOutputRate: number;
|
|
41
|
-
exactAccuracy: number;
|
|
42
|
-
accepted: number;
|
|
43
|
-
acceptedAccuracy: number | null;
|
|
44
|
-
abstentionRate: number;
|
|
45
|
-
correctAbstentions: number;
|
|
46
|
-
failures: {
|
|
47
|
-
input: string;
|
|
48
|
-
expected: unknown;
|
|
49
|
-
actual: unknown;
|
|
50
|
-
}[];
|
|
51
|
-
} | null;
|
|
39
|
+
supervisionSha256: string;
|
|
52
40
|
supervisedTokens: number;
|
|
53
41
|
loss: number[];
|
|
54
42
|
exportParity: {
|
|
@@ -70,49 +58,7 @@ export declare function runSequence(command: "train" | "eval", project: Awaited<
|
|
|
70
58
|
actual: unknown;
|
|
71
59
|
}[];
|
|
72
60
|
};
|
|
73
|
-
|
|
74
|
-
examples: number;
|
|
75
|
-
invalidOutputRate: number;
|
|
76
|
-
exactAccuracy: number;
|
|
77
|
-
accepted: number;
|
|
78
|
-
acceptedAccuracy: number | null;
|
|
79
|
-
abstentionRate: number;
|
|
80
|
-
correctAbstentions: number;
|
|
81
|
-
failures: {
|
|
82
|
-
input: string;
|
|
83
|
-
expected: unknown;
|
|
84
|
-
actual: unknown;
|
|
85
|
-
}[];
|
|
86
|
-
};
|
|
87
|
-
untrained: {
|
|
88
|
-
examples: number;
|
|
89
|
-
invalidOutputRate: number;
|
|
90
|
-
exactAccuracy: number;
|
|
91
|
-
accepted: number;
|
|
92
|
-
acceptedAccuracy: number | null;
|
|
93
|
-
abstentionRate: number;
|
|
94
|
-
correctAbstentions: number;
|
|
95
|
-
failures: {
|
|
96
|
-
input: string;
|
|
97
|
-
expected: unknown;
|
|
98
|
-
actual: unknown;
|
|
99
|
-
}[];
|
|
100
|
-
};
|
|
101
|
-
float: {
|
|
102
|
-
examples: number;
|
|
103
|
-
invalidOutputRate: number;
|
|
104
|
-
exactAccuracy: number;
|
|
105
|
-
accepted: number;
|
|
106
|
-
acceptedAccuracy: number | null;
|
|
107
|
-
abstentionRate: number;
|
|
108
|
-
correctAbstentions: number;
|
|
109
|
-
failures: {
|
|
110
|
-
input: string;
|
|
111
|
-
expected: unknown;
|
|
112
|
-
actual: unknown;
|
|
113
|
-
}[];
|
|
114
|
-
};
|
|
115
|
-
quantized: {
|
|
61
|
+
evaluation: {
|
|
116
62
|
examples: number;
|
|
117
63
|
invalidOutputRate: number;
|
|
118
64
|
exactAccuracy: number;
|
|
@@ -140,20 +86,6 @@ export declare function runSequence(command: "train" | "eval", project: Awaited<
|
|
|
140
86
|
actual: unknown;
|
|
141
87
|
}[];
|
|
142
88
|
} | null;
|
|
143
|
-
baseline: {
|
|
144
|
-
examples: number;
|
|
145
|
-
invalidOutputRate: number;
|
|
146
|
-
exactAccuracy: number;
|
|
147
|
-
accepted: number;
|
|
148
|
-
acceptedAccuracy: number | null;
|
|
149
|
-
abstentionRate: number;
|
|
150
|
-
correctAbstentions: number;
|
|
151
|
-
failures: {
|
|
152
|
-
input: string;
|
|
153
|
-
expected: unknown;
|
|
154
|
-
actual: unknown;
|
|
155
|
-
}[];
|
|
156
|
-
} | null;
|
|
157
89
|
trainingMs: number;
|
|
158
90
|
notes: string;
|
|
159
91
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/run-sequence.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAMzD,wBAAsB,WAAW,CAC/B,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,EAChD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"run-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/run-sequence.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAMzD,wBAAsB,WAAW,CAC/B,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,EAChD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAmG5C,KAAK;YACL,UAAU;YACV,UAAU;YACV,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsBT"}
|
|
@@ -2,6 +2,10 @@ import type { DatasetExample } from "@matchbox-ai/core";
|
|
|
2
2
|
import type { Token } from "@matchbox-ai/core/runtime";
|
|
3
3
|
/** Experimental supervision adapter. Application-facing datasets remain input/output JSONL. */
|
|
4
4
|
export interface SequenceRecipe {
|
|
5
|
+
/** Optional supervised rejection rows. Their annotations must decode to null. */
|
|
6
|
+
rejections?: readonly DatasetExample<null>[];
|
|
7
|
+
/** Train the unknown-token embedding by masking this fraction of training token IDs. */
|
|
8
|
+
tokenDropout?: number;
|
|
5
9
|
tokenizer: "characters" | "words";
|
|
6
10
|
readout: "all" | "last";
|
|
7
11
|
labels: readonly string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,+FAA+F;AAC/F,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,YAAY,GAAG,OAAO,CAAC;IAClC,OAAO,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;CAClG"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,+FAA+F;AAC/F,MAAM,WAAW,cAAc;IAC7B,iFAAiF;IACjF,UAAU,CAAC,EAAE,SAAS,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,YAAY,GAAG,OAAO,CAAC;IAClC,OAAO,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;CAClG"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
interface NativeResult {
|
|
2
|
+
parameters: number;
|
|
3
|
+
weights: Uint8Array;
|
|
4
|
+
loss: number[];
|
|
5
|
+
}
|
|
6
|
+
export declare function predict(config: {
|
|
7
|
+
vocabularySize: number;
|
|
8
|
+
labelCount: number;
|
|
9
|
+
}, weights: Uint8Array, inputs: number[][]): number[];
|
|
10
|
+
export declare function fit(config: {
|
|
11
|
+
vocabularySize: number;
|
|
12
|
+
labelCount: number;
|
|
13
|
+
}, inputs: number[][], labels: number[], progress?: (epoch: number, loss: number) => void): Promise<NativeResult>;
|
|
14
|
+
export {};
|
|
15
|
+
//# sourceMappingURL=fit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fit.d.ts","sourceRoot":"","sources":["../../src/native/fit.ts"],"names":[],"mappings":"AACA,UAAU,YAAY;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,wBAAgB,OAAO,CACrB,MAAM,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EACtD,OAAO,EAAE,UAAU,EACnB,MAAM,EAAE,MAAM,EAAE,EAAE,GACjB,MAAM,EAAE,CASV;AACD,wBAAgB,GAAG,CACjB,MAAM,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EACtD,MAAM,EAAE,MAAM,EAAE,EAAE,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GAC/C,OAAO,CAAC,YAAY,CAAC,CAmBvB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/native/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAExC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
type Config = {
|
|
2
|
+
vocabularySize: number;
|
|
3
|
+
fields: number[];
|
|
4
|
+
};
|
|
5
|
+
export declare function fitNativeRecord(config: Config, inputs: number[][], labels: number[], progress?: (epoch: number, loss: number) => void): Promise<{
|
|
6
|
+
parameters: number;
|
|
7
|
+
weights: Uint8Array;
|
|
8
|
+
loss: number[];
|
|
9
|
+
}>;
|
|
10
|
+
export declare function predictNativeRecord(config: Config, weights: Uint8Array, inputs: number[]): number[];
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=record.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"record.d.ts","sourceRoot":"","sources":["../../src/native/record.ts"],"names":[],"mappings":"AAEA,KAAK,MAAM,GAAG;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;AAU3D,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EAAE,EAAE,EAClB,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;gBAPvB,MAAM;aAAW,UAAU;UAAQ,MAAM,EAAE;GAoBrE;AACD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,YAMxF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package-model-CPj0qnQs.js","names":[],"sources":["../src/packaging/package-model.ts"],"sourcesContent":["import { mkdir, writeFile } from \"node:fs/promises\";\nimport { dirname, resolve } from \"node:path\";\nimport type { SequenceArtifact, RecordArtifact } from \"@matchbox-ai/core/internal\";\nexport async function packageModel(\n output: string,\n model: SequenceArtifact | RecordArtifact,\n report: unknown,\n) {\n await mkdir(dirname(output), { recursive: true });\n const declaration = output.replace(/\\.matchbox$/, \".d.matchbox.ts\");\n if (declaration === output) throw new Error(\"The output filename must end in .matchbox.\");\n const decoderImport =\n model.decoderModule === null\n ? \"\"\n : `import decode from ${JSON.stringify(model.decoderModule.replace(/\\.ts$/, \"\"))};\\n`;\n const taskImport = model.taskModule.replace(/\\.ts$/, \"\");\n await writeFile(output, JSON.stringify(model));\n await writeFile(\n declaration,\n `import task from ${JSON.stringify(taskImport)};\\nimport type { InferOutput } from \"@matchbox-ai/core\";\\nimport type { MatchboxParser } from \"@matchbox-ai/core/runtime\";\\ndeclare const parser: MatchboxParser<InferOutput<typeof task>> & { load(): Promise<void>; dispose(): void };\\nexport default parser;\\n`,\n );\n await writeFile(resolve(dirname(output), \"report.json\"), JSON.stringify(report, null, 2) + \"\\n\");\n // A portable wrapper for bundlers without the Vite plugin; inference still remains local.\n await writeFile(\n output.replace(/\\.matchbox$/, \".ts\"),\n `import task from ${JSON.stringify(taskImport)};\\n${decoderImport}import { createParser } from \"@matchbox-ai/core/runtime\";\\nconst model = ${JSON.stringify(model)};\\nexport default createParser(model, task${model.decoderModule === null ? \"\" : \", decode\"});\\n`,\n );\n}\n"],"mappings":";;;AAGA,eAAsB,aACpB,QACA,OACA,QACA;CACA,MAAM,MAAM,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;CAChD,MAAM,cAAc,OAAO,QAAQ,eAAe,gBAAgB;CAClE,IAAI,gBAAgB,
|
|
1
|
+
{"version":3,"file":"package-model-CPj0qnQs.js","names":[],"sources":["../src/packaging/package-model.ts"],"sourcesContent":["import { mkdir, writeFile } from \"node:fs/promises\";\nimport { dirname, resolve } from \"node:path\";\nimport type { SequenceArtifact, RecordArtifact } from \"@matchbox-ai/core/internal\";\nexport async function packageModel(\n output: string,\n model: SequenceArtifact | RecordArtifact,\n report: unknown,\n) {\n await mkdir(dirname(output), { recursive: true });\n const declaration = output.replace(/\\.matchbox$/, \".d.matchbox.ts\");\n if (declaration === output) {\n throw new Error(\"The output filename must end in .matchbox.\");\n }\n const decoderImport =\n model.decoderModule === null\n ? \"\"\n : `import decode from ${JSON.stringify(model.decoderModule.replace(/\\.ts$/, \"\"))};\\n`;\n const taskImport = model.taskModule.replace(/\\.ts$/, \"\");\n await writeFile(output, JSON.stringify(model));\n await writeFile(\n declaration,\n `import task from ${JSON.stringify(taskImport)};\\nimport type { InferOutput } from \"@matchbox-ai/core\";\\nimport type { MatchboxParser } from \"@matchbox-ai/core/runtime\";\\ndeclare const parser: MatchboxParser<InferOutput<typeof task>> & { load(): Promise<void>; dispose(): void };\\nexport default parser;\\n`,\n );\n await writeFile(resolve(dirname(output), \"report.json\"), JSON.stringify(report, null, 2) + \"\\n\");\n // A portable wrapper for bundlers without the Vite plugin; inference still remains local.\n await writeFile(\n output.replace(/\\.matchbox$/, \".ts\"),\n `import task from ${JSON.stringify(taskImport)};\\n${decoderImport}import { createParser } from \"@matchbox-ai/core/runtime\";\\nconst model = ${JSON.stringify(model)};\\nexport default createParser(model, task${model.decoderModule === null ? \"\" : \", decode\"});\\n`,\n );\n}\n"],"mappings":";;;AAGA,eAAsB,aACpB,QACA,OACA,QACA;CACA,MAAM,MAAM,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;CAChD,MAAM,cAAc,OAAO,QAAQ,eAAe,gBAAgB;CAClE,IAAI,gBAAgB,QAClB,MAAM,IAAI,MAAM,4CAA4C;CAE9D,MAAM,gBACJ,MAAM,kBAAkB,OACpB,KACA,sBAAsB,KAAK,UAAU,MAAM,cAAc,QAAQ,SAAS,EAAE,CAAC,EAAE;CACrF,MAAM,aAAa,MAAM,WAAW,QAAQ,SAAS,EAAE;CACvD,MAAM,UAAU,QAAQ,KAAK,UAAU,KAAK,CAAC;CAC7C,MAAM,UACJ,aACA,oBAAoB,KAAK,UAAU,UAAU,EAAE,mQACjD;CACA,MAAM,UAAU,QAAQ,QAAQ,MAAM,GAAG,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;CAE/F,MAAM,UACJ,OAAO,QAAQ,eAAe,KAAK,GACnC,oBAAoB,KAAK,UAAU,UAAU,EAAE,KAAK,cAAc,2EAA2E,KAAK,UAAU,KAAK,EAAE,4CAA4C,MAAM,kBAAkB,OAAO,KAAK,WAAW,KAChQ;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package-model.d.ts","sourceRoot":"","sources":["../../src/packaging/package-model.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACnF,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,gBAAgB,GAAG,cAAc,EACxC,MAAM,EAAE,OAAO,
|
|
1
|
+
{"version":3,"file":"package-model.d.ts","sourceRoot":"","sources":["../../src/packaging/package-model.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACnF,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,gBAAgB,GAAG,cAAc,EACxC,MAAM,EAAE,OAAO,iBAuBhB"}
|
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
export declare const pipelineSchema: z.ZodObject<{
|
|
3
|
-
input: z.ZodOptional<z.ZodObject<{
|
|
4
|
-
kind: z.ZodLiteral<"words">;
|
|
5
|
-
}, z.core.$strict>>;
|
|
6
3
|
prediction: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
7
4
|
kind: z.ZodLiteral<"field-classifier">;
|
|
8
5
|
}, z.core.$strict>, z.ZodObject<{
|
|
9
6
|
kind: z.ZodLiteral<"token-classifier">;
|
|
10
|
-
recipe: z.ZodString
|
|
11
|
-
decode: z.ZodString
|
|
7
|
+
recipe: z.ZodDefault<z.ZodString>;
|
|
8
|
+
decode: z.ZodDefault<z.ZodString>;
|
|
12
9
|
}, z.core.$strict>], "kind">;
|
|
13
10
|
acceptance: z.ZodOptional<z.ZodObject<{
|
|
14
11
|
minAccuracy: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"define-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/define-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,eAAO,MAAM,cAAc
|
|
1
|
+
{"version":3,"file":"define-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/define-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,eAAO,MAAM,cAAc;;;;;;;;;;;;kBAezB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAE3D"}
|
|
@@ -3,49 +3,8 @@ import type { ParserDefinition } from "@matchbox-ai/core";
|
|
|
3
3
|
import type { z } from "zod";
|
|
4
4
|
export declare function loadArtifact(configPath: string): Promise<{
|
|
5
5
|
artifact: {
|
|
6
|
-
formatVersion:
|
|
7
|
-
|
|
8
|
-
kind: "sequence-parser";
|
|
9
|
-
architecture: "embedding-window-mlp";
|
|
10
|
-
taskModule: string;
|
|
11
|
-
decoderModule: string;
|
|
12
|
-
taskMetadata: unknown;
|
|
13
|
-
readout: "all" | "last";
|
|
14
|
-
tokenizer: "characters" | "words";
|
|
15
|
-
vocabulary: string[];
|
|
16
|
-
labels: string[];
|
|
17
|
-
radius: number;
|
|
18
|
-
threshold: number;
|
|
19
|
-
precision: "float32" | "int8";
|
|
20
|
-
weights: [{
|
|
21
|
-
name: string;
|
|
22
|
-
shape: number[];
|
|
23
|
-
values: number[];
|
|
24
|
-
scale: number;
|
|
25
|
-
}, {
|
|
26
|
-
name: string;
|
|
27
|
-
shape: number[];
|
|
28
|
-
values: number[];
|
|
29
|
-
scale: number;
|
|
30
|
-
}, {
|
|
31
|
-
name: string;
|
|
32
|
-
shape: number[];
|
|
33
|
-
values: number[];
|
|
34
|
-
scale: number;
|
|
35
|
-
}, {
|
|
36
|
-
name: string;
|
|
37
|
-
shape: number[];
|
|
38
|
-
values: number[];
|
|
39
|
-
scale: number;
|
|
40
|
-
}, {
|
|
41
|
-
name: string;
|
|
42
|
-
shape: number[];
|
|
43
|
-
values: number[];
|
|
44
|
-
scale: number;
|
|
45
|
-
}];
|
|
46
|
-
} | {
|
|
47
|
-
formatVersion: 2;
|
|
48
|
-
modelTopology: Record<string, unknown>;
|
|
6
|
+
formatVersion: 3;
|
|
7
|
+
engine: "burn-0.21";
|
|
49
8
|
kind: "record-parser";
|
|
50
9
|
architecture: "bag-of-words-mlp";
|
|
51
10
|
taskModule: string;
|
|
@@ -57,28 +16,25 @@ export declare function loadArtifact(configPath: string): Promise<{
|
|
|
57
16
|
values: (string | number | boolean | null)[];
|
|
58
17
|
}[];
|
|
59
18
|
threshold: number;
|
|
60
|
-
precision: "float32"
|
|
61
|
-
weights:
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
values: number[];
|
|
80
|
-
scale: number;
|
|
81
|
-
}];
|
|
19
|
+
precision: "float32";
|
|
20
|
+
weights: string;
|
|
21
|
+
} | {
|
|
22
|
+
formatVersion: 3;
|
|
23
|
+
engine: "burn-0.21";
|
|
24
|
+
kind: "sequence-parser";
|
|
25
|
+
architecture: "embedding-window-mlp";
|
|
26
|
+
taskModule: string;
|
|
27
|
+
decoderModule: string;
|
|
28
|
+
taskMetadata: unknown;
|
|
29
|
+
readout: "all" | "last";
|
|
30
|
+
tokenizer: "characters" | "words";
|
|
31
|
+
vocabulary: string[];
|
|
32
|
+
labels: string[];
|
|
33
|
+
radius: 1;
|
|
34
|
+
unknownTokens: "abstain" | "predict";
|
|
35
|
+
threshold: number;
|
|
36
|
+
precision: "float32";
|
|
37
|
+
weights: string;
|
|
82
38
|
};
|
|
83
39
|
task: ParserDefinition<z.ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>>;
|
|
84
40
|
decode: SequenceDecoder | undefined;
|
|
@@ -109,7 +65,6 @@ export declare function loadArtifact(configPath: string): Promise<{
|
|
|
109
65
|
}>;
|
|
110
66
|
output: string;
|
|
111
67
|
config: {
|
|
112
|
-
formatVersion: 1;
|
|
113
68
|
task: string;
|
|
114
69
|
train: string;
|
|
115
70
|
validation: string;
|
|
@@ -121,7 +76,6 @@ export declare function loadArtifact(configPath: string): Promise<{
|
|
|
121
76
|
recipe: string;
|
|
122
77
|
decoder: string;
|
|
123
78
|
} | undefined;
|
|
124
|
-
baseline?: string | undefined;
|
|
125
79
|
challenges?: string | undefined;
|
|
126
80
|
};
|
|
127
81
|
root: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artifact.d.ts","sourceRoot":"","sources":["../../src/project/artifact.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM
|
|
1
|
+
{"version":3,"file":"artifact.d.ts","sourceRoot":"","sources":["../../src/project/artifact.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAoB1B,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgBhC"}
|
package/dist/project/config.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
export declare const configSchema: z.ZodObject<{
|
|
3
|
-
formatVersion: z.ZodDefault<z.ZodLiteral<1>>;
|
|
4
3
|
task: z.ZodDefault<z.ZodString>;
|
|
5
4
|
train: z.ZodDefault<z.ZodString>;
|
|
6
5
|
validation: z.ZodDefault<z.ZodString>;
|
|
@@ -12,12 +11,10 @@ export declare const configSchema: z.ZodObject<{
|
|
|
12
11
|
recipe: z.ZodString;
|
|
13
12
|
decoder: z.ZodString;
|
|
14
13
|
}, z.core.$strict>>;
|
|
15
|
-
baseline: z.ZodOptional<z.ZodString>;
|
|
16
14
|
challenges: z.ZodOptional<z.ZodString>;
|
|
17
15
|
}, z.core.$strict>;
|
|
18
16
|
export declare function loadConfig(path: string): Promise<{
|
|
19
17
|
config: {
|
|
20
|
-
formatVersion: 1;
|
|
21
18
|
task: string;
|
|
22
19
|
train: string;
|
|
23
20
|
validation: string;
|
|
@@ -29,9 +26,9 @@ export declare function loadConfig(path: string): Promise<{
|
|
|
29
26
|
recipe: string;
|
|
30
27
|
decoder: string;
|
|
31
28
|
} | undefined;
|
|
32
|
-
baseline?: string | undefined;
|
|
33
29
|
challenges?: string | undefined;
|
|
34
30
|
};
|
|
35
31
|
root: string;
|
|
32
|
+
pipelinePath: string | undefined;
|
|
36
33
|
}>;
|
|
37
34
|
//# sourceMappingURL=config.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/project/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/project/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,YAAY;;;;;;;;;;;;;kBAUvB,CAAC;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;GA2C5C"}
|
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
export declare function discover(target?: string, cwd?: string): Promise<string>;
|
|
2
|
+
/** Lists conventional tasks immediately beneath an application’s matchbox directory. */
|
|
3
|
+
export declare function listTasks(root: string): Promise<string[]>;
|
|
2
4
|
//# sourceMappingURL=discover.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../../src/project/discover.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../../src/project/discover.ts"],"names":[],"mappings":"AAgCA,wBAAsB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,SAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAmCpF;AAED,wFAAwF;AACxF,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAU/D"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Named task modules have one flat or directory-backed TypeScript entry point. */
|
|
2
|
+
export declare function findEntry(root: string, name: string): Promise<string | undefined>;
|
|
3
|
+
/** Explicit filenames stay exact; extensionless references use the named entry convention. */
|
|
4
|
+
export declare function resolveModule(root: string, reference: string): Promise<string>;
|
|
5
|
+
//# sourceMappingURL=entry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"entry.d.ts","sourceRoot":"","sources":["../../src/project/entry.ts"],"names":[],"mappings":"AAYA,mFAAmF;AACnF,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAUvF;AACD,8FAA8F;AAC9F,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMpF"}
|
package/dist/project/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/project/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/project/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/project/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { n as discover, r as
|
|
2
|
-
export { discover, loadArtifact, loadConfig };
|
|
1
|
+
import { i as loadConfig, n as discover, r as listTasks, t as loadArtifact } from "../project-BR13wXHr.js";
|
|
2
|
+
export { discover, listTasks, loadArtifact, loadConfig };
|
|
@@ -1,14 +1,36 @@
|
|
|
1
|
-
import { n as pipelineSchema } from "./define-pipeline-
|
|
1
|
+
import { n as pipelineSchema } from "./define-pipeline-D6I-jAE_.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { readArtifact, tensorPredictor } from "@matchbox-ai/core/internal";
|
|
4
4
|
import { access, readFile, readdir, stat } from "node:fs/promises";
|
|
5
|
-
import { basename, dirname, resolve } from "node:path";
|
|
5
|
+
import { basename, dirname, extname, resolve } from "node:path";
|
|
6
6
|
import { pathToFileURL } from "node:url";
|
|
7
7
|
import { createParser } from "@matchbox-ai/core/runtime";
|
|
8
|
+
//#region src/project/entry.ts
|
|
9
|
+
async function isFile(path) {
|
|
10
|
+
try {
|
|
11
|
+
return (await stat(path)).isFile();
|
|
12
|
+
} catch (error) {
|
|
13
|
+
if (["ENOENT", "ENOTDIR"].includes(error.code ?? "")) return false;
|
|
14
|
+
throw error;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/** Named task modules have one flat or directory-backed TypeScript entry point. */
|
|
18
|
+
async function findEntry(root, name) {
|
|
19
|
+
const base = resolve(root, name);
|
|
20
|
+
const candidates = [`${base}.ts`, resolve(base, `${basename(base)}.ts`)];
|
|
21
|
+
const found = (await Promise.all(candidates.map(isFile))).map((exists, index) => exists ? candidates[index] : void 0).filter((path) => path !== void 0);
|
|
22
|
+
if (found.length > 1) throw new Error(`Conflicting task entry points: ${found.join(" and ")}. Keep only one.`);
|
|
23
|
+
return found[0];
|
|
24
|
+
}
|
|
25
|
+
/** Explicit filenames stay exact; extensionless references use the named entry convention. */
|
|
26
|
+
async function resolveModule(root, reference) {
|
|
27
|
+
if (extname(reference)) return resolve(root, reference);
|
|
28
|
+
return await findEntry(root, reference) ?? resolve(root, `${reference}.ts`);
|
|
29
|
+
}
|
|
30
|
+
//#endregion
|
|
8
31
|
//#region src/project/config.ts
|
|
9
32
|
const configSchema = z.strictObject({
|
|
10
|
-
|
|
11
|
-
task: z.string().default("./parser.ts"),
|
|
33
|
+
task: z.string().default("./parser"),
|
|
12
34
|
train: z.string().default("./data/train.jsonl"),
|
|
13
35
|
validation: z.string().default("./evals/validation.jsonl"),
|
|
14
36
|
eval: z.string().default("./evals/test.jsonl"),
|
|
@@ -19,7 +41,6 @@ const configSchema = z.strictObject({
|
|
|
19
41
|
recipe: z.string(),
|
|
20
42
|
decoder: z.string()
|
|
21
43
|
}).optional(),
|
|
22
|
-
baseline: z.string().optional(),
|
|
23
44
|
challenges: z.string().optional()
|
|
24
45
|
});
|
|
25
46
|
async function loadConfig(path) {
|
|
@@ -29,37 +50,41 @@ async function loadConfig(path) {
|
|
|
29
50
|
const configPath = directory ? resolve(root, "matchbox.config.ts") : resolve(path);
|
|
30
51
|
const authored = await access(configPath).then(async () => (await import(pathToFileURL(configPath).href)).default, () => ({}));
|
|
31
52
|
let defaults = {};
|
|
32
|
-
|
|
33
|
-
|
|
53
|
+
const pipelinePath = await findEntry(root, "pipeline");
|
|
54
|
+
if (pipelinePath) {
|
|
55
|
+
const pipeline = pipelineSchema.parse((await import(pathToFileURL(pipelinePath).href)).default);
|
|
34
56
|
defaults = { ...pipeline.acceptance };
|
|
35
57
|
if (pipeline.prediction.kind === "token-classifier") defaults.sequence = {
|
|
36
58
|
recipe: pipeline.prediction.recipe,
|
|
37
59
|
decoder: pipeline.prediction.decode
|
|
38
60
|
};
|
|
39
|
-
} else if (!authored.sequence) throw new Error(`Missing
|
|
61
|
+
} else if (!authored.sequence) throw new Error(`Missing pipeline.ts or pipeline/pipeline.ts in ${root}. Author an explicit pipeline before training.`);
|
|
40
62
|
const config = configSchema.parse({
|
|
41
63
|
output: resolve(root, "../../.matchbox", basename(root), "model.matchbox"),
|
|
42
64
|
...defaults,
|
|
43
65
|
...authored
|
|
44
66
|
});
|
|
45
|
-
if (!config.baseline && await exists("evals/baseline.ts")) config.baseline = "./evals/baseline.ts";
|
|
46
67
|
if (!config.challenges && await exists("evals/challenges.json")) config.challenges = "./evals/challenges.json";
|
|
68
|
+
config.task = await resolveModule(root, config.task);
|
|
69
|
+
if (config.sequence) {
|
|
70
|
+
config.sequence.recipe = await resolveModule(root, config.sequence.recipe);
|
|
71
|
+
config.sequence.decoder = await resolveModule(root, config.sequence.decoder);
|
|
72
|
+
}
|
|
47
73
|
return {
|
|
48
74
|
config,
|
|
49
|
-
root
|
|
75
|
+
root,
|
|
76
|
+
pipelinePath
|
|
50
77
|
};
|
|
51
78
|
}
|
|
52
79
|
//#endregion
|
|
53
80
|
//#region src/project/discover.ts
|
|
54
81
|
const exists = (path) => access(path).then(() => true, () => false);
|
|
55
82
|
async function select(root) {
|
|
56
|
-
if (await
|
|
83
|
+
if (basename(root) === "parser" && await findEntry(dirname(root), "pipeline") && await findEntry(dirname(root), "parser") === resolve(root, "parser.ts")) return dirname(root);
|
|
84
|
+
if (await findEntry(root, "parser")) return root;
|
|
57
85
|
if (await exists(resolve(root, "matchbox.config.ts"))) return resolve(root, "matchbox.config.ts");
|
|
86
|
+
const tasks = await listTasks(root);
|
|
58
87
|
const directory = resolve(root, "matchbox");
|
|
59
|
-
const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
|
|
60
|
-
const tasks = [];
|
|
61
|
-
for (const entry of entries) if (entry.isDirectory() && await exists(resolve(directory, entry.name, "parser.ts"))) tasks.push(entry.name);
|
|
62
|
-
tasks.sort();
|
|
63
88
|
if (tasks.length > 1) throw new Error(`Choose a task: ${tasks.join(", ")}. For example, matchbox-ai train ${tasks[0]}.`);
|
|
64
89
|
return tasks.length ? resolve(directory, tasks[0]) : void 0;
|
|
65
90
|
}
|
|
@@ -75,7 +100,7 @@ async function discover(target, cwd = process.cwd()) {
|
|
|
75
100
|
for (;;) {
|
|
76
101
|
if (target) {
|
|
77
102
|
const task = resolve(root, "matchbox", target);
|
|
78
|
-
if (await
|
|
103
|
+
if (await findEntry(task, "parser")) return task;
|
|
79
104
|
} else {
|
|
80
105
|
const selected = await select(root);
|
|
81
106
|
if (selected) return selected;
|
|
@@ -85,6 +110,14 @@ async function discover(target, cwd = process.cwd()) {
|
|
|
85
110
|
root = parent;
|
|
86
111
|
}
|
|
87
112
|
}
|
|
113
|
+
/** Lists conventional tasks immediately beneath an application’s matchbox directory. */
|
|
114
|
+
async function listTasks(root) {
|
|
115
|
+
const directory = resolve(root, "matchbox");
|
|
116
|
+
const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
|
|
117
|
+
const tasks = [];
|
|
118
|
+
for (const entry of entries) if (entry.isDirectory() && await findEntry(resolve(directory, entry.name), "parser")) tasks.push(entry.name);
|
|
119
|
+
return tasks.sort();
|
|
120
|
+
}
|
|
88
121
|
//#endregion
|
|
89
122
|
//#region src/project/artifact.ts
|
|
90
123
|
async function loadArtifact(configPath) {
|
|
@@ -120,6 +153,6 @@ async function loadArtifact(configPath) {
|
|
|
120
153
|
};
|
|
121
154
|
}
|
|
122
155
|
//#endregion
|
|
123
|
-
export { discover as n,
|
|
156
|
+
export { loadConfig as i, discover as n, listTasks as r, loadArtifact as t };
|
|
124
157
|
|
|
125
|
-
//# sourceMappingURL=project-
|
|
158
|
+
//# sourceMappingURL=project-BR13wXHr.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"project-BR13wXHr.js","names":[],"sources":["../src/project/entry.ts","../src/project/config.ts","../src/project/discover.ts","../src/project/artifact.ts"],"sourcesContent":["import { stat } from \"node:fs/promises\";\nimport { basename, extname, resolve } from \"node:path\";\nasync function isFile(path: string): Promise<boolean> {\n try {\n return (await stat(path)).isFile();\n } catch (error) {\n if ([\"ENOENT\", \"ENOTDIR\"].includes((error as NodeJS.ErrnoException).code ?? \"\")) {\n return false;\n }\n throw error;\n }\n}\n/** Named task modules have one flat or directory-backed TypeScript entry point. */\nexport async function findEntry(root: string, name: string): Promise<string | undefined> {\n const base = resolve(root, name);\n const candidates = [`${base}.ts`, resolve(base, `${basename(base)}.ts`)];\n const found = (await Promise.all(candidates.map(isFile)))\n .map((exists, index) => (exists ? candidates[index] : undefined))\n .filter((path): path is string => path !== undefined);\n if (found.length > 1) {\n throw new Error(`Conflicting task entry points: ${found.join(\" and \")}. Keep only one.`);\n }\n return found[0];\n}\n/** Explicit filenames stay exact; extensionless references use the named entry convention. */\nexport async function resolveModule(root: string, reference: string): Promise<string> {\n if (extname(reference)) {\n return resolve(root, reference);\n }\n // Missing training modules are reported when imported, so artifact evaluation needs no recipe.\n return (await findEntry(root, reference)) ?? resolve(root, `${reference}.ts`);\n}\n","import { access, stat } from \"node:fs/promises\";\nimport { basename, dirname, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { z } from \"zod\";\nimport { findEntry, resolveModule } from \"./entry.js\";\nimport { pipelineSchema } from \"../pipeline/index.js\";\nexport const configSchema = z.strictObject({\n task: z.string().default(\"./parser\"),\n train: z.string().default(\"./data/train.jsonl\"),\n validation: z.string().default(\"./evals/validation.jsonl\"),\n eval: z.string().default(\"./evals/test.jsonl\"),\n output: z.string().endsWith(\".matchbox\"),\n minAccuracy: z.number().min(0).max(1).default(0.95),\n maxBytes: z.number().positive().default(64000),\n sequence: z.strictObject({ recipe: z.string(), decoder: z.string() }).optional(),\n challenges: z.string().optional(),\n});\nexport async function loadConfig(path: string) {\n const directory = (await stat(path)).isDirectory();\n const root = directory ? resolve(path) : dirname(resolve(path));\n const exists = (file: string) =>\n access(resolve(root, file)).then(\n () => true,\n () => false,\n );\n const configPath = directory ? resolve(root, \"matchbox.config.ts\") : resolve(path);\n const authored = await access(configPath).then(\n async () => (await import(pathToFileURL(configPath).href)).default,\n () => ({}),\n );\n let defaults: Record<string, unknown> = {};\n const pipelinePath = await findEntry(root, \"pipeline\");\n if (pipelinePath) {\n const pipeline = pipelineSchema.parse((await import(pathToFileURL(pipelinePath).href)).default);\n defaults = { ...pipeline.acceptance };\n if (pipeline.prediction.kind === \"token-classifier\") {\n defaults.sequence = {\n recipe: pipeline.prediction.recipe,\n decoder: pipeline.prediction.decode,\n };\n }\n } else if (!authored.sequence) {\n throw new Error(\n `Missing pipeline.ts or pipeline/pipeline.ts in ${root}. Author an explicit pipeline before training.`,\n );\n }\n const config = configSchema.parse({\n output: resolve(root, \"../../.matchbox\", basename(root), \"model.matchbox\"),\n ...defaults,\n ...authored,\n });\n if (!config.challenges && (await exists(\"evals/challenges.json\"))) {\n config.challenges = \"./evals/challenges.json\";\n }\n config.task = await resolveModule(root, config.task);\n if (config.sequence) {\n config.sequence.recipe = await resolveModule(root, config.sequence.recipe);\n config.sequence.decoder = await resolveModule(root, config.sequence.decoder);\n }\n return { config, root, pipelinePath };\n}\n","import { access, readdir, stat } from \"node:fs/promises\";\nimport { findEntry } from \"./entry.js\";\nimport { basename, dirname, resolve } from \"node:path\";\nconst exists = (path: string) =>\n access(path).then(\n () => true,\n () => false,\n );\nasync function select(root: string): Promise<string | undefined> {\n // A parser/parser.ts module directory is inside its task, not a second task.\n if (\n basename(root) === \"parser\" &&\n (await findEntry(dirname(root), \"pipeline\")) &&\n (await findEntry(dirname(root), \"parser\")) === resolve(root, \"parser.ts\")\n ) {\n return dirname(root);\n }\n if (await findEntry(root, \"parser\")) {\n return root;\n }\n if (await exists(resolve(root, \"matchbox.config.ts\"))) {\n return resolve(root, \"matchbox.config.ts\");\n }\n const tasks = await listTasks(root);\n const directory = resolve(root, \"matchbox\");\n if (tasks.length > 1) {\n throw new Error(\n `Choose a task: ${tasks.join(\", \")}. For example, matchbox-ai train ${tasks[0]}.`,\n );\n }\n return tasks.length ? resolve(directory, tasks[0]!) : undefined;\n}\nexport async function discover(target?: string, cwd = process.cwd()): Promise<string> {\n if (target && (await exists(resolve(cwd, target)))) {\n const path = resolve(cwd, target);\n if ((await stat(path)).isFile()) {\n return path;\n }\n const selected = await select(path);\n if (selected) {\n return selected;\n }\n throw new Error(`No Matchbox task found in ${path}.`);\n }\n let root = resolve(cwd);\n for (;;) {\n if (target) {\n const task = resolve(root, \"matchbox\", target);\n if (await findEntry(task, \"parser\")) {\n return task;\n }\n } else {\n const selected = await select(root);\n if (selected) {\n return selected;\n }\n }\n const parent = dirname(root);\n if (parent === root) {\n throw new Error(\n target\n ? `Task not found: ${target}.`\n : \"No Matchbox project found. Run matchbox-ai init <name>.\",\n );\n }\n root = parent;\n }\n}\n\n/** Lists conventional tasks immediately beneath an application’s matchbox directory. */\nexport async function listTasks(root: string): Promise<string[]> {\n const directory = resolve(root, \"matchbox\");\n const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);\n const tasks: string[] = [];\n for (const entry of entries) {\n if (entry.isDirectory() && (await findEntry(resolve(directory, entry.name), \"parser\"))) {\n tasks.push(entry.name);\n }\n }\n return tasks.sort();\n}\n","import { readFile } from \"node:fs/promises\";\nimport { dirname, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { createParser } from \"@matchbox-ai/core/runtime\";\nimport { readArtifact, tensorPredictor } from \"@matchbox-ai/core/internal\";\nimport type { SequenceDecoder } from \"@matchbox-ai/core/runtime\";\nimport type { ParserDefinition } from \"@matchbox-ai/core\";\nimport type { z } from \"zod\";\nimport { loadConfig } from \"./config.js\";\nexport async function loadArtifact(configPath: string) {\n const { config, root } = await loadConfig(configPath);\n const output = resolve(root, config.output);\n const text = await readFile(output, \"utf8\").catch(() => {\n throw new Error(`No readable model at ${output}. Run matchbox-ai train first.`);\n });\n const artifact = readArtifact(JSON.parse(text));\n const task: ParserDefinition<z.ZodType> = (\n await import(pathToFileURL(resolve(dirname(output), artifact.taskModule)).href)\n ).default;\n const decode: SequenceDecoder | undefined =\n artifact.decoderModule === null\n ? undefined\n : (await import(pathToFileURL(resolve(dirname(output), artifact.decoderModule)).href))\n .default;\n return {\n artifact,\n task,\n decode,\n parser: createParser(artifact, task, decode),\n inspect: async (input: string) => {\n const predictor = await tensorPredictor(artifact);\n try {\n if (artifact.kind === \"record-parser\") {\n return predictor.record(input);\n }\n const tokens = predictor.sequence(input);\n return { tokens, candidate: decode!(tokens, input) };\n } finally {\n predictor.dispose();\n }\n },\n output,\n config,\n root,\n };\n}\n"],"mappings":";;;;;;;;AAEA,eAAe,OAAO,MAAgC;CACpD,IAAI;EACF,QAAQ,MAAM,KAAK,IAAI,EAAA,CAAG,OAAO;CACnC,SAAS,OAAO;EACd,IAAI,CAAC,UAAU,SAAS,CAAC,CAAC,SAAU,MAAgC,QAAQ,EAAE,GAC5E,OAAO;EAET,MAAM;CACR;AACF;;AAEA,eAAsB,UAAU,MAAc,MAA2C;CACvF,MAAM,OAAO,QAAQ,MAAM,IAAI;CAC/B,MAAM,aAAa,CAAC,GAAG,KAAK,MAAM,QAAQ,MAAM,GAAG,SAAS,IAAI,EAAE,IAAI,CAAC;CACvE,MAAM,SAAS,MAAM,QAAQ,IAAI,WAAW,IAAI,MAAM,CAAC,EAAA,CACpD,KAAK,QAAQ,UAAW,SAAS,WAAW,SAAS,KAAA,CAAU,CAAC,CAChE,QAAQ,SAAyB,SAAS,KAAA,CAAS;CACtD,IAAI,MAAM,SAAS,GACjB,MAAM,IAAI,MAAM,kCAAkC,MAAM,KAAK,OAAO,EAAE,iBAAiB;CAEzF,OAAO,MAAM;AACf;;AAEA,eAAsB,cAAc,MAAc,WAAoC;CACpF,IAAI,QAAQ,SAAS,GACnB,OAAO,QAAQ,MAAM,SAAS;CAGhC,OAAQ,MAAM,UAAU,MAAM,SAAS,KAAM,QAAQ,MAAM,GAAG,UAAU,IAAI;AAC9E;;;ACzBA,MAAa,eAAe,EAAE,aAAa;CACzC,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,UAAU;CACnC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,oBAAoB;CAC9C,YAAY,EAAE,OAAO,CAAC,CAAC,QAAQ,0BAA0B;CACzD,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,oBAAoB;CAC7C,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,WAAW;CACvC,aAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAI;CAClD,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,IAAK;CAC7C,UAAU,EAAE,aAAa;EAAE,QAAQ,EAAE,OAAO;EAAG,SAAS,EAAE,OAAO;CAAE,CAAC,CAAC,CAAC,SAAS;CAC/E,YAAY,EAAE,OAAO,CAAC,CAAC,SAAS;AAClC,CAAC;AACD,eAAsB,WAAW,MAAc;CAC7C,MAAM,aAAa,MAAM,KAAK,IAAI,EAAA,CAAG,YAAY;CACjD,MAAM,OAAO,YAAY,QAAQ,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC;CAC9D,MAAM,UAAU,SACd,OAAO,QAAQ,MAAM,IAAI,CAAC,CAAC,CAAC,WACpB,YACA,KACR;CACF,MAAM,aAAa,YAAY,QAAQ,MAAM,oBAAoB,IAAI,QAAQ,IAAI;CACjF,MAAM,WAAW,MAAM,OAAO,UAAU,CAAC,CAAC,KACxC,aAAa,MAAM,OAAO,cAAc,UAAU,CAAC,CAAC,MAAA,CAAO,gBACpD,CAAC,EACV;CACA,IAAI,WAAoC,CAAC;CACzC,MAAM,eAAe,MAAM,UAAU,MAAM,UAAU;CACrD,IAAI,cAAc;EAChB,MAAM,WAAW,eAAe,OAAO,MAAM,OAAO,cAAc,YAAY,CAAC,CAAC,MAAA,CAAO,OAAO;EAC9F,WAAW,EAAE,GAAG,SAAS,WAAW;EACpC,IAAI,SAAS,WAAW,SAAS,oBAC/B,SAAS,WAAW;GAClB,QAAQ,SAAS,WAAW;GAC5B,SAAS,SAAS,WAAW;EAC/B;CAEJ,OAAO,IAAI,CAAC,SAAS,UACnB,MAAM,IAAI,MACR,kDAAkD,KAAK,+CACzD;CAEF,MAAM,SAAS,aAAa,MAAM;EAChC,QAAQ,QAAQ,MAAM,mBAAmB,SAAS,IAAI,GAAG,gBAAgB;EACzE,GAAG;EACH,GAAG;CACL,CAAC;CACD,IAAI,CAAC,OAAO,cAAe,MAAM,OAAO,uBAAuB,GAC7D,OAAO,aAAa;CAEtB,OAAO,OAAO,MAAM,cAAc,MAAM,OAAO,IAAI;CACnD,IAAI,OAAO,UAAU;EACnB,OAAO,SAAS,SAAS,MAAM,cAAc,MAAM,OAAO,SAAS,MAAM;EACzE,OAAO,SAAS,UAAU,MAAM,cAAc,MAAM,OAAO,SAAS,OAAO;CAC7E;CACA,OAAO;EAAE;EAAQ;EAAM;CAAa;AACtC;;;ACzDA,MAAM,UAAU,SACd,OAAO,IAAI,CAAC,CAAC,WACL,YACA,KACR;AACF,eAAe,OAAO,MAA2C;CAE/D,IACE,SAAS,IAAI,MAAM,YAClB,MAAM,UAAU,QAAQ,IAAI,GAAG,UAAU,KACzC,MAAM,UAAU,QAAQ,IAAI,GAAG,QAAQ,MAAO,QAAQ,MAAM,WAAW,GAExE,OAAO,QAAQ,IAAI;CAErB,IAAI,MAAM,UAAU,MAAM,QAAQ,GAChC,OAAO;CAET,IAAI,MAAM,OAAO,QAAQ,MAAM,oBAAoB,CAAC,GAClD,OAAO,QAAQ,MAAM,oBAAoB;CAE3C,MAAM,QAAQ,MAAM,UAAU,IAAI;CAClC,MAAM,YAAY,QAAQ,MAAM,UAAU;CAC1C,IAAI,MAAM,SAAS,GACjB,MAAM,IAAI,MACR,kBAAkB,MAAM,KAAK,IAAI,EAAE,mCAAmC,MAAM,GAAG,EACjF;CAEF,OAAO,MAAM,SAAS,QAAQ,WAAW,MAAM,EAAG,IAAI,KAAA;AACxD;AACA,eAAsB,SAAS,QAAiB,MAAM,QAAQ,IAAI,GAAoB;CACpF,IAAI,UAAW,MAAM,OAAO,QAAQ,KAAK,MAAM,CAAC,GAAI;EAClD,MAAM,OAAO,QAAQ,KAAK,MAAM;EAChC,KAAK,MAAM,KAAK,IAAI,EAAA,CAAG,OAAO,GAC5B,OAAO;EAET,MAAM,WAAW,MAAM,OAAO,IAAI;EAClC,IAAI,UACF,OAAO;EAET,MAAM,IAAI,MAAM,6BAA6B,KAAK,EAAE;CACtD;CACA,IAAI,OAAO,QAAQ,GAAG;CACtB,SAAS;EACP,IAAI,QAAQ;GACV,MAAM,OAAO,QAAQ,MAAM,YAAY,MAAM;GAC7C,IAAI,MAAM,UAAU,MAAM,QAAQ,GAChC,OAAO;EAEX,OAAO;GACL,MAAM,WAAW,MAAM,OAAO,IAAI;GAClC,IAAI,UACF,OAAO;EAEX;EACA,MAAM,SAAS,QAAQ,IAAI;EAC3B,IAAI,WAAW,MACb,MAAM,IAAI,MACR,SACI,mBAAmB,OAAO,KAC1B,yDACN;EAEF,OAAO;CACT;AACF;;AAGA,eAAsB,UAAU,MAAiC;CAC/D,MAAM,YAAY,QAAQ,MAAM,UAAU;CAC1C,MAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChF,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,SAAS,SAClB,IAAI,MAAM,YAAY,KAAM,MAAM,UAAU,QAAQ,WAAW,MAAM,IAAI,GAAG,QAAQ,GAClF,MAAM,KAAK,MAAM,IAAI;CAGzB,OAAO,MAAM,KAAK;AACpB;;;ACvEA,eAAsB,aAAa,YAAoB;CACrD,MAAM,EAAE,QAAQ,SAAS,MAAM,WAAW,UAAU;CACpD,MAAM,SAAS,QAAQ,MAAM,OAAO,MAAM;CAC1C,MAAM,OAAO,MAAM,SAAS,QAAQ,MAAM,CAAC,CAAC,YAAY;EACtD,MAAM,IAAI,MAAM,wBAAwB,OAAO,+BAA+B;CAChF,CAAC;CACD,MAAM,WAAW,aAAa,KAAK,MAAM,IAAI,CAAC;CAC9C,MAAM,QACJ,MAAM,OAAO,cAAc,QAAQ,QAAQ,MAAM,GAAG,SAAS,UAAU,CAAC,CAAC,CAAC,MAAA,CAC1E;CACF,MAAM,SACJ,SAAS,kBAAkB,OACvB,KAAA,KACC,MAAM,OAAO,cAAc,QAAQ,QAAQ,MAAM,GAAG,SAAS,aAAa,CAAC,CAAC,CAAC,MAAA,CAC3E;CACT,OAAO;EACL;EACA;EACA;EACA,QAAQ,aAAa,UAAU,MAAM,MAAM;EAC3C,SAAS,OAAO,UAAkB;GAChC,MAAM,YAAY,MAAM,gBAAgB,QAAQ;GAChD,IAAI;IACF,IAAI,SAAS,SAAS,iBACpB,OAAO,UAAU,OAAO,KAAK;IAE/B,MAAM,SAAS,UAAU,SAAS,KAAK;IACvC,OAAO;KAAE;KAAQ,WAAW,OAAQ,QAAQ,KAAK;IAAE;GACrD,UAAU;IACR,UAAU,QAAQ;GACpB;EACF;EACA;EACA;EACA;CACF;AACF"}
|