@matchbox-ai/train 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +3 -0
- package/THIRD-PARTY-NOTICES.md +17 -0
- package/dist/codecs/index.d.ts +2 -0
- package/dist/codecs/index.d.ts.map +1 -0
- package/dist/define-pipeline-Colvwt7u.js +32 -0
- package/dist/define-pipeline-Colvwt7u.js.map +1 -0
- package/dist/encoders/index.d.ts +3 -0
- package/dist/encoders/index.d.ts.map +1 -0
- package/dist/encoders/word-tokens.d.ts +5 -0
- package/dist/encoders/word-tokens.d.ts.map +1 -0
- package/dist/evaluate-sequence-ZCwyomzQ.js +42 -0
- package/dist/evaluate-sequence-ZCwyomzQ.js.map +1 -0
- package/dist/evaluation/evaluate-sequence.d.ts +17 -0
- package/dist/evaluation/evaluate-sequence.d.ts.map +1 -0
- package/dist/evaluation/same-output.d.ts +2 -0
- package/dist/evaluation/same-output.d.ts.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +33 -0
- package/dist/index.js.map +1 -0
- package/dist/load-project.d.ts +34 -0
- package/dist/load-project.d.ts.map +1 -0
- package/dist/models/index.d.ts +2 -0
- package/dist/models/index.d.ts.map +1 -0
- package/dist/models/presets.d.ts +14 -0
- package/dist/models/presets.d.ts.map +1 -0
- package/dist/models/record/fit.d.ts +124 -0
- package/dist/models/record/fit.d.ts.map +1 -0
- package/dist/models/record/index.d.ts +2 -0
- package/dist/models/record/index.d.ts.map +1 -0
- package/dist/models/record/run.d.ts +115 -0
- package/dist/models/record/run.d.ts.map +1 -0
- package/dist/models/record/verify-export.d.ts +8 -0
- package/dist/models/record/verify-export.d.ts.map +1 -0
- package/dist/models/sequence/create-network.d.ts +3 -0
- package/dist/models/sequence/create-network.d.ts.map +1 -0
- package/dist/models/sequence/fit-sequence.d.ts +139 -0
- package/dist/models/sequence/fit-sequence.d.ts.map +1 -0
- package/dist/models/sequence/index.d.ts +3 -0
- package/dist/models/sequence/index.d.ts.map +1 -0
- package/dist/models/sequence/run-sequence.d.ts +162 -0
- package/dist/models/sequence/run-sequence.d.ts.map +1 -0
- package/dist/models/sequence/types.d.ts +10 -0
- package/dist/models/sequence/types.d.ts.map +1 -0
- package/dist/package-model-CPj0qnQs.js +18 -0
- package/dist/package-model-CPj0qnQs.js.map +1 -0
- package/dist/packaging/package-model.d.ts +3 -0
- package/dist/packaging/package-model.d.ts.map +1 -0
- package/dist/pipeline/define-pipeline.d.ts +20 -0
- package/dist/pipeline/define-pipeline.d.ts.map +1 -0
- package/dist/pipeline/index.d.ts +3 -0
- package/dist/pipeline/index.d.ts.map +1 -0
- package/dist/project/artifact.d.ts +129 -0
- package/dist/project/artifact.d.ts.map +1 -0
- package/dist/project/config.d.ts +37 -0
- package/dist/project/config.d.ts.map +1 -0
- package/dist/project/discover.d.ts +2 -0
- package/dist/project/discover.d.ts.map +1 -0
- package/dist/project/index.d.ts +4 -0
- package/dist/project/index.d.ts.map +1 -0
- package/dist/project/index.js +2 -0
- package/dist/project-BP331m20.js +125 -0
- package/dist/project-BP331m20.js.map +1 -0
- package/dist/record-BTZ5LWHF.js +205 -0
- package/dist/record-BTZ5LWHF.js.map +1 -0
- package/dist/run-Cwa7vD_T.js +72 -0
- package/dist/run-Cwa7vD_T.js.map +1 -0
- package/dist/run.d.ts +273 -0
- package/dist/run.d.ts.map +1 -0
- package/dist/sequence-C_WfVGO1.js +251 -0
- package/dist/sequence-C_WfVGO1.js.map +1 -0
- package/dist/train.d.ts +276 -0
- package/dist/train.d.ts.map +1 -0
- package/dist/types.d.ts +17 -0
- package/dist/types.d.ts.map +1 -0
- package/docs/README.md +9 -0
- package/docs/agent-skill.md +22 -0
- package/docs/cli.md +54 -0
- package/docs/dataset-format.md +58 -0
- package/docs/end-to-end.md +3 -0
- package/docs/evaluation.md +9 -0
- package/docs/filter-research.md +63 -0
- package/docs/getting-started.md +88 -0
- package/docs/neural-training.md +3 -0
- package/docs/parser-api.md +86 -0
- package/docs/pipelines.md +30 -0
- package/docs/primitives/README.md +19 -0
- package/docs/project-structure.md +41 -0
- package/docs/react.md +16 -0
- package/docs/releases.md +50 -0
- package/docs/repository-design.md +11 -0
- package/docs/runtime-backends.md +18 -0
- package/package.json +57 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import type { SequenceArtifact } from "@matchbox-ai/core/internal";
|
|
2
|
+
import type { DatasetExample } from "@matchbox-ai/core";
|
|
3
|
+
import type { SequenceRecipe } from "./types.js";
|
|
4
|
+
export declare function fitSequence(examples: readonly DatasetExample<unknown>[], recipe: SequenceRecipe, metadata: Pick<SequenceArtifact, "taskModule" | "taskMetadata" | "decoderModule">, probes?: readonly string[], shuffleLabels?: boolean, progress?: (epoch: number, loss: number) => void): Promise<{
|
|
5
|
+
untrained: {
|
|
6
|
+
formatVersion: 2;
|
|
7
|
+
modelTopology: Record<string, unknown>;
|
|
8
|
+
kind: "sequence-parser";
|
|
9
|
+
architecture: "embedding-window-mlp";
|
|
10
|
+
taskModule: string;
|
|
11
|
+
decoderModule: string;
|
|
12
|
+
taskMetadata: unknown;
|
|
13
|
+
readout: "all" | "last";
|
|
14
|
+
tokenizer: "characters" | "words";
|
|
15
|
+
vocabulary: string[];
|
|
16
|
+
labels: string[];
|
|
17
|
+
radius: number;
|
|
18
|
+
threshold: number;
|
|
19
|
+
precision: "float32" | "int8";
|
|
20
|
+
weights: [{
|
|
21
|
+
name: string;
|
|
22
|
+
shape: number[];
|
|
23
|
+
values: number[];
|
|
24
|
+
scale: number;
|
|
25
|
+
}, {
|
|
26
|
+
name: string;
|
|
27
|
+
shape: number[];
|
|
28
|
+
values: number[];
|
|
29
|
+
scale: number;
|
|
30
|
+
}, {
|
|
31
|
+
name: string;
|
|
32
|
+
shape: number[];
|
|
33
|
+
values: number[];
|
|
34
|
+
scale: number;
|
|
35
|
+
}, {
|
|
36
|
+
name: string;
|
|
37
|
+
shape: number[];
|
|
38
|
+
values: number[];
|
|
39
|
+
scale: number;
|
|
40
|
+
}, {
|
|
41
|
+
name: string;
|
|
42
|
+
shape: number[];
|
|
43
|
+
values: number[];
|
|
44
|
+
scale: number;
|
|
45
|
+
}];
|
|
46
|
+
};
|
|
47
|
+
float: {
|
|
48
|
+
formatVersion: 2;
|
|
49
|
+
modelTopology: Record<string, unknown>;
|
|
50
|
+
kind: "sequence-parser";
|
|
51
|
+
architecture: "embedding-window-mlp";
|
|
52
|
+
taskModule: string;
|
|
53
|
+
decoderModule: string;
|
|
54
|
+
taskMetadata: unknown;
|
|
55
|
+
readout: "all" | "last";
|
|
56
|
+
tokenizer: "characters" | "words";
|
|
57
|
+
vocabulary: string[];
|
|
58
|
+
labels: string[];
|
|
59
|
+
radius: number;
|
|
60
|
+
threshold: number;
|
|
61
|
+
precision: "float32" | "int8";
|
|
62
|
+
weights: [{
|
|
63
|
+
name: string;
|
|
64
|
+
shape: number[];
|
|
65
|
+
values: number[];
|
|
66
|
+
scale: number;
|
|
67
|
+
}, {
|
|
68
|
+
name: string;
|
|
69
|
+
shape: number[];
|
|
70
|
+
values: number[];
|
|
71
|
+
scale: number;
|
|
72
|
+
}, {
|
|
73
|
+
name: string;
|
|
74
|
+
shape: number[];
|
|
75
|
+
values: number[];
|
|
76
|
+
scale: number;
|
|
77
|
+
}, {
|
|
78
|
+
name: string;
|
|
79
|
+
shape: number[];
|
|
80
|
+
values: number[];
|
|
81
|
+
scale: number;
|
|
82
|
+
}, {
|
|
83
|
+
name: string;
|
|
84
|
+
shape: number[];
|
|
85
|
+
values: number[];
|
|
86
|
+
scale: number;
|
|
87
|
+
}];
|
|
88
|
+
};
|
|
89
|
+
quantized: {
|
|
90
|
+
formatVersion: 2;
|
|
91
|
+
modelTopology: Record<string, unknown>;
|
|
92
|
+
kind: "sequence-parser";
|
|
93
|
+
architecture: "embedding-window-mlp";
|
|
94
|
+
taskModule: string;
|
|
95
|
+
decoderModule: string;
|
|
96
|
+
taskMetadata: unknown;
|
|
97
|
+
readout: "all" | "last";
|
|
98
|
+
tokenizer: "characters" | "words";
|
|
99
|
+
vocabulary: string[];
|
|
100
|
+
labels: string[];
|
|
101
|
+
radius: number;
|
|
102
|
+
threshold: number;
|
|
103
|
+
precision: "float32" | "int8";
|
|
104
|
+
weights: [{
|
|
105
|
+
name: string;
|
|
106
|
+
shape: number[];
|
|
107
|
+
values: number[];
|
|
108
|
+
scale: number;
|
|
109
|
+
}, {
|
|
110
|
+
name: string;
|
|
111
|
+
shape: number[];
|
|
112
|
+
values: number[];
|
|
113
|
+
scale: number;
|
|
114
|
+
}, {
|
|
115
|
+
name: string;
|
|
116
|
+
shape: number[];
|
|
117
|
+
values: number[];
|
|
118
|
+
scale: number;
|
|
119
|
+
}, {
|
|
120
|
+
name: string;
|
|
121
|
+
shape: number[];
|
|
122
|
+
values: number[];
|
|
123
|
+
scale: number;
|
|
124
|
+
}, {
|
|
125
|
+
name: string;
|
|
126
|
+
shape: number[];
|
|
127
|
+
values: number[];
|
|
128
|
+
scale: number;
|
|
129
|
+
}];
|
|
130
|
+
};
|
|
131
|
+
history: number[];
|
|
132
|
+
parity: {
|
|
133
|
+
examples: number;
|
|
134
|
+
maxConfidenceError: number;
|
|
135
|
+
labelDisagreements: number;
|
|
136
|
+
};
|
|
137
|
+
supervisedTokens: number;
|
|
138
|
+
}>;
|
|
139
|
+
//# sourceMappingURL=fit-sequence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fit-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/fit-sequence.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,SAAS,cAAc,CAAC,OAAO,CAAC,EAAE,EAC5C,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,YAAY,GAAG,cAAc,GAAG,eAAe,CAAC,EACjF,MAAM,GAAE,SAAS,MAAM,EAAO,EAC9B,aAAa,UAAQ,EACrB,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6HjD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import type { loadProject } from "../../load-project.js";
|
|
2
|
+
export declare function runSequence(command: "train" | "eval", project: Awaited<ReturnType<typeof loadProject>>, progress?: (epoch: number, loss: number) => void): Promise<{
|
|
3
|
+
evaluation: {
|
|
4
|
+
examples: number;
|
|
5
|
+
invalidOutputRate: number;
|
|
6
|
+
exactAccuracy: number;
|
|
7
|
+
accepted: number;
|
|
8
|
+
acceptedAccuracy: number | null;
|
|
9
|
+
abstentionRate: number;
|
|
10
|
+
correctAbstentions: number;
|
|
11
|
+
failures: {
|
|
12
|
+
input: string;
|
|
13
|
+
expected: unknown;
|
|
14
|
+
actual: unknown;
|
|
15
|
+
}[];
|
|
16
|
+
};
|
|
17
|
+
report?: never;
|
|
18
|
+
output?: never;
|
|
19
|
+
} | {
|
|
20
|
+
evaluation?: never;
|
|
21
|
+
report: {
|
|
22
|
+
formatVersion: number;
|
|
23
|
+
architecture: "embedding-window-mlp";
|
|
24
|
+
backend: string;
|
|
25
|
+
seed: number;
|
|
26
|
+
artifactSha256: string;
|
|
27
|
+
bytes: number;
|
|
28
|
+
parameters: number;
|
|
29
|
+
datasetSha256: {
|
|
30
|
+
source: string;
|
|
31
|
+
sha256: string;
|
|
32
|
+
}[];
|
|
33
|
+
examples: {
|
|
34
|
+
train: number;
|
|
35
|
+
validation: number;
|
|
36
|
+
eval: number;
|
|
37
|
+
};
|
|
38
|
+
shuffledLabelsUngated: {
|
|
39
|
+
examples: number;
|
|
40
|
+
invalidOutputRate: number;
|
|
41
|
+
exactAccuracy: number;
|
|
42
|
+
accepted: number;
|
|
43
|
+
acceptedAccuracy: number | null;
|
|
44
|
+
abstentionRate: number;
|
|
45
|
+
correctAbstentions: number;
|
|
46
|
+
failures: {
|
|
47
|
+
input: string;
|
|
48
|
+
expected: unknown;
|
|
49
|
+
actual: unknown;
|
|
50
|
+
}[];
|
|
51
|
+
} | null;
|
|
52
|
+
supervisedTokens: number;
|
|
53
|
+
loss: number[];
|
|
54
|
+
exportParity: {
|
|
55
|
+
examples: number;
|
|
56
|
+
maxConfidenceError: number;
|
|
57
|
+
labelDisagreements: number;
|
|
58
|
+
};
|
|
59
|
+
validation: {
|
|
60
|
+
examples: number;
|
|
61
|
+
invalidOutputRate: number;
|
|
62
|
+
exactAccuracy: number;
|
|
63
|
+
accepted: number;
|
|
64
|
+
acceptedAccuracy: number | null;
|
|
65
|
+
abstentionRate: number;
|
|
66
|
+
correctAbstentions: number;
|
|
67
|
+
failures: {
|
|
68
|
+
input: string;
|
|
69
|
+
expected: unknown;
|
|
70
|
+
actual: unknown;
|
|
71
|
+
}[];
|
|
72
|
+
};
|
|
73
|
+
untrainedUngated: {
|
|
74
|
+
examples: number;
|
|
75
|
+
invalidOutputRate: number;
|
|
76
|
+
exactAccuracy: number;
|
|
77
|
+
accepted: number;
|
|
78
|
+
acceptedAccuracy: number | null;
|
|
79
|
+
abstentionRate: number;
|
|
80
|
+
correctAbstentions: number;
|
|
81
|
+
failures: {
|
|
82
|
+
input: string;
|
|
83
|
+
expected: unknown;
|
|
84
|
+
actual: unknown;
|
|
85
|
+
}[];
|
|
86
|
+
};
|
|
87
|
+
untrained: {
|
|
88
|
+
examples: number;
|
|
89
|
+
invalidOutputRate: number;
|
|
90
|
+
exactAccuracy: number;
|
|
91
|
+
accepted: number;
|
|
92
|
+
acceptedAccuracy: number | null;
|
|
93
|
+
abstentionRate: number;
|
|
94
|
+
correctAbstentions: number;
|
|
95
|
+
failures: {
|
|
96
|
+
input: string;
|
|
97
|
+
expected: unknown;
|
|
98
|
+
actual: unknown;
|
|
99
|
+
}[];
|
|
100
|
+
};
|
|
101
|
+
float: {
|
|
102
|
+
examples: number;
|
|
103
|
+
invalidOutputRate: number;
|
|
104
|
+
exactAccuracy: number;
|
|
105
|
+
accepted: number;
|
|
106
|
+
acceptedAccuracy: number | null;
|
|
107
|
+
abstentionRate: number;
|
|
108
|
+
correctAbstentions: number;
|
|
109
|
+
failures: {
|
|
110
|
+
input: string;
|
|
111
|
+
expected: unknown;
|
|
112
|
+
actual: unknown;
|
|
113
|
+
}[];
|
|
114
|
+
};
|
|
115
|
+
quantized: {
|
|
116
|
+
examples: number;
|
|
117
|
+
invalidOutputRate: number;
|
|
118
|
+
exactAccuracy: number;
|
|
119
|
+
accepted: number;
|
|
120
|
+
acceptedAccuracy: number | null;
|
|
121
|
+
abstentionRate: number;
|
|
122
|
+
correctAbstentions: number;
|
|
123
|
+
failures: {
|
|
124
|
+
input: string;
|
|
125
|
+
expected: unknown;
|
|
126
|
+
actual: unknown;
|
|
127
|
+
}[];
|
|
128
|
+
};
|
|
129
|
+
challenges: {
|
|
130
|
+
examples: number;
|
|
131
|
+
invalidOutputRate: number;
|
|
132
|
+
exactAccuracy: number;
|
|
133
|
+
accepted: number;
|
|
134
|
+
acceptedAccuracy: number | null;
|
|
135
|
+
abstentionRate: number;
|
|
136
|
+
correctAbstentions: number;
|
|
137
|
+
failures: {
|
|
138
|
+
input: string;
|
|
139
|
+
expected: unknown;
|
|
140
|
+
actual: unknown;
|
|
141
|
+
}[];
|
|
142
|
+
} | null;
|
|
143
|
+
baseline: {
|
|
144
|
+
examples: number;
|
|
145
|
+
invalidOutputRate: number;
|
|
146
|
+
exactAccuracy: number;
|
|
147
|
+
accepted: number;
|
|
148
|
+
acceptedAccuracy: number | null;
|
|
149
|
+
abstentionRate: number;
|
|
150
|
+
correctAbstentions: number;
|
|
151
|
+
failures: {
|
|
152
|
+
input: string;
|
|
153
|
+
expected: unknown;
|
|
154
|
+
actual: unknown;
|
|
155
|
+
}[];
|
|
156
|
+
} | null;
|
|
157
|
+
trainingMs: number;
|
|
158
|
+
notes: string;
|
|
159
|
+
};
|
|
160
|
+
output: string;
|
|
161
|
+
}>;
|
|
162
|
+
//# sourceMappingURL=run-sequence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-sequence.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/run-sequence.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAMzD,wBAAsB,WAAW,CAC/B,OAAO,EAAE,OAAO,GAAG,MAAM,EACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC,EAChD,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAqF5C,KAAK;YACL,UAAU;YACV,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BT"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { DatasetExample } from "@matchbox-ai/core";
|
|
2
|
+
import type { Token } from "@matchbox-ai/core/runtime";
|
|
3
|
+
/** Experimental supervision adapter. Application-facing datasets remain input/output JSONL. */
|
|
4
|
+
export interface SequenceRecipe {
|
|
5
|
+
tokenizer: "characters" | "words";
|
|
6
|
+
readout: "all" | "last";
|
|
7
|
+
labels: readonly string[];
|
|
8
|
+
annotate(example: DatasetExample<unknown>, tokens: readonly Token[]): readonly (string | null)[];
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/models/sequence/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACvD,+FAA+F;AAC/F,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,YAAY,GAAG,OAAO,CAAC;IAClC,OAAO,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;CAClG"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname, resolve } from "node:path";
|
|
3
|
+
//#region src/packaging/package-model.ts
|
|
4
|
+
async function packageModel(output, model, report) {
|
|
5
|
+
await mkdir(dirname(output), { recursive: true });
|
|
6
|
+
const declaration = output.replace(/\.matchbox$/, ".d.matchbox.ts");
|
|
7
|
+
if (declaration === output) throw new Error("The output filename must end in .matchbox.");
|
|
8
|
+
const decoderImport = model.decoderModule === null ? "" : `import decode from ${JSON.stringify(model.decoderModule.replace(/\.ts$/, ""))};\n`;
|
|
9
|
+
const taskImport = model.taskModule.replace(/\.ts$/, "");
|
|
10
|
+
await writeFile(output, JSON.stringify(model));
|
|
11
|
+
await writeFile(declaration, `import task from ${JSON.stringify(taskImport)};\nimport type { InferOutput } from "@matchbox-ai/core";\nimport type { MatchboxParser } from "@matchbox-ai/core/runtime";\ndeclare const parser: MatchboxParser<InferOutput<typeof task>> & { load(): Promise<void>; dispose(): void };\nexport default parser;\n`);
|
|
12
|
+
await writeFile(resolve(dirname(output), "report.json"), JSON.stringify(report, null, 2) + "\n");
|
|
13
|
+
await writeFile(output.replace(/\.matchbox$/, ".ts"), `import task from ${JSON.stringify(taskImport)};\n${decoderImport}import { createParser } from "@matchbox-ai/core/runtime";\nconst model = ${JSON.stringify(model)};\nexport default createParser(model, task${model.decoderModule === null ? "" : ", decode"});\n`);
|
|
14
|
+
}
|
|
15
|
+
//#endregion
|
|
16
|
+
export { packageModel as t };
|
|
17
|
+
|
|
18
|
+
//# sourceMappingURL=package-model-CPj0qnQs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"package-model-CPj0qnQs.js","names":[],"sources":["../src/packaging/package-model.ts"],"sourcesContent":["import { mkdir, writeFile } from \"node:fs/promises\";\nimport { dirname, resolve } from \"node:path\";\nimport type { SequenceArtifact, RecordArtifact } from \"@matchbox-ai/core/internal\";\nexport async function packageModel(\n output: string,\n model: SequenceArtifact | RecordArtifact,\n report: unknown,\n) {\n await mkdir(dirname(output), { recursive: true });\n const declaration = output.replace(/\\.matchbox$/, \".d.matchbox.ts\");\n if (declaration === output) throw new Error(\"The output filename must end in .matchbox.\");\n const decoderImport =\n model.decoderModule === null\n ? \"\"\n : `import decode from ${JSON.stringify(model.decoderModule.replace(/\\.ts$/, \"\"))};\\n`;\n const taskImport = model.taskModule.replace(/\\.ts$/, \"\");\n await writeFile(output, JSON.stringify(model));\n await writeFile(\n declaration,\n `import task from ${JSON.stringify(taskImport)};\\nimport type { InferOutput } from \"@matchbox-ai/core\";\\nimport type { MatchboxParser } from \"@matchbox-ai/core/runtime\";\\ndeclare const parser: MatchboxParser<InferOutput<typeof task>> & { load(): Promise<void>; dispose(): void };\\nexport default parser;\\n`,\n );\n await writeFile(resolve(dirname(output), \"report.json\"), JSON.stringify(report, null, 2) + \"\\n\");\n // A portable wrapper for bundlers without the Vite plugin; inference still remains local.\n await writeFile(\n output.replace(/\\.matchbox$/, \".ts\"),\n `import task from ${JSON.stringify(taskImport)};\\n${decoderImport}import { createParser } from \"@matchbox-ai/core/runtime\";\\nconst model = ${JSON.stringify(model)};\\nexport default createParser(model, task${model.decoderModule === null ? \"\" : \", decode\"});\\n`,\n );\n}\n"],"mappings":";;;AAGA,eAAsB,aACpB,QACA,OACA,QACA;CACA,MAAM,MAAM,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;CAChD,MAAM,cAAc,OAAO,QAAQ,eAAe,gBAAgB;CAClE,IAAI,gBAAgB,QAAQ,MAAM,IAAI,MAAM,4CAA4C;CACxF,MAAM,gBACJ,MAAM,kBAAkB,OACpB,KACA,sBAAsB,KAAK,UAAU,MAAM,cAAc,QAAQ,SAAS,EAAE,CAAC,EAAE;CACrF,MAAM,aAAa,MAAM,WAAW,QAAQ,SAAS,EAAE;CACvD,MAAM,UAAU,QAAQ,KAAK,UAAU,KAAK,CAAC;CAC7C,MAAM,UACJ,aACA,oBAAoB,KAAK,UAAU,UAAU,EAAE,mQACjD;CACA,MAAM,UAAU,QAAQ,QAAQ,MAAM,GAAG,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;CAE/F,MAAM,UACJ,OAAO,QAAQ,eAAe,KAAK,GACnC,oBAAoB,KAAK,UAAU,UAAU,EAAE,KAAK,cAAc,2EAA2E,KAAK,UAAU,KAAK,EAAE,4CAA4C,MAAM,kBAAkB,OAAO,KAAK,WAAW,KAChQ;AACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"package-model.d.ts","sourceRoot":"","sources":["../../src/packaging/package-model.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACnF,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,gBAAgB,GAAG,cAAc,EACxC,MAAM,EAAE,OAAO,iBAqBhB"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const pipelineSchema: z.ZodObject<{
|
|
3
|
+
input: z.ZodOptional<z.ZodObject<{
|
|
4
|
+
kind: z.ZodLiteral<"words">;
|
|
5
|
+
}, z.core.$strict>>;
|
|
6
|
+
prediction: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
7
|
+
kind: z.ZodLiteral<"field-classifier">;
|
|
8
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
9
|
+
kind: z.ZodLiteral<"token-classifier">;
|
|
10
|
+
recipe: z.ZodString;
|
|
11
|
+
decode: z.ZodString;
|
|
12
|
+
}, z.core.$strict>], "kind">;
|
|
13
|
+
acceptance: z.ZodOptional<z.ZodObject<{
|
|
14
|
+
minAccuracy: z.ZodOptional<z.ZodNumber>;
|
|
15
|
+
maxBytes: z.ZodOptional<z.ZodNumber>;
|
|
16
|
+
}, z.core.$strict>>;
|
|
17
|
+
}, z.core.$strict>;
|
|
18
|
+
export type Pipeline = z.infer<typeof pipelineSchema>;
|
|
19
|
+
export declare function definePipeline(pipeline: Pipeline): Pipeline;
|
|
20
|
+
//# sourceMappingURL=define-pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"define-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/define-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;kBA+BvB,CAAC;AACL,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAE3D"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtE,YAAY,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import type { SequenceDecoder } from "@matchbox-ai/core/runtime";
|
|
2
|
+
import type { ParserDefinition } from "@matchbox-ai/core";
|
|
3
|
+
import type { z } from "zod";
|
|
4
|
+
export declare function loadArtifact(configPath: string): Promise<{
|
|
5
|
+
artifact: {
|
|
6
|
+
formatVersion: 2;
|
|
7
|
+
modelTopology: Record<string, unknown>;
|
|
8
|
+
kind: "sequence-parser";
|
|
9
|
+
architecture: "embedding-window-mlp";
|
|
10
|
+
taskModule: string;
|
|
11
|
+
decoderModule: string;
|
|
12
|
+
taskMetadata: unknown;
|
|
13
|
+
readout: "all" | "last";
|
|
14
|
+
tokenizer: "characters" | "words";
|
|
15
|
+
vocabulary: string[];
|
|
16
|
+
labels: string[];
|
|
17
|
+
radius: number;
|
|
18
|
+
threshold: number;
|
|
19
|
+
precision: "float32" | "int8";
|
|
20
|
+
weights: [{
|
|
21
|
+
name: string;
|
|
22
|
+
shape: number[];
|
|
23
|
+
values: number[];
|
|
24
|
+
scale: number;
|
|
25
|
+
}, {
|
|
26
|
+
name: string;
|
|
27
|
+
shape: number[];
|
|
28
|
+
values: number[];
|
|
29
|
+
scale: number;
|
|
30
|
+
}, {
|
|
31
|
+
name: string;
|
|
32
|
+
shape: number[];
|
|
33
|
+
values: number[];
|
|
34
|
+
scale: number;
|
|
35
|
+
}, {
|
|
36
|
+
name: string;
|
|
37
|
+
shape: number[];
|
|
38
|
+
values: number[];
|
|
39
|
+
scale: number;
|
|
40
|
+
}, {
|
|
41
|
+
name: string;
|
|
42
|
+
shape: number[];
|
|
43
|
+
values: number[];
|
|
44
|
+
scale: number;
|
|
45
|
+
}];
|
|
46
|
+
} | {
|
|
47
|
+
formatVersion: 2;
|
|
48
|
+
modelTopology: Record<string, unknown>;
|
|
49
|
+
kind: "record-parser";
|
|
50
|
+
architecture: "bag-of-words-mlp";
|
|
51
|
+
taskModule: string;
|
|
52
|
+
decoderModule: null;
|
|
53
|
+
taskMetadata: unknown;
|
|
54
|
+
vocabulary: string[];
|
|
55
|
+
fields: {
|
|
56
|
+
name: string;
|
|
57
|
+
values: (string | number | boolean | null)[];
|
|
58
|
+
}[];
|
|
59
|
+
threshold: number;
|
|
60
|
+
precision: "float32" | "int8";
|
|
61
|
+
weights: [{
|
|
62
|
+
name: string;
|
|
63
|
+
shape: number[];
|
|
64
|
+
values: number[];
|
|
65
|
+
scale: number;
|
|
66
|
+
}, {
|
|
67
|
+
name: string;
|
|
68
|
+
shape: number[];
|
|
69
|
+
values: number[];
|
|
70
|
+
scale: number;
|
|
71
|
+
}, {
|
|
72
|
+
name: string;
|
|
73
|
+
shape: number[];
|
|
74
|
+
values: number[];
|
|
75
|
+
scale: number;
|
|
76
|
+
}, {
|
|
77
|
+
name: string;
|
|
78
|
+
shape: number[];
|
|
79
|
+
values: number[];
|
|
80
|
+
scale: number;
|
|
81
|
+
}];
|
|
82
|
+
};
|
|
83
|
+
task: ParserDefinition<z.ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>>;
|
|
84
|
+
decode: SequenceDecoder | undefined;
|
|
85
|
+
parser: import("@matchbox-ai/core/runtime").MatchboxParser<unknown> & {
|
|
86
|
+
load(): Promise<void>;
|
|
87
|
+
dispose(): void;
|
|
88
|
+
};
|
|
89
|
+
inspect: (input: string) => Promise<{
|
|
90
|
+
fields: {
|
|
91
|
+
field: string;
|
|
92
|
+
value: string | number | boolean | null | undefined;
|
|
93
|
+
confidence: number;
|
|
94
|
+
}[];
|
|
95
|
+
value: {
|
|
96
|
+
[k: string]: string | number | boolean | null | undefined;
|
|
97
|
+
};
|
|
98
|
+
confidence: number;
|
|
99
|
+
} | {
|
|
100
|
+
tokens: {
|
|
101
|
+
text: string;
|
|
102
|
+
key: string;
|
|
103
|
+
start: number;
|
|
104
|
+
end: number;
|
|
105
|
+
label: string;
|
|
106
|
+
confidence: number;
|
|
107
|
+
}[];
|
|
108
|
+
candidate: unknown;
|
|
109
|
+
}>;
|
|
110
|
+
output: string;
|
|
111
|
+
config: {
|
|
112
|
+
formatVersion: 1;
|
|
113
|
+
task: string;
|
|
114
|
+
train: string;
|
|
115
|
+
validation: string;
|
|
116
|
+
eval: string;
|
|
117
|
+
output: string;
|
|
118
|
+
minAccuracy: number;
|
|
119
|
+
maxBytes: number;
|
|
120
|
+
sequence?: {
|
|
121
|
+
recipe: string;
|
|
122
|
+
decoder: string;
|
|
123
|
+
} | undefined;
|
|
124
|
+
baseline?: string | undefined;
|
|
125
|
+
challenges?: string | undefined;
|
|
126
|
+
};
|
|
127
|
+
root: string;
|
|
128
|
+
}>;
|
|
129
|
+
//# sourceMappingURL=artifact.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifact.d.ts","sourceRoot":"","sources":["../../src/project/artifact.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAoB1B,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAchC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const configSchema: z.ZodObject<{
|
|
3
|
+
formatVersion: z.ZodDefault<z.ZodLiteral<1>>;
|
|
4
|
+
task: z.ZodDefault<z.ZodString>;
|
|
5
|
+
train: z.ZodDefault<z.ZodString>;
|
|
6
|
+
validation: z.ZodDefault<z.ZodString>;
|
|
7
|
+
eval: z.ZodDefault<z.ZodString>;
|
|
8
|
+
output: z.ZodString;
|
|
9
|
+
minAccuracy: z.ZodDefault<z.ZodNumber>;
|
|
10
|
+
maxBytes: z.ZodDefault<z.ZodNumber>;
|
|
11
|
+
sequence: z.ZodOptional<z.ZodObject<{
|
|
12
|
+
recipe: z.ZodString;
|
|
13
|
+
decoder: z.ZodString;
|
|
14
|
+
}, z.core.$strict>>;
|
|
15
|
+
baseline: z.ZodOptional<z.ZodString>;
|
|
16
|
+
challenges: z.ZodOptional<z.ZodString>;
|
|
17
|
+
}, z.core.$strict>;
|
|
18
|
+
export declare function loadConfig(path: string): Promise<{
|
|
19
|
+
config: {
|
|
20
|
+
formatVersion: 1;
|
|
21
|
+
task: string;
|
|
22
|
+
train: string;
|
|
23
|
+
validation: string;
|
|
24
|
+
eval: string;
|
|
25
|
+
output: string;
|
|
26
|
+
minAccuracy: number;
|
|
27
|
+
maxBytes: number;
|
|
28
|
+
sequence?: {
|
|
29
|
+
recipe: string;
|
|
30
|
+
decoder: string;
|
|
31
|
+
} | undefined;
|
|
32
|
+
baseline?: string | undefined;
|
|
33
|
+
challenges?: string | undefined;
|
|
34
|
+
};
|
|
35
|
+
root: string;
|
|
36
|
+
}>;
|
|
37
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/project/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;kBAYvB,CAAC;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;GAsC5C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../../src/project/discover.ts"],"names":[],"mappings":"AAuBA,wBAAsB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,SAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BpF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/project/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC"}
|