doppler-gpu 0.4.9 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/README.md +89 -12
  3. package/models/catalog.json +612 -7
  4. package/package.json +30 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/gpu/device.js +100 -32
  35. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  36. package/src/inference/pipelines/text/attention/projections.js +161 -24
  37. package/src/inference/pipelines/text/attention/record.js +67 -9
  38. package/src/inference/pipelines/text/attention/run.js +60 -7
  39. package/src/inference/pipelines/text/attention/types.js +18 -9
  40. package/src/inference/pipelines/text/config.d.ts +5 -0
  41. package/src/inference/pipelines/text/config.js +91 -0
  42. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  43. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  44. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  45. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  46. package/src/inference/pipelines/text/generator-steps.js +3 -1
  47. package/src/inference/pipelines/text/generator.js +90 -3
  48. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  49. package/src/inference/pipelines/text/init.d.ts +2 -0
  50. package/src/inference/pipelines/text/init.js +2 -0
  51. package/src/inference/pipelines/text/layer.js +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  53. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  54. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  55. package/src/inference/pipelines/text/logits/index.js +85 -11
  56. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  57. package/src/inference/pipelines/text/ops.js +32 -0
  58. package/src/inference/pipelines/text/types.d.ts +38 -0
  59. package/src/inference/pipelines/text/weights.d.ts +10 -0
  60. package/src/inference/pipelines/text/weights.js +33 -1
  61. package/src/inference/pipelines/text.d.ts +4 -2
  62. package/src/inference/pipelines/text.js +112 -0
  63. package/src/inference/tokenizers/behavior-flags.js +5 -1
  64. package/src/inference/tokenizers/bundled.js +99 -8
  65. package/src/loader/doppler-loader.d.ts +2 -0
  66. package/src/loader/doppler-loader.js +9 -0
  67. package/src/loader/final-weights-loader.d.ts +8 -0
  68. package/src/loader/final-weights-loader.js +42 -0
  69. package/src/loader/layer-loader.d.ts +6 -0
  70. package/src/loader/layer-loader.js +102 -3
  71. package/src/loader/loader-types.d.ts +15 -2
  72. package/src/tooling/node-converter.js +33 -1
  73. package/src/tooling/node-webgpu.js +85 -2
  74. package/src/tooling/runtime-optimization.d.ts +109 -0
  75. package/src/tooling/runtime-optimization.js +872 -0
  76. package/src/tooling-exports.shared.d.ts +19 -0
  77. package/src/tooling-exports.shared.js +11 -0
  78. package/src/version.js +1 -1
@@ -91,6 +91,9 @@ export const DEFAULT_CONVERTER_CONFIG = {
91
91
  manifest: DEFAULT_CONVERTER_MANIFEST_CONFIG,
92
92
  inference: DEFAULT_CONVERTER_INFERENCE_CONFIG,
93
93
  output: DEFAULT_CONVERTER_OUTPUT_CONFIG,
94
+ architecture: null,
95
+ sourceTensors: null,
96
+ sourceTokenizer: null,
94
97
  moeConfig: null,
95
98
  };
96
99
 
@@ -105,6 +108,9 @@ export function createConverterConfig(overrides) {
105
108
  manifest: { ...DEFAULT_CONVERTER_MANIFEST_CONFIG },
106
109
  inference: { ...DEFAULT_CONVERTER_INFERENCE_CONFIG },
107
110
  output: { ...DEFAULT_CONVERTER_OUTPUT_CONFIG },
111
+ architecture: null,
112
+ sourceTensors: null,
113
+ sourceTokenizer: null,
108
114
  };
109
115
  }
110
116
 
@@ -133,6 +139,15 @@ export function createConverterConfig(overrides) {
133
139
  output: overrides.output
134
140
  ? { ...DEFAULT_CONVERTER_OUTPUT_CONFIG, ...overrides.output }
135
141
  : { ...DEFAULT_CONVERTER_OUTPUT_CONFIG },
142
+ architecture: Object.prototype.hasOwnProperty.call(overrides, 'architecture')
143
+ ? (overrides.architecture ?? null)
144
+ : null,
145
+ sourceTensors: Object.prototype.hasOwnProperty.call(overrides, 'sourceTensors')
146
+ ? (overrides.sourceTensors ?? null)
147
+ : null,
148
+ sourceTokenizer: Object.prototype.hasOwnProperty.call(overrides, 'sourceTokenizer')
149
+ ? (overrides.sourceTokenizer ?? null)
150
+ : null,
136
151
  moeConfig: Object.prototype.hasOwnProperty.call(overrides, 'moeConfig')
137
152
  ? (overrides.moeConfig ?? null)
138
153
  : null,
@@ -84,6 +84,10 @@ export interface AttentionSchema {
84
84
  attnLogitSoftcapping?: number | null;
85
85
  /** Use query-key normalization */
86
86
  queryKeyNorm?: boolean;
87
+ /** Numerical family used for Q/K normalization. */
88
+ queryKeyNormType?: 'rmsnorm' | 'layernorm';
89
+ /** Normalize each head independently or the complete projection before head reshaping. */
90
+ queryKeyNormAxis?: 'head' | 'projection';
87
91
  /** Layers that apply query-key normalization; null means all layers when queryKeyNorm=true */
88
92
  queryKeyNormLayers?: number[] | null;
89
93
  /** Layers that have explicit Q/K RMSNorm scale tensors; null means every normalized layer must have weights */
@@ -100,6 +104,8 @@ export interface AttentionSchema {
100
104
 
101
105
  /** Normalization configuration */
102
106
  export interface NormalizationSchema {
107
+ /** Transformer normalization family. */
108
+ type?: 'rmsnorm' | 'layernorm';
103
109
  /** Add 1.0 to RMSNorm weights (Gemma-style) */
104
110
  rmsNormWeightOffset?: boolean;
105
111
  /** RMSNorm epsilon */
@@ -108,6 +114,8 @@ export interface NormalizationSchema {
108
114
  postAttentionNorm?: boolean;
109
115
  /** Use pre-feedforward norm */
110
116
  preFeedforwardNorm?: boolean;
117
+ /** Canonical final LayerNorm bias tensor, or null for a bias-free LayerNorm. */
118
+ finalNormBiasTensor?: string | null;
111
119
  /** Use post-feedforward norm */
112
120
  postFeedforwardNorm?: boolean;
113
121
  }
@@ -260,6 +260,10 @@ export interface ManifestAttentionSchema {
260
260
  slidingWindow: number | null;
261
261
  /** Query-key normalization */
262
262
  queryKeyNorm: boolean;
263
+ /** Numerical family used for Q/K normalization. */
264
+ queryKeyNormType: 'rmsnorm' | 'layernorm';
265
+ /** Normalize each head independently or the complete projection before head reshaping. */
266
+ queryKeyNormAxis: 'head' | 'projection';
263
267
  /** Layers that apply query-key normalization; null means all layers when queryKeyNorm=true */
264
268
  queryKeyNormLayers: number[] | null;
265
269
  /** Layers that have explicit Q/K RMSNorm scale tensors; null means every normalized layer must have weights */
@@ -276,9 +280,11 @@ export interface ManifestAttentionSchema {
276
280
 
277
281
  /**
278
282
  * Normalization configuration for inference.
279
- * Controls RMSNorm behavior and sandwich norm architecture.
283
+ * Controls transformer normalization and sandwich norm architecture.
280
284
  */
281
285
  export interface ManifestNormalizationSchema {
286
+ /** Normalization family used by transformer and final norms. */
287
+ type: 'rmsnorm' | 'layernorm';
282
288
  /** RMSNorm epsilon for numerical stability (default: 1e-5) */
283
289
  rmsNormEps: number;
284
290
  /** Use (1 + weight) pattern for RMSNorm (Gemma models) */
@@ -289,6 +295,8 @@ export interface ManifestNormalizationSchema {
289
295
  preFeedforwardNorm: boolean;
290
296
  /** Has post-feedforward normalization (sandwich norm) */
291
297
  postFeedforwardNorm: boolean;
298
+ /** Exact final LayerNorm bias tensor; null for RMSNorm or bias-free LayerNorm. */
299
+ finalNormBiasTensor: string | null;
292
300
  }
293
301
 
294
302
  /**
@@ -380,6 +388,20 @@ export interface ManifestOutputSchema {
380
388
  embeddingVocabSize: number | null;
381
389
  /** Optional embedding-only postprocessor stack applied after pooled hidden states. */
382
390
  embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
391
+ /** Optional exact tensor name for an additive decoder/LM-head bias. */
392
+ lmHeadBiasTensor: string | null;
393
+ }
394
+
395
+ export interface ManifestSequencePoolingSchema {
396
+ mode: 'mean' | 'last';
397
+ excludeTokenIds: number[];
398
+ }
399
+
400
+ export interface ManifestSequenceSchema {
401
+ alphabet: 'amino_acid' | 'nucleotide';
402
+ tokenEmbeddings: boolean;
403
+ pooledEmbedding: ManifestSequencePoolingSchema | null;
404
+ logits: boolean;
383
405
  }
384
406
 
385
407
  export interface ManifestEmbeddingProjectionSchema {
@@ -511,12 +533,16 @@ export interface ManifestInferenceSchema {
511
533
  chatTemplate: ManifestChatTemplateSchema;
512
534
  /** Whether this artifact exposes embedding workload support through pipeline.embed(). */
513
535
  supportsEmbedding: boolean;
536
+ /** Whether this artifact exposes typed biological sequence outputs. */
537
+ supportsSequence: boolean;
514
538
  /** Whether this artifact exposes rerank workload support through prefillWithLogits(). */
515
539
  supportsRerank: boolean;
516
540
  /** DiffusionGemma block-diffusion runtime contract, null for non-DiffusionGemma models. */
517
541
  diffusionGemma: ManifestDiffusionGemmaSchema | null;
518
542
  /** Manifest-owned rerank scoring contract, null when rerank is unsupported. */
519
543
  rerank: ManifestRerankSchema | null;
544
+ /** Biological sequence input/output contract, null when unsupported. */
545
+ sequence: ManifestSequenceSchema | null;
520
546
  /** Layer pipeline override (null = use optimized hardcoded path) */
521
547
  pipeline: LayerPipelineSchema | null;
522
548
  /** Explicit session policy for execution v1 manifests */
@@ -793,6 +819,11 @@ export declare function modelSupportsEmbedding(
793
819
  manifest: Partial<ManifestSchema> | null | undefined
794
820
  ): boolean;
795
821
 
822
+ /** Check if a manifest supports typed biological sequence workloads. */
823
+ export declare function modelSupportsSequence(
824
+ manifest: Partial<ManifestSchema> | null | undefined
825
+ ): boolean;
826
+
796
827
  /** Check if a manifest supports rerank workloads. */
797
828
  export declare function modelSupportsRerank(
798
829
  manifest: Partial<ManifestSchema> | null | undefined
@@ -49,6 +49,11 @@ export const DEFAULT_MANIFEST_INFERENCE = {
49
49
  // of modelType: dedicated embedding models (modelType="embedding") always
50
50
  // support embedding workloads; text-generation models opt in explicitly.
51
51
  supportsEmbedding: false,
52
+ // When true, exposes residue/base-level encoder outputs through
53
+ // pipeline.encodeSequence(). The exact alphabet and output surfaces are
54
+ // declared by inference.sequence; the runtime never infers them from a
55
+ // model family name.
56
+ supportsSequence: false,
52
57
  // When true, allows workload="rerank" to dispatch through prefillWithLogits()
53
58
  // using manifest-owned scoring metadata in inference.rerank.
54
59
  supportsRerank: false,
@@ -67,6 +72,11 @@ export const DEFAULT_MANIFEST_INFERENCE = {
67
72
  attnLogitSoftcapping: null, // No softcapping (null = disabled)
68
73
  slidingWindow: null, // Full attention (null = no sliding window)
69
74
  queryKeyNorm: false,
75
+ // Q/K normalization is normally per attention head RMSNorm (Qwen/Gemma).
76
+ // Encoder families such as ESMC normalize the complete Q/K projection with
77
+ // LayerNorm before reshaping into heads, so both semantics are explicit.
78
+ queryKeyNormType: 'rmsnorm',
79
+ queryKeyNormAxis: 'head',
70
80
  queryKeyNormLayers: null,
71
81
  queryKeyNormWeightLayers: null,
72
82
  valueNorm: false,
@@ -78,11 +88,17 @@ export const DEFAULT_MANIFEST_INFERENCE = {
78
88
  outputGateType: null,
79
89
  },
80
90
  normalization: {
91
+ // Transformer normalization family. Existing decoder manifests resolve to
92
+ // rmsnorm; encoder checkpoints such as ESM-2 must declare layernorm.
93
+ type: 'rmsnorm',
81
94
  rmsNormEps: DEFAULT_RMS_NORM_EPS,
82
95
  rmsNormWeightOffset: false,
83
96
  postAttentionNorm: false,
84
97
  preFeedforwardNorm: false,
85
98
  postFeedforwardNorm: false,
99
+ // Exact optional tensor for the final LayerNorm bias. Per-layer norm bias
100
+ // tensors use canonical *.bias names and are loaded with their layer.
101
+ finalNormBiasTensor: null,
86
102
  },
87
103
  ffn: {
88
104
  activation: 'silu',
@@ -127,6 +143,8 @@ export const DEFAULT_MANIFEST_INFERENCE = {
127
143
  embeddingTranspose: false,
128
144
  embeddingVocabSize: null,
129
145
  embeddingPostprocessor: null,
146
+ // Optional exact tensor name for an additive decoder/LM-head bias.
147
+ lmHeadBiasTensor: null,
130
148
  },
131
149
  layerPattern: {
132
150
  type: 'uniform', // All layers same type
@@ -142,6 +160,7 @@ export const DEFAULT_MANIFEST_INFERENCE = {
142
160
  },
143
161
  diffusionGemma: null,
144
162
  rerank: null,
163
+ sequence: null,
145
164
  pipeline: null,
146
165
  session: null,
147
166
  execution: null,
@@ -201,6 +220,13 @@ export function modelSupportsEmbedding(manifest) {
201
220
  return manifest?.inference?.supportsEmbedding === true;
202
221
  }
203
222
 
223
+ export function modelSupportsSequence(manifest) {
224
+ return manifest?.inference?.supportsSequence === true
225
+ && manifest?.inference?.sequence != null
226
+ && typeof manifest.inference.sequence === 'object'
227
+ && !Array.isArray(manifest.inference.sequence);
228
+ }
229
+
204
230
  export function modelSupportsRerank(manifest) {
205
231
  return manifest?.inference?.supportsRerank === true
206
232
  && manifest?.inference?.rerank != null
@@ -123,7 +123,19 @@ function resolveConversionPlanV1(options) {
123
123
  const rawConfig = options?.rawConfig || {};
124
124
  const tensors = Array.isArray(options?.tensors) ? options.tensors : [];
125
125
  const converterConfig = options.converterConfig;
126
- const inference = converterConfig.inference;
126
+ const rawInference = converterConfig.inference;
127
+ const inference = rawInference && typeof rawInference === 'object'
128
+ ? {
129
+ ...rawInference,
130
+ attention: rawInference.attention && typeof rawInference.attention === 'object'
131
+ ? {
132
+ queryKeyNormType: 'rmsnorm',
133
+ queryKeyNormAxis: 'head',
134
+ ...rawInference.attention,
135
+ }
136
+ : rawInference.attention,
137
+ }
138
+ : rawInference;
127
139
  const execution = converterConfig.execution;
128
140
  const session = converterConfig.session;
129
141
  const modelType = converterConfig?.modelType ?? rawConfig?.model_type ?? 'transformer';
@@ -217,10 +229,12 @@ function resolveConversionPlanV1(options) {
217
229
  layerPattern: inference.layerPattern ?? { type: 'uniform', globalPattern: null, period: null, offset: null, layerTypes: null },
218
230
  chatTemplate: inference.chatTemplate,
219
231
  supportsEmbedding: inference.supportsEmbedding ?? false,
232
+ supportsSequence: inference.supportsSequence ?? false,
220
233
  supportsRerank: inference.supportsRerank ?? false,
221
234
  supportsTranscription: inference.supportsTranscription ?? false,
222
235
  supportsVision: inference.supportsVision ?? false,
223
236
  rerank: inference.rerank ?? null,
237
+ sequence: inference.sequence ?? null,
224
238
  diffusionGemma: inference.diffusionGemma ?? null,
225
239
  pipeline: inference.pipeline ?? null,
226
240
  session,
@@ -778,7 +778,7 @@ function resolveIntermediateSizeFromTensors(architecture, model, tensorLocations
778
778
  return architecture;
779
779
  }
780
780
  const modelType = String(rawConfig?.model_type ?? getNestedTextConfig(rawConfig)?.model_type ?? '').toLowerCase();
781
- if (modelType !== 'lfm2') {
781
+ if (modelType !== 'lfm2' && modelType !== 'amplify') {
782
782
  return architecture;
783
783
  }
784
784
  const entries = Array.isArray(model?.tensors) && model.tensors.length > 0
@@ -1576,7 +1576,7 @@ export function extractArchitecture(config, ggufConfig) {
1576
1576
  'vocab_size'
1577
1577
  );
1578
1578
  const maxSeqLen = requireNumber(
1579
- fromConfig('max_position_embeddings', 'n_positions', 'max_seq_len'),
1579
+ fromConfig('max_position_embeddings', 'n_positions', 'max_seq_len', 'max_length'),
1580
1580
  'max_position_embeddings'
1581
1581
  );
1582
1582
  const ropeTheta = fromConfig('rope_theta') ?? undefined;
@@ -2264,6 +2264,7 @@ export function createManifest(
2264
2264
  throw new Error('Missing modelType for manifest');
2265
2265
  }
2266
2266
  const isDiffusion = resolvedModelType === 'diffusion';
2267
+ const isEmbedding = resolvedModelType === 'embedding';
2267
2268
  const architecture = options.architecture ?? model.architecture ?? (
2268
2269
  isDiffusion ? 'diffusion' : extractArchitecture(model.config, model.ggufConfig)
2269
2270
  );
@@ -2300,8 +2301,8 @@ export function createManifest(
2300
2301
  inference = {
2301
2302
  ...inference,
2302
2303
  output: {
2303
- ...inference.output,
2304
2304
  ...embeddingOutput,
2305
+ ...inference.output,
2305
2306
  embeddingPostprocessor,
2306
2307
  },
2307
2308
  };
@@ -2309,7 +2310,7 @@ export function createManifest(
2309
2310
 
2310
2311
  const eosTokenId = options.eosTokenId !== undefined
2311
2312
  ? options.eosTokenId
2312
- : isDiffusion
2313
+ : isDiffusion || isEmbedding
2313
2314
  ? null
2314
2315
  : resolveEosTokenId({
2315
2316
  config: rawConfig,
@@ -0,0 +1,50 @@
1
+ export interface SourceTensorRenameRule {
2
+ kind: 'rename';
3
+ match: string;
4
+ replace: string;
5
+ expectedMatches: number;
6
+ role?: string;
7
+ group?: string;
8
+ }
9
+
10
+ export interface SourceTensorSplitPart {
11
+ replace: string;
12
+ size: number;
13
+ role?: string;
14
+ group?: string;
15
+ }
16
+
17
+ export interface SourceTensorSplitRule {
18
+ kind: 'split';
19
+ match: string;
20
+ axis: 0;
21
+ parts: SourceTensorSplitPart[];
22
+ expectedMatches: number;
23
+ }
24
+
25
+ export interface SourceTensorIgnoreRule {
26
+ kind: 'ignore';
27
+ match: string;
28
+ reason: string;
29
+ expectedMatches: number;
30
+ }
31
+
32
+ export interface SourceTensorPolicy {
33
+ requireAll: boolean;
34
+ rules: Array<SourceTensorRenameRule | SourceTensorSplitRule | SourceTensorIgnoreRule>;
35
+ }
36
+
37
+ export interface SourceTensorDescriptor {
38
+ name: string;
39
+ shape: number[];
40
+ offset: number;
41
+ size: number;
42
+ role?: string;
43
+ group?: string;
44
+ [key: string]: unknown;
45
+ }
46
+
47
+ export declare function applySourceTensorRules<T extends SourceTensorDescriptor>(
48
+ tensors: T[],
49
+ policy: SourceTensorPolicy | null
50
+ ): T[];
@@ -0,0 +1,180 @@
1
+ function requirePlainObject(value, label) {
2
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
3
+ throw new Error(`${label} must be an object.`);
4
+ }
5
+ return value;
6
+ }
7
+
8
+ function requireString(value, label) {
9
+ if (typeof value !== 'string' || value.length === 0) {
10
+ throw new Error(`${label} must be a non-empty string.`);
11
+ }
12
+ return value;
13
+ }
14
+
15
+ function compileRulePattern(rule, index) {
16
+ const source = requireString(rule.match, `sourceTensors.rules[${index}].match`);
17
+ try {
18
+ return new RegExp(source);
19
+ } catch (error) {
20
+ throw new Error(
21
+ `sourceTensors.rules[${index}].match is not a valid regular expression: ${error.message}`
22
+ );
23
+ }
24
+ }
25
+
26
+ function requireExpectedMatches(rule, index) {
27
+ const value = rule.expectedMatches;
28
+ if (!Number.isInteger(value) || value < 0) {
29
+ throw new Error(
30
+ `sourceTensors.rules[${index}].expectedMatches must be a non-negative integer.`
31
+ );
32
+ }
33
+ return value;
34
+ }
35
+
36
+ function applyDescriptorOverrides(tensor, rule) {
37
+ return {
38
+ ...tensor,
39
+ ...(rule.role ? { role: requireString(rule.role, 'source tensor role') } : {}),
40
+ ...(rule.group ? { group: requireString(rule.group, 'source tensor group') } : {}),
41
+ };
42
+ }
43
+
44
+ function renameTensor(tensor, rule, pattern, index) {
45
+ const replacement = requireString(rule.replace, `sourceTensors.rules[${index}].replace`);
46
+ const name = tensor.name.replace(pattern, replacement);
47
+ if (!name || name === tensor.name && !pattern.test(tensor.name)) {
48
+ throw new Error(`sourceTensors.rules[${index}] did not produce a target tensor name.`);
49
+ }
50
+ return [applyDescriptorOverrides({ ...tensor, name }, rule)];
51
+ }
52
+
53
+ function splitTensor(tensor, rule, pattern, index) {
54
+ if (rule.axis !== 0) {
55
+ throw new Error(`sourceTensors.rules[${index}].axis must be 0 for contiguous SafeTensors splits.`);
56
+ }
57
+ if (!Array.isArray(tensor.shape) || tensor.shape.length < 1) {
58
+ throw new Error(`Cannot split source tensor "${tensor.name}" without a shape.`);
59
+ }
60
+ if (!Array.isArray(rule.parts) || rule.parts.length < 2) {
61
+ throw new Error(`sourceTensors.rules[${index}].parts must contain at least two parts.`);
62
+ }
63
+ const leadingSize = Number(tensor.shape[0]);
64
+ const rowByteSize = tensor.size / leadingSize;
65
+ if (!Number.isInteger(leadingSize) || leadingSize <= 0 || !Number.isInteger(rowByteSize)) {
66
+ throw new Error(`Source tensor "${tensor.name}" cannot be split into contiguous rows.`);
67
+ }
68
+
69
+ let rowOffset = 0;
70
+ const outputs = rule.parts.map((rawPart, partIndex) => {
71
+ const part = requirePlainObject(
72
+ rawPart,
73
+ `sourceTensors.rules[${index}].parts[${partIndex}]`
74
+ );
75
+ const rows = part.size;
76
+ if (!Number.isInteger(rows) || rows <= 0) {
77
+ throw new Error(
78
+ `sourceTensors.rules[${index}].parts[${partIndex}].size must be a positive integer.`
79
+ );
80
+ }
81
+ const replacement = requireString(
82
+ part.replace,
83
+ `sourceTensors.rules[${index}].parts[${partIndex}].replace`
84
+ );
85
+ const output = applyDescriptorOverrides({
86
+ ...tensor,
87
+ name: tensor.name.replace(pattern, replacement),
88
+ shape: [rows, ...tensor.shape.slice(1)],
89
+ offset: tensor.offset + rowOffset * rowByteSize,
90
+ size: rows * rowByteSize,
91
+ }, part);
92
+ rowOffset += rows;
93
+ return output;
94
+ });
95
+
96
+ if (rowOffset !== leadingSize) {
97
+ throw new Error(
98
+ `sourceTensors.rules[${index}] split ${rowOffset} rows from "${tensor.name}", expected ${leadingSize}.`
99
+ );
100
+ }
101
+ return outputs;
102
+ }
103
+
104
+ function applyRule(tensor, compiledRule, index) {
105
+ const { rule, pattern } = compiledRule;
106
+ if (rule.kind === 'rename') {
107
+ return renameTensor(tensor, rule, pattern, index);
108
+ }
109
+ if (rule.kind === 'split') {
110
+ return splitTensor(tensor, rule, pattern, index);
111
+ }
112
+ if (rule.kind === 'ignore') {
113
+ requireString(rule.reason, `sourceTensors.rules[${index}].reason`);
114
+ return [];
115
+ }
116
+ throw new Error(
117
+ `sourceTensors.rules[${index}].kind must be "rename", "split", or "ignore".`
118
+ );
119
+ }
120
+
121
+ function assertUniqueTargetNames(tensors) {
122
+ const names = new Set();
123
+ for (const tensor of tensors) {
124
+ if (names.has(tensor.name)) {
125
+ throw new Error(`sourceTensors rules produced duplicate target tensor "${tensor.name}".`);
126
+ }
127
+ names.add(tensor.name);
128
+ }
129
+ }
130
+
131
+ export function applySourceTensorRules(tensors, policy) {
132
+ if (policy == null) return tensors;
133
+ requirePlainObject(policy, 'sourceTensors');
134
+ if (typeof policy.requireAll !== 'boolean') {
135
+ throw new Error('sourceTensors.requireAll must be boolean.');
136
+ }
137
+ if (!Array.isArray(policy.rules) || policy.rules.length === 0) {
138
+ throw new Error('sourceTensors.rules must be a non-empty array.');
139
+ }
140
+ const compiledRules = policy.rules.map((rawRule, index) => {
141
+ const rule = requirePlainObject(rawRule, `sourceTensors.rules[${index}]`);
142
+ return {
143
+ rule,
144
+ pattern: compileRulePattern(rule, index),
145
+ expectedMatches: requireExpectedMatches(rule, index),
146
+ matches: 0,
147
+ };
148
+ });
149
+
150
+ const outputs = [];
151
+ for (const tensor of tensors) {
152
+ const matches = compiledRules
153
+ .map((entry, index) => ({ entry, index }))
154
+ .filter(({ entry }) => entry.pattern.test(tensor.name));
155
+ if (matches.length > 1) {
156
+ throw new Error(`Source tensor "${tensor.name}" matches multiple sourceTensors rules.`);
157
+ }
158
+ if (matches.length === 0) {
159
+ if (policy.requireAll) {
160
+ throw new Error(`Source tensor "${tensor.name}" is not covered by sourceTensors rules.`);
161
+ }
162
+ outputs.push(tensor);
163
+ continue;
164
+ }
165
+ const [{ entry, index }] = matches;
166
+ entry.matches += 1;
167
+ outputs.push(...applyRule(tensor, entry, index));
168
+ }
169
+
170
+ for (let index = 0; index < compiledRules.length; index += 1) {
171
+ const entry = compiledRules[index];
172
+ if (entry.matches !== entry.expectedMatches) {
173
+ throw new Error(
174
+ `sourceTensors.rules[${index}] matched ${entry.matches} tensors, expected ${entry.expectedMatches}.`
175
+ );
176
+ }
177
+ }
178
+ assertUniqueTargetNames(outputs);
179
+ return outputs;
180
+ }
@@ -0,0 +1,45 @@
1
+ export interface CharacterVocabSourceTokenizerPolicy {
2
+ kind: 'character_vocab';
3
+ vocabFile: string;
4
+ specialTokens: {
5
+ pad: string;
6
+ bos: string;
7
+ eos: string;
8
+ unk: string;
9
+ mask?: string | null;
10
+ };
11
+ addBosToken: boolean;
12
+ addEosToken: boolean;
13
+ }
14
+
15
+ export interface GreedyVocabSourceTokenizerPolicy {
16
+ kind: 'greedy_vocab';
17
+ vocabFile: string;
18
+ specialTokens: {
19
+ pad: string;
20
+ bos: string;
21
+ eos?: string | null;
22
+ unk: string;
23
+ mask?: string | null;
24
+ };
25
+ addBosToken: boolean;
26
+ addEosToken: boolean;
27
+ }
28
+
29
+ export type SourceTokenizerPolicy =
30
+ | CharacterVocabSourceTokenizerPolicy
31
+ | GreedyVocabSourceTokenizerPolicy;
32
+
33
+ export declare function validateSourceTokenizerPolicy(
34
+ policy: SourceTokenizerPolicy | null | undefined
35
+ ): SourceTokenizerPolicy | null;
36
+
37
+ export declare function buildCharacterTokenizerJson(
38
+ policy: CharacterVocabSourceTokenizerPolicy,
39
+ vocabText: string
40
+ ): Record<string, unknown>;
41
+
42
+ export declare function buildSourceTokenizerJson(
43
+ policy: SourceTokenizerPolicy,
44
+ vocabText: string
45
+ ): Record<string, unknown>;