doppler-gpu 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +98 -137
- package/assets/doppler.svg +1 -3
- package/models/catalog.json +1575 -0
- package/package.json +59 -26
- package/src/cli/doppler-cli.js +28 -7
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +52 -3
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +50 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +35 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +548 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +41 -0
- package/src/config/param-validator.js +67 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +117 -292
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +10 -2
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +2 -2
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +414 -21
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +589 -117
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +486 -13
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +23 -1
- package/src/inference/pipelines/text/config.js +235 -112
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +45 -2
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +254 -25
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.js +145 -39
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +4 -1
- package/src/inference/pipelines/text/generator-runtime.js +54 -20
- package/src/inference/pipelines/text/generator-steps.js +5 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +303 -20
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +65 -17
- package/src/inference/pipelines/text/layer.js +111 -5
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +185 -24
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +78 -115
- package/src/inference/pipelines/text/logits/index.js +105 -87
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.js +48 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +105 -21
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +60 -14
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +159 -22
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-readme-hero.svg +0 -84
- package/assets/doppler-runtime-map.svg +0 -84
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -35
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -58
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -318
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -28
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -26
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -300
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -159
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -124
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -68
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -25
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -89
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -40
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -219
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { log as debugLog } from '../debug/index.js';
|
|
2
1
|
import { readBuffer } from '../memory/buffer-pool.js';
|
|
3
2
|
import { CAPTURE_LEVELS, createDefaultCaptureConfig } from '../debug/capture-policy.js';
|
|
4
3
|
import { selectRuleValue } from '../rules/rule-registry.js';
|
|
@@ -8,47 +7,12 @@ import { cloneJsonValue } from '../utils/clone-json.js';
|
|
|
8
7
|
import { sha256BytesHex } from '../utils/sha256.js';
|
|
9
8
|
import { resolvePromptInput } from './pipelines/text/generator-prefill-helpers.js';
|
|
10
9
|
|
|
11
|
-
const DEFAULT_SAMPLING_DEFAULTS = Object.freeze({
|
|
12
|
-
temperature: 1.0,
|
|
13
|
-
topP: 0.95,
|
|
14
|
-
topK: 50,
|
|
15
|
-
repetitionPenalty: 1.1,
|
|
16
|
-
greedyThreshold: 0.01,
|
|
17
|
-
repetitionPenaltyWindow: 100,
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
const DEFAULT_HARNESS_PROMPT = 'The color of the sky is';
|
|
21
|
-
const DEFAULT_RUNTIME_PLACEHOLDER_PROMPT = 'Hello from Doppler.';
|
|
22
|
-
const DEFAULT_QWEN_PROMPT = Object.freeze({
|
|
23
|
-
messages: Object.freeze([
|
|
24
|
-
Object.freeze({
|
|
25
|
-
role: 'user',
|
|
26
|
-
content: 'Answer in one short sentence: What color is the sky on a clear day?',
|
|
27
|
-
}),
|
|
28
|
-
]),
|
|
29
|
-
});
|
|
30
|
-
const DEFAULT_TRANSLATEGEMMA_PROMPT = Object.freeze({
|
|
31
|
-
messages: Object.freeze([
|
|
32
|
-
Object.freeze({
|
|
33
|
-
role: 'user',
|
|
34
|
-
content: Object.freeze([
|
|
35
|
-
Object.freeze({
|
|
36
|
-
type: 'text',
|
|
37
|
-
source_lang_code: 'en',
|
|
38
|
-
target_lang_code: 'fr',
|
|
39
|
-
text: 'Hello world.',
|
|
40
|
-
}),
|
|
41
|
-
]),
|
|
42
|
-
}),
|
|
43
|
-
]),
|
|
44
|
-
});
|
|
45
10
|
const DEFAULT_IMAGE_TRANSCRIPTION_PROMPT = 'Describe the image in one short sentence.';
|
|
46
11
|
const DEFAULT_IMAGE_TRANSCRIPTION_SOFT_TOKEN_BUDGET = 70;
|
|
47
|
-
const DEFAULT_HARNESS_MAX_TOKENS = 32;
|
|
48
12
|
const EMBEDDING_PREVIEW_LENGTH = 16;
|
|
49
13
|
const GENERATION_TOKEN_DIAGNOSTIC_LIMIT = 32;
|
|
50
14
|
const DECODE_RECORD_TOP_OP_LIMIT = 20;
|
|
51
|
-
|
|
15
|
+
const RERANK_SCORE_POLICIES = new Set(['logit_difference', 'true_logit']);
|
|
52
16
|
|
|
53
17
|
export function normalizeDecodeRecordOpLabels(value) {
|
|
54
18
|
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
@@ -355,54 +319,18 @@ export async function captureKvCacheByteProof(pipeline, enabled) {
|
|
|
355
319
|
};
|
|
356
320
|
}
|
|
357
321
|
|
|
358
|
-
function hasPromptInput(value) {
|
|
359
|
-
return (typeof value === 'string' && value.trim().length > 0)
|
|
360
|
-
|| (Array.isArray(value) && value.length > 0)
|
|
361
|
-
|| isStructuredPromptInput(value);
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
function warnIfUsingDefaults(runtimeConfig, runOverrides = null) {
|
|
365
|
-
if (defaultsWarningEmitted) return;
|
|
366
|
-
const promptOverride = runtimeConfig?.inference?.prompt;
|
|
367
|
-
const hasPrompt = hasPromptInput(runOverrides?.prompt)
|
|
368
|
-
|| hasPromptInput(promptOverride);
|
|
369
|
-
const hasRuntimeSampling = isPlainObject(runtimeConfig?.inference?.sampling)
|
|
370
|
-
&& Object.keys(runtimeConfig.inference.sampling).length > 0;
|
|
371
|
-
const hasRunSampling = isPlainObject(runOverrides?.sampling)
|
|
372
|
-
&& Object.keys(runOverrides.sampling).length > 0;
|
|
373
|
-
const hasSampling = hasRuntimeSampling || hasRunSampling;
|
|
374
|
-
const hasMaxTokens = Number.isFinite(runOverrides?.maxTokens)
|
|
375
|
-
|| Number.isFinite(runtimeConfig?.inference?.generation?.maxTokens);
|
|
376
|
-
if (hasPrompt && hasSampling && hasMaxTokens) return;
|
|
377
|
-
defaultsWarningEmitted = true;
|
|
378
|
-
const missingFields = [];
|
|
379
|
-
const defaults = [];
|
|
380
|
-
if (!hasPrompt) {
|
|
381
|
-
missingFields.push('prompt');
|
|
382
|
-
defaults.push(` prompt: "${DEFAULT_HARNESS_PROMPT}"`);
|
|
383
|
-
}
|
|
384
|
-
if (!hasMaxTokens) {
|
|
385
|
-
missingFields.push('generation.maxTokens');
|
|
386
|
-
defaults.push(` maxTokens: ${DEFAULT_HARNESS_MAX_TOKENS}`);
|
|
387
|
-
}
|
|
388
|
-
if (!hasSampling) {
|
|
389
|
-
missingFields.push('sampling');
|
|
390
|
-
defaults.push(` temperature: ${DEFAULT_SAMPLING_DEFAULTS.temperature}`);
|
|
391
|
-
defaults.push(` topK: ${DEFAULT_SAMPLING_DEFAULTS.topK}`);
|
|
392
|
-
defaults.push(` topP: ${DEFAULT_SAMPLING_DEFAULTS.topP}`);
|
|
393
|
-
}
|
|
394
|
-
debugLog.warn('Harness',
|
|
395
|
-
`Running with default inference parameters for missing fields: ${missingFields.join(', ')}.\n`
|
|
396
|
-
+ defaults.join('\n')
|
|
397
|
-
+ '\n Provide explicit runtime.inference.sampling and generation.maxTokens if you want harness-stable settings.'
|
|
398
|
-
);
|
|
399
|
-
}
|
|
400
322
|
const embeddingSemanticFixtureAsset = await loadJson(
|
|
401
323
|
'./fixtures/embedding-semantic-fixtures.json',
|
|
402
324
|
import.meta.url,
|
|
403
325
|
'Failed to load embedding semantic fixtures'
|
|
404
326
|
);
|
|
405
327
|
|
|
328
|
+
const rerankSemanticFixtureAsset = await loadJson(
|
|
329
|
+
'./fixtures/rerank-semantic-fixtures.json',
|
|
330
|
+
import.meta.url,
|
|
331
|
+
'Failed to load rerank semantic fixtures'
|
|
332
|
+
);
|
|
333
|
+
|
|
406
334
|
function asText(value) {
|
|
407
335
|
if (typeof value !== 'string') return null;
|
|
408
336
|
const trimmed = value.trim();
|
|
@@ -482,6 +410,59 @@ function normalizeThroughputCorpus(corpus) {
|
|
|
482
410
|
return normalized.length > 0 ? normalized : null;
|
|
483
411
|
}
|
|
484
412
|
|
|
413
|
+
function normalizeRerankCases(cases) {
|
|
414
|
+
if (!Array.isArray(cases)) return null;
|
|
415
|
+
const normalized = [];
|
|
416
|
+
for (let i = 0; i < cases.length; i++) {
|
|
417
|
+
const entry = cases[i];
|
|
418
|
+
if (!entry || typeof entry !== 'object') continue;
|
|
419
|
+
|
|
420
|
+
const query = asText(entry.query);
|
|
421
|
+
const positive = asText(entry.positive);
|
|
422
|
+
const negative = asText(entry.negative);
|
|
423
|
+
if (!query || !positive || !negative) {
|
|
424
|
+
continue;
|
|
425
|
+
}
|
|
426
|
+
normalized.push({
|
|
427
|
+
id: asText(entry.id) ?? `rerank-${i + 1}`,
|
|
428
|
+
query,
|
|
429
|
+
positive,
|
|
430
|
+
negative,
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
return normalized.length > 0 ? normalized : null;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function resolveDefaultRerankSemanticFixtures() {
|
|
437
|
+
const defaults = rerankSemanticFixtureAsset?.defaults;
|
|
438
|
+
if (!isPlainObject(defaults)) {
|
|
439
|
+
throw new Error('Rerank semantic fixture asset must define defaults.');
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
const cases = normalizeRerankCases(defaults.cases);
|
|
443
|
+
if (!cases) {
|
|
444
|
+
throw new Error('Rerank semantic fixture asset must define cases.');
|
|
445
|
+
}
|
|
446
|
+
if (!Number.isFinite(defaults.minPairAcc)) {
|
|
447
|
+
throw new Error('Rerank semantic fixture asset must define minPairAcc.');
|
|
448
|
+
}
|
|
449
|
+
if (!Number.isFinite(defaults.minScoreMargin)) {
|
|
450
|
+
throw new Error('Rerank semantic fixture asset must define minScoreMargin.');
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
return {
|
|
454
|
+
cases,
|
|
455
|
+
minPairAcc: Math.max(0, Math.min(1, Number(defaults.minPairAcc))),
|
|
456
|
+
minScoreMargin: Number(defaults.minScoreMargin),
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
const DEFAULT_RERANK_SEMANTIC_FIXTURES = resolveDefaultRerankSemanticFixtures();
|
|
461
|
+
|
|
462
|
+
export function getDefaultRerankSemanticFixtures() {
|
|
463
|
+
return cloneJsonValue(DEFAULT_RERANK_SEMANTIC_FIXTURES);
|
|
464
|
+
}
|
|
465
|
+
|
|
485
466
|
function resolveDefaultEmbeddingSemanticFixtures() {
|
|
486
467
|
const defaults = embeddingSemanticFixtureAsset?.defaults;
|
|
487
468
|
if (!isPlainObject(defaults)) {
|
|
@@ -565,12 +546,13 @@ function resolveEmbeddingSemanticFixtures(runtimeConfig, options = null) {
|
|
|
565
546
|
function resolveEmbeddingSemanticStyle(pipeline) {
|
|
566
547
|
const manifest = pipeline?.manifest ?? null;
|
|
567
548
|
const style = selectRuleValue('inference', 'config', 'embeddingSemanticStyle', {
|
|
568
|
-
|
|
549
|
+
modelType: String(manifest?.modelType ?? '').toLowerCase(),
|
|
569
550
|
manifestModelType: String(
|
|
570
551
|
manifest?.config?.model_type
|
|
571
552
|
?? manifest?.config?.text_config?.model_type
|
|
572
553
|
?? ''
|
|
573
554
|
).toLowerCase(),
|
|
555
|
+
sourceCheckpointId: String(manifest?.artifactIdentity?.sourceCheckpointId ?? ''),
|
|
574
556
|
});
|
|
575
557
|
if (typeof style === 'string' && style.length > 0) {
|
|
576
558
|
return style;
|
|
@@ -587,6 +569,12 @@ function formatEmbeddingSemanticText(text, kind, style) {
|
|
|
587
569
|
return `title: None | text: ${text}`;
|
|
588
570
|
}
|
|
589
571
|
}
|
|
572
|
+
if (style === 'qwen3_embedding') {
|
|
573
|
+
if (kind === 'query') {
|
|
574
|
+
return `Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: ${text}`;
|
|
575
|
+
}
|
|
576
|
+
return text;
|
|
577
|
+
}
|
|
590
578
|
return text;
|
|
591
579
|
}
|
|
592
580
|
|
|
@@ -595,7 +583,506 @@ export function resolvePrompt(runtimeConfig) {
|
|
|
595
583
|
if (typeof runtimePrompt === 'string' && runtimePrompt.trim()) {
|
|
596
584
|
return runtimePrompt.trim();
|
|
597
585
|
}
|
|
598
|
-
|
|
586
|
+
throw new Error('Harness embedding requires explicit runtime.inference.prompt.');
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function assertRerankTokenId(value, label) {
|
|
590
|
+
const tokenId = Number(value);
|
|
591
|
+
if (!Number.isInteger(tokenId) || tokenId < 0) {
|
|
592
|
+
throw new Error(`Manifest rerank config requires non-negative integer ${label}.`);
|
|
593
|
+
}
|
|
594
|
+
return tokenId;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function assertRerankText(value, label, preserve = false) {
|
|
598
|
+
if (typeof value !== 'string') {
|
|
599
|
+
throw new Error(`Manifest rerank config requires non-empty ${label}.`);
|
|
600
|
+
}
|
|
601
|
+
const text = value.trim();
|
|
602
|
+
if (!text) {
|
|
603
|
+
throw new Error(`Manifest rerank config requires non-empty ${label}.`);
|
|
604
|
+
}
|
|
605
|
+
return preserve ? value : text;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
export function resolveRerankScoringConfig(pipeline) {
|
|
609
|
+
const config = pipeline?.manifest?.inference?.rerank;
|
|
610
|
+
if (!isPlainObject(config)) {
|
|
611
|
+
throw new Error('Rerank workload requires manifest.inference.rerank scoring config.');
|
|
612
|
+
}
|
|
613
|
+
const format = assertRerankText(config.format, 'format');
|
|
614
|
+
if (format !== 'qwen3_yes_no_logit') {
|
|
615
|
+
throw new Error(`Unsupported rerank scoring format "${format}".`);
|
|
616
|
+
}
|
|
617
|
+
const trueTokenId = assertRerankTokenId(config.trueTokenId, 'trueTokenId');
|
|
618
|
+
const falseTokenId = assertRerankTokenId(config.falseTokenId, 'falseTokenId');
|
|
619
|
+
if (trueTokenId === falseTokenId) {
|
|
620
|
+
throw new Error('Manifest rerank config trueTokenId and falseTokenId must be distinct.');
|
|
621
|
+
}
|
|
622
|
+
const score = assertRerankText(config.score, 'score');
|
|
623
|
+
if (!RERANK_SCORE_POLICIES.has(score)) {
|
|
624
|
+
throw new Error(`Unsupported rerank score policy "${score}".`);
|
|
625
|
+
}
|
|
626
|
+
const probability = assertRerankText(config.probability, 'probability');
|
|
627
|
+
if (probability !== 'sigmoid') {
|
|
628
|
+
throw new Error(`Unsupported rerank probability policy "${probability}".`);
|
|
629
|
+
}
|
|
630
|
+
return {
|
|
631
|
+
format,
|
|
632
|
+
instruction: assertRerankText(config.instruction, 'instruction'),
|
|
633
|
+
inputTemplate: assertRerankText(config.inputTemplate, 'inputTemplate', true),
|
|
634
|
+
prefix: assertRerankText(config.prefix, 'prefix', true),
|
|
635
|
+
suffix: assertRerankText(config.suffix, 'suffix', true),
|
|
636
|
+
trueToken: assertRerankText(config.trueToken, 'trueToken'),
|
|
637
|
+
trueTokenId,
|
|
638
|
+
falseToken: assertRerankText(config.falseToken, 'falseToken'),
|
|
639
|
+
falseTokenId,
|
|
640
|
+
score,
|
|
641
|
+
probability,
|
|
642
|
+
};
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function replaceRerankTemplate(template, values) {
|
|
646
|
+
let output = template;
|
|
647
|
+
for (const [key, value] of Object.entries(values)) {
|
|
648
|
+
const placeholder = `{${key}}`;
|
|
649
|
+
if (!output.includes(placeholder)) {
|
|
650
|
+
throw new Error(`Manifest rerank inputTemplate is missing ${placeholder}.`);
|
|
651
|
+
}
|
|
652
|
+
output = output.split(placeholder).join(value);
|
|
653
|
+
}
|
|
654
|
+
return output;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
export function formatRerankPrompt(query, document, scoringConfig) {
|
|
658
|
+
const instruction = assertRerankText(scoringConfig?.instruction, 'instruction');
|
|
659
|
+
const normalizedQuery = assertRerankText(query, 'query');
|
|
660
|
+
const normalizedDocument = assertRerankText(document, 'document');
|
|
661
|
+
const input = replaceRerankTemplate(
|
|
662
|
+
assertRerankText(scoringConfig?.inputTemplate, 'inputTemplate', true),
|
|
663
|
+
{
|
|
664
|
+
instruction,
|
|
665
|
+
query: normalizedQuery,
|
|
666
|
+
document: normalizedDocument,
|
|
667
|
+
}
|
|
668
|
+
);
|
|
669
|
+
return `${assertRerankText(scoringConfig?.prefix, 'prefix', true)}${input}${assertRerankText(scoringConfig?.suffix, 'suffix', true)}`;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function sigmoid(value) {
|
|
673
|
+
return 1 / (1 + Math.exp(-value));
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function computeRerankScore(scoringConfig, trueLogit, falseLogit) {
|
|
677
|
+
if (scoringConfig.score === 'logit_difference') {
|
|
678
|
+
return trueLogit - falseLogit;
|
|
679
|
+
}
|
|
680
|
+
if (scoringConfig.score === 'true_logit') {
|
|
681
|
+
return trueLogit;
|
|
682
|
+
}
|
|
683
|
+
throw new Error(`Unsupported rerank score policy "${scoringConfig.score}".`);
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
function assertLogitsVector(value) {
|
|
687
|
+
if (!ArrayBuffer.isView(value) && !Array.isArray(value)) {
|
|
688
|
+
throw new Error('Rerank prefillWithLogits result must include a logits vector.');
|
|
689
|
+
}
|
|
690
|
+
return value;
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
function buildRerankScoreRecord(query, document, prompt, tokenCount, trueLogit, falseLogit, config, scoringPath, phase = null) {
|
|
694
|
+
if (!Number.isFinite(trueLogit) || !Number.isFinite(falseLogit)) {
|
|
695
|
+
throw new Error(
|
|
696
|
+
`Rerank logits missing finite yes/no scores at token IDs ${config.trueTokenId}/${config.falseTokenId}.`
|
|
697
|
+
);
|
|
698
|
+
}
|
|
699
|
+
const score = computeRerankScore(config, trueLogit, falseLogit);
|
|
700
|
+
const probability = sigmoid(score);
|
|
701
|
+
return {
|
|
702
|
+
query,
|
|
703
|
+
document,
|
|
704
|
+
prompt,
|
|
705
|
+
tokenCount,
|
|
706
|
+
score,
|
|
707
|
+
probability,
|
|
708
|
+
trueLogit,
|
|
709
|
+
falseLogit,
|
|
710
|
+
trueTokenId: config.trueTokenId,
|
|
711
|
+
falseTokenId: config.falseTokenId,
|
|
712
|
+
scoringPath,
|
|
713
|
+
phase,
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
export async function scoreRerankDocument(pipeline, query, document, scoringConfig = null, options = {}) {
|
|
718
|
+
if (!pipeline || (typeof pipeline.prefillWithTokenLogits !== 'function' && typeof pipeline.prefillWithLogits !== 'function')) {
|
|
719
|
+
throw new Error('Rerank workload requires pipeline.prefillWithTokenLogits() or pipeline.prefillWithLogits().');
|
|
720
|
+
}
|
|
721
|
+
const config = scoringConfig ?? resolveRerankScoringConfig(pipeline);
|
|
722
|
+
const prompt = formatRerankPrompt(query, document, config);
|
|
723
|
+
pipeline.reset?.();
|
|
724
|
+
let trueLogit;
|
|
725
|
+
let falseLogit;
|
|
726
|
+
let tokenCount = 0;
|
|
727
|
+
let scoringPath = 'full-logits';
|
|
728
|
+
let resultPhase = null;
|
|
729
|
+
const totalStart = performance.now();
|
|
730
|
+
if (typeof pipeline.prefillWithTokenLogits === 'function') {
|
|
731
|
+
const prefillCallStart = performance.now();
|
|
732
|
+
const result = await pipeline.prefillWithTokenLogits(
|
|
733
|
+
prompt,
|
|
734
|
+
[config.trueTokenId, config.falseTokenId],
|
|
735
|
+
{
|
|
736
|
+
useChatTemplate: false,
|
|
737
|
+
benchmark: options.benchmark === true,
|
|
738
|
+
}
|
|
739
|
+
);
|
|
740
|
+
const prefillCallMs = performance.now() - prefillCallStart;
|
|
741
|
+
const logits = assertLogitsVector(result?.logits);
|
|
742
|
+
trueLogit = Number(logits[0]);
|
|
743
|
+
falseLogit = Number(logits[1]);
|
|
744
|
+
tokenCount = Number.isFinite(result?.tokens?.length) ? result.tokens.length : 0;
|
|
745
|
+
scoringPath = 'selected-token-logits';
|
|
746
|
+
resultPhase = {
|
|
747
|
+
...(isPlainObject(result?.phase) ? result.phase : {}),
|
|
748
|
+
prefillCallMs,
|
|
749
|
+
};
|
|
750
|
+
} else {
|
|
751
|
+
const prefillCallStart = performance.now();
|
|
752
|
+
const result = await pipeline.prefillWithLogits(prompt, {
|
|
753
|
+
useChatTemplate: false,
|
|
754
|
+
benchmark: options.benchmark === true,
|
|
755
|
+
});
|
|
756
|
+
const prefillCallMs = performance.now() - prefillCallStart;
|
|
757
|
+
const logits = assertLogitsVector(result?.logits);
|
|
758
|
+
trueLogit = Number(logits[config.trueTokenId]);
|
|
759
|
+
falseLogit = Number(logits[config.falseTokenId]);
|
|
760
|
+
tokenCount = Number.isFinite(result?.tokens?.length) ? result.tokens.length : 0;
|
|
761
|
+
resultPhase = {
|
|
762
|
+
...(isPlainObject(result?.phase) ? result.phase : {}),
|
|
763
|
+
prefillCallMs,
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
return buildRerankScoreRecord(
|
|
767
|
+
query,
|
|
768
|
+
document,
|
|
769
|
+
prompt,
|
|
770
|
+
tokenCount,
|
|
771
|
+
trueLogit,
|
|
772
|
+
falseLogit,
|
|
773
|
+
config,
|
|
774
|
+
scoringPath,
|
|
775
|
+
{
|
|
776
|
+
...(resultPhase ?? {}),
|
|
777
|
+
totalMs: performance.now() - totalStart,
|
|
778
|
+
promptChars: prompt.length,
|
|
779
|
+
}
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
function resolveRerankInput(runtimeConfig, runOverrides = null) {
|
|
784
|
+
const source = isPlainObject(runOverrides?.rerank)
|
|
785
|
+
? runOverrides.rerank
|
|
786
|
+
: runtimeConfig?.inference?.rerank;
|
|
787
|
+
if (!isPlainObject(source)) {
|
|
788
|
+
throw new Error('Harness rerank requires explicit runtime.inference.rerank.');
|
|
789
|
+
}
|
|
790
|
+
const query = asText(source.query);
|
|
791
|
+
if (!query) {
|
|
792
|
+
throw new Error('Harness rerank requires non-empty runtime.inference.rerank.query.');
|
|
793
|
+
}
|
|
794
|
+
const documents = Array.isArray(source.documents)
|
|
795
|
+
? source.documents.map(asText).filter(Boolean)
|
|
796
|
+
: [];
|
|
797
|
+
if (documents.length === 0) {
|
|
798
|
+
throw new Error('Harness rerank requires non-empty runtime.inference.rerank.documents.');
|
|
799
|
+
}
|
|
800
|
+
return { query, documents };
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
function summarizeRerankScores(scores) {
|
|
804
|
+
const sorted = [...scores].sort((a, b) => {
|
|
805
|
+
const scoreDelta = b.score - a.score;
|
|
806
|
+
return scoreDelta !== 0 ? scoreDelta : a.index - b.index;
|
|
807
|
+
});
|
|
808
|
+
return {
|
|
809
|
+
ranking: sorted.map((entry, rank) => ({
|
|
810
|
+
rank: rank + 1,
|
|
811
|
+
index: entry.index,
|
|
812
|
+
document: entry.document,
|
|
813
|
+
score: Number(entry.score.toFixed(6)),
|
|
814
|
+
probability: Number(entry.probability.toFixed(6)),
|
|
815
|
+
trueLogit: Number(entry.trueLogit.toFixed(6)),
|
|
816
|
+
falseLogit: Number(entry.falseLogit.toFixed(6)),
|
|
817
|
+
tokenCount: entry.tokenCount,
|
|
818
|
+
scoringPath: entry.scoringPath,
|
|
819
|
+
})),
|
|
820
|
+
top: sorted[0] ?? null,
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
function getPipelineTokenizer(pipeline) {
|
|
825
|
+
const tokenizer = pipeline?.tokenizer;
|
|
826
|
+
return tokenizer && typeof tokenizer.encode === 'function' ? tokenizer : null;
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
function longestCommonTokenPrefixLength(rows) {
|
|
830
|
+
if (!Array.isArray(rows) || rows.length === 0) {
|
|
831
|
+
return 0;
|
|
832
|
+
}
|
|
833
|
+
const first = rows[0];
|
|
834
|
+
let length = Array.isArray(first) ? first.length : 0;
|
|
835
|
+
for (let rowIndex = 1; rowIndex < rows.length; rowIndex += 1) {
|
|
836
|
+
const row = rows[rowIndex];
|
|
837
|
+
if (!Array.isArray(row)) {
|
|
838
|
+
return 0;
|
|
839
|
+
}
|
|
840
|
+
length = Math.min(length, row.length);
|
|
841
|
+
for (let index = 0; index < length; index += 1) {
|
|
842
|
+
if (row[index] !== first[index]) {
|
|
843
|
+
length = index;
|
|
844
|
+
break;
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
return length;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
async function createRerankPrefixContext(pipeline, query, documents, config, options = {}) {
|
|
852
|
+
const totalStart = performance.now();
|
|
853
|
+
if (
|
|
854
|
+
typeof pipeline?.prefillKVOnly !== 'function'
|
|
855
|
+
|| typeof pipeline?.prefillWithTokenLogits !== 'function'
|
|
856
|
+
|| typeof pipeline?.resetToSeqLen !== 'function'
|
|
857
|
+
) {
|
|
858
|
+
return null;
|
|
859
|
+
}
|
|
860
|
+
const tokenizer = getPipelineTokenizer(pipeline);
|
|
861
|
+
if (!tokenizer) {
|
|
862
|
+
return null;
|
|
863
|
+
}
|
|
864
|
+
const prompts = documents.map((document) => formatRerankPrompt(query, document, config));
|
|
865
|
+
const tokenRows = prompts.map((prompt, index) => normalizeTokenIdArray(
|
|
866
|
+
tokenizer.encode(prompt),
|
|
867
|
+
`rerank.prompt[${index}].tokenIds`
|
|
868
|
+
));
|
|
869
|
+
const prefixLength = longestCommonTokenPrefixLength(tokenRows);
|
|
870
|
+
if (prefixLength <= 0 || tokenRows.some((row) => row.length <= prefixLength)) {
|
|
871
|
+
return null;
|
|
872
|
+
}
|
|
873
|
+
const prefixTokens = tokenRows[0].slice(0, prefixLength);
|
|
874
|
+
pipeline.reset?.();
|
|
875
|
+
const prefillStart = performance.now();
|
|
876
|
+
const prefix = await pipeline.prefillKVOnly('', {
|
|
877
|
+
useChatTemplate: false,
|
|
878
|
+
inputIds: prefixTokens,
|
|
879
|
+
benchmark: options.benchmark === true,
|
|
880
|
+
});
|
|
881
|
+
const prefillMs = performance.now() - prefillStart;
|
|
882
|
+
return {
|
|
883
|
+
prefix,
|
|
884
|
+
seqLen: prefix.seqLen,
|
|
885
|
+
prefixTokens,
|
|
886
|
+
prompts,
|
|
887
|
+
tokenRows,
|
|
888
|
+
suffixRows: tokenRows.map((row) => row.slice(prefixLength)),
|
|
889
|
+
phase: {
|
|
890
|
+
totalMs: performance.now() - totalStart,
|
|
891
|
+
prefillMs,
|
|
892
|
+
prefixTokens: prefixTokens.length,
|
|
893
|
+
documentCount: documents.length,
|
|
894
|
+
},
|
|
895
|
+
};
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
async function scoreRerankDocumentFromPrefix(pipeline, query, document, config, prefixContext, index, options = {}) {
|
|
899
|
+
const suffixTokens = prefixContext.suffixRows[index];
|
|
900
|
+
const totalStart = performance.now();
|
|
901
|
+
try {
|
|
902
|
+
const prefillCallStart = performance.now();
|
|
903
|
+
const result = await pipeline.prefillWithTokenLogits(
|
|
904
|
+
'',
|
|
905
|
+
[config.trueTokenId, config.falseTokenId],
|
|
906
|
+
{
|
|
907
|
+
useChatTemplate: false,
|
|
908
|
+
inputIds: suffixTokens,
|
|
909
|
+
benchmark: options.benchmark === true,
|
|
910
|
+
}
|
|
911
|
+
);
|
|
912
|
+
const prefillCallMs = performance.now() - prefillCallStart;
|
|
913
|
+
const logits = assertLogitsVector(result?.logits);
|
|
914
|
+
const trueLogit = Number(logits[0]);
|
|
915
|
+
const falseLogit = Number(logits[1]);
|
|
916
|
+
return buildRerankScoreRecord(
|
|
917
|
+
query,
|
|
918
|
+
document,
|
|
919
|
+
prefixContext.prompts[index],
|
|
920
|
+
prefixContext.prefixTokens.length + suffixTokens.length,
|
|
921
|
+
trueLogit,
|
|
922
|
+
falseLogit,
|
|
923
|
+
config,
|
|
924
|
+
'prefix-selected-token-logits',
|
|
925
|
+
{
|
|
926
|
+
...(isPlainObject(result?.phase) ? result.phase : {}),
|
|
927
|
+
totalMs: performance.now() - totalStart,
|
|
928
|
+
prefillCallMs,
|
|
929
|
+
suffixTokens: suffixTokens.length,
|
|
930
|
+
}
|
|
931
|
+
);
|
|
932
|
+
} finally {
|
|
933
|
+
pipeline.resetToSeqLen(prefixContext.seqLen);
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
export async function runRerank(pipeline, runtimeConfig, runOverrides = null) {
|
|
938
|
+
const input = resolveRerankInput(runtimeConfig, runOverrides);
|
|
939
|
+
const config = resolveRerankScoringConfig(pipeline);
|
|
940
|
+
const start = performance.now();
|
|
941
|
+
const scores = [];
|
|
942
|
+
const prefixStart = performance.now();
|
|
943
|
+
const prefixContext = await createRerankPrefixContext(pipeline, input.query, input.documents, config, {
|
|
944
|
+
benchmark: runOverrides?.benchmark === true,
|
|
945
|
+
});
|
|
946
|
+
const prefixMs = performance.now() - prefixStart;
|
|
947
|
+
for (let i = 0; i < input.documents.length; i++) {
|
|
948
|
+
const scored = prefixContext
|
|
949
|
+
? await scoreRerankDocumentFromPrefix(
|
|
950
|
+
pipeline,
|
|
951
|
+
input.query,
|
|
952
|
+
input.documents[i],
|
|
953
|
+
config,
|
|
954
|
+
prefixContext,
|
|
955
|
+
i,
|
|
956
|
+
{
|
|
957
|
+
benchmark: runOverrides?.benchmark === true,
|
|
958
|
+
}
|
|
959
|
+
)
|
|
960
|
+
: await scoreRerankDocument(
|
|
961
|
+
pipeline,
|
|
962
|
+
input.query,
|
|
963
|
+
input.documents[i],
|
|
964
|
+
config,
|
|
965
|
+
{
|
|
966
|
+
benchmark: runOverrides?.benchmark === true,
|
|
967
|
+
}
|
|
968
|
+
);
|
|
969
|
+
scores.push({
|
|
970
|
+
index: i,
|
|
971
|
+
...scored,
|
|
972
|
+
});
|
|
973
|
+
}
|
|
974
|
+
const summary = summarizeRerankScores(scores);
|
|
975
|
+
const durationMs = Math.max(1, performance.now() - start);
|
|
976
|
+
const documentDurations = scores
|
|
977
|
+
.map((entry) => Number(entry.phase?.totalMs))
|
|
978
|
+
.filter((value) => Number.isFinite(value));
|
|
979
|
+
const documentTotalMs = documentDurations.reduce((sum, value) => sum + value, 0);
|
|
980
|
+
return {
|
|
981
|
+
query: input.query,
|
|
982
|
+
documents: input.documents,
|
|
983
|
+
documentCount: input.documents.length,
|
|
984
|
+
scores,
|
|
985
|
+
ranking: summary.ranking,
|
|
986
|
+
topDocument: summary.top
|
|
987
|
+
? {
|
|
988
|
+
index: summary.top.index,
|
|
989
|
+
document: summary.top.document,
|
|
990
|
+
score: summary.top.score,
|
|
991
|
+
probability: summary.top.probability,
|
|
992
|
+
}
|
|
993
|
+
: null,
|
|
994
|
+
phase: {
|
|
995
|
+
totalMs: durationMs,
|
|
996
|
+
prefixMs,
|
|
997
|
+
prefixApplied: prefixContext != null,
|
|
998
|
+
prefixTokens: Number.isFinite(prefixContext?.prefixTokens?.length)
|
|
999
|
+
? prefixContext.prefixTokens.length
|
|
1000
|
+
: 0,
|
|
1001
|
+
prefix: prefixContext?.phase ?? null,
|
|
1002
|
+
documentCount: input.documents.length,
|
|
1003
|
+
documentTotalMs,
|
|
1004
|
+
maxDocumentMs: documentDurations.length > 0 ? Math.max(...documentDurations) : 0,
|
|
1005
|
+
avgDocumentMs: documentDurations.length > 0 ? documentTotalMs / documentDurations.length : 0,
|
|
1006
|
+
documents: scores.map((entry) => ({
|
|
1007
|
+
index: entry.index,
|
|
1008
|
+
scoringPath: entry.scoringPath,
|
|
1009
|
+
tokenCount: entry.tokenCount,
|
|
1010
|
+
phase: entry.phase ?? null,
|
|
1011
|
+
})),
|
|
1012
|
+
},
|
|
1013
|
+
durationMs,
|
|
1014
|
+
};
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
function resolveRerankSemanticFixtures(runtimeConfig, options = null) {
|
|
1018
|
+
const overrides = isPlainObject(options?.rerankSemantic)
|
|
1019
|
+
? options.rerankSemantic
|
|
1020
|
+
: null;
|
|
1021
|
+
const runtimeOverrides = runtimeConfig?.shared?.benchmark?.run?.rerankSemantic;
|
|
1022
|
+
const source = overrides ?? (isPlainObject(runtimeOverrides) ? runtimeOverrides : null);
|
|
1023
|
+
const cases = normalizeRerankCases(source?.cases)
|
|
1024
|
+
?? DEFAULT_RERANK_SEMANTIC_FIXTURES.cases;
|
|
1025
|
+
const minPairAcc = Number.isFinite(source?.minPairAcc)
|
|
1026
|
+
? Math.max(0, Math.min(1, Number(source.minPairAcc)))
|
|
1027
|
+
: DEFAULT_RERANK_SEMANTIC_FIXTURES.minPairAcc;
|
|
1028
|
+
const minScoreMargin = Number.isFinite(source?.minScoreMargin)
|
|
1029
|
+
? Number(source.minScoreMargin)
|
|
1030
|
+
: DEFAULT_RERANK_SEMANTIC_FIXTURES.minScoreMargin;
|
|
1031
|
+
return {
|
|
1032
|
+
cases,
|
|
1033
|
+
minPairAcc,
|
|
1034
|
+
minScoreMargin,
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
export async function runRerankSemanticChecks(pipeline, options = null) {
|
|
1039
|
+
const fixture = resolveRerankSemanticFixtures(pipeline?.runtimeConfig ?? {}, options);
|
|
1040
|
+
const config = resolveRerankScoringConfig(pipeline);
|
|
1041
|
+
const start = performance.now();
|
|
1042
|
+
const pairs = [];
|
|
1043
|
+
let pairPassed = 0;
|
|
1044
|
+
for (const testCase of fixture.cases) {
|
|
1045
|
+
const positive = await scoreRerankDocument(
|
|
1046
|
+
pipeline,
|
|
1047
|
+
testCase.query,
|
|
1048
|
+
testCase.positive,
|
|
1049
|
+
config
|
|
1050
|
+
);
|
|
1051
|
+
const negative = await scoreRerankDocument(
|
|
1052
|
+
pipeline,
|
|
1053
|
+
testCase.query,
|
|
1054
|
+
testCase.negative,
|
|
1055
|
+
config
|
|
1056
|
+
);
|
|
1057
|
+
const margin = positive.score - negative.score;
|
|
1058
|
+
const passed = Number.isFinite(margin) && margin > fixture.minScoreMargin;
|
|
1059
|
+
if (passed) pairPassed++;
|
|
1060
|
+
pairs.push({
|
|
1061
|
+
id: testCase.id,
|
|
1062
|
+
query: testCase.query,
|
|
1063
|
+
positive: testCase.positive,
|
|
1064
|
+
negative: testCase.negative,
|
|
1065
|
+
passed,
|
|
1066
|
+
positiveScore: Number(positive.score.toFixed(6)),
|
|
1067
|
+
negativeScore: Number(negative.score.toFixed(6)),
|
|
1068
|
+
positiveProbability: Number(positive.probability.toFixed(6)),
|
|
1069
|
+
negativeProbability: Number(negative.probability.toFixed(6)),
|
|
1070
|
+
margin: Number.isFinite(margin) ? Number(margin.toFixed(6)) : null,
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
const pairAcc = pairs.length > 0 ? pairPassed / pairs.length : 0;
|
|
1074
|
+
const passed = pairAcc >= fixture.minPairAcc;
|
|
1075
|
+
return {
|
|
1076
|
+
passed,
|
|
1077
|
+
pairAcc,
|
|
1078
|
+
pairPassed,
|
|
1079
|
+
pairTotal: pairs.length,
|
|
1080
|
+
minPairAcc: Number(fixture.minPairAcc.toFixed(4)),
|
|
1081
|
+
minScoreMargin: Number(fixture.minScoreMargin.toFixed(4)),
|
|
1082
|
+
failedCaseIds: pairs.filter((item) => !item.passed).map((item) => item.id),
|
|
1083
|
+
pairs,
|
|
1084
|
+
durationMs: Math.max(1, performance.now() - start),
|
|
1085
|
+
};
|
|
599
1086
|
}
|
|
600
1087
|
|
|
601
1088
|
function isStructuredPromptInput(value) {
|
|
@@ -624,26 +1111,6 @@ function resolvePromptTemplateType(source) {
|
|
|
624
1111
|
return asText(source?.manifest?.inference?.chatTemplate?.type);
|
|
625
1112
|
}
|
|
626
1113
|
|
|
627
|
-
function buildDefaultGenerationPrompt(templateType) {
|
|
628
|
-
if (templateType === 'qwen') {
|
|
629
|
-
return clonePromptInput(DEFAULT_QWEN_PROMPT);
|
|
630
|
-
}
|
|
631
|
-
if (templateType === 'translategemma') {
|
|
632
|
-
return clonePromptInput(DEFAULT_TRANSLATEGEMMA_PROMPT);
|
|
633
|
-
}
|
|
634
|
-
return DEFAULT_HARNESS_PROMPT;
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
function shouldPreferModelDefaultPrompt(runtimePrompt, templateType) {
|
|
638
|
-
if (templateType !== 'translategemma' && templateType !== 'qwen') {
|
|
639
|
-
return false;
|
|
640
|
-
}
|
|
641
|
-
if (typeof runtimePrompt !== 'string') {
|
|
642
|
-
return false;
|
|
643
|
-
}
|
|
644
|
-
return runtimePrompt.trim() === DEFAULT_RUNTIME_PLACEHOLDER_PROMPT;
|
|
645
|
-
}
|
|
646
|
-
|
|
647
1114
|
function assertPromptContract(runtimePrompt, templateType, source = 'runtime.inference.prompt') {
|
|
648
1115
|
if (templateType !== 'translategemma') {
|
|
649
1116
|
return;
|
|
@@ -652,10 +1119,6 @@ function assertPromptContract(runtimePrompt, templateType, source = 'runtime.inf
|
|
|
652
1119
|
return;
|
|
653
1120
|
}
|
|
654
1121
|
if (typeof runtimePrompt === 'string') {
|
|
655
|
-
const trimmed = runtimePrompt.trim();
|
|
656
|
-
if (!trimmed || trimmed === DEFAULT_RUNTIME_PLACEHOLDER_PROMPT) {
|
|
657
|
-
return;
|
|
658
|
-
}
|
|
659
1122
|
throw new Error(
|
|
660
1123
|
`TranslateGemma harness prompt contract violation: ${source} must be ` +
|
|
661
1124
|
'{ messages: [...] } with source_lang_code/target_lang_code blocks, not a plain string.'
|
|
@@ -671,7 +1134,7 @@ function assertPromptContract(runtimePrompt, templateType, source = 'runtime.inf
|
|
|
671
1134
|
|
|
672
1135
|
function describePromptInput(promptInput) {
|
|
673
1136
|
if (typeof promptInput === 'string') {
|
|
674
|
-
return promptInput.trim() ||
|
|
1137
|
+
return promptInput.trim() || '[empty prompt]';
|
|
675
1138
|
}
|
|
676
1139
|
if (isPlainObject(promptInput?.image) && typeof promptInput?.prompt === 'string') {
|
|
677
1140
|
const width = Number.isFinite(promptInput.image.width) ? promptInput.image.width : '?';
|
|
@@ -924,9 +1387,6 @@ function resolveGenerationPromptInput(runtimeConfig, runOverrides = null, source
|
|
|
924
1387
|
|
|
925
1388
|
const runtimePrompt = runtimeConfig?.inference?.prompt;
|
|
926
1389
|
assertPromptContract(runtimePrompt, templateType, 'runtimeConfig.inference.prompt');
|
|
927
|
-
if (shouldPreferModelDefaultPrompt(runtimePrompt, templateType)) {
|
|
928
|
-
return buildDefaultGenerationPrompt(templateType);
|
|
929
|
-
}
|
|
930
1390
|
if (typeof runtimePrompt === 'string' && runtimePrompt.trim()) {
|
|
931
1391
|
return runtimePrompt;
|
|
932
1392
|
}
|
|
@@ -934,15 +1394,15 @@ function resolveGenerationPromptInput(runtimeConfig, runOverrides = null, source
|
|
|
934
1394
|
return clonePromptInput(runtimePrompt);
|
|
935
1395
|
}
|
|
936
1396
|
|
|
937
|
-
|
|
1397
|
+
throw new Error('Harness generation requires explicit runOverrides.prompt or runtime.inference.prompt.');
|
|
938
1398
|
}
|
|
939
1399
|
|
|
940
1400
|
function resolveMaxTokens(runtimeConfig) {
|
|
941
1401
|
const runtimeMax = runtimeConfig?.inference?.generation?.maxTokens;
|
|
942
|
-
if (Number.isFinite(runtimeMax)) {
|
|
943
|
-
return Math.
|
|
1402
|
+
if (Number.isFinite(runtimeMax) && runtimeMax > 0) {
|
|
1403
|
+
return Math.floor(runtimeMax);
|
|
944
1404
|
}
|
|
945
|
-
|
|
1405
|
+
throw new Error('Harness generation requires explicit runtime.inference.generation.maxTokens.');
|
|
946
1406
|
}
|
|
947
1407
|
|
|
948
1408
|
function resolveAutomaticGenerationDiagnostics(runtimeConfig, runOverrides = null) {
|
|
@@ -1413,6 +1873,17 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
|
|
|
1413
1873
|
if (Number.isFinite(stats.decodeReadbackCleanupMs)) gpu.decodeReadbackCleanupMs = stats.decodeReadbackCleanupMs;
|
|
1414
1874
|
if (Number.isFinite(stats.decodeReadbackCopyMs)) gpu.decodeReadbackCopyMs = stats.decodeReadbackCopyMs;
|
|
1415
1875
|
if (Number.isFinite(stats.prefillRecordMs)) gpu.prefillRecordMs = stats.prefillRecordMs;
|
|
1876
|
+
if (Number.isFinite(stats.prefillRecordOps)) gpu.prefillRecordOps = stats.prefillRecordOps;
|
|
1877
|
+
if (Number.isFinite(stats.prefillRecordPasses)) gpu.prefillRecordPasses = stats.prefillRecordPasses;
|
|
1878
|
+
const prefillRecordOpLabels = normalizeDecodeRecordOpLabels(stats.prefillRecordOpLabels);
|
|
1879
|
+
if (prefillRecordOpLabels) {
|
|
1880
|
+
const prefillRecordTopOps = buildDecodeRecordTopOps(prefillRecordOpLabels, stats.prefillRecordOps);
|
|
1881
|
+
const prefillRecordTopOpGroups = buildDecodeRecordTopOpGroups(prefillRecordOpLabels, stats.prefillRecordOps);
|
|
1882
|
+
gpu.prefillRecordOpLabels = prefillRecordOpLabels;
|
|
1883
|
+
gpu.prefillRecordUniqueOpLabels = Object.keys(prefillRecordOpLabels).length;
|
|
1884
|
+
gpu.prefillRecordTopOps = prefillRecordTopOps;
|
|
1885
|
+
gpu.prefillRecordTopOpGroups = prefillRecordTopOpGroups;
|
|
1886
|
+
}
|
|
1416
1887
|
if (Number.isFinite(stats.prefillSubmitWaitMs)) gpu.prefillSubmitWaitMs = stats.prefillSubmitWaitMs;
|
|
1417
1888
|
if (
|
|
1418
1889
|
Number.isFinite(decodeMs) &&
|
|
@@ -1545,7 +2016,6 @@ export function isCoherentOutput(tokens, output) {
|
|
|
1545
2016
|
}
|
|
1546
2017
|
|
|
1547
2018
|
export async function runGeneration(pipeline, runtimeConfig, runOverrides = null) {
|
|
1548
|
-
warnIfUsingDefaults(runtimeConfig, runOverrides);
|
|
1549
2019
|
const tokens = [];
|
|
1550
2020
|
const tokenIds = [];
|
|
1551
2021
|
const tokenRecords = [];
|
|
@@ -1558,7 +2028,6 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
|
|
|
1558
2028
|
? Math.max(1, Math.floor(runOverrides.maxTokens))
|
|
1559
2029
|
: resolveMaxTokens(runtimeConfig);
|
|
1560
2030
|
const sampling = {
|
|
1561
|
-
...DEFAULT_SAMPLING_DEFAULTS,
|
|
1562
2031
|
...(runtimeConfig.inference?.sampling || {}),
|
|
1563
2032
|
...(isPlainObject(runOverrides?.sampling) ? runOverrides.sampling : {}),
|
|
1564
2033
|
};
|
|
@@ -1586,6 +2055,7 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
|
|
|
1586
2055
|
repetitionPenalty: sampling.repetitionPenalty,
|
|
1587
2056
|
greedyThreshold: sampling.greedyThreshold,
|
|
1588
2057
|
useChatTemplate,
|
|
2058
|
+
benchmark: runOverrides?.benchmark === true,
|
|
1589
2059
|
profile,
|
|
1590
2060
|
...(disableCommandBatchingForDiagnostics ? { disableCommandBatching: true } : {}),
|
|
1591
2061
|
diagnostics,
|
|
@@ -1706,12 +2176,13 @@ export async function runTextInference(pipeline, runtimeConfig, runOverrides = n
|
|
|
1706
2176
|
}
|
|
1707
2177
|
|
|
1708
2178
|
export async function runEmbedding(pipeline, runtimeConfig, runOverrides = null) {
|
|
1709
|
-
warnIfUsingDefaults(runtimeConfig);
|
|
1710
2179
|
const prompt = typeof runOverrides?.prompt === 'string' && runOverrides.prompt.trim()
|
|
1711
2180
|
? runOverrides.prompt.trim()
|
|
1712
2181
|
: resolvePrompt(runtimeConfig);
|
|
1713
2182
|
const start = performance.now();
|
|
1714
|
-
const result = await pipeline.embed(prompt
|
|
2183
|
+
const result = await pipeline.embed(prompt, {
|
|
2184
|
+
benchmark: runOverrides?.benchmark === true,
|
|
2185
|
+
});
|
|
1715
2186
|
const durationMs = Math.max(1, performance.now() - start);
|
|
1716
2187
|
const tokenCount = Number.isFinite(result?.tokens?.length) ? result.tokens.length : 0;
|
|
1717
2188
|
const stats = summarizeEmbeddingValues(result?.embedding);
|
|
@@ -1719,6 +2190,7 @@ export async function runEmbedding(pipeline, runtimeConfig, runOverrides = null)
|
|
|
1719
2190
|
prompt,
|
|
1720
2191
|
tokenCount,
|
|
1721
2192
|
durationMs,
|
|
2193
|
+
phase: result?.phase ?? null,
|
|
1722
2194
|
...stats,
|
|
1723
2195
|
};
|
|
1724
2196
|
}
|