doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "doppler-gpu",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "AI-native model release foundry and evidence-backed WebGPU runtime for JavaScript",
|
|
5
5
|
"main": "src/capsule-runtime.js",
|
|
6
6
|
"types": "src/capsule-runtime.d.ts",
|
|
@@ -107,11 +107,13 @@
|
|
|
107
107
|
"catscan:sync": "node tools/sync-catscan-index.js",
|
|
108
108
|
"catscan:check": "node tools/sync-catscan-index.js --check",
|
|
109
109
|
"conflicts:check": "node tools/check-merge-markers.js",
|
|
110
|
-
"config:single-source:check": "node tools/check-config-single-source.js",
|
|
110
|
+
"config:single-source:check": "node tools/check-config-single-source.js && node tools/sync-generation-contract.js --check",
|
|
111
111
|
"rules:bundle:sync": "node tools/sync-rule-bundle.js",
|
|
112
112
|
"rules:bundle:check": "node tools/sync-rule-bundle.js --check",
|
|
113
113
|
"evidence:workflows:check": "node tools/check-evidence-workflow-ownership.js",
|
|
114
114
|
"inference:boundaries:check": "node tools/check-inference-boundaries.js",
|
|
115
|
+
"generation:contract:check": "node tools/sync-generation-contract.js --check",
|
|
116
|
+
"generation:contract:sync": "node tools/sync-generation-contract.js",
|
|
115
117
|
"source:style:sync": "node tools/check-source-style.js --write",
|
|
116
118
|
"source:style:check": "node tools/check-source-style.js",
|
|
117
119
|
"source:architecture:check": "node tools/check-source-architecture.js",
|
|
@@ -219,6 +221,10 @@
|
|
|
219
221
|
"types": "./src/capsule-runtime.d.ts",
|
|
220
222
|
"import": "./src/capsule-runtime.js"
|
|
221
223
|
},
|
|
224
|
+
"./generation-contract": {
|
|
225
|
+
"types": "./src/config/generation-contract.d.ts",
|
|
226
|
+
"import": "./src/config/generation-contract.js"
|
|
227
|
+
},
|
|
222
228
|
"./host": {
|
|
223
229
|
"types": "./src/client/capsule-host.d.ts",
|
|
224
230
|
"browser": "./src/client/capsule-host.browser.js",
|
package/src/capsule-runtime.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { DOPPLER_VERSION } from './version.js';
|
|
2
|
+
export { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from './config/generation-contract.js';
|
|
3
|
+
export type { GenerationInput, GenerationOptions, ResolvedGenerationOptions, GenerationOutput, GenerationCompletion } from './config/generation-contract.js';
|
|
2
4
|
export { createDopplerRuntime, createForecastProgramFactory, RUNTIME_CORE_VERSION } from './client/runtime/composition-root.js';
|
|
3
5
|
export { createFetchCapsuleArtifactStore } from './client/runtime/fetch-capsule-artifact-store.js';
|
|
4
6
|
|
package/src/capsule-runtime.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { DOPPLER_VERSION } from './version.js';
|
|
2
|
+
export { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from './config/generation-contract.js';
|
|
2
3
|
export { createDopplerRuntime, createForecastProgramFactory, RUNTIME_CORE_VERSION } from './client/runtime/composition-root.js';
|
|
3
4
|
export { createFetchCapsuleArtifactStore } from './client/runtime/fetch-capsule-artifact-store.js';
|
|
4
5
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
export { openCapsule } from './doppler-api.js';
|
|
2
|
+
export { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from '../config/generation-contract.js';
|
|
3
|
+
export type { GenerationInput, GenerationOptions, ResolvedGenerationOptions, GenerationOutput, GenerationCompletion } from '../config/generation-contract.js';
|
|
2
4
|
export type { DopplerCapsuleOpenOptions } from './runtime/index.js';
|
|
3
5
|
export type { DopplerRuntimeSession } from './runtime/composition-root.js';
|
|
@@ -123,3 +123,5 @@ export type {
|
|
|
123
123
|
DopplerSignedRevocationEnvelope,
|
|
124
124
|
DopplerSignedRevocationStatus,
|
|
125
125
|
} from '../config/revocation-updates.js';
|
|
126
|
+
export { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from '../config/generation-contract.js';
|
|
127
|
+
export type { GenerationOptions, GenerationInput, GenerationOutput, GenerationCompletion, ResolvedGenerationOptions } from '../config/generation-contract.js';
|
|
@@ -22,6 +22,7 @@ export type DopplerEvidenceStats = Omit<PipelineStats, 'gpuTimePrefillMs' | 'gpu
|
|
|
22
22
|
};
|
|
23
23
|
|
|
24
24
|
export interface DopplerGenerationConfigEvidence {
|
|
25
|
+
logitMaskIdentity?: { id: string; contentDigest: string };
|
|
25
26
|
maxTokens: number;
|
|
26
27
|
temperature: number;
|
|
27
28
|
topP: number;
|
|
@@ -138,7 +138,19 @@ function resolveGenerationConfigEvidence(pipeline, options) {
|
|
|
138
138
|
if (seed !== null && (!Number.isFinite(seed) || seed < 0)) {
|
|
139
139
|
throw new Error('Resolved Doppler seed must be null or a non-negative number.');
|
|
140
140
|
}
|
|
141
|
+
let logitMaskIdentity;
|
|
142
|
+
if (options.logitMaskFn != null) {
|
|
143
|
+
const identity = options.logitMaskIdentity;
|
|
144
|
+
if (typeof options.logitMaskFn !== 'function' || !identity || typeof identity.id !== 'string' || !identity.id.trim()
|
|
145
|
+
|| !/^sha256:[a-f0-9]{64}$/.test(identity.contentDigest || '')) {
|
|
146
|
+
throw new Error('Generation evidence for logitMaskFn requires logitMaskIdentity with id and SHA-256 contentDigest.');
|
|
147
|
+
}
|
|
148
|
+
logitMaskIdentity = { id: identity.id, contentDigest: identity.contentDigest };
|
|
149
|
+
} else if (options.logitMaskIdentity != null) {
|
|
150
|
+
throw new Error('logitMaskIdentity requires an executed logitMaskFn.');
|
|
151
|
+
}
|
|
141
152
|
return {
|
|
153
|
+
...(logitMaskIdentity ? { logitMaskIdentity } : {}),
|
|
142
154
|
maxTokens,
|
|
143
155
|
temperature: sampling.temperature,
|
|
144
156
|
topP: sampling.topP,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolveGenerationOptions, validateGenerationInput } from '../../config/generation-contract.js';
|
|
2
2
|
|
|
3
3
|
const text = (value) => typeof value === 'string' && value.trim().length > 0;
|
|
4
4
|
const texts = (value) => Array.isArray(value) && value.length > 0 && value.every(text);
|
|
@@ -8,22 +8,23 @@ export function createCapsuleOperationAdapters({ program, generate, rerank, embe
|
|
|
8
8
|
return {
|
|
9
9
|
generate: {
|
|
10
10
|
validate({ input, options }) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if (Object.hasOwn(input, 'promptTokens')) requireValue(Array.isArray(input.promptTokens) && input.promptTokens.length > 0 && input.promptTokens.every((id) => Number.isSafeInteger(id) && id >= 0), 'Invalid prompt token IDs.');
|
|
14
|
-
requireGenerationOptions(options);
|
|
15
|
-
if (options.stopSequences !== undefined) requireValue(Array.isArray(options.stopSequences) && options.stopSequences.every(text), 'Invalid stop sequences.');
|
|
16
|
-
if (options.suppressTokenIds !== undefined) requireValue(Array.isArray(options.suppressTokenIds) && options.suppressTokenIds.every((id) => Number.isSafeInteger(id) && id >= 0), 'Invalid suppressed token IDs.');
|
|
11
|
+
validateGenerationInput(input);
|
|
12
|
+
resolveGenerationOptions(options);
|
|
17
13
|
},
|
|
18
14
|
async *execute({ input, options }, signal) {
|
|
19
15
|
const tokenIds = [];
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
16
|
+
const iterator = generate({ ...input, ...options, signal });
|
|
17
|
+
try {
|
|
18
|
+
while (true) {
|
|
19
|
+
const next = await iterator.next();
|
|
20
|
+
if (next.done) return { text: program.decodeTokens(tokenIds), tokenIds, ...next.value };
|
|
21
|
+
const tokenId = next.value;
|
|
22
|
+
requireValue(Number.isSafeInteger(tokenId) && tokenId >= 0, 'Invalid generated token ID.');
|
|
23
|
+
tokenIds.push(tokenId);
|
|
24
|
+
const output = { text: program.decodeTokens(tokenIds), tokenIds: [...tokenIds] };
|
|
25
|
+
yield { delta: { tokenId }, output };
|
|
26
|
+
}
|
|
27
|
+
} finally { await iterator.return?.(); }
|
|
27
28
|
},
|
|
28
29
|
},
|
|
29
30
|
embed: {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DOPPLER_VERSION } from '../../version.js';
|
|
2
|
+
import { GenerationError } from '../../config/generation-contract.js';
|
|
2
3
|
import { freezeCapsuleV2 } from '../../config/capsule-v2.js';
|
|
3
4
|
import { CAPSULE_OPERATION_EVENT_SCHEMA, CAPSULE_OPERATION_RECEIPT_SCHEMA, hashCapsuleObservation, normalizeCapsuleObservation, snapshotCapsuleOperationRequest } from '../../config/capsule-operation.js';
|
|
4
5
|
|
|
@@ -16,8 +17,8 @@ export function createCapsuleOperationExecutor({ adapters, identity, assertCurre
|
|
|
16
17
|
if (active) throw new Error('Capsule operation already active; submit a distinct job after it finishes.');
|
|
17
18
|
active = true;
|
|
18
19
|
const controller = new AbortController();
|
|
19
|
-
const cancel = () => controller.abort(externalSignal.reason
|
|
20
|
-
const deadline = () => controller.abort(new
|
|
20
|
+
const cancel = () => controller.abort(new GenerationError('aborted', externalSignal.reason?.message || 'Capsule operation cancelled.', { cause: externalSignal.reason }));
|
|
21
|
+
const deadline = () => controller.abort(new GenerationError('deadline', 'Capsule operation deadline exceeded.'));
|
|
21
22
|
let timer;
|
|
22
23
|
let iterator;
|
|
23
24
|
let failure;
|
|
@@ -11,10 +11,12 @@ function arraysEqual(left, right) {
|
|
|
11
11
|
function toPipelineOptions(options, signal) {
|
|
12
12
|
return {
|
|
13
13
|
maxTokens: options.maxTokens,
|
|
14
|
+
maxSeqLen: options.maxSeqLen,
|
|
14
15
|
temperature: options.temperature,
|
|
15
16
|
topP: options.topP,
|
|
16
17
|
topK: options.topK,
|
|
17
18
|
repetitionPenalty: options.repetitionPenalty,
|
|
19
|
+
presencePenalty: options.presencePenalty,
|
|
18
20
|
repetitionPenaltyWindow: options.repetitionPenaltyWindow,
|
|
19
21
|
...(Number.isFinite(options.seed) ? { seed: options.seed } : {}),
|
|
20
22
|
suppressSpecialTokens: options.suppressSpecialTokens,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// One loaded program owns mutable generation and adapter state. Cancellation
|
|
2
2
|
// requests cooperation; disposal waits until the current operation has drained.
|
|
3
|
+
import { GenerationError } from '../../config/generation-contract.js';
|
|
3
4
|
export function createCapsuleSessionExecution() {
|
|
4
5
|
let active = null;
|
|
5
6
|
let closing = false;
|
|
@@ -13,7 +14,7 @@ export function createCapsuleSessionExecution() {
|
|
|
13
14
|
|| typeof signal.aborted !== 'boolean')) throw new Error('Capsule operation signal must be an AbortSignal.');
|
|
14
15
|
let finish;
|
|
15
16
|
const done = new Promise(resolve => { finish = resolve; });
|
|
16
|
-
const cancel = () => controller.abort(signal.reason);
|
|
17
|
+
const cancel = () => controller.abort(new GenerationError('aborted', signal.reason?.message || 'Capsule operation cancelled.', { cause: signal.reason }));
|
|
17
18
|
const lease = { controller, done, stop: null, release() {
|
|
18
19
|
signal?.removeEventListener('abort', cancel);
|
|
19
20
|
active = null;
|
|
@@ -56,12 +57,15 @@ export function createCapsuleSessionExecution() {
|
|
|
56
57
|
lease.stop = () => output.return();
|
|
57
58
|
try {
|
|
58
59
|
controller.signal.throwIfAborted();
|
|
59
|
-
|
|
60
|
+
while (true) {
|
|
61
|
+
const step = await source.next();
|
|
60
62
|
controller.signal.throwIfAborted();
|
|
61
|
-
|
|
63
|
+
if (step.done) return step.value;
|
|
64
|
+
yield step.value;
|
|
62
65
|
}
|
|
63
|
-
|
|
64
|
-
|
|
66
|
+
} finally {
|
|
67
|
+
try { await source.return?.(); } finally { lease.release(); }
|
|
68
|
+
}
|
|
65
69
|
})();
|
|
66
70
|
return output;
|
|
67
71
|
},
|
|
@@ -5,7 +5,9 @@ import type { CapsuleReleaseEvent, CapsuleReleasePolicy, ReleaseCheckpoint } fro
|
|
|
5
5
|
import type { TargetPlan, TargetPlanSelectionPolicy } from '../../config/target-plan.js';
|
|
6
6
|
import type { InitialExecutionIdentity } from '../../config/initial-execution-identity.js';
|
|
7
7
|
import type { DeviceProfile } from './target-selector.js';
|
|
8
|
-
import type { GenerationRunOptions } from './session-controller.js';
|
|
8
|
+
import type { GenerationRunOptions, GenerationResult } from './session-controller.js';
|
|
9
|
+
import type { GenerationOutput } from '../../config/generation-contract.js';
|
|
10
|
+
import type { GENERATION_CONTRACT } from '../../config/generation-contract.js';
|
|
9
11
|
import type { CapsuleRerankReceipt, CapsuleRerankRequest } from './capsule-rerank.js';
|
|
10
12
|
import type { CapsuleOperationRequest } from '../../config/capsule-operation.js';
|
|
11
13
|
import type { CapsuleOperationEvent } from './capsule-operation-executor.js';
|
|
@@ -39,6 +41,7 @@ export interface RuntimePorts {
|
|
|
39
41
|
}
|
|
40
42
|
|
|
41
43
|
export interface DopplerRuntimeSession {
|
|
44
|
+
readonly generationContract: typeof GENERATION_CONTRACT;
|
|
42
45
|
schema: 'doppler.capsule-session/v1';
|
|
43
46
|
readonly loaded: boolean;
|
|
44
47
|
readonly closed: boolean;
|
|
@@ -54,8 +57,8 @@ export interface DopplerRuntimeSession {
|
|
|
54
57
|
observedInitialExecutionIdentity: InitialExecutionIdentity | null;
|
|
55
58
|
deviceProfile: DeviceProfile;
|
|
56
59
|
verification: Awaited<ReturnType<typeof verifyCapsule>>;
|
|
57
|
-
generate(options: GenerationRunOptions): AsyncGenerator<number,
|
|
58
|
-
generateText(options: GenerationRunOptions): Promise<{
|
|
60
|
+
generate(options: GenerationRunOptions): AsyncGenerator<number, GenerationResult, void>;
|
|
61
|
+
generateText(options: GenerationRunOptions): Promise<GenerationOutput & { modelId: string }>;
|
|
59
62
|
rerank(request: CapsuleRerankRequest): Promise<CapsuleRerankReceipt>;
|
|
60
63
|
forecast(request: CapsuleForecastRequest): Promise<CapsuleForecastResult>;
|
|
61
64
|
embed(request: CapsuleEmbeddingRequest): Promise<CapsuleEmbeddingResult>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { hashTargetPlan, assertQualifiedTargetOperation, normalizeTargetPlanSelectionPolicy } from '../../config/target-plan.js';
|
|
2
|
+
import { GENERATION_CONTRACT } from '../../config/generation-contract.js';
|
|
2
3
|
import { assertInitialExecutionIdentity } from '../../config/initial-execution-identity.js';
|
|
3
4
|
import { freezeCapsuleV2, verifyCapsuleV2Artifacts } from '../../config/capsule-v2.js';
|
|
4
5
|
import { verifyCapsuleMetadata, getCapsuleIdentity } from '../../config/capsule.js';
|
|
@@ -157,6 +158,7 @@ export function createDopplerRuntime(ports) {
|
|
|
157
158
|
}
|
|
158
159
|
|
|
159
160
|
const session = {
|
|
161
|
+
get generationContract() { return GENERATION_CONTRACT; },
|
|
160
162
|
modelId: capsule.modelId,
|
|
161
163
|
capsuleId: capsule.capsuleId,
|
|
162
164
|
semanticRoot: capsule.semanticRoot,
|
|
@@ -243,7 +245,7 @@ export function createDopplerRuntime(ports) {
|
|
|
243
245
|
await assertPlanUnchanged();
|
|
244
246
|
assertQualifiedTargetOperation(selectedPlan, deviceProfile.surface, 'generate');
|
|
245
247
|
try {
|
|
246
|
-
yield* sessionController.generateTokens(selectedPlan, { ...generationOptions, modules });
|
|
248
|
+
return yield* sessionController.generateTokens(selectedPlan, { ...generationOptions, modules });
|
|
247
249
|
} finally {
|
|
248
250
|
await assertPlanUnchanged();
|
|
249
251
|
}
|
|
@@ -251,8 +253,14 @@ export function createDopplerRuntime(ports) {
|
|
|
251
253
|
|
|
252
254
|
async generateText(generationOptions = {}) {
|
|
253
255
|
const tokens = [];
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
const iterator = this.generate(generationOptions);
|
|
257
|
+
try {
|
|
258
|
+
while (true) {
|
|
259
|
+
const step = await iterator.next();
|
|
260
|
+
if (step.done) return { text: program.decodeTokens(tokens), tokenIds: tokens, modelId: capsule.modelId, ...step.value };
|
|
261
|
+
tokens.push(step.value);
|
|
262
|
+
}
|
|
263
|
+
} finally { await iterator.return?.(); }
|
|
256
264
|
},
|
|
257
265
|
|
|
258
266
|
async rerank(request) {
|
|
@@ -310,7 +318,7 @@ export function createDopplerRuntime(ports) {
|
|
|
310
318
|
emit(observer, { type: 'capsule-load-complete', capsuleId: capsule.capsuleId, artifactMetrics: verifiedStore.getMetrics() });
|
|
311
319
|
const executeOperation = createCapsuleOperationExecutor({ adapters,
|
|
312
320
|
prepareExecution: createCapsuleAdapterExecution({ program, capsule: { ...verification.identity, modelId: capsule.modelId }, targetPlan: selectedPlan }),
|
|
313
|
-
identity: { capsule: verification.identity, targetId: selectedPlan.targetId, targetPlanDigest,
|
|
321
|
+
identity: { capsule: verification.identity, modelId: capsule.modelId, targetId: selectedPlan.targetId, targetPlanDigest,
|
|
314
322
|
artifactReceipts: verification.artifactReceipts, releaseEventDigest: verification.lifecycle?.event.digest ?? null,
|
|
315
323
|
...releaseAuthorization.receiptFields },
|
|
316
324
|
async assertCurrent(request) {
|
|
@@ -322,7 +330,7 @@ export function createDopplerRuntime(ports) {
|
|
|
322
330
|
return Object.assign(session, {
|
|
323
331
|
generate: (options = {}) => execution.stream(async function* (signal) {
|
|
324
332
|
requireBaseProgram();
|
|
325
|
-
yield* local.generate({ ...options, signal });
|
|
333
|
+
return yield* local.generate({ ...options, signal });
|
|
326
334
|
}, options.signal),
|
|
327
335
|
generateText: async (options = {}) => runLocal(signal => local.generateText({ ...options, signal }), options.signal),
|
|
328
336
|
forecast: async request => runLocal(signal => local.forecast({ ...request, signal }), request?.signal),
|
|
@@ -1,30 +1,19 @@
|
|
|
1
1
|
import type { TargetPlan } from '../../config/target-plan.js';
|
|
2
2
|
import type { CommandExecutor } from './command-executor.js';
|
|
3
3
|
import type { ResourceBinder } from './resource-binder.js';
|
|
4
|
+
import type { GenerationOptions, GenerationInput, GenerationCompletion, ResolvedGenerationOptions } from '../../config/generation-contract.js';
|
|
4
5
|
|
|
5
|
-
export
|
|
6
|
-
prompt?: unknown;
|
|
7
|
-
promptTokens?: number[];
|
|
8
|
-
maxTokens: number;
|
|
9
|
-
maxSeqLen: number;
|
|
10
|
-
temperature: number;
|
|
11
|
-
topP: number;
|
|
12
|
-
topK: number;
|
|
13
|
-
repetitionPenalty: number;
|
|
14
|
-
repetitionPenaltyWindow: number;
|
|
15
|
-
useChatTemplate: boolean;
|
|
16
|
-
seed?: number;
|
|
17
|
-
suppressTokenIds?: number[];
|
|
18
|
-
stopSequences?: string[];
|
|
6
|
+
export type GenerationRunOptions = GenerationInput & GenerationOptions & {
|
|
19
7
|
signal?: AbortSignal | null;
|
|
20
|
-
}
|
|
8
|
+
};
|
|
9
|
+
export interface GenerationResult { sampling: ResolvedGenerationOptions; completion: GenerationCompletion; }
|
|
21
10
|
|
|
22
11
|
export interface SessionController {
|
|
23
|
-
generateTokens(targetPlan: TargetPlan, options: GenerationRunOptions): AsyncGenerator<number,
|
|
12
|
+
generateTokens(targetPlan: TargetPlan, options: GenerationRunOptions): AsyncGenerator<number, GenerationResult, void>;
|
|
24
13
|
close(): Promise<void>;
|
|
25
14
|
}
|
|
26
15
|
|
|
27
|
-
export
|
|
16
|
+
export { resolveGenerationOptions as requireGenerationOptions } from '../../config/generation-contract.js';
|
|
28
17
|
|
|
29
18
|
export declare function sampleCapsuleLogits(logits: ArrayLike<number>, contextTokens: number[], options: GenerationRunOptions, tokenContract?: Record<string, unknown>): number;
|
|
30
19
|
export declare function createSessionController(commandExecutor: CommandExecutor, resourceBinder: ResourceBinder, program: object): SessionController;
|
|
@@ -1,103 +1,24 @@
|
|
|
1
|
+
import { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from '../../config/generation-contract.js';
|
|
2
|
+
import { applyRepetitionPenalty, applyPresencePenalty, sample } from '../../inference/token-sampling.js';
|
|
1
3
|
|
|
2
|
-
export
|
|
3
|
-
const requiredNumbers = ['maxTokens', 'temperature', 'topP', 'topK', 'repetitionPenalty', 'repetitionPenaltyWindow'];
|
|
4
|
-
for (const field of requiredNumbers) {
|
|
5
|
-
if (!Number.isFinite(options[field])) throw new Error(`Capsule generation requires explicit ${field}.`);
|
|
6
|
-
}
|
|
7
|
-
if (!Number.isInteger(options.maxTokens) || options.maxTokens < 1) throw new Error('Capsule generation maxTokens must be a positive integer.');
|
|
8
|
-
if (!Number.isInteger(options.topK) || options.topK < 0) throw new Error('Capsule generation topK must be a non-negative integer.');
|
|
9
|
-
if (!Number.isInteger(options.repetitionPenaltyWindow) || options.repetitionPenaltyWindow < 1) {
|
|
10
|
-
throw new Error('Capsule generation repetitionPenaltyWindow must be a positive integer.');
|
|
11
|
-
}
|
|
12
|
-
if (options.temperature < 0 || options.topP <= 0 || options.topP > 1 || options.repetitionPenalty <= 0) {
|
|
13
|
-
throw new Error('Capsule generation sampling values are outside their valid ranges.');
|
|
14
|
-
}
|
|
15
|
-
if (options.temperature > 0 && !Number.isFinite(options.seed)) {
|
|
16
|
-
throw new Error('Capsule stochastic generation requires an explicit numeric seed.');
|
|
17
|
-
}
|
|
18
|
-
if (typeof options.useChatTemplate !== 'boolean') {
|
|
19
|
-
throw new Error('Capsule generation requires explicit useChatTemplate.');
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function applyRepetitionPenalty(logits, tokens, penalty, windowSize) {
|
|
24
|
-
if (penalty === 1) return;
|
|
25
|
-
for (const token of new Set(tokens.slice(-windowSize))) {
|
|
26
|
-
if (token >= 0 && token < logits.length) logits[token] = logits[token] > 0 ? logits[token] / penalty : logits[token] * penalty;
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
function seededRandom(seed) {
|
|
31
|
-
const value = Math.sin(seed) * 10000;
|
|
32
|
-
return value - Math.floor(value);
|
|
33
|
-
}
|
|
4
|
+
export const requireGenerationOptions = resolveGenerationOptions;
|
|
34
5
|
|
|
35
6
|
export function sampleCapsuleLogits(sourceLogits, contextTokens, options, tokenContract = {}) {
|
|
36
7
|
const logits = Float32Array.from(sourceLogits || []);
|
|
37
8
|
if (logits.length === 0) throw new Error('Capsule execution returned empty logits.');
|
|
38
9
|
applyRepetitionPenalty(logits, contextTokens, options.repetitionPenalty, options.repetitionPenaltyWindow);
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
for (const tokenId of suppressed) {
|
|
42
|
-
if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < logits.length) logits[tokenId] = -Infinity;
|
|
43
|
-
}
|
|
44
|
-
const candidates = [];
|
|
45
|
-
let nanCount = 0;
|
|
46
|
-
let positiveInfinityCount = 0;
|
|
47
|
-
let negativeInfinityCount = 0;
|
|
48
|
-
for (let tokenId = 0; tokenId < logits.length; tokenId += 1) {
|
|
49
|
-
const logit = logits[tokenId];
|
|
50
|
-
if (Number.isFinite(logit)) {
|
|
51
|
-
candidates.push({ tokenId, logit });
|
|
52
|
-
} else if (Number.isNaN(logit)) {
|
|
53
|
-
nanCount += 1;
|
|
54
|
-
} else if (logit === Infinity) {
|
|
55
|
-
positiveInfinityCount += 1;
|
|
56
|
-
} else {
|
|
57
|
-
negativeInfinityCount += 1;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
if (candidates.length === 0) {
|
|
61
|
-
throw new Error(
|
|
62
|
-
'Capsule execution returned no finite sampling candidates '
|
|
63
|
-
+ `(logits=${logits.length}, nan=${nanCount}, +inf=${positiveInfinityCount}, -inf=${negativeInfinityCount}).`
|
|
64
|
-
);
|
|
65
|
-
}
|
|
66
|
-
candidates.sort((left, right) => right.logit - left.logit || left.tokenId - right.tokenId);
|
|
67
|
-
if (options.temperature === 0) return candidates[0].tokenId;
|
|
68
|
-
const topK = options.topK > 0 ? Math.min(options.topK, candidates.length) : candidates.length;
|
|
69
|
-
let filtered = candidates.slice(0, topK);
|
|
70
|
-
const maxScaled = filtered[0].logit / options.temperature;
|
|
71
|
-
let total = 0;
|
|
72
|
-
for (const candidate of filtered) {
|
|
73
|
-
candidate.probability = Math.exp(candidate.logit / options.temperature - maxScaled);
|
|
74
|
-
total += candidate.probability;
|
|
75
|
-
}
|
|
76
|
-
for (const candidate of filtered) candidate.probability /= total;
|
|
77
|
-
if (options.topP < 1) {
|
|
78
|
-
let cumulative = 0;
|
|
79
|
-
filtered = filtered.filter((candidate) => {
|
|
80
|
-
if (cumulative >= options.topP) return false;
|
|
81
|
-
cumulative += candidate.probability;
|
|
82
|
-
return true;
|
|
83
|
-
});
|
|
84
|
-
const filteredTotal = filtered.reduce((sum, candidate) => sum + candidate.probability, 0);
|
|
85
|
-
for (const candidate of filtered) candidate.probability /= filteredTotal;
|
|
86
|
-
}
|
|
87
|
-
const random = seededRandom(options.seed);
|
|
88
|
-
let cumulative = 0;
|
|
89
|
-
for (const candidate of filtered) {
|
|
90
|
-
cumulative += candidate.probability;
|
|
91
|
-
if (random < cumulative) return candidate.tokenId;
|
|
92
|
-
}
|
|
93
|
-
return filtered.at(-1).tokenId;
|
|
10
|
+
applyPresencePenalty(logits, contextTokens, options.presencePenalty, options.repetitionPenaltyWindow);
|
|
11
|
+
return sample(logits, { ...options, padTokenId: tokenContract.padTokenId });
|
|
94
12
|
}
|
|
95
13
|
|
|
96
|
-
function
|
|
97
|
-
if (
|
|
98
|
-
if (
|
|
99
|
-
|
|
100
|
-
|
|
14
|
+
function stoppingReason(tokenId, generatedTokens, options, tokenContract, program) {
|
|
15
|
+
if (tokenId === tokenContract.eosTokenId) return 'eos-token';
|
|
16
|
+
if (tokenContract.stopTokenIds?.includes(tokenId)) return 'stop-token';
|
|
17
|
+
if (options.stopSequences.length > 0) {
|
|
18
|
+
const text = program.decodeTokens(generatedTokens);
|
|
19
|
+
if (options.stopSequences.some(sequence => text.endsWith(sequence))) return 'stop-sequence';
|
|
20
|
+
}
|
|
21
|
+
return generatedTokens.length >= options.maxTokens ? 'max-tokens' : null;
|
|
101
22
|
}
|
|
102
23
|
|
|
103
24
|
export function createSessionController(commandExecutor, resourceBinder, program) {
|
|
@@ -107,50 +28,51 @@ export function createSessionController(commandExecutor, resourceBinder, program
|
|
|
107
28
|
let closed = false;
|
|
108
29
|
|
|
109
30
|
return {
|
|
110
|
-
async *generateTokens(targetPlan,
|
|
31
|
+
async *generateTokens(targetPlan, request = {}) {
|
|
111
32
|
if (closed) throw new Error('Capsule runtime session is closed.');
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
if (
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
if (!Number.isInteger(dimensions.maxSeqLen) || dimensions.maxSeqLen < promptTokens.length + options.maxTokens) {
|
|
124
|
-
throw new Error('Capsule generation requires maxSeqLen large enough for prompt and generated tokens.');
|
|
33
|
+
const { prompt, promptTokens: inputTokens, signal, modules, ...requestedSampling } = request;
|
|
34
|
+
const input = Object.fromEntries(Object.entries({ prompt, promptTokens: inputTokens }).filter(([, value]) => value !== undefined));
|
|
35
|
+
validateGenerationInput(input);
|
|
36
|
+
const sampling = resolveGenerationOptions(requestedSampling);
|
|
37
|
+
const options = { ...input, ...sampling, signal, modules };
|
|
38
|
+
if (signal?.aborted) throw new GenerationError('aborted', 'Generation aborted before prefill.', { cause: signal.reason });
|
|
39
|
+
const promptTokens = inputTokens ? [...inputTokens] : program.tokenize(prompt, { useChatTemplate: sampling.useChatTemplate });
|
|
40
|
+
if (promptTokens.length === 0) throw new GenerationError('invalidRequest', 'Capsule generation prompt must produce at least one token.');
|
|
41
|
+
const dimensions = { seqLen: promptTokens.length, maxSeqLen: sampling.maxSeqLen, batchSize: 1 };
|
|
42
|
+
if (dimensions.maxSeqLen < promptTokens.length + sampling.maxTokens) {
|
|
43
|
+
throw new GenerationError('invalidRequest', 'Capsule generation requires maxSeqLen large enough for prompt and generated tokens.');
|
|
125
44
|
}
|
|
126
|
-
program.reset();
|
|
127
|
-
resourceBinder.bindSlots(targetPlan.memoryLayout, dimensions);
|
|
128
|
-
resourceBinder.writeSlot('input_tokens', Uint32Array.from(promptTokens));
|
|
129
45
|
const contextTokens = [...promptTokens];
|
|
130
46
|
const generatedTokens = [];
|
|
131
47
|
const tokenContract = program.getTokenContract();
|
|
132
48
|
let stepResult = null;
|
|
133
49
|
try {
|
|
50
|
+
program.reset();
|
|
51
|
+
resourceBinder.bindSlots(targetPlan.memoryLayout, dimensions);
|
|
52
|
+
resourceBinder.writeSlot('input_tokens', Uint32Array.from(promptTokens));
|
|
134
53
|
const prefill = await commandExecutor.executePhase('prefill', targetPlan.phases.prefill, {
|
|
135
|
-
signal: options
|
|
136
|
-
modules: options.modules,
|
|
137
|
-
context: { prompt: options.prompt ?? '', promptTokens, generationOptions: options },
|
|
54
|
+
signal, modules, context: { prompt: prompt ?? '', promptTokens, generationOptions: options },
|
|
138
55
|
});
|
|
139
56
|
stepResult = prefill.results.at(-1);
|
|
140
|
-
for (let step = 0; step <
|
|
141
|
-
if (
|
|
142
|
-
const tokenId = sampleCapsuleLogits(stepResult?.logits, contextTokens,
|
|
57
|
+
for (let step = 0; step < sampling.maxTokens; step += 1) {
|
|
58
|
+
if (signal?.aborted) throw new GenerationError('aborted', 'Generation aborted during decode.', { cause: signal.reason });
|
|
59
|
+
const tokenId = sampleCapsuleLogits(stepResult?.logits, contextTokens, sampling, tokenContract);
|
|
143
60
|
program.releaseStepResult(stepResult);
|
|
144
61
|
stepResult = null;
|
|
145
62
|
generatedTokens.push(tokenId);
|
|
146
63
|
contextTokens.push(tokenId);
|
|
64
|
+
const stopReason = stoppingReason(tokenId, generatedTokens, sampling, tokenContract, program);
|
|
147
65
|
yield tokenId;
|
|
148
|
-
if (
|
|
149
|
-
if (
|
|
66
|
+
if (signal?.aborted) throw new GenerationError('aborted', 'Generation aborted after token delivery.', { cause: signal.reason });
|
|
67
|
+
if (stopReason) return {
|
|
68
|
+
sampling,
|
|
69
|
+
completion: {
|
|
70
|
+
schema: GENERATION_CONTRACT.completionSchema, stopReason,
|
|
71
|
+
promptTokenCount: promptTokens.length, generatedTokenCount: generatedTokens.length,
|
|
72
|
+
},
|
|
73
|
+
};
|
|
150
74
|
const decode = await commandExecutor.executePhase('decode', targetPlan.phases.decode, {
|
|
151
|
-
signal: options
|
|
152
|
-
modules: options.modules,
|
|
153
|
-
context: { contextTokens, generationOptions: options },
|
|
75
|
+
signal, modules, context: { contextTokens, generationOptions: options },
|
|
154
76
|
});
|
|
155
77
|
stepResult = decode.results.at(-1);
|
|
156
78
|
}
|