@independo/inderun-web 0.2.0-dev.1 → 0.2.0-dev.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +187 -1
- package/dist/core/browser-host.d.ts.map +1 -0
- package/dist/{engine.d.ts → core/engine.d.ts} +6 -0
- package/dist/core/engine.d.ts.map +1 -0
- package/dist/{engine.js → core/engine.js} +14 -1
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/host.d.ts.map +1 -0
- package/dist/{provider.d.ts → core/provider.d.ts} +9 -0
- package/dist/core/provider.d.ts.map +1 -0
- package/dist/core/registry.d.ts.map +1 -0
- package/dist/{route-planner.d.ts → core/route-planner.d.ts} +31 -8
- package/dist/core/route-planner.d.ts.map +1 -0
- package/dist/{route-planner.js → core/route-planner.js} +46 -17
- package/dist/{router.d.ts → core/router.d.ts} +11 -1
- package/dist/core/router.d.ts.map +1 -0
- package/dist/{router.js → core/router.js} +26 -20
- package/dist/core/telemetry.d.ts.map +1 -0
- package/dist/index.d.ts +9 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -9
- package/dist/onnx.d.ts +13 -0
- package/dist/onnx.d.ts.map +1 -0
- package/dist/onnx.js +12 -0
- package/dist/openai.d.ts +1 -1
- package/dist/openai.d.ts.map +1 -1
- package/dist/openai.js +1 -1
- package/dist/providers/onnx/provider.d.ts +78 -0
- package/dist/providers/onnx/provider.d.ts.map +1 -0
- package/dist/providers/onnx/provider.js +217 -0
- package/dist/providers/onnx/runtime.d.ts +110 -0
- package/dist/providers/onnx/runtime.d.ts.map +1 -0
- package/dist/providers/onnx/runtime.js +76 -0
- package/dist/providers/onnx/transformers-runtime.d.ts +72 -0
- package/dist/providers/onnx/transformers-runtime.d.ts.map +1 -0
- package/dist/providers/onnx/transformers-runtime.js +210 -0
- package/dist/{openai-provider.d.ts → providers/openai/provider.d.ts} +21 -3
- package/dist/providers/openai/provider.d.ts.map +1 -0
- package/dist/{openai-provider.js → providers/openai/provider.js} +42 -2
- package/dist/providers/system-model/chrome-runtime.d.ts +48 -0
- package/dist/providers/system-model/chrome-runtime.d.ts.map +1 -0
- package/dist/providers/system-model/chrome-runtime.js +137 -0
- package/dist/providers/system-model/provider.d.ts +67 -0
- package/dist/providers/system-model/provider.d.ts.map +1 -0
- package/dist/providers/system-model/provider.js +190 -0
- package/dist/providers/system-model/runtime.d.ts +110 -0
- package/dist/providers/system-model/runtime.d.ts.map +1 -0
- package/dist/providers/system-model/runtime.js +73 -0
- package/dist/system-model.d.ts +13 -0
- package/dist/system-model.d.ts.map +1 -0
- package/dist/system-model.js +12 -0
- package/dist/web-sdk.d.ts +23 -6
- package/dist/web-sdk.d.ts.map +1 -1
- package/dist/web-sdk.js +29 -12
- package/package.json +21 -5
- package/dist/browser-host.d.ts.map +0 -1
- package/dist/engine.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/host.d.ts.map +0 -1
- package/dist/openai-provider.d.ts.map +0 -1
- package/dist/provider.d.ts.map +0 -1
- package/dist/registry.d.ts.map +0 -1
- package/dist/route-planner.d.ts.map +0 -1
- package/dist/router.d.ts.map +0 -1
- package/dist/telemetry.d.ts.map +0 -1
- /package/dist/{browser-host.d.ts → core/browser-host.d.ts} +0 -0
- /package/dist/{browser-host.js → core/browser-host.js} +0 -0
- /package/dist/{errors.d.ts → core/errors.d.ts} +0 -0
- /package/dist/{errors.js → core/errors.js} +0 -0
- /package/dist/{host.d.ts → core/host.d.ts} +0 -0
- /package/dist/{host.js → core/host.js} +0 -0
- /package/dist/{provider.js → core/provider.js} +0 -0
- /package/dist/{registry.d.ts → core/registry.d.ts} +0 -0
- /package/dist/{registry.js → core/registry.js} +0 -0
- /package/dist/{telemetry.d.ts → core/telemetry.d.ts} +0 -0
- /package/dist/{telemetry.js → core/telemetry.js} +0 -0
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { OnnxRuntimeError } from "./runtime.js";
|
|
2
|
+
const TEXT_GENERATION_TASK = "text-generation";
|
|
3
|
+
/**
|
|
4
|
+
* Creates the default Web ONNX runtime, backed by Transformers.js (which runs ONNX models through
|
|
5
|
+
* `onnxruntime-web`).
|
|
6
|
+
*
|
|
7
|
+
* ONNX Runtime GenAI has no browser build, so Transformers.js fills the generative-loop role
|
|
8
|
+
* (tokenization, sampling, KV cache) behind the `OnnxTextGenerationRuntime` seam. Applications that
|
|
9
|
+
* need a different runtime implement the seam themselves and pass it to the provider.
|
|
10
|
+
*
|
|
11
|
+
* `@huggingface/transformers` is an optional dependency the application installs: when it is absent
|
|
12
|
+
* the runtime
|
|
13
|
+
* reports itself unavailable instead of throwing, so routing degrades to an explainable
|
|
14
|
+
* `capability_unavailable` rejection.
|
|
15
|
+
*
|
|
16
|
+
* @param options - Loader, backend, and model-resolution overrides.
|
|
17
|
+
*/
|
|
18
|
+
export function createTransformersJsRuntime(options = {}) {
|
|
19
|
+
let modulePromise;
|
|
20
|
+
let generatorPromise;
|
|
21
|
+
async function loadModule() {
|
|
22
|
+
modulePromise ??= (options.load ?? loadTransformersModule)();
|
|
23
|
+
return modulePromise;
|
|
24
|
+
}
|
|
25
|
+
async function loadGenerator(modelPackage) {
|
|
26
|
+
generatorPromise ??= (async () => {
|
|
27
|
+
const module = await loadModule();
|
|
28
|
+
const device = resolveDevice(options.device);
|
|
29
|
+
if (options.createGenerator) {
|
|
30
|
+
return options.createGenerator({ modelPackage, module, device });
|
|
31
|
+
}
|
|
32
|
+
const modelRef = getModelRef(modelPackage);
|
|
33
|
+
if (!modelRef) {
|
|
34
|
+
throw new OnnxRuntimeError("capability", "model source unavailable: the model package does not declare source.ref.");
|
|
35
|
+
}
|
|
36
|
+
applyEnvironment(module, modelPackage, options.localModelPath);
|
|
37
|
+
const pipelineOptions = {
|
|
38
|
+
device,
|
|
39
|
+
dtype: options.dtype ?? defaultDtype(device),
|
|
40
|
+
...(options.pipelineOptions ?? {})
|
|
41
|
+
};
|
|
42
|
+
return module.pipeline(TEXT_GENERATION_TASK, modelRef, pipelineOptions);
|
|
43
|
+
})();
|
|
44
|
+
return generatorPromise;
|
|
45
|
+
}
|
|
46
|
+
return {
|
|
47
|
+
async prepare(modelPackage) {
|
|
48
|
+
let module;
|
|
49
|
+
try {
|
|
50
|
+
module = await loadModule();
|
|
51
|
+
}
|
|
52
|
+
catch (err) {
|
|
53
|
+
modulePromise = undefined;
|
|
54
|
+
return {
|
|
55
|
+
available: false,
|
|
56
|
+
reason: `runtime package unavailable: install the optional dependency @huggingface/transformers (${getErrorMessage(err)}).`
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
if (typeof module.pipeline !== "function") {
|
|
60
|
+
return {
|
|
61
|
+
available: false,
|
|
62
|
+
reason: "runtime initialization failed: @huggingface/transformers did not expose pipeline()."
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
if (modelPackage.format !== "onnx") {
|
|
66
|
+
return {
|
|
67
|
+
available: false,
|
|
68
|
+
reason: `unsupported model format: the Transformers.js runtime consumes ONNX weights, received '${modelPackage.format}'.`
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
const sourceType = modelPackage.source?.sourceType;
|
|
72
|
+
if (sourceType === "programmatic" && !options.createGenerator) {
|
|
73
|
+
return {
|
|
74
|
+
available: false,
|
|
75
|
+
reason: "model source unavailable: programmatic model packages require a createGenerator option."
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
if (!options.createGenerator && !getModelRef(modelPackage)) {
|
|
79
|
+
return {
|
|
80
|
+
available: false,
|
|
81
|
+
reason: "model source unavailable: the model package does not declare source.ref."
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
return { available: true };
|
|
85
|
+
},
|
|
86
|
+
async generate(input, signal) {
|
|
87
|
+
let generator;
|
|
88
|
+
try {
|
|
89
|
+
generator = await loadGenerator(input.modelPackage);
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
generatorPromise = undefined;
|
|
93
|
+
if (err instanceof OnnxRuntimeError)
|
|
94
|
+
throw err;
|
|
95
|
+
if (isResourceExhaustion(err)) {
|
|
96
|
+
throw new OnnxRuntimeError("unavailable", `insufficient memory to load the model: ${getErrorMessage(err)}. Try a smaller model or a more aggressive dtype (for example 'q4').`, err);
|
|
97
|
+
}
|
|
98
|
+
throw new OnnxRuntimeError("capability", `model files missing or unreadable: ${getErrorMessage(err)}`, err);
|
|
99
|
+
}
|
|
100
|
+
if (signal?.aborted) {
|
|
101
|
+
throw new OnnxRuntimeError("timeout", "ONNX generation was aborted before it started.");
|
|
102
|
+
}
|
|
103
|
+
let raw;
|
|
104
|
+
try {
|
|
105
|
+
raw = await generator(input.messages, createGenerationOptions(input, signal));
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
throw new OnnxRuntimeError("internal", `ONNX generation failed: ${getErrorMessage(err)}`, err);
|
|
109
|
+
}
|
|
110
|
+
const text = extractGeneratedText(raw);
|
|
111
|
+
if (text === undefined) {
|
|
112
|
+
throw new OnnxRuntimeError("internal", "model output malformed: the Transformers.js pipeline returned no text.");
|
|
113
|
+
}
|
|
114
|
+
return { text };
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
async function loadTransformersModule() {
|
|
119
|
+
// The specifier stays a literal so bundlers and dev servers resolve the bare package name; the
|
|
120
|
+
// `as string` cast keeps TypeScript from requiring the optional dependency's types here. Apps that
|
|
121
|
+
// do not install Transformers.js should pass their own `load` (or a different runtime) rather than
|
|
122
|
+
// relying on this default, which their bundler will fail to resolve.
|
|
123
|
+
return (await import("@huggingface/transformers"));
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Browser-viable weight variant. Transformers.js defaults to `fp32`, which makes ONNX Runtime
|
|
127
|
+
* allocate the full-precision model and fail session creation with an allocation error for
|
|
128
|
+
* anything but tiny models.
|
|
129
|
+
*/
|
|
130
|
+
function defaultDtype(device) {
|
|
131
|
+
return device === "webgpu" ? "q4f16" : "q4";
|
|
132
|
+
}
|
|
133
|
+
function resolveDevice(configured) {
|
|
134
|
+
if (configured !== undefined)
|
|
135
|
+
return configured;
|
|
136
|
+
const gpu = globalThis.navigator?.gpu;
|
|
137
|
+
return gpu ? "webgpu" : "wasm";
|
|
138
|
+
}
|
|
139
|
+
function getModelRef(modelPackage) {
|
|
140
|
+
const ref = modelPackage.source?.ref;
|
|
141
|
+
return typeof ref === "string" && ref.length > 0 ? ref : undefined;
|
|
142
|
+
}
|
|
143
|
+
function applyEnvironment(module, modelPackage, localModelPath) {
|
|
144
|
+
const env = module.env;
|
|
145
|
+
if (!env)
|
|
146
|
+
return;
|
|
147
|
+
const sourceType = modelPackage.source?.sourceType;
|
|
148
|
+
if (sourceType === "bundled" || sourceType === "app_managed") {
|
|
149
|
+
env.allowLocalModels = true;
|
|
150
|
+
env.allowRemoteModels = false;
|
|
151
|
+
if (localModelPath !== undefined) {
|
|
152
|
+
env.localModelPath = localModelPath;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function createGenerationOptions(input, signal) {
|
|
157
|
+
const generation = input.generation;
|
|
158
|
+
const generationOptions = {};
|
|
159
|
+
if (generation?.maxOutputTokens !== undefined) {
|
|
160
|
+
generationOptions.max_new_tokens = generation.maxOutputTokens;
|
|
161
|
+
}
|
|
162
|
+
if (generation?.temperature !== undefined) {
|
|
163
|
+
generationOptions.temperature = generation.temperature;
|
|
164
|
+
generationOptions.do_sample = generation.temperature > 0;
|
|
165
|
+
}
|
|
166
|
+
if (generation?.topP !== undefined) {
|
|
167
|
+
generationOptions.top_p = generation.topP;
|
|
168
|
+
}
|
|
169
|
+
if (generation?.stop !== undefined && generation.stop.length > 0) {
|
|
170
|
+
generationOptions.stop_strings = generation.stop;
|
|
171
|
+
}
|
|
172
|
+
if (signal !== undefined) {
|
|
173
|
+
generationOptions.signal = signal;
|
|
174
|
+
}
|
|
175
|
+
return generationOptions;
|
|
176
|
+
}
|
|
177
|
+
function extractGeneratedText(raw) {
|
|
178
|
+
const first = Array.isArray(raw) ? raw[0] : raw;
|
|
179
|
+
if (!isRecord(first)) {
|
|
180
|
+
return typeof first === "string" ? first : undefined;
|
|
181
|
+
}
|
|
182
|
+
const generated = first.generated_text;
|
|
183
|
+
if (typeof generated === "string") {
|
|
184
|
+
return generated;
|
|
185
|
+
}
|
|
186
|
+
if (Array.isArray(generated)) {
|
|
187
|
+
const last = generated[generated.length - 1];
|
|
188
|
+
if (isRecord(last) && typeof last.content === "string") {
|
|
189
|
+
return last.content;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Detects out-of-memory failures surfaced by ONNX Runtime session creation, which arrive as
|
|
196
|
+
* allocation errors from the WASM/native layer rather than as typed errors.
|
|
197
|
+
*/
|
|
198
|
+
function isResourceExhaustion(error) {
|
|
199
|
+
const message = getErrorMessage(error).toLowerCase();
|
|
200
|
+
return (message.includes("bad_alloc") ||
|
|
201
|
+
message.includes("out of memory") ||
|
|
202
|
+
message.includes("failed to allocate") ||
|
|
203
|
+
message.includes("allocation failed"));
|
|
204
|
+
}
|
|
205
|
+
function isRecord(value) {
|
|
206
|
+
return typeof value === "object" && value !== null;
|
|
207
|
+
}
|
|
208
|
+
function getErrorMessage(error) {
|
|
209
|
+
return error instanceof Error ? error.message : String(error);
|
|
210
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { TaskRequest, TaskResult } from "@independo/inderun-contracts";
|
|
2
|
-
import type { HostServices } from "
|
|
3
|
-
import type { ProviderAdapter, ProviderDescriptor, ProviderDynamicCapabilities, RunContext } from "
|
|
2
|
+
import type { HostServices } from "../../core/host.js";
|
|
3
|
+
import type { ProviderAdapter, ProviderDescriptor, ProviderDynamicCapabilities, RunContext } from "../../core/provider.js";
|
|
4
4
|
export declare const DEFAULT_OPENAI_RESPONSES_ENDPOINT = "https://api.openai.com/v1/responses";
|
|
5
5
|
/**
|
|
6
6
|
* Configuration for the OpenAI Responses provider.
|
|
@@ -31,6 +31,17 @@ export interface OpenAIProviderOptions {
|
|
|
31
31
|
* Optional HTTP timeout for the host transport attempt.
|
|
32
32
|
*/
|
|
33
33
|
timeoutMs?: number;
|
|
34
|
+
/**
|
|
35
|
+
* Timeout for the endpoint reachability probe issued from `capabilities()`.
|
|
36
|
+
* Defaults to 3000ms. The OpenAI API has no dedicated health endpoint, so this
|
|
37
|
+
* is a cheap unauthenticated `GET` against the configured endpoint.
|
|
38
|
+
*/
|
|
39
|
+
healthCheckTimeoutMs?: number;
|
|
40
|
+
/**
|
|
41
|
+
* How long a reachability probe result is cached before `capabilities()` re-probes.
|
|
42
|
+
* Defaults to 5000ms. Avoids re-probing on every routing decision and UI capability check.
|
|
43
|
+
*/
|
|
44
|
+
healthCheckCacheMs?: number;
|
|
34
45
|
}
|
|
35
46
|
/**
|
|
36
47
|
* Cloud provider adapter that executes Mode-1 text-to-text requests through the OpenAI Responses API.
|
|
@@ -44,6 +55,7 @@ export declare class OpenAIResponsesProvider implements ProviderAdapter {
|
|
|
44
55
|
private readonly id;
|
|
45
56
|
private readonly endpointUrl;
|
|
46
57
|
private readonly auth;
|
|
58
|
+
private cachedHealth?;
|
|
47
59
|
/**
|
|
48
60
|
* Creates an OpenAI Responses provider.
|
|
49
61
|
* @param options - Provider configuration, including required OpenAI model id.
|
|
@@ -55,9 +67,15 @@ export declare class OpenAIResponsesProvider implements ProviderAdapter {
|
|
|
55
67
|
describe(): ProviderDescriptor;
|
|
56
68
|
/**
|
|
57
69
|
* Reports dynamic provider availability for the current host.
|
|
70
|
+
*
|
|
71
|
+
* After the static host-service checks pass, this probes endpoint reachability with a
|
|
72
|
+
* cheap unauthenticated `GET` against the configured endpoint (the OpenAI API has no
|
|
73
|
+
* dedicated health endpoint). The result is cached for `healthCheckCacheMs` so routing
|
|
74
|
+
* decisions and repeated UI capability checks don't re-probe on every call.
|
|
58
75
|
* @param host - Host services available to the engine.
|
|
59
76
|
*/
|
|
60
77
|
capabilities(host: HostServices): Promise<ProviderDynamicCapabilities>;
|
|
78
|
+
private checkEndpointReachable;
|
|
61
79
|
/**
|
|
62
80
|
* Executes a normalized text-to-text task using the OpenAI Responses API.
|
|
63
81
|
* @param request - Canonical IndeRun task request.
|
|
@@ -77,4 +95,4 @@ export declare class OpenAIResponsesProvider implements ProviderAdapter {
|
|
|
77
95
|
*/
|
|
78
96
|
private mapHttpError;
|
|
79
97
|
}
|
|
80
|
-
//# sourceMappingURL=
|
|
98
|
+
//# sourceMappingURL=provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/providers/openai/provider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAQ5E,OAAO,KAAK,EAAE,YAAY,EAAe,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,2BAA2B,EAC3B,UAAU,EACX,MAAM,wBAAwB,CAAC;AAEhC,eAAO,MAAM,iCAAiC,wCAAwC,CAAC;AAKvF;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,IAAI,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAAC;IACjC;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAwBD;;;;;;GAMG;AACH,qBAAa,uBAAwB,YAAW,eAAe;IAUjD,OAAO,CAAC,QAAQ,CAAC,OAAO;IATpC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA4B;IACjD,OAAO,CAAC,YAAY,CAAC,CAA6D;IAElF;;;OAGG;gBAC0B,OAAO,EAAE,qBAAqB;IAM3D;;OAEG;IACH,QAAQ,IAAI,kBAAkB;IAsB9B;;;;;;;;OAQG;IACG,YAAY,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,2BAA2B,CAAC;YAkB9D,sBAAsB;IAuCpC;;;;;;OAMG;IACG,GAAG,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IAkHzE,OAAO,CAAC,iBAAiB;IAuBzB;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;CAmErB"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { createAuthError, createInternal, createRateLimited, createTimeout, createUnavailable } from "
|
|
1
|
+
import { createAuthError, createInternal, createRateLimited, createTimeout, createUnavailable } from "../../core/errors.js";
|
|
2
2
|
export const DEFAULT_OPENAI_RESPONSES_ENDPOINT = "https://api.openai.com/v1/responses";
|
|
3
|
+
const DEFAULT_HEALTH_CHECK_TIMEOUT_MS = 3000;
|
|
4
|
+
const DEFAULT_HEALTH_CHECK_CACHE_MS = 5000;
|
|
3
5
|
/**
|
|
4
6
|
* Cloud provider adapter that executes Mode-1 text-to-text requests through the OpenAI Responses API.
|
|
5
7
|
*
|
|
@@ -12,6 +14,7 @@ export class OpenAIResponsesProvider {
|
|
|
12
14
|
id;
|
|
13
15
|
endpointUrl;
|
|
14
16
|
auth;
|
|
17
|
+
cachedHealth;
|
|
15
18
|
/**
|
|
16
19
|
* Creates an OpenAI Responses provider.
|
|
17
20
|
* @param options - Provider configuration, including required OpenAI model id.
|
|
@@ -48,6 +51,11 @@ export class OpenAIResponsesProvider {
|
|
|
48
51
|
}
|
|
49
52
|
/**
|
|
50
53
|
* Reports dynamic provider availability for the current host.
|
|
54
|
+
*
|
|
55
|
+
* After the static host-service checks pass, this probes endpoint reachability with a
|
|
56
|
+
* cheap unauthenticated `GET` against the configured endpoint (the OpenAI API has no
|
|
57
|
+
* dedicated health endpoint). The result is cached for `healthCheckCacheMs` so routing
|
|
58
|
+
* decisions and repeated UI capability checks don't re-probe on every call.
|
|
51
59
|
* @param host - Host services available to the engine.
|
|
52
60
|
*/
|
|
53
61
|
async capabilities(host) {
|
|
@@ -63,7 +71,39 @@ export class OpenAIResponsesProvider {
|
|
|
63
71
|
reason: "OpenAI Responses provider requires a SecureStorageService when auth is enabled."
|
|
64
72
|
};
|
|
65
73
|
}
|
|
66
|
-
return
|
|
74
|
+
return this.checkEndpointReachable(host.httpClient, host.clock);
|
|
75
|
+
}
|
|
76
|
+
async checkEndpointReachable(httpClient, clock) {
|
|
77
|
+
const now = clock ? clock.now() : Date.now();
|
|
78
|
+
const cacheMs = this.options.healthCheckCacheMs ?? DEFAULT_HEALTH_CHECK_CACHE_MS;
|
|
79
|
+
if (this.cachedHealth && now - this.cachedHealth.checkedAt < cacheMs) {
|
|
80
|
+
return this.cachedHealth.result;
|
|
81
|
+
}
|
|
82
|
+
let result;
|
|
83
|
+
try {
|
|
84
|
+
const response = await httpClient.send({
|
|
85
|
+
method: "GET",
|
|
86
|
+
url: this.endpointUrl,
|
|
87
|
+
timeoutMs: this.options.healthCheckTimeoutMs ?? DEFAULT_HEALTH_CHECK_TIMEOUT_MS
|
|
88
|
+
});
|
|
89
|
+
result =
|
|
90
|
+
response.status >= 500
|
|
91
|
+
? {
|
|
92
|
+
available: false,
|
|
93
|
+
reason: `OpenAI Responses endpoint returned HTTP ${response.status}.`
|
|
94
|
+
}
|
|
95
|
+
: { available: true };
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
result = {
|
|
99
|
+
available: false,
|
|
100
|
+
reason: isAbortError(err)
|
|
101
|
+
? "OpenAI Responses endpoint health check timed out."
|
|
102
|
+
: "OpenAI Responses endpoint is unreachable."
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
this.cachedHealth = { result, checkedAt: now };
|
|
106
|
+
return result;
|
|
67
107
|
}
|
|
68
108
|
/**
|
|
69
109
|
* Executes a normalized text-to-text task using the OpenAI Responses API.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { type SystemModelRuntime } from "./runtime.js";
|
|
2
|
+
/**
|
|
3
|
+
* Structural view of the Chrome/Edge Prompt API's `LanguageModel` global this runtime uses.
|
|
4
|
+
*
|
|
5
|
+
* Typed structurally (not imported from `@types/dom`) since the Prompt API is not yet part of
|
|
6
|
+
* stable browser type definitions. See https://developer.chrome.com/docs/ai/prompt-api.
|
|
7
|
+
*/
|
|
8
|
+
export interface PromptApiLanguageModel {
|
|
9
|
+
availability: (options?: Record<string, unknown>) => Promise<string>;
|
|
10
|
+
create: (options?: Record<string, unknown>) => Promise<PromptApiSession>;
|
|
11
|
+
}
|
|
12
|
+
/** Structural view of a Prompt API session. */
|
|
13
|
+
export interface PromptApiSession {
|
|
14
|
+
prompt: (input: string, options?: {
|
|
15
|
+
signal?: AbortSignal;
|
|
16
|
+
}) => Promise<string>;
|
|
17
|
+
destroy: () => void;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Generation controls forwarded to `LanguageModel.availability()` and `LanguageModel.create()`.
|
|
21
|
+
*
|
|
22
|
+
* Chrome's docs require passing the same options to both calls, since availability can depend on
|
|
23
|
+
* the requested configuration.
|
|
24
|
+
*/
|
|
25
|
+
export interface ChromePromptApiGenerationOptions {
|
|
26
|
+
temperature?: number;
|
|
27
|
+
topK?: number;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Configuration for the Chrome Prompt API-backed system-model runtime.
|
|
31
|
+
*/
|
|
32
|
+
export interface ChromePromptApiRuntimeOptions {
|
|
33
|
+
/** Generation controls passed to both `availability()` and `create()`. */
|
|
34
|
+
generation?: ChromePromptApiGenerationOptions;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Creates the default Web system-model runtime, backed by Chrome's built-in Prompt API
|
|
38
|
+
* (`LanguageModel`).
|
|
39
|
+
*
|
|
40
|
+
* The browser owns model availability, download, and execution; this runtime never bundles or
|
|
41
|
+
* fetches a model itself. When `LanguageModel` is not defined (unsupported browser, disabled
|
|
42
|
+
* flag, or origin trial not enrolled) the runtime reports itself unavailable instead of throwing,
|
|
43
|
+
* so routing degrades to an explainable `capability_unavailable` rejection.
|
|
44
|
+
*
|
|
45
|
+
* @param options - Generation control overrides forwarded to the Prompt API.
|
|
46
|
+
*/
|
|
47
|
+
export declare function createChromePromptApiRuntime(options?: ChromePromptApiRuntimeOptions): SystemModelRuntime;
|
|
48
|
+
//# sourceMappingURL=chrome-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chrome-runtime.d.ts","sourceRoot":"","sources":["../../../src/providers/system-model/chrome-runtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,kBAAkB,EACxB,MAAM,cAAc,CAAC;AAEtB;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACrC,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC1E;AAED,+CAA+C;AAC/C,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/E,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gCAAgC;IAC/C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC5C,0EAA0E;IAC1E,UAAU,CAAC,EAAE,gCAAgC,CAAC;CAC/C;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,GAAE,6BAAkC,GAC1C,kBAAkB,CA+EpB"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { SystemModelRuntimeError } from "./runtime.js";
|
|
2
|
+
/**
|
|
3
|
+
* Creates the default Web system-model runtime, backed by Chrome's built-in Prompt API
|
|
4
|
+
* (`LanguageModel`).
|
|
5
|
+
*
|
|
6
|
+
* The browser owns model availability, download, and execution; this runtime never bundles or
|
|
7
|
+
* fetches a model itself. When `LanguageModel` is not defined (unsupported browser, disabled
|
|
8
|
+
* flag, or origin trial not enrolled) the runtime reports itself unavailable instead of throwing,
|
|
9
|
+
* so routing degrades to an explainable `capability_unavailable` rejection.
|
|
10
|
+
*
|
|
11
|
+
* @param options - Generation control overrides forwarded to the Prompt API.
|
|
12
|
+
*/
|
|
13
|
+
export function createChromePromptApiRuntime(options = {}) {
|
|
14
|
+
const promptOptions = toPromptApiOptions(options.generation);
|
|
15
|
+
return {
|
|
16
|
+
async availability() {
|
|
17
|
+
const languageModel = getLanguageModel();
|
|
18
|
+
if (!languageModel) {
|
|
19
|
+
return {
|
|
20
|
+
kind: "api_missing",
|
|
21
|
+
reason: "system model API missing: 'LanguageModel' is not defined on this browser. See https://developer.chrome.com/docs/ai/prompt-api."
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
let result;
|
|
25
|
+
try {
|
|
26
|
+
result = await languageModel.availability(promptOptions);
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
return mapAvailabilityError(err);
|
|
30
|
+
}
|
|
31
|
+
switch (result) {
|
|
32
|
+
case "available":
|
|
33
|
+
return { kind: "available" };
|
|
34
|
+
case "downloadable":
|
|
35
|
+
return {
|
|
36
|
+
kind: "downloadable",
|
|
37
|
+
reason: "model downloadable: the model can be downloaded but is not installed yet."
|
|
38
|
+
};
|
|
39
|
+
case "downloading":
|
|
40
|
+
return {
|
|
41
|
+
kind: "downloading",
|
|
42
|
+
reason: "model downloading: the model is currently downloading."
|
|
43
|
+
};
|
|
44
|
+
case "unavailable":
|
|
45
|
+
return {
|
|
46
|
+
kind: "model_unavailable",
|
|
47
|
+
reason: "model unavailable: the model is unavailable on this device."
|
|
48
|
+
};
|
|
49
|
+
default:
|
|
50
|
+
return {
|
|
51
|
+
kind: "unavailable",
|
|
52
|
+
reason: `provider temporarily unavailable: unrecognized availability result '${result}'.`
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
async generate(input, signal) {
|
|
57
|
+
const languageModel = getLanguageModel();
|
|
58
|
+
if (!languageModel) {
|
|
59
|
+
throw new SystemModelRuntimeError("capability", "system model API missing: 'LanguageModel' is not defined on this browser.");
|
|
60
|
+
}
|
|
61
|
+
let session;
|
|
62
|
+
try {
|
|
63
|
+
session = await languageModel.create({ ...promptOptions, signal });
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
throw mapGenerateError(err);
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
const text = await session.prompt(toPromptText(input), signal !== undefined ? { signal } : {});
|
|
70
|
+
return { text };
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
throw mapGenerateError(err);
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
session.destroy();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
function getLanguageModel() {
|
|
82
|
+
return globalThis.LanguageModel;
|
|
83
|
+
}
|
|
84
|
+
function toPromptApiOptions(generation) {
|
|
85
|
+
const promptOptions = {};
|
|
86
|
+
if (generation?.temperature !== undefined)
|
|
87
|
+
promptOptions.temperature = generation.temperature;
|
|
88
|
+
if (generation?.topK !== undefined)
|
|
89
|
+
promptOptions.topK = generation.topK;
|
|
90
|
+
return promptOptions;
|
|
91
|
+
}
|
|
92
|
+
function toPromptText(input) {
|
|
93
|
+
return input.messages.map((message) => `${message.role}: ${message.content}`).join("\n");
|
|
94
|
+
}
|
|
95
|
+
function mapAvailabilityError(error) {
|
|
96
|
+
const name = getDomExceptionName(error);
|
|
97
|
+
if (name === "NotSupportedError") {
|
|
98
|
+
return {
|
|
99
|
+
kind: "browser_unsupported",
|
|
100
|
+
reason: `browser unsupported: ${getErrorMessage(error)}`
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
if (name === "NotAllowedError" || name === "SecurityError") {
|
|
104
|
+
return {
|
|
105
|
+
kind: "feature_disabled",
|
|
106
|
+
reason: `browser feature disabled: ${getErrorMessage(error)}`
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
kind: "unavailable",
|
|
111
|
+
reason: `provider temporarily unavailable: ${getErrorMessage(error)}`
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function mapGenerateError(error) {
|
|
115
|
+
if (error instanceof SystemModelRuntimeError)
|
|
116
|
+
return error;
|
|
117
|
+
const name = getDomExceptionName(error);
|
|
118
|
+
switch (name) {
|
|
119
|
+
case "NotSupportedError":
|
|
120
|
+
return new SystemModelRuntimeError("capability", `hardware unsupported: ${getErrorMessage(error)}`, error);
|
|
121
|
+
case "NotAllowedError":
|
|
122
|
+
case "SecurityError":
|
|
123
|
+
return new SystemModelRuntimeError("capability", `browser feature disabled: ${getErrorMessage(error)}`, error);
|
|
124
|
+
case "QuotaExceededError":
|
|
125
|
+
return new SystemModelRuntimeError("unavailable", `storage or network constraints: ${getErrorMessage(error)}`, error);
|
|
126
|
+
case "AbortError":
|
|
127
|
+
return new SystemModelRuntimeError("timeout", `system model generation was aborted: ${getErrorMessage(error)}`, error);
|
|
128
|
+
default:
|
|
129
|
+
return new SystemModelRuntimeError("internal", `system model generation failed: ${getErrorMessage(error)}`, error);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function getDomExceptionName(error) {
|
|
133
|
+
return error instanceof DOMException ? error.name : undefined;
|
|
134
|
+
}
|
|
135
|
+
function getErrorMessage(error) {
|
|
136
|
+
return error instanceof Error ? error.message : String(error);
|
|
137
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { TaskRequest, TaskResult } from "@independo/inderun-contracts";
|
|
2
|
+
import type { HostServices } from "../../core/host.js";
|
|
3
|
+
import { type SystemModelRuntime } from "./runtime.js";
|
|
4
|
+
import type { ProviderAdapter, ProviderDescriptor, ProviderDynamicCapabilities, RunContext } from "../../core/provider.js";
|
|
5
|
+
/** Default provider id of the Web member of the system-model provider family. */
|
|
6
|
+
export declare const DEFAULT_SYSTEM_MODEL_WEB_PROVIDER_ID = "local.system-model.web";
|
|
7
|
+
/**
|
|
8
|
+
* Configuration for the Web system-model provider.
|
|
9
|
+
*/
|
|
10
|
+
export interface SystemModelProviderOptions {
|
|
11
|
+
/**
|
|
12
|
+
* Provider id exposed in telemetry and routing explanations.
|
|
13
|
+
*/
|
|
14
|
+
id?: string;
|
|
15
|
+
/**
|
|
16
|
+
* Runtime implementation used to check availability and execute generation. Defaults to
|
|
17
|
+
* `createChromePromptApiRuntime()`; pass `createFixtureSystemModelRuntime()` from
|
|
18
|
+
* `system-model.js` for tests and offline demos.
|
|
19
|
+
*/
|
|
20
|
+
runtime?: SystemModelRuntime;
|
|
21
|
+
/**
|
|
22
|
+
* Optional generation timeout in milliseconds. A request-level `constraints.timeoutMs` wins.
|
|
23
|
+
*/
|
|
24
|
+
timeoutMs?: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Local provider adapter that executes Mode-1 text-to-text requests against a browser-managed
|
|
28
|
+
* on-device model (for example Chrome's Prompt API / Gemini Nano).
|
|
29
|
+
*
|
|
30
|
+
* Unlike the ONNX Runtime provider family, this provider takes no developer-supplied model: the
|
|
31
|
+
* browser owns model availability, download, and execution. The adapter owns the IndeRun-facing
|
|
32
|
+
* contract (descriptor, capability checks, error taxonomy) and delegates everything else to an
|
|
33
|
+
* injectable {@link SystemModelRuntime}.
|
|
34
|
+
*/
|
|
35
|
+
export declare class SystemModelWebProvider implements ProviderAdapter {
|
|
36
|
+
private readonly options;
|
|
37
|
+
private readonly id;
|
|
38
|
+
private readonly runtime;
|
|
39
|
+
/**
|
|
40
|
+
* Creates a Web system-model provider.
|
|
41
|
+
* @param options - Optional id, runtime, and timeout overrides.
|
|
42
|
+
*/
|
|
43
|
+
constructor(options?: SystemModelProviderOptions);
|
|
44
|
+
/**
|
|
45
|
+
* Returns static metadata used by the router for deterministic provider selection.
|
|
46
|
+
*/
|
|
47
|
+
describe(): ProviderDescriptor;
|
|
48
|
+
/**
|
|
49
|
+
* Reports dynamic provider availability for the current host.
|
|
50
|
+
*
|
|
51
|
+
* Every failure below is provider-internal detail: the shared route planner flattens it into a
|
|
52
|
+
* single `capability_unavailable` rejection carrying the returned reason.
|
|
53
|
+
*
|
|
54
|
+
* @param _host - Host services available to the engine (unused; the browser owns execution).
|
|
55
|
+
*/
|
|
56
|
+
capabilities(_host: HostServices): Promise<ProviderDynamicCapabilities>;
|
|
57
|
+
/**
|
|
58
|
+
* Executes a normalized text-to-text task against the browser-managed on-device model.
|
|
59
|
+
* @param request - Canonical IndeRun task request.
|
|
60
|
+
* @param context - Engine execution context containing run id and host services.
|
|
61
|
+
* @returns Normalized IndeRun task result.
|
|
62
|
+
* @throws IndeRunException mapped to the standard error taxonomy.
|
|
63
|
+
*/
|
|
64
|
+
run(request: TaskRequest, context: RunContext): Promise<TaskResult>;
|
|
65
|
+
private mapRuntimeError;
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/providers/system-model/provider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAS5E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,OAAO,EAIL,KAAK,kBAAkB,EACxB,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,2BAA2B,EAC3B,UAAU,EACX,MAAM,wBAAwB,CAAC;AAEhC,iFAAiF;AACjF,eAAO,MAAM,oCAAoC,2BAA2B,CAAC;AAK7E;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,OAAO,CAAC,EAAE,kBAAkB,CAAC;IAC7B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;GAQG;AACH,qBAAa,sBAAuB,YAAW,eAAe;IAQhD,OAAO,CAAC,QAAQ,CAAC,OAAO;IAPpC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAE7C;;;OAGG;gBAC0B,OAAO,GAAE,0BAA+B;IAKrE;;OAEG;IACH,QAAQ,IAAI,kBAAkB;IAsB9B;;;;;;;OAOG;IACG,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,2BAA2B,CAAC;IAK7E;;;;;;OAMG;IACG,GAAG,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IA8DzE,OAAO,CAAC,eAAe;CA2BxB"}
|