@qvac/inference 0.20.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +90 -24
- package/README.md +8 -5
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +8 -20
- package/dist/handlers/assess-model-fit.js +195 -77
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +37 -2
- package/dist/resources/model-fit/assess.js +24 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +269 -85
- package/dist/schemas/assess-model-fit.js +78 -54
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +708 -66
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +104 -36
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +25 -18
- package/dist/schemas/llamacpp-config.js +26 -16
- package/dist/schemas/load-model.d.ts +2608 -469
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +183 -6
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +43 -29
- package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
- package/dist/resources/model-fit/fit-stub/index.js +0 -2
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { QVACRegistryClient } from '@qvac/registry-client';
|
|
2
|
+
import type { RegistryItem } from '../../models/registry/models.js';
|
|
3
|
+
type BundleSpec = {
|
|
4
|
+
sourceName: string;
|
|
5
|
+
targetSuffix: string;
|
|
6
|
+
};
|
|
7
|
+
export type CoremlSidecarSpec = {
|
|
8
|
+
root: string;
|
|
9
|
+
modelStem: string;
|
|
10
|
+
bundles: readonly BundleSpec[];
|
|
11
|
+
};
|
|
12
|
+
type CompanionSet = NonNullable<RegistryItem['companionSet']>;
|
|
13
|
+
export declare function getCoremlSidecarSpec(registryPath: string, registrySource: string, platform: string | undefined): CoremlSidecarSpec | undefined;
|
|
14
|
+
export declare function getCoremlSetKey(model: RegistryItem, spec: CoremlSidecarSpec): string;
|
|
15
|
+
export declare function readCachedCoremlCompanionSet(model: RegistryItem, platform: string | undefined, cacheDir?: string): Promise<CompanionSet | undefined>;
|
|
16
|
+
export declare function writeCachedCoremlCompanionSet(set: CompanionSet, cacheDir?: string): Promise<void>;
|
|
17
|
+
export declare function findCatalogCoremlCompanionSet(model: RegistryItem, platform: string | undefined): CompanionSet | undefined;
|
|
18
|
+
export declare function findLocallyCachedCoremlCompanionSet(model: RegistryItem, platform: string | undefined, cacheDir?: string): Promise<CompanionSet | undefined>;
|
|
19
|
+
/**
|
|
20
|
+
* Build the existing companion-set shape from live registry metadata. The
|
|
21
|
+
* sidecars are optional and are not part of the generated GGUF constants;
|
|
22
|
+
* this lets the SDK use them as soon as the registry has ingested the files.
|
|
23
|
+
*/
|
|
24
|
+
export declare function findCoremlCompanionSet(client: Pick<QVACRegistryClient, 'findModels'>, model: RegistryItem, platform: string | undefined): Promise<CompanionSet | undefined>;
|
|
25
|
+
export {};
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import { models } from '../../models/registry/models.js';
|
|
2
|
+
import { generateShortHash } from '../../utils/formatting.js';
|
|
3
|
+
import { getModelsCacheDir } from '../../utils/cache/paths.js';
|
|
4
|
+
import { validateAndJoinPath } from '../../utils/path-security.js';
|
|
5
|
+
import { promises as fsPromises } from 'bare-fs';
|
|
6
|
+
import { z } from 'zod';
|
|
7
|
+
// Apple Core ML sidecars for registry GGUFs. A native addon runs a stage on a
|
|
8
|
+
// compiled `.mlmodelc` bundle when one sits next to the GGUF and falls back to
|
|
9
|
+
// ggml otherwise, so on macOS and iOS the bundle is downloaded with its GGUF
|
|
10
|
+
// into one companion-set directory. The registry publishes a bundle as its
|
|
11
|
+
// individual component files, none of which is a model constant of its own.
|
|
12
|
+
const COREML_COMPONENTS = [
|
|
13
|
+
'analytics/coremldata.bin',
|
|
14
|
+
'coremldata.bin',
|
|
15
|
+
'metadata.json',
|
|
16
|
+
'model.mil',
|
|
17
|
+
'weights/weight.bin'
|
|
18
|
+
];
|
|
19
|
+
const SIDECAR_FAMILIES = [
|
|
20
|
+
{
|
|
21
|
+
// The pinned ASR addon supports these encoder sidecars. The other bundles
|
|
22
|
+
// in the registry (CTC, Indic CTC, Nemotron, and TDT 1.1B) must not be
|
|
23
|
+
// downloaded until the addon can actually use them.
|
|
24
|
+
root: 'qvac_models_compiled/ggml/parakeet/2026-09-28/',
|
|
25
|
+
pathPattern: /\/ggml\/parakeet\//,
|
|
26
|
+
stemPattern: /^(.+)\.(?:f16|q4_0|q8_0)\.gguf$/,
|
|
27
|
+
bundlesByStem: {
|
|
28
|
+
'parakeet-tdt-0.6b-v3': [
|
|
29
|
+
{ sourceName: 'parakeet-tdt-0.6b-v3-encoder', targetSuffix: '-encoder' }
|
|
30
|
+
],
|
|
31
|
+
'parakeet-unified-en-0.6b': [
|
|
32
|
+
{ sourceName: 'parakeet-unified-en-0.6b-encoder', targetSuffix: '-encoder' }
|
|
33
|
+
],
|
|
34
|
+
'parakeet-eou-120m-v1': [
|
|
35
|
+
{ sourceName: 'parakeet_realtime_eou_120m-v1-encoder', targetSuffix: '-encoder' }
|
|
36
|
+
],
|
|
37
|
+
'diar_streaming_sortformer_4spk-v2.1': [
|
|
38
|
+
{ sourceName: 'diar_streaming_sortformer_4spk-v2.1-encoder', targetSuffix: '-encoder' },
|
|
39
|
+
{
|
|
40
|
+
sourceName: 'diar_streaming_sortformer_4spk-v2.1-encoder-bypass-pre-encode',
|
|
41
|
+
targetSuffix: '-encoder-bypass-pre-encode'
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
// The TTS addon runs the Audio8 codec's synthesis stack on this bundle when
|
|
48
|
+
// it sits beside the codec decoder GGUF. Every quantization tier shares the
|
|
49
|
+
// one bundle, which is why its name drops the tier.
|
|
50
|
+
root: 'qvac_models_compiled/ggml/audio8/2026-09-30/',
|
|
51
|
+
pathPattern: /\/ggml\/audio-?8\//,
|
|
52
|
+
stemPattern: /^(.+)-(?:f16|f32|q8_0)\.gguf$/,
|
|
53
|
+
bundlesByStem: {
|
|
54
|
+
'audio8-codec-decoder': [{ sourceName: 'audio8-codec-decoder', targetSuffix: '' }]
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
];
|
|
58
|
+
const CACHE_METADATA_NAME = '.coreml-companions.json';
|
|
59
|
+
const checksumSchema = z.string().regex(/^[a-f0-9]{64}$/i);
|
|
60
|
+
const cachedFileSchema = z.object({
|
|
61
|
+
key: z.string(),
|
|
62
|
+
registryPath: z.string(),
|
|
63
|
+
registrySource: z.literal('s3'),
|
|
64
|
+
targetName: z.string(),
|
|
65
|
+
expectedSize: z.number().int().nonnegative(),
|
|
66
|
+
sha256Checksum: checksumSchema,
|
|
67
|
+
blobCoreKey: z.string(),
|
|
68
|
+
blobBlockOffset: z.number(),
|
|
69
|
+
blobBlockLength: z.number(),
|
|
70
|
+
blobByteOffset: z.number(),
|
|
71
|
+
primary: z.boolean().default(false)
|
|
72
|
+
});
|
|
73
|
+
const cachedSetSchema = z.object({
|
|
74
|
+
setKey: z.string(),
|
|
75
|
+
primaryKey: z.literal('modelPath'),
|
|
76
|
+
files: z.array(cachedFileSchema)
|
|
77
|
+
});
|
|
78
|
+
export function getCoremlSidecarSpec(registryPath, registrySource, platform) {
|
|
79
|
+
if ((platform !== 'darwin' && platform !== 'ios') || registrySource !== 's3')
|
|
80
|
+
return undefined;
|
|
81
|
+
const filename = registryPath.split('/').pop() || '';
|
|
82
|
+
for (const family of SIDECAR_FAMILIES) {
|
|
83
|
+
if (!family.pathPattern.test(registryPath))
|
|
84
|
+
continue;
|
|
85
|
+
const modelStem = filename.match(family.stemPattern)?.[1];
|
|
86
|
+
const bundles = modelStem ? family.bundlesByStem[modelStem] : undefined;
|
|
87
|
+
if (modelStem && bundles)
|
|
88
|
+
return { root: family.root, modelStem, bundles };
|
|
89
|
+
}
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
export function getCoremlSetKey(model, spec) {
|
|
93
|
+
return generateShortHash(`${model.registrySource}:${model.registryPath}:coreml:${spec.root}`);
|
|
94
|
+
}
|
|
95
|
+
function getCachedSetMetadataPath(model, spec, cacheDir) {
|
|
96
|
+
return validateAndJoinPath(cacheDir, 'sets', getCoremlSetKey(model, spec), CACHE_METADATA_NAME);
|
|
97
|
+
}
|
|
98
|
+
function expectedSidecarPaths(spec) {
|
|
99
|
+
const paths = new Map();
|
|
100
|
+
for (const bundle of spec.bundles) {
|
|
101
|
+
for (const component of COREML_COMPONENTS) {
|
|
102
|
+
paths.set(`${spec.modelStem}${bundle.targetSuffix}.mlmodelc/${component}`, `${spec.root}${bundle.sourceName}.mlmodelc/${component}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return paths;
|
|
106
|
+
}
|
|
107
|
+
function isValidCachedSet(set, model, spec) {
|
|
108
|
+
if (set.setKey !== getCoremlSetKey(model, spec) || set.primaryKey !== 'modelPath') {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
const primary = set.files.find((file) => file.key === 'modelPath');
|
|
112
|
+
const filename = model.registryPath.split('/').pop();
|
|
113
|
+
if (!primary ||
|
|
114
|
+
primary.registryPath !== model.registryPath ||
|
|
115
|
+
primary.targetName !== filename ||
|
|
116
|
+
primary.expectedSize !== model.expectedSize ||
|
|
117
|
+
primary.sha256Checksum !== model.sha256Checksum) {
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
const expected = expectedSidecarPaths(spec);
|
|
121
|
+
const actual = set.files.filter((file) => file.key !== 'modelPath');
|
|
122
|
+
if (actual.length === 0 ||
|
|
123
|
+
new Set(set.files.map((file) => file.targetName)).size !== set.files.length) {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
if (actual.some((file) => expected.get(file.targetName) !== file.registryPath))
|
|
127
|
+
return false;
|
|
128
|
+
for (const bundle of spec.bundles) {
|
|
129
|
+
const prefix = `${spec.modelStem}${bundle.targetSuffix}.mlmodelc/`;
|
|
130
|
+
const count = actual.filter((file) => file.targetName.startsWith(prefix)).length;
|
|
131
|
+
if (count !== 0 && count !== COREML_COMPONENTS.length)
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
export async function readCachedCoremlCompanionSet(model, platform, cacheDir = getModelsCacheDir()) {
|
|
137
|
+
const spec = getCoremlSidecarSpec(model.registryPath, model.registrySource, platform);
|
|
138
|
+
if (!spec)
|
|
139
|
+
return undefined;
|
|
140
|
+
try {
|
|
141
|
+
const raw = await fsPromises.readFile(getCachedSetMetadataPath(model, spec, cacheDir), 'utf8');
|
|
142
|
+
const parsed = cachedSetSchema.safeParse(JSON.parse(raw));
|
|
143
|
+
if (!parsed.success || !isValidCachedSet(parsed.data, model, spec))
|
|
144
|
+
return undefined;
|
|
145
|
+
return parsed.data;
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
return undefined;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
export async function writeCachedCoremlCompanionSet(set, cacheDir = getModelsCacheDir()) {
|
|
152
|
+
const setDir = validateAndJoinPath(cacheDir, 'sets', set.setKey);
|
|
153
|
+
await fsPromises.mkdir(setDir, { recursive: true });
|
|
154
|
+
await fsPromises.writeFile(validateAndJoinPath(setDir, CACHE_METADATA_NAME), JSON.stringify(set));
|
|
155
|
+
}
|
|
156
|
+
export function findCatalogCoremlCompanionSet(model, platform) {
|
|
157
|
+
const spec = getCoremlSidecarSpec(model.registryPath, model.registrySource, platform);
|
|
158
|
+
if (!spec)
|
|
159
|
+
return undefined;
|
|
160
|
+
const sidecarFiles = [];
|
|
161
|
+
for (const bundle of spec.bundles) {
|
|
162
|
+
const targetPrefix = `${spec.modelStem}${bundle.targetSuffix}.mlmodelc/`;
|
|
163
|
+
const registryPrefix = `${spec.root}${bundle.sourceName}.mlmodelc/`;
|
|
164
|
+
const entries = COREML_COMPONENTS.map((component) => models.find((entry) => entry.registrySource === 's3' && entry.registryPath === registryPrefix + component));
|
|
165
|
+
if (entries.some((entry) => !entry))
|
|
166
|
+
continue;
|
|
167
|
+
for (let index = 0; index < entries.length; index++) {
|
|
168
|
+
const entry = entries[index];
|
|
169
|
+
sidecarFiles.push({
|
|
170
|
+
key: `${bundle.sourceName}/${COREML_COMPONENTS[index]}`,
|
|
171
|
+
registryPath: entry.registryPath,
|
|
172
|
+
registrySource: entry.registrySource,
|
|
173
|
+
targetName: targetPrefix + COREML_COMPONENTS[index],
|
|
174
|
+
expectedSize: entry.expectedSize,
|
|
175
|
+
sha256Checksum: entry.sha256Checksum,
|
|
176
|
+
blobCoreKey: entry.blobCoreKey,
|
|
177
|
+
blobBlockOffset: entry.blobBlockOffset,
|
|
178
|
+
blobBlockLength: entry.blobBlockLength,
|
|
179
|
+
blobByteOffset: entry.blobByteOffset
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return createCompanionSet(model, spec, sidecarFiles);
|
|
184
|
+
}
|
|
185
|
+
export async function findLocallyCachedCoremlCompanionSet(model, platform, cacheDir = getModelsCacheDir()) {
|
|
186
|
+
const set = (await readCachedCoremlCompanionSet(model, platform, cacheDir)) ??
|
|
187
|
+
findCatalogCoremlCompanionSet(model, platform);
|
|
188
|
+
if (!set)
|
|
189
|
+
return undefined;
|
|
190
|
+
for (const file of set.files) {
|
|
191
|
+
try {
|
|
192
|
+
const path = validateAndJoinPath(cacheDir, 'sets', set.setKey, file.targetName);
|
|
193
|
+
const stat = await fsPromises.stat(path);
|
|
194
|
+
if (!stat.isFile() || stat.size !== file.expectedSize)
|
|
195
|
+
return undefined;
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
return undefined;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return set;
|
|
202
|
+
}
|
|
203
|
+
function createCompanionSet(model, spec, sidecarFiles) {
|
|
204
|
+
if (sidecarFiles.length === 0)
|
|
205
|
+
return undefined;
|
|
206
|
+
const filename = model.registryPath.split('/').pop() || model.registryPath;
|
|
207
|
+
return {
|
|
208
|
+
setKey: getCoremlSetKey(model, spec),
|
|
209
|
+
primaryKey: 'modelPath',
|
|
210
|
+
files: [
|
|
211
|
+
{
|
|
212
|
+
key: 'modelPath',
|
|
213
|
+
registryPath: model.registryPath,
|
|
214
|
+
registrySource: model.registrySource,
|
|
215
|
+
targetName: filename,
|
|
216
|
+
expectedSize: model.expectedSize,
|
|
217
|
+
sha256Checksum: model.sha256Checksum,
|
|
218
|
+
blobCoreKey: model.blobCoreKey,
|
|
219
|
+
blobBlockOffset: model.blobBlockOffset,
|
|
220
|
+
blobBlockLength: model.blobBlockLength,
|
|
221
|
+
blobByteOffset: model.blobByteOffset,
|
|
222
|
+
primary: true
|
|
223
|
+
},
|
|
224
|
+
...sidecarFiles
|
|
225
|
+
]
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Build the existing companion-set shape from live registry metadata. The
|
|
230
|
+
* sidecars are optional and are not part of the generated GGUF constants;
|
|
231
|
+
* this lets the SDK use them as soon as the registry has ingested the files.
|
|
232
|
+
*/
|
|
233
|
+
export async function findCoremlCompanionSet(client, model, platform) {
|
|
234
|
+
const spec = getCoremlSidecarSpec(model.registryPath, model.registrySource, platform);
|
|
235
|
+
if (!spec)
|
|
236
|
+
return undefined;
|
|
237
|
+
const sidecarFiles = [];
|
|
238
|
+
for (const bundle of spec.bundles) {
|
|
239
|
+
const sourcePrefix = `${spec.root}${bundle.sourceName}.mlmodelc/`;
|
|
240
|
+
const registryEntries = await client.findModels({
|
|
241
|
+
gte: { path: sourcePrefix },
|
|
242
|
+
lte: { path: `${sourcePrefix}` }
|
|
243
|
+
});
|
|
244
|
+
const byPath = new Map();
|
|
245
|
+
for (const entry of registryEntries) {
|
|
246
|
+
if (entry.source === 's3' && entry.path.startsWith(sourcePrefix)) {
|
|
247
|
+
byPath.set(entry.path, entry);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
// A partially ingested .mlmodelc is not a usable bundle. Other complete
|
|
251
|
+
// bundles (notably Sortformer's batch/AOSC pair) may still be staged.
|
|
252
|
+
const entries = COREML_COMPONENTS.map((component) => byPath.get(sourcePrefix + component));
|
|
253
|
+
if (entries.some((entry) => !entry?.blobBinding?.sha256))
|
|
254
|
+
continue;
|
|
255
|
+
for (let i = 0; i < COREML_COMPONENTS.length; i++) {
|
|
256
|
+
const component = COREML_COMPONENTS[i];
|
|
257
|
+
const entry = entries[i];
|
|
258
|
+
const binding = entry.blobBinding;
|
|
259
|
+
sidecarFiles.push({
|
|
260
|
+
key: `${bundle.sourceName}/${component}`,
|
|
261
|
+
registryPath: entry.path,
|
|
262
|
+
registrySource: entry.source,
|
|
263
|
+
targetName: `${spec.modelStem}${bundle.targetSuffix}.mlmodelc/${component}`,
|
|
264
|
+
expectedSize: binding.byteLength,
|
|
265
|
+
sha256Checksum: binding.sha256,
|
|
266
|
+
blobCoreKey: typeof binding.coreKey === 'string' ? binding.coreKey : binding.coreKey.toString('hex'),
|
|
267
|
+
blobBlockOffset: binding.blockOffset,
|
|
268
|
+
blobBlockLength: binding.blockLength,
|
|
269
|
+
blobByteOffset: binding.byteOffset
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return createCompanionSet(model, spec, sidecarFiles);
|
|
274
|
+
}
|
|
@@ -4,7 +4,7 @@ import Corestore from 'corestore';
|
|
|
4
4
|
import Hyperswarm from 'hyperswarm';
|
|
5
5
|
import Hyperdrive from 'hyperdrive';
|
|
6
6
|
import { getQvacPath } from '../../utils/qvac-paths.js';
|
|
7
|
-
import { getModelsCacheDir, generateShortHash, detectShardedModel, getShardedModelCacheDir, getShardPath, checkShardCompleteness, measureChecksum, extractTensorsFromShards, calculatePercentage } from '../../utils/index.js';
|
|
7
|
+
import { getModelsCacheDir, generateShortHash, detectShardedModel, getShardedModelCacheDir, getShardPath, checkShardCompleteness, measureChecksum, extractTensorsFromShards, calculatePercentage, joinWithinBase } from '../../utils/index.js';
|
|
8
8
|
import { getModelBySrc } from '../../models/registry/index.js';
|
|
9
9
|
import { createHyperdriveDownloadKey, startOrJoinDownload, applyJoinedDownloadStats } from './download-manager.js';
|
|
10
10
|
import { getConfig } from '../../runtime/state.js';
|
|
@@ -529,7 +529,7 @@ export async function downloadModelFromHyperdrive(hyperdriveKey, modelFileName,
|
|
|
529
529
|
const corestoreDir = getCorestoreDir(hyperdriveKey);
|
|
530
530
|
const cacheDir = getModelsCacheDir();
|
|
531
531
|
const sourceHash = generateShortHash(`${hyperdriveKey}/${modelFileName}`);
|
|
532
|
-
const modelPath =
|
|
532
|
+
const modelPath = joinWithinBase(cacheDir, `${sourceHash}_${modelFileName}`);
|
|
533
533
|
// First, check if we already have a valid cached file (only if we have model metadata)
|
|
534
534
|
if (model) {
|
|
535
535
|
const cachedPath = await validateCachedFile(modelPath, modelFileName, model.expectedSize, expectedChecksum || model.sha256Checksum, hooks);
|
|
@@ -3,9 +3,11 @@ import { generateShortHash, detectShardedModel, getShardedModelCacheDir, getShar
|
|
|
3
3
|
import { getSingleFileCachePath } from '../../utils/cache/paths.js';
|
|
4
4
|
import { getModelByPath } from '../../models/registry/index.js';
|
|
5
5
|
import { getRegistryClient } from '../../runtime/registry-client.js';
|
|
6
|
+
import { getRuntimeContext } from '../../runtime/state.js';
|
|
6
7
|
import { createRegistryDownloadKey, startOrJoinDownload, applyJoinedDownloadStats } from './download-manager.js';
|
|
7
8
|
import { buildBlobBinding, validateCachedFile, downloadSingleFileFromRegistry } from './registry-download-utils.js';
|
|
8
9
|
import { downloadCompanionSetFromRegistry } from './registry-companion-set.js';
|
|
10
|
+
import { findCoremlCompanionSet, findLocallyCachedCoremlCompanionSet, getCoremlSidecarSpec, writeCachedCoremlCompanionSet } from './coreml-sidecars.js';
|
|
9
11
|
import { DownloadCancelledError, ModelNotFoundError, RegistryDownloadFailedError } from '../../errors/index.js';
|
|
10
12
|
import { getEngineLogger } from '../../logging/index.js';
|
|
11
13
|
import { resolveRegistryDownloadMetadata } from './registry-metadata.js';
|
|
@@ -233,6 +235,63 @@ export async function downloadModelFromRegistry(registryPath, registrySource, pr
|
|
|
233
235
|
throw error;
|
|
234
236
|
}
|
|
235
237
|
}
|
|
238
|
+
// Apple Core ML sidecars are optional registry assets. Discover them
|
|
239
|
+
// before the ordinary single-file path so the GGUF and its complete
|
|
240
|
+
// .mlmodelc directories share the companion-set cache layout.
|
|
241
|
+
const platform = getRuntimeContext().platform;
|
|
242
|
+
if (modelMetadata && getCoremlSidecarSpec(registryPath, registrySource, platform)) {
|
|
243
|
+
const companionHooks = {
|
|
244
|
+
...hooks,
|
|
245
|
+
markCacheHit: () => {
|
|
246
|
+
hooks?.markCacheHit?.();
|
|
247
|
+
ctx.setCacheHit(true);
|
|
248
|
+
},
|
|
249
|
+
markCacheMiss: () => {
|
|
250
|
+
hooks?.markCacheMiss?.();
|
|
251
|
+
ctx.setCacheHit(false);
|
|
252
|
+
}
|
|
253
|
+
};
|
|
254
|
+
try {
|
|
255
|
+
const cachedSet = await findLocallyCachedCoremlCompanionSet(modelMetadata, platform);
|
|
256
|
+
if (cachedSet) {
|
|
257
|
+
return await downloadCompanionSetFromRegistry({
|
|
258
|
+
companionSet: cachedSet,
|
|
259
|
+
downloadKey,
|
|
260
|
+
progressCallback: ctx.broadcastProgress,
|
|
261
|
+
signal: ctx.signal,
|
|
262
|
+
hooks: companionHooks,
|
|
263
|
+
shouldClearCache: ctx.shouldClearCache
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
const coremlSet = await findCoremlCompanionSet(await getRegistryClient(), modelMetadata, platform);
|
|
267
|
+
if (coremlSet) {
|
|
268
|
+
const modelPath = await downloadCompanionSetFromRegistry({
|
|
269
|
+
companionSet: coremlSet,
|
|
270
|
+
downloadKey,
|
|
271
|
+
progressCallback: ctx.broadcastProgress,
|
|
272
|
+
signal: ctx.signal,
|
|
273
|
+
hooks: companionHooks,
|
|
274
|
+
shouldClearCache: ctx.shouldClearCache
|
|
275
|
+
});
|
|
276
|
+
try {
|
|
277
|
+
await writeCachedCoremlCompanionSet(coremlSet);
|
|
278
|
+
}
|
|
279
|
+
catch (error) {
|
|
280
|
+
logger.warn('Unable to save Core ML cache metadata', { registryPath, error });
|
|
281
|
+
}
|
|
282
|
+
return modelPath;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
if (ctx.signal.aborted || error instanceof DownloadCancelledError) {
|
|
287
|
+
throw new DownloadCancelledError();
|
|
288
|
+
}
|
|
289
|
+
logger.warn('Core ML sidecar unavailable; loading the GGUF without it', {
|
|
290
|
+
registryPath,
|
|
291
|
+
error
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
}
|
|
236
295
|
// Generic companion set via generated metadata
|
|
237
296
|
if (modelMetadata?.companionSet) {
|
|
238
297
|
const companionHooks = {
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { StartRpcServerRequest, StopRpcServerRequest, DiscoverRpcServersRequest } from '../schemas/rpc-server.js';
|
|
2
|
+
export declare function handleStartRpcServer(request: StartRpcServerRequest): Promise<{
|
|
3
|
+
serverId: string;
|
|
4
|
+
url: string;
|
|
5
|
+
rdmaCapable: boolean;
|
|
6
|
+
type: "startRpcServer";
|
|
7
|
+
}>;
|
|
8
|
+
export declare function handleStopRpcServer(request: StopRpcServerRequest): Promise<{
|
|
9
|
+
type: "stopRpcServer";
|
|
10
|
+
}>;
|
|
11
|
+
export declare function handleDiscoverRpcServers(request: DiscoverRpcServersRequest): Promise<{
|
|
12
|
+
type: "discoverRpcServers";
|
|
13
|
+
servers: {
|
|
14
|
+
url: string;
|
|
15
|
+
rdmaAvailable: boolean;
|
|
16
|
+
devices: {
|
|
17
|
+
index: number;
|
|
18
|
+
freeMemory: number;
|
|
19
|
+
totalMemory: number;
|
|
20
|
+
}[];
|
|
21
|
+
}[];
|
|
22
|
+
}>;
|
|
23
|
+
export declare function closeRpcResources(): Promise<void>;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
+
if (value !== null && value !== void 0) {
|
|
3
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
+
var dispose, inner;
|
|
5
|
+
if (async) {
|
|
6
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
+
dispose = value[Symbol.asyncDispose];
|
|
8
|
+
}
|
|
9
|
+
if (dispose === void 0) {
|
|
10
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
+
dispose = value[Symbol.dispose];
|
|
12
|
+
if (async) inner = dispose;
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
+
}
|
|
18
|
+
else if (async) {
|
|
19
|
+
env.stack.push({ async: true });
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
};
|
|
23
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
+
return function (env) {
|
|
25
|
+
function fail(e) {
|
|
26
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
+
env.hasError = true;
|
|
28
|
+
}
|
|
29
|
+
var r, s = 0;
|
|
30
|
+
function next() {
|
|
31
|
+
while (r = env.stack.pop()) {
|
|
32
|
+
try {
|
|
33
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
+
if (r.dispose) {
|
|
35
|
+
var result = r.dispose.call(r.value);
|
|
36
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
+
}
|
|
38
|
+
else s |= 1;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
fail(e);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
+
if (env.hasError) throw env.error;
|
|
46
|
+
}
|
|
47
|
+
return next();
|
|
48
|
+
};
|
|
49
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
+
var e = new Error(message);
|
|
51
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
+
});
|
|
53
|
+
import { getRequestRegistry } from '../runtime/request-context.js';
|
|
54
|
+
import { generateRandomRequestId } from '../runtime/request-id.js';
|
|
55
|
+
import { rpcServers } from '../rpc/instance.js';
|
|
56
|
+
import { discoverRpcEndpoints } from '../rpc/discovery.js';
|
|
57
|
+
import { clearRpcServerProvider } from '../rpc/provider.js';
|
|
58
|
+
import { RpcServerOperationError, InferenceCancelledError, RequestIdConflictError } from '../errors/index.js';
|
|
59
|
+
const requests = new Map();
|
|
60
|
+
let closing;
|
|
61
|
+
async function run(operation, work) {
|
|
62
|
+
if (closing)
|
|
63
|
+
throw new RpcServerOperationError(operation, 'Engine is closing');
|
|
64
|
+
try {
|
|
65
|
+
return await work();
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
if (error instanceof InferenceCancelledError ||
|
|
69
|
+
error instanceof RpcServerOperationError ||
|
|
70
|
+
error instanceof RequestIdConflictError) {
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
throw new RpcServerOperationError(operation, error instanceof Error ? error.message : String(error), error);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
async function withContext(kind, requestId, work) {
|
|
77
|
+
const task = (async () => {
|
|
78
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
79
|
+
try {
|
|
80
|
+
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({ requestId, kind }), true);
|
|
81
|
+
return await work(ctx);
|
|
82
|
+
}
|
|
83
|
+
catch (e_1) {
|
|
84
|
+
env_1.error = e_1;
|
|
85
|
+
env_1.hasError = true;
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
const result_1 = __disposeResources(env_1);
|
|
89
|
+
if (result_1)
|
|
90
|
+
await result_1;
|
|
91
|
+
}
|
|
92
|
+
})();
|
|
93
|
+
requests.set(task, requestId);
|
|
94
|
+
try {
|
|
95
|
+
return await task;
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
if (error instanceof InferenceCancelledError) {
|
|
99
|
+
throw new InferenceCancelledError(requestId, error.partial, error);
|
|
100
|
+
}
|
|
101
|
+
throw error;
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
requests.delete(task);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
export async function handleStartRpcServer(request) {
|
|
108
|
+
return run('startRpcServer', () => withContext('rpcServer', request.requestId ?? generateRandomRequestId(), async (ctx) => ({
|
|
109
|
+
type: 'startRpcServer',
|
|
110
|
+
...(await rpcServers.start(request, ctx))
|
|
111
|
+
})));
|
|
112
|
+
}
|
|
113
|
+
export async function handleStopRpcServer(request) {
|
|
114
|
+
return run('stopRpcServer', async () => {
|
|
115
|
+
await rpcServers.stop(request.serverId);
|
|
116
|
+
return { type: 'stopRpcServer' };
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
export async function handleDiscoverRpcServers(request) {
|
|
120
|
+
return run('discoverRpcServers', () => withContext('rpcDiscovery', request.requestId ?? generateRandomRequestId(), async (ctx) => ({
|
|
121
|
+
type: 'discoverRpcServers',
|
|
122
|
+
servers: await discoverRpcEndpoints(request.topic, request.timeoutMs ?? 5000, ctx)
|
|
123
|
+
})));
|
|
124
|
+
}
|
|
125
|
+
export function closeRpcResources() {
|
|
126
|
+
if (closing)
|
|
127
|
+
return closing;
|
|
128
|
+
closing = (async () => {
|
|
129
|
+
for (const requestId of requests.values())
|
|
130
|
+
getRequestRegistry().cancel({ requestId });
|
|
131
|
+
const stopping = rpcServers.close();
|
|
132
|
+
const results = await Promise.allSettled([stopping, ...requests.keys()]);
|
|
133
|
+
const stopResult = results[0];
|
|
134
|
+
if (stopResult.status === 'rejected')
|
|
135
|
+
throw stopResult.reason;
|
|
136
|
+
clearRpcServerProvider();
|
|
137
|
+
})();
|
|
138
|
+
return closing.finally(() => {
|
|
139
|
+
closing = undefined;
|
|
140
|
+
});
|
|
141
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { registerRpcServerProvider } from './rpc/provider.js';
|
|
2
|
+
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, createVectorIndex, loadVectorIndex, type VectorIndex, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, startRpcServer, stopRpcServer, discoverRpcServers, getSystemResources, assessModelFit, loggingStream, subscribeServerLogs, type ServerLogHandler, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, audioGen, audioEdit, audioUnderstand, classify, video, type VideoProgressTick, upscale, worldCreateScene, worldStep, type WorldStepResult, type WorldStepProgressTick, type WorldSceneResult, type WorldSceneResultWithPack, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, type FinetuneHandle } from './api/index.js';
|
|
2
3
|
export { close } from './dispatch.js';
|
|
3
4
|
export { plugins, registerPlugin, registerPlugins, getPlugin, hasPlugin, getAllPlugins, unregisterPlugin } from './plugins/index.js';
|
|
4
5
|
export * from './surface.js';
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/// <reference types="bare" />
|
|
2
|
+
export { registerRpcServerProvider } from './rpc/provider.js';
|
|
2
3
|
// Public API: the operations, engine lifecycle and explicit plugin assembly, and
|
|
3
4
|
// the value-clean surface (schemas / types / consts / errors / model registry)
|
|
4
5
|
// re-exported from `./surface`.
|
|
5
|
-
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, createVectorIndex, loadVectorIndex, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, assessModelFit, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, audioGen, audioEdit, audioUnderstand, classify, video, upscale, worldCreateScene, worldStep, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './api/index.js';
|
|
6
|
+
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, createVectorIndex, loadVectorIndex, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, startRpcServer, stopRpcServer, discoverRpcServers, getSystemResources, assessModelFit, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, audioGen, audioEdit, audioUnderstand, classify, video, upscale, worldCreateScene, worldStep, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './api/index.js';
|
|
6
7
|
// Engine lifecycle and explicit plugin assembly. No plugins are registered by
|
|
7
8
|
// default: an app assembles the engines it needs via `plugins([...])` or the
|
|
8
9
|
// `registerPlugin`/`registerPlugins` primitives, then calls the operations.
|