@qvac/inference 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/NOTICE +95 -39
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/delete-cache.d.ts +7 -1
  5. package/dist/api/delete-cache.js +11 -1
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/delete-cache.js +3 -0
  16. package/dist/handlers/vector-index.d.ts +2 -0
  17. package/dist/handlers/vector-index.js +40 -0
  18. package/dist/index.d.ts +1 -1
  19. package/dist/index.js +1 -1
  20. package/dist/models/registry/models.d.ts +351 -12
  21. package/dist/models/registry/models.js +2271 -1957
  22. package/dist/models/registry/resource-profiles.js +135 -0
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  34. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  36. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  37. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  38. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  39. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  40. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  41. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  44. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  46. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  47. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
  49. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
  50. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  52. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
  53. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  54. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  55. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  56. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  57. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  58. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  59. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  60. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  61. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  62. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  63. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  64. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  65. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  66. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  67. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  68. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  70. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  71. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  72. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  73. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  74. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  75. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  76. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  77. package/dist/plugins/index.d.ts +1 -1
  78. package/dist/plugins/index.js +1 -1
  79. package/dist/plugins/ops/load-model.js +22 -2
  80. package/dist/plugins/ops/translate.js +58 -12
  81. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  82. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  83. package/dist/plugins/registry.d.ts +15 -0
  84. package/dist/plugins/registry.js +133 -15
  85. package/dist/plugins/turbovec-backend.d.ts +17 -0
  86. package/dist/plugins/turbovec-backend.js +99 -0
  87. package/dist/registry.js +4 -0
  88. package/dist/resources/model-fit/assess.js +169 -68
  89. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  90. package/dist/resources/model-fit/estimators/llm.js +3 -1
  91. package/dist/resources/model-fit/floor.d.ts +31 -0
  92. package/dist/resources/model-fit/floor.js +65 -0
  93. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  94. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  95. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  96. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  97. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  98. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  99. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  100. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  101. package/dist/resources/native.js +5 -4
  102. package/dist/resources/types.d.ts +1 -3
  103. package/dist/runtime/env.d.ts +2 -0
  104. package/dist/runtime/env.js +9 -2
  105. package/dist/runtime/lifecycle.js +2 -0
  106. package/dist/runtime/model-registry.d.ts +3 -1
  107. package/dist/runtime/model-registry.js +2 -1
  108. package/dist/runtime/request-context.js +17 -0
  109. package/dist/runtime/vector-index-backend.d.ts +25 -0
  110. package/dist/runtime/vector-index-backend.js +1 -0
  111. package/dist/runtime/vector-index-registry.d.ts +76 -0
  112. package/dist/runtime/vector-index-registry.js +161 -0
  113. package/dist/schemas/assess-model-fit.d.ts +84 -0
  114. package/dist/schemas/assess-model-fit.js +72 -3
  115. package/dist/schemas/audio-gen.d.ts +482 -7
  116. package/dist/schemas/audio-gen.js +376 -17
  117. package/dist/schemas/batch-completion-stream.d.ts +10 -0
  118. package/dist/schemas/batch-completion-stream.js +5 -3
  119. package/dist/schemas/bci.d.ts +1 -1
  120. package/dist/schemas/common.d.ts +782 -19
  121. package/dist/schemas/common.js +8 -1
  122. package/dist/schemas/completion-event.d.ts +14 -1
  123. package/dist/schemas/completion-event.js +4 -0
  124. package/dist/schemas/completion-stream.d.ts +22 -0
  125. package/dist/schemas/completion-stream.js +41 -4
  126. package/dist/schemas/config.d.ts +48 -30
  127. package/dist/schemas/errors.d.ts +4 -0
  128. package/dist/schemas/errors.js +22 -0
  129. package/dist/schemas/index.d.ts +1 -0
  130. package/dist/schemas/index.js +1 -0
  131. package/dist/schemas/llamacpp-config.d.ts +4 -8
  132. package/dist/schemas/llamacpp-config.js +4 -4
  133. package/dist/schemas/load-model.d.ts +2604 -180
  134. package/dist/schemas/model-config-schemas.d.ts +228 -5
  135. package/dist/schemas/model-ops.d.ts +3 -0
  136. package/dist/schemas/model-ops.js +6 -0
  137. package/dist/schemas/plugin.d.ts +21 -11
  138. package/dist/schemas/sdcpp-config.d.ts +221 -4
  139. package/dist/schemas/sdcpp-config.js +128 -22
  140. package/dist/schemas/text-to-speech.d.ts +780 -19
  141. package/dist/schemas/text-to-speech.js +298 -44
  142. package/dist/schemas/transcription-config.js +7 -4
  143. package/dist/schemas/vector-index.d.ts +194 -0
  144. package/dist/schemas/vector-index.js +185 -0
  145. package/dist/utils/addon-responses.d.ts +30 -3
  146. package/dist/utils/aggregate-events.d.ts +3 -1
  147. package/dist/utils/aggregate-events.js +7 -1
  148. package/dist/utils/tts-stats.d.ts +15 -11
  149. package/dist/utils/tts-stats.js +33 -11
  150. package/package.json +29 -26
@@ -0,0 +1,192 @@
1
+ import fs from 'bare-fs';
2
+ import path from 'bare-path';
3
+ import { generateShortHash } from '../../../utils/formatting.js';
4
+ /**
5
+ * Stable `ggml_type` ordinals from `ggml.h` (the header `fitParams` is compiled
6
+ * against). Names llama.cpp has kept numbered since the enum was frozen:
7
+ * F32=0, F16=1, Q4_0=2, Q4_1=3, (4/5 removed Q4_2/Q4_3), Q5_0=6, Q5_1=7,
8
+ * Q8_0=8, Q8_1=9, IQ4_NL=20, BF16=30. Unmapped names (TurboQuant/PolarQuant)
9
+ * fail the conversion instead of silently projecting against llama defaults.
10
+ */
11
+ const GGML_TYPE = {
12
+ f32: 0,
13
+ f16: 1,
14
+ q4_0: 2,
15
+ q4_1: 3,
16
+ q5_0: 6,
17
+ q5_1: 7,
18
+ q8_0: 8,
19
+ q8_1: 9,
20
+ iq4_nl: 20,
21
+ bf16: 30
22
+ };
23
+ /** `enum llama_flash_attn_type` — domain checked in `@qvac/model-fit` index.js. */
24
+ const FLASH_ATTN_TYPE = {
25
+ auto: -1,
26
+ off: 0,
27
+ on: 1
28
+ };
29
+ /** `enum llama_split_mode`. */
30
+ const SPLIT_MODE = {
31
+ none: 0,
32
+ layer: 1,
33
+ tensor: 3
34
+ };
35
+ function parseOptionalInt(value) {
36
+ if (value === undefined)
37
+ return undefined;
38
+ const parsed = Number(value);
39
+ if (!Number.isSafeInteger(parsed))
40
+ return undefined;
41
+ return parsed;
42
+ }
43
+ function lookup(table, value, label) {
44
+ if (value === undefined)
45
+ return undefined;
46
+ const mapped = table[value.toLowerCase()];
47
+ if (mapped === undefined) {
48
+ throw new TypeError(`Unsupported ${label} for in-process fit: ${value}`);
49
+ }
50
+ return mapped;
51
+ }
52
+ function param(config, ...keys) {
53
+ for (const key of keys) {
54
+ const value = config.params[key];
55
+ if (value !== undefined)
56
+ return value;
57
+ }
58
+ return undefined;
59
+ }
60
+ export function toFitConfig(config) {
61
+ const nCtx = parseOptionalInt(param(config, 'ctx_size', 'ctx-size'));
62
+ const nGpuLayers = parseOptionalInt(param(config, 'gpu_layers', 'gpu-layers', 'n-gpu-layers', 'n_gpu_layers'));
63
+ const nBatch = parseOptionalInt(param(config, 'batch_size', 'batch-size'));
64
+ const mainGpu = parseOptionalInt(param(config, 'main-gpu', 'main_gpu'));
65
+ const typeK = lookup(GGML_TYPE, param(config, 'cache-type-k'), 'KV cache type');
66
+ const typeV = lookup(GGML_TYPE, param(config, 'cache-type-v'), 'KV cache type');
67
+ const flashAttnType = lookup(FLASH_ATTN_TYPE, param(config, 'flash-attn', 'flash_attn'), 'flash-attn');
68
+ const splitMode = lookup(SPLIT_MODE, param(config, 'split-mode', 'split_mode'), 'split-mode');
69
+ // device, load_mode, parallel, and tensor-split are fit evidence on the
70
+ // process path but are not fields on public FitConfig — omitting them here
71
+ // is the API limit, not a silent default.
72
+ return {
73
+ modelPath: config.modelPath,
74
+ ...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
75
+ ...(config.marginMiB !== undefined && { marginMiB: config.marginMiB }),
76
+ ...(config.nCtxMin !== undefined && { nCtxMin: config.nCtxMin }),
77
+ ...(nCtx !== undefined && nCtx > 0 && { nCtx }),
78
+ ...(nGpuLayers !== undefined && { nGpuLayers }),
79
+ ...(nBatch !== undefined && { nBatch }),
80
+ ...(mainGpu !== undefined && { mainGpu }),
81
+ ...(typeK !== undefined && { typeK }),
82
+ ...(typeV !== undefined && { typeV }),
83
+ ...(flashAttnType !== undefined && { flashAttnType }),
84
+ ...(splitMode !== undefined && { splitMode })
85
+ };
86
+ }
87
+ function markerKey(config) {
88
+ return generateShortHash(JSON.stringify(config));
89
+ }
90
+ function runningMarkerPath(stateDir, config) {
91
+ return path.join(stateDir, `${markerKey(config)}.running`);
92
+ }
93
+ function crashedMarkerPathForConfig(stateDir, config) {
94
+ return path.join(stateDir, `${markerKey(config)}.crashed`);
95
+ }
96
+ export function crashMarkerPath(stateDir, config) {
97
+ return runningMarkerPath(stateDir, toFitConfig(config));
98
+ }
99
+ export function crashedMarkerPath(stateDir, config) {
100
+ return crashedMarkerPathForConfig(stateDir, toFitConfig(config));
101
+ }
102
+ function exists(file) {
103
+ try {
104
+ fs.accessSync(file);
105
+ return true;
106
+ }
107
+ catch {
108
+ return false;
109
+ }
110
+ }
111
+ function writeMarker(file) {
112
+ fs.mkdirSync(path.dirname(file), { recursive: true });
113
+ fs.writeFileSync(file, '');
114
+ }
115
+ function clearMarker(file) {
116
+ try {
117
+ fs.unlinkSync(file);
118
+ }
119
+ catch {
120
+ // Marker cleanup must not turn an advisory fit into a load failure.
121
+ }
122
+ }
123
+ function persistCrash(running, crashed) {
124
+ try {
125
+ writeMarker(crashed);
126
+ clearMarker(running);
127
+ }
128
+ catch {
129
+ // Keep `.running` if `.crashed` could not be written so the next launch
130
+ // still skips native instead of retrying the abort.
131
+ }
132
+ }
133
+ function unknown(reason, message) {
134
+ return { status: 'unknown', reason, message };
135
+ }
136
+ function formatError(error) {
137
+ return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
138
+ }
139
+ async function defaultFit(config) {
140
+ const { fitParamsAsync } = await import('@qvac/model-fit');
141
+ return fitParamsAsync(config);
142
+ }
143
+ async function resolveStateDir(explicit) {
144
+ if (explicit !== undefined)
145
+ return explicit;
146
+ const { getCacheDir } = await import('../../../utils/cache/paths.js');
147
+ return getCacheDir('model-fit');
148
+ }
149
+ /**
150
+ * In-process llama.cpp fit for hosts with no disposable child (Android/iOS).
151
+ *
152
+ * The fit runs on a model-fit worker thread, so the JS loop stays free, but a
153
+ * thread is not a process boundary: a leftover `.running` marker means a
154
+ * previous call aborted the process. That is converted to `.crashed` so later
155
+ * launches skip native for the same path+config instead of retrying the abort.
156
+ */
157
+ export async function runInProcessFit(_loadKind, config, options = {}) {
158
+ if (options.signal?.aborted === true) {
159
+ return unknown('cancelled', 'In-process fit was cancelled');
160
+ }
161
+ let fitConfig;
162
+ try {
163
+ fitConfig = toFitConfig(config);
164
+ }
165
+ catch (error) {
166
+ return unknown('invocation-error', formatError(error));
167
+ }
168
+ const stateDir = await resolveStateDir(options.stateDir);
169
+ const running = runningMarkerPath(stateDir, fitConfig);
170
+ const crashed = crashedMarkerPathForConfig(stateDir, fitConfig);
171
+ if (exists(crashed) || exists(running)) {
172
+ persistCrash(running, crashed);
173
+ return unknown('crashed', 'Previous in-process fit did not finish; treating this model as unknown');
174
+ }
175
+ try {
176
+ writeMarker(running);
177
+ }
178
+ catch (error) {
179
+ return unknown('invocation-error', `Fit crash marker could not be written: ${formatError(error)}`);
180
+ }
181
+ try {
182
+ const fit = options.fit ?? defaultFit;
183
+ const result = await fit(fitConfig);
184
+ return { status: 'completed', result };
185
+ }
186
+ catch (error) {
187
+ return unknown('invocation-error', formatError(error));
188
+ }
189
+ finally {
190
+ clearMarker(running);
191
+ }
192
+ }
@@ -0,0 +1,70 @@
1
+ import { type FitLlamaProcessConfig, type FitLlamaResult, type LlamaLoadKind } from '@qvac/model-fit/process';
2
+ import type { AbortSignal } from 'bare-abort-controller';
3
+ export type IsolatedFitUnknownReason = 'unsupported-platform' | 'spawn-failed' | 'timeout' | 'cancelled' | 'crashed' | 'invalid-response' | 'invocation-error';
4
+ export type IsolatedFitResult = {
5
+ status: 'completed';
6
+ result: FitLlamaResult;
7
+ } | {
8
+ status: 'unknown';
9
+ reason: IsolatedFitUnknownReason;
10
+ message: string;
11
+ stderrTail?: string;
12
+ };
13
+ export interface SpawnContext {
14
+ command: string;
15
+ options: {
16
+ args: string[];
17
+ platform: string;
18
+ arch: string;
19
+ stdio: string[];
20
+ env: Record<string, string>;
21
+ };
22
+ }
23
+ export interface RuntimeContext {
24
+ platform: string;
25
+ arch: string;
26
+ isAndroid: boolean;
27
+ isBrowser: boolean;
28
+ isIOS: boolean;
29
+ }
30
+ export interface RunIsolatedFitOptions {
31
+ timeoutMs?: number;
32
+ signal?: AbortSignal;
33
+ spawnProcess?: (context: SpawnContext) => ChildProcess;
34
+ runtime?: RuntimeContext;
35
+ environment?: Record<string, string | undefined>;
36
+ runnerPath?: string;
37
+ runnerArgs?: string[];
38
+ terminationGraceMs?: number;
39
+ finalKillGraceMs?: number;
40
+ drainGraceMs?: number;
41
+ }
42
+ export interface ErrorEmitter {
43
+ on(event: 'error', listener: (error: Error) => void): unknown;
44
+ off(event: 'error', listener: (error: Error) => void): unknown;
45
+ }
46
+ export interface ReadableChildStream {
47
+ setEncoding(encoding: 'utf8'): void;
48
+ destroy(): void;
49
+ on(event: 'error', listener: (error: Error) => void): unknown;
50
+ on(event: 'data', listener: (chunk: string) => void): unknown;
51
+ off(event: 'error', listener: (error: Error) => void): unknown;
52
+ off(event: 'data', listener: (chunk: string) => void): unknown;
53
+ }
54
+ export interface WritableChildStream {
55
+ end(data?: string): void;
56
+ destroy(): void;
57
+ on(event: 'error', listener: (error: Error) => void): unknown;
58
+ off(event: 'error', listener: (error: Error) => void): unknown;
59
+ }
60
+ export interface ChildProcess {
61
+ stdin: WritableChildStream | null;
62
+ stdout: ReadableChildStream | null;
63
+ stderr: ReadableChildStream | null;
64
+ kill(signal?: string): boolean;
65
+ on(event: 'error', listener: (error: Error) => void): unknown;
66
+ on(event: 'exit' | 'close', listener: (code: number | null, signal: string | number | null) => void): unknown;
67
+ off(event: 'error', listener: (error: Error) => void): unknown;
68
+ off(event: 'exit' | 'close', listener: (code: number | null, signal: string | number | null) => void): unknown;
69
+ }
70
+ export declare function runIsolatedFit(loadKind: LlamaLoadKind, config: FitLlamaProcessConfig, options?: RunIsolatedFitOptions): Promise<IsolatedFitResult>;
@@ -0,0 +1,372 @@
1
+ import { encodeFitLlamaProcessRequest, FIT_PROCESS_MAX_RESPONSE_BYTES, parseFitProcessResponse, resolveFitProcessRunnerPath } from '@qvac/model-fit/process';
2
+ import env from 'bare-env';
3
+ import spawnBare from 'bare-runtime/spawn';
4
+ import { arch, isAndroid, isBrowser, isIOS, platform } from 'which-runtime';
5
+ const DEFAULT_TIMEOUT_MS = 60_000;
6
+ const TERMINATION_GRACE_MS = 1_000;
7
+ const FINAL_KILL_GRACE_MS = 1_000;
8
+ const DRAIN_GRACE_MS = 1_000;
9
+ const STDERR_TAIL_BYTES = 16 * 1024;
10
+ const FIT_ENVIRONMENT_KEYS = [
11
+ 'HOME',
12
+ 'PATH',
13
+ 'TMPDIR',
14
+ 'TMP',
15
+ 'TEMP',
16
+ 'SystemRoot',
17
+ 'WINDIR',
18
+ 'LD_LIBRARY_PATH',
19
+ 'DYLD_LIBRARY_PATH',
20
+ 'VK_ICD_FILENAMES',
21
+ 'VK_DRIVER_FILES',
22
+ 'CUDA_VISIBLE_DEVICES',
23
+ 'HIP_VISIBLE_DEVICES',
24
+ 'ROCR_VISIBLE_DEVICES'
25
+ ];
26
+ function ignoreLateError() { }
27
+ function allowedEnvironment(environment) {
28
+ const allowed = {};
29
+ for (const key of FIT_ENVIRONMENT_KEYS) {
30
+ const value = environment[key];
31
+ if (value !== undefined)
32
+ allowed[key] = value;
33
+ }
34
+ return allowed;
35
+ }
36
+ function appendTail(current, chunk) {
37
+ const combined = Buffer.concat([current, Buffer.from(chunk)]);
38
+ if (combined.length <= STDERR_TAIL_BYTES)
39
+ return combined;
40
+ let start = combined.length - STDERR_TAIL_BYTES;
41
+ // A byte-offset trim can open mid-codepoint; walk forward past UTF-8
42
+ // continuation bytes (0b10xxxxxx) so toString() never leads with U+FFFD.
43
+ while (start < combined.length && (combined[start] & 0xc0) === 0x80)
44
+ start++;
45
+ return combined.subarray(start);
46
+ }
47
+ function unknown(reason, message, stderrTail) {
48
+ return stderrTail.length === 0
49
+ ? { status: 'unknown', reason, message }
50
+ : { status: 'unknown', reason, message, stderrTail: stderrTail.toString() };
51
+ }
52
+ function formatError(error) {
53
+ return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
54
+ }
55
+ function normalizeExitSignal(signal) {
56
+ return signal === null || signal === 0 ? null : String(signal);
57
+ }
58
+ function parseResponse(line, stderrTail) {
59
+ try {
60
+ const response = parseFitProcessResponse(JSON.parse(line));
61
+ if (response.status === 'invocation-error') {
62
+ return unknown('invocation-error', `${response.error.name}: ${response.error.message}`, stderrTail);
63
+ }
64
+ return { status: 'completed', result: response.result };
65
+ }
66
+ catch (error) {
67
+ return unknown('invalid-response', formatError(error), stderrTail);
68
+ }
69
+ }
70
+ export function runIsolatedFit(loadKind, config, options = {}) {
71
+ const runtime = options.runtime ?? {
72
+ platform,
73
+ arch,
74
+ isAndroid,
75
+ isBrowser,
76
+ isIOS
77
+ };
78
+ if (runtime.isAndroid || runtime.isBrowser || runtime.isIOS) {
79
+ return Promise.resolve({
80
+ status: 'unknown',
81
+ reason: 'unsupported-platform',
82
+ message: `Fit subprocess isolation is unavailable on ${runtime.platform}`
83
+ });
84
+ }
85
+ const spawnProcess = options.spawnProcess ??
86
+ ((context) => spawnBare(context.command, context.options));
87
+ const environment = options.environment ?? env;
88
+ const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
89
+ const terminationGraceMs = options.terminationGraceMs ?? TERMINATION_GRACE_MS;
90
+ const finalKillGraceMs = options.finalKillGraceMs ?? FINAL_KILL_GRACE_MS;
91
+ const drainGraceMs = options.drainGraceMs ?? DRAIN_GRACE_MS;
92
+ return new Promise((resolve) => {
93
+ let child;
94
+ const streams = {
95
+ stdin: undefined,
96
+ stdout: undefined,
97
+ stderr: undefined
98
+ };
99
+ const timers = {
100
+ timeout: undefined,
101
+ forceKill: undefined,
102
+ finalKill: undefined,
103
+ drain: undefined
104
+ };
105
+ let settled = false;
106
+ let destroyingStreams = false;
107
+ let stdout = '';
108
+ let stdoutFailure;
109
+ let stderrTail = Buffer.alloc(0);
110
+ let pendingTermination;
111
+ let exitStatus;
112
+ function safely(action) {
113
+ try {
114
+ action();
115
+ }
116
+ catch { }
117
+ }
118
+ // A destroyed pipe or a reaped child can still emit EPIPE/ERR_STREAM_DESTROYED
119
+ // after the promise settles; without a listener that would take down the host,
120
+ // so the active handler is swapped for an inert one instead of removed.
121
+ function silenceErrors(emitter, active) {
122
+ if (emitter === undefined)
123
+ return;
124
+ safely(() => emitter.on('error', ignoreLateError));
125
+ safely(() => emitter.off('error', active));
126
+ }
127
+ function cleanup() {
128
+ if (timers.timeout !== undefined)
129
+ clearTimeout(timers.timeout);
130
+ if (timers.forceKill !== undefined)
131
+ clearTimeout(timers.forceKill);
132
+ if (timers.finalKill !== undefined)
133
+ clearTimeout(timers.finalKill);
134
+ if (timers.drain !== undefined)
135
+ clearTimeout(timers.drain);
136
+ options.signal?.removeEventListener('abort', onAbort);
137
+ safely(() => child?.off('exit', onExit));
138
+ safely(() => child?.off('close', onClose));
139
+ safely(() => streams.stdout?.off('data', onStdoutData));
140
+ safely(() => streams.stderr?.off('data', onStderrData));
141
+ silenceErrors(child, onChildError);
142
+ silenceErrors(streams.stdin, onStdinError);
143
+ silenceErrors(streams.stdout, onStdoutError);
144
+ silenceErrors(streams.stderr, onStderrError);
145
+ }
146
+ function settle(result) {
147
+ if (settled)
148
+ return;
149
+ settled = true;
150
+ cleanup();
151
+ resolve(result);
152
+ }
153
+ function kill(signal) {
154
+ safely(() => {
155
+ child?.kill(signal);
156
+ });
157
+ }
158
+ function requestTermination(reason, message, crashCanOverride = false) {
159
+ if (settled || exitStatus !== undefined || pendingTermination !== undefined)
160
+ return;
161
+ pendingTermination = { reason, message, crashCanOverride };
162
+ timers.forceKill = setTimeout(() => {
163
+ if (settled || exitStatus !== undefined)
164
+ return;
165
+ kill('SIGKILL');
166
+ if (exitStatus !== undefined)
167
+ return;
168
+ timers.finalKill = setTimeout(() => {
169
+ if (settled || exitStatus !== undefined || pendingTermination === undefined)
170
+ return;
171
+ // Bound liveness when the child cannot be observed as reaped after SIGKILL.
172
+ destroyLocalStreams();
173
+ settle(unknown(pendingTermination.reason, `${pendingTermination.message}; child did not report exit`, stderrTail));
174
+ }, finalKillGraceMs);
175
+ }, terminationGraceMs);
176
+ kill('SIGTERM');
177
+ }
178
+ function destroyLocalStreams() {
179
+ destroyingStreams = true;
180
+ safely(() => streams.stdin?.destroy());
181
+ safely(() => streams.stdout?.destroy());
182
+ safely(() => streams.stderr?.destroy());
183
+ }
184
+ function finalize(code, signal) {
185
+ const newlineIndex = stdout.indexOf('\n');
186
+ const hasTrailingTerminator = newlineIndex === stdout.length - 1;
187
+ const invalidFraming = newlineIndex !== -1 && !hasTrailingTerminator;
188
+ const line = hasTrailingTerminator ? stdout.slice(0, -1) : stdout;
189
+ const response = stdoutFailure === undefined && !invalidFraming && line !== ''
190
+ ? parseResponse(line, stderrTail)
191
+ : undefined;
192
+ if (pendingTermination !== undefined && pendingTermination.crashCanOverride === false) {
193
+ settle(unknown(pendingTermination.reason, pendingTermination.message, stderrTail));
194
+ return;
195
+ }
196
+ // A signalled death outranks the runner's own diagnosis: the response was
197
+ // written before whatever killed the child, so it cannot describe the exit.
198
+ if (signal === null &&
199
+ response?.status === 'unknown' &&
200
+ response.reason === 'invocation-error') {
201
+ settle(response);
202
+ return;
203
+ }
204
+ if (code !== 0 || signal !== null) {
205
+ settle(unknown('crashed', `Fit subprocess exited with code ${String(code)} and signal ${String(signal)}`, stderrTail));
206
+ return;
207
+ }
208
+ if (pendingTermination !== undefined) {
209
+ settle(unknown(pendingTermination.reason, pendingTermination.message, stderrTail));
210
+ return;
211
+ }
212
+ if (stdoutFailure !== undefined) {
213
+ settle(unknown('invalid-response', stdoutFailure, stderrTail));
214
+ return;
215
+ }
216
+ if (invalidFraming || line === '') {
217
+ settle(unknown('invalid-response', invalidFraming
218
+ ? 'Fit subprocess returned invalid line framing'
219
+ : `Fit subprocess exited with code ${String(code)} and signal ${String(signal)}`, stderrTail));
220
+ return;
221
+ }
222
+ settle(response ?? parseResponse(line, stderrTail));
223
+ }
224
+ function onAbort() {
225
+ requestTermination('cancelled', 'Fit subprocess was cancelled');
226
+ }
227
+ function onChildError(error) {
228
+ requestTermination('spawn-failed', formatError(error));
229
+ }
230
+ function onExit(code, rawSignal) {
231
+ if (exitStatus !== undefined)
232
+ return;
233
+ const signal = normalizeExitSignal(rawSignal);
234
+ exitStatus = { code, signal };
235
+ if (timers.timeout !== undefined) {
236
+ clearTimeout(timers.timeout);
237
+ timers.timeout = undefined;
238
+ }
239
+ if (timers.forceKill !== undefined) {
240
+ clearTimeout(timers.forceKill);
241
+ timers.forceKill = undefined;
242
+ }
243
+ if (timers.finalKill !== undefined) {
244
+ clearTimeout(timers.finalKill);
245
+ timers.finalKill = undefined;
246
+ }
247
+ timers.drain = setTimeout(() => {
248
+ destroyLocalStreams();
249
+ finalize(code, signal);
250
+ }, drainGraceMs);
251
+ }
252
+ function onClose(code, rawSignal) {
253
+ const status = exitStatus ?? { code, signal: normalizeExitSignal(rawSignal) };
254
+ finalize(status.code, status.signal);
255
+ }
256
+ function onStdoutData(chunk) {
257
+ if (settled || pendingTermination !== undefined || destroyingStreams) {
258
+ return;
259
+ }
260
+ if (Buffer.byteLength(stdout) + Buffer.byteLength(chunk) > FIT_PROCESS_MAX_RESPONSE_BYTES) {
261
+ if (exitStatus !== undefined) {
262
+ stdoutFailure = 'Fit subprocess response exceeds 1 MiB';
263
+ return;
264
+ }
265
+ requestTermination('invalid-response', 'Fit subprocess response exceeds 1 MiB');
266
+ return;
267
+ }
268
+ stdout += chunk;
269
+ const newlineIndex = stdout.indexOf('\n');
270
+ if (newlineIndex !== -1 && newlineIndex !== stdout.length - 1) {
271
+ if (exitStatus !== undefined) {
272
+ stdoutFailure = 'Fit subprocess returned invalid line framing';
273
+ }
274
+ else {
275
+ requestTermination('invalid-response', 'Fit subprocess returned invalid line framing');
276
+ }
277
+ }
278
+ }
279
+ function onStderrData(chunk) {
280
+ if (settled || destroyingStreams)
281
+ return;
282
+ stderrTail = appendTail(stderrTail, chunk);
283
+ }
284
+ function onStdinError(error) {
285
+ if (destroyingStreams)
286
+ return;
287
+ requestTermination('invalid-response', `Fit subprocess stdin failed: ${error.message}`, true);
288
+ }
289
+ function onStdoutError(error) {
290
+ if (destroyingStreams)
291
+ return;
292
+ requestTermination('invalid-response', `Fit subprocess stdout failed: ${error.message}`);
293
+ }
294
+ function onStderrError(error) {
295
+ if (destroyingStreams)
296
+ return;
297
+ requestTermination('invalid-response', `Fit subprocess stderr failed: ${error.message}`);
298
+ }
299
+ try {
300
+ const stdio = runtime.platform === 'win32'
301
+ ? ['overlapped', 'overlapped', 'overlapped']
302
+ : ['pipe', 'pipe', 'pipe'];
303
+ child = spawnProcess({
304
+ command: 'bare',
305
+ options: {
306
+ args: [
307
+ options.runnerPath ?? resolveFitProcessRunnerPath(),
308
+ ...(options.runnerArgs ?? [])
309
+ ],
310
+ platform: runtime.platform,
311
+ arch: runtime.arch,
312
+ stdio,
313
+ env: allowedEnvironment(environment)
314
+ }
315
+ });
316
+ }
317
+ catch (error) {
318
+ settle(unknown('spawn-failed', formatError(error), stderrTail));
319
+ return;
320
+ }
321
+ child.on('error', onChildError);
322
+ child.on('exit', onExit);
323
+ child.on('close', onClose);
324
+ const stdin = child.stdin;
325
+ const stdoutStream = child.stdout;
326
+ const stderrStream = child.stderr;
327
+ streams.stdin = stdin ?? undefined;
328
+ streams.stdout = stdoutStream ?? undefined;
329
+ streams.stderr = stderrStream ?? undefined;
330
+ options.signal?.addEventListener('abort', onAbort, { once: true });
331
+ timers.timeout = setTimeout(() => {
332
+ requestTermination('timeout', `Fit subprocess exceeded ${timeoutMs}ms`);
333
+ }, timeoutMs);
334
+ try {
335
+ streams.stdin?.on('error', onStdinError);
336
+ streams.stdout?.on('data', onStdoutData);
337
+ streams.stdout?.on('error', onStdoutError);
338
+ streams.stderr?.on('data', onStderrData);
339
+ streams.stderr?.on('error', onStderrError);
340
+ }
341
+ catch (error) {
342
+ requestTermination('invalid-response', `Fit subprocess stream setup failed: ${formatError(error)}`);
343
+ destroyLocalStreams();
344
+ return;
345
+ }
346
+ if (stdin === null || stdoutStream === null || stderrStream === null) {
347
+ requestTermination('spawn-failed', 'Fit subprocess stdio pipes are unavailable');
348
+ destroyLocalStreams();
349
+ return;
350
+ }
351
+ try {
352
+ stdoutStream.setEncoding('utf8');
353
+ stderrStream.setEncoding('utf8');
354
+ }
355
+ catch (error) {
356
+ requestTermination('invalid-response', `Fit subprocess stream setup failed: ${formatError(error)}`);
357
+ destroyLocalStreams();
358
+ return;
359
+ }
360
+ if (options.signal?.aborted === true) {
361
+ onAbort();
362
+ return;
363
+ }
364
+ try {
365
+ stdin.end(encodeFitLlamaProcessRequest(loadKind, config));
366
+ }
367
+ catch (error) {
368
+ requestTermination('invalid-response', `Fit subprocess request write failed: ${formatError(error)}`);
369
+ destroyLocalStreams();
370
+ }
371
+ });
372
+ }
@@ -14,6 +14,8 @@ const cpuArchitectures = cpuModule.status === 'fulfilled'
14
14
  const gpuTypes = gpuModule.status === 'fulfilled'
15
15
  ? supportedEnumValues(gpuModule.value.default.constants.gpuType, gpuModule.value.default.constants.gpuType.UNKNOWN)
16
16
  : [];
17
+ // `availableMemory()` reports the constrained memory on iOS.
18
+ const reportsProcessAllowance = os.platform() === 'ios';
17
19
  export const nativeResourceCollectorDependencies = {
18
20
  cpuArchitectures,
19
21
  gpuTypes,
@@ -38,12 +40,11 @@ export const nativeResourceCollectorDependencies = {
38
40
  // A throw here is a sampling failure, not a platform gap: let it reach the
39
41
  // collector so the metric reports `failed` rather than `unavailable`.
40
42
  const usage = os.memoryUsage();
41
- // No platform exposes the per-process allowance yet. On iOS this is
42
- // `os_proc_available_memory()` — the limit jetsam actually enforces — and
43
- // it needs a native source before the metric can report a value.
43
+ const available = reportsProcessAllowance ? os.availableMemory() : 0;
44
44
  return {
45
45
  usedBytes: usage && usage.rss > 0 ? usage.rss : undefined,
46
- availableBytes: undefined
46
+ // Zero states no headroom, which is not a budget.
47
+ availableBytes: available > 0 ? available : undefined
47
48
  };
48
49
  }
49
50
  };
@@ -55,9 +55,7 @@ export interface GPUInfoContext {
55
55
  *
56
56
  * @property usedBytes - The process's resident footprint (RSS).
57
57
  * @property availableBytes - How much more this process may allocate before
58
- * the OS intervenes. Only some platforms can state this — on iOS it is
59
- * `os_proc_available_memory()`, the number jetsam actually enforces.
60
- * `undefined` where no such per-process limit is exposed.
58
+ * the OS intervenes. `undefined` where the platform states no such limit.
61
59
  */
62
60
  export interface ProcessMemorySample {
63
61
  usedBytes: number | undefined;
@@ -7,10 +7,12 @@ export declare function initEnv(): void;
7
7
  */
8
8
  export declare function getEnv(): {
9
9
  HOME_DIR: string;
10
+ QVAC_ADVISORY_MODEL_FIT?: string | undefined;
10
11
  };
11
12
  /**
12
13
  * Get the validated env config. Must call initEnv() first.
13
14
  */
14
15
  export declare function getValidatedEnv(): {
15
16
  HOME_DIR: string;
17
+ QVAC_ADVISORY_MODEL_FIT?: string | undefined;
16
18
  };