@reftrixmcp/ml 0.1.7 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/worker-resource-limits.d.ts +89 -0
- package/dist/config/worker-resource-limits.d.ts.map +1 -0
- package/dist/config/worker-resource-limits.js +156 -0
- package/dist/config/worker-resource-limits.js.map +1 -0
- package/dist/dinov2/service.d.ts +13 -0
- package/dist/dinov2/service.d.ts.map +1 -1
- package/dist/dinov2/service.js +86 -8
- package/dist/dinov2/service.js.map +1 -1
- package/dist/dinov2/worker-thread.js +72 -7
- package/dist/dinov2/worker-thread.js.map +1 -1
- package/dist/embeddings/service.d.ts +19 -0
- package/dist/embeddings/service.d.ts.map +1 -1
- package/dist/embeddings/service.js +87 -1
- package/dist/embeddings/service.js.map +1 -1
- package/dist/embeddings/worker-thread-types.d.ts +1 -1
- package/dist/embeddings/worker-thread.d.ts.map +1 -1
- package/dist/embeddings/worker-thread.js +10 -85
- package/dist/embeddings/worker-thread.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/onnx-availability.d.ts +29 -0
- package/dist/onnx-availability.d.ts.map +1 -0
- package/dist/onnx-availability.js +69 -0
- package/dist/onnx-availability.js.map +1 -0
- package/dist/onnx-provider-detect.d.ts +47 -0
- package/dist/onnx-provider-detect.d.ts.map +1 -0
- package/dist/onnx-provider-detect.js +111 -0
- package/dist/onnx-provider-detect.js.map +1 -0
- package/package.json +8 -3
- package/src/config/worker-resource-limits.ts +176 -0
- package/src/dinov2/service.ts +105 -8
- package/src/dinov2/worker-thread.ts +85 -8
- package/src/embeddings/service.ts +99 -1
- package/src/embeddings/worker-thread-types.ts +1 -1
- package/src/embeddings/worker-thread.ts +15 -104
- package/src/index.ts +15 -0
- package/src/onnx-availability.ts +78 -0
- package/src/onnx-provider-detect.ts +134 -0
|
@@ -28,16 +28,20 @@ import type {
|
|
|
28
28
|
} from "./worker-thread-types.js";
|
|
29
29
|
|
|
30
30
|
// =====================================================
|
|
31
|
-
// Dynamic import of onnxruntime-node
|
|
31
|
+
// Dynamic import of onnxruntime-node (optional dependency)
|
|
32
32
|
// =====================================================
|
|
33
33
|
|
|
34
|
+
import { safeImportOnnx } from "../onnx-availability.js";
|
|
35
|
+
import { detectExecutionProvider, isLdLibraryPathSetAtOsLevel } from "../onnx-provider-detect.js";
|
|
36
|
+
import type { ExecutionProvider } from "../onnx-provider-detect.js";
|
|
37
|
+
|
|
34
38
|
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
|
|
35
39
|
type OrtModule = typeof import("onnxruntime-node");
|
|
36
40
|
let ort: OrtModule | null = null;
|
|
37
41
|
|
|
38
42
|
async function getOrt(): Promise<OrtModule> {
|
|
39
43
|
if (!ort) {
|
|
40
|
-
ort = await
|
|
44
|
+
ort = await safeImportOnnx();
|
|
41
45
|
}
|
|
42
46
|
return ort;
|
|
43
47
|
}
|
|
@@ -137,16 +141,71 @@ function l2Normalize(vec: Float32Array): Float32Array {
|
|
|
137
141
|
// Session management
|
|
138
142
|
// =====================================================
|
|
139
143
|
|
|
144
|
+
/** Resolved execution provider for the current session. */
|
|
145
|
+
let resolvedProvider: ExecutionProvider = "cpu";
|
|
146
|
+
|
|
140
147
|
async function initializeSession(modelPath: string): Promise<void> {
|
|
141
148
|
if (session) return;
|
|
142
149
|
|
|
150
|
+
const provider = detectExecutionProvider("DINOv2Worker");
|
|
151
|
+
|
|
152
|
+
// Safety check: if CUDA is requested but LD_LIBRARY_PATH was not set at
|
|
153
|
+
// the OS level (only set via loadEnvLocal at runtime), CUDA init will crash
|
|
154
|
+
// because dlopen() can't find CUDA shared libraries. Fall back to CPU.
|
|
155
|
+
let effectiveProvider = provider;
|
|
156
|
+
if (effectiveProvider === "cuda" && !isLdLibraryPathSetAtOsLevel()) {
|
|
157
|
+
console.warn(
|
|
158
|
+
"[DINOv2Worker] CUDA requested but LD_LIBRARY_PATH not set at OS level. " +
|
|
159
|
+
"dlopen() cannot find CUDA libraries. Falling back to CPU. " +
|
|
160
|
+
"To use CUDA, set LD_LIBRARY_PATH before starting the Node.js process."
|
|
161
|
+
);
|
|
162
|
+
effectiveProvider = "cpu";
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const sessionOptions =
|
|
166
|
+
effectiveProvider === "cuda"
|
|
167
|
+
? {
|
|
168
|
+
executionProviders: ["CUDAExecutionProvider", "CPUExecutionProvider"],
|
|
169
|
+
enableMemPattern: false,
|
|
170
|
+
}
|
|
171
|
+
: {
|
|
172
|
+
executionProviders: ["cpu"],
|
|
173
|
+
enableCpuMemArena: false,
|
|
174
|
+
enableMemPattern: false,
|
|
175
|
+
};
|
|
176
|
+
|
|
143
177
|
const ortModule = await getOrt();
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
178
|
+
|
|
179
|
+
// Attempt to create session with the resolved provider.
|
|
180
|
+
// If CUDA init fails (e.g. library not found), catch and retry with CPU.
|
|
181
|
+
try {
|
|
182
|
+
session = (await ortModule.InferenceSession.create(
|
|
183
|
+
modelPath,
|
|
184
|
+
sessionOptions
|
|
185
|
+
)) as unknown as OrtSession;
|
|
186
|
+
resolvedProvider = effectiveProvider;
|
|
187
|
+
} catch (providerError) {
|
|
188
|
+
if (effectiveProvider !== "cpu") {
|
|
189
|
+
// CUDA init failed — fallback to CPU
|
|
190
|
+
const errorMsg =
|
|
191
|
+
providerError instanceof Error ? providerError.message : String(providerError);
|
|
192
|
+
console.warn(
|
|
193
|
+
"[DINOv2Worker] CUDA session creation failed, falling back to CPU: %s",
|
|
194
|
+
errorMsg
|
|
195
|
+
);
|
|
196
|
+
session = (await ortModule.InferenceSession.create(modelPath, {
|
|
197
|
+
executionProviders: ["cpu"],
|
|
198
|
+
enableCpuMemArena: false,
|
|
199
|
+
enableMemPattern: false,
|
|
200
|
+
})) as unknown as OrtSession;
|
|
201
|
+
resolvedProvider = "cpu";
|
|
202
|
+
} else {
|
|
203
|
+
throw providerError;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
147
206
|
|
|
148
207
|
// eslint-disable-next-line no-console
|
|
149
|
-
console.log("[DINOv2Worker] Session initialized:", modelPath);
|
|
208
|
+
console.log("[DINOv2Worker] Session initialized (provider: %s):", resolvedProvider, modelPath);
|
|
150
209
|
}
|
|
151
210
|
|
|
152
211
|
async function disposeSession(): Promise<void> {
|
|
@@ -185,8 +244,18 @@ async function runInference(
|
|
|
185
244
|
// Create input tensor [1, 3, height, width]
|
|
186
245
|
const inputTensor = new ortModule.Tensor("float32", inputData, [1, 3, height, width]);
|
|
187
246
|
|
|
188
|
-
// Run inference
|
|
189
|
-
|
|
247
|
+
// Run inference — use try/finally to ensure tensor disposal (OOM-FIX-1)
|
|
248
|
+
// ONNX Tensor holds native C++ memory that V8 GC cannot reclaim.
|
|
249
|
+
// Without explicit dispose(), ~1.4MB/inference leaks as native memory.
|
|
250
|
+
let output: Record<string, { data: Float32Array; dims: readonly number[] }>;
|
|
251
|
+
try {
|
|
252
|
+
output = await session.run({ pixel_values: inputTensor });
|
|
253
|
+
} finally {
|
|
254
|
+
// Dispose input tensor immediately after inference completes
|
|
255
|
+
if (typeof (inputTensor as unknown as { dispose?: () => void }).dispose === "function") {
|
|
256
|
+
(inputTensor as unknown as { dispose: () => void }).dispose();
|
|
257
|
+
}
|
|
258
|
+
}
|
|
190
259
|
|
|
191
260
|
// Extract CLS token from last_hidden_state [1, 257, 768]
|
|
192
261
|
const lastHiddenState = output.last_hidden_state;
|
|
@@ -210,6 +279,14 @@ async function runInference(
|
|
|
210
279
|
clsToken[i] = data[i]!;
|
|
211
280
|
}
|
|
212
281
|
|
|
282
|
+
// Dispose output tensors after data extraction (OOM-FIX-1)
|
|
283
|
+
for (const key of Object.keys(output)) {
|
|
284
|
+
const tensor = output[key] as unknown as { dispose?: () => void };
|
|
285
|
+
if (typeof tensor?.dispose === "function") {
|
|
286
|
+
tensor.dispose();
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
213
290
|
// L2 normalize with NaN/Infinity defense
|
|
214
291
|
const normalized = l2Normalize(clsToken);
|
|
215
292
|
|
|
@@ -21,6 +21,8 @@ import { fileURLToPath } from "node:url";
|
|
|
21
21
|
import path from "node:path";
|
|
22
22
|
import type { EmbeddingTextType, EmbeddingServiceConfig, CacheStats } from "./types.js";
|
|
23
23
|
import type { WorkerMessage, WorkerResponse } from "./worker-thread-types.js";
|
|
24
|
+
import { OnnxRuntimeUnavailableError } from "../onnx-availability.js";
|
|
25
|
+
import { getMLWorkerThreadOptions } from "../config/worker-resource-limits.js";
|
|
24
26
|
|
|
25
27
|
/**
|
|
26
28
|
* Disposable pipeline interface (used only in in-process fallback mode).
|
|
@@ -306,7 +308,13 @@ export class EmbeddingService {
|
|
|
306
308
|
// worker_threads Workers, so we must explicitly pass an empty array
|
|
307
309
|
// to prevent the parent's flags (set by WorkerSupervisor fork())
|
|
308
310
|
// from propagating.
|
|
309
|
-
|
|
311
|
+
//
|
|
312
|
+
// PR7e-β1: Add explicit resourceLimits to prevent OOM when ONNX
|
|
313
|
+
// falls back to CPU (onnxruntime-node). Without resourceLimits,
|
|
314
|
+
// V8 old-space defaults to < 4GB and e5-base CPU inference OOMs.
|
|
315
|
+
// See ADR-0012 §3 BLOCKER 4 and config/worker-resource-limits.ts.
|
|
316
|
+
const workerOptions = { execArgv: [], ...getMLWorkerThreadOptions() };
|
|
317
|
+
this.worker = new Worker(scriptPath, workerOptions);
|
|
310
318
|
|
|
311
319
|
// Handle worker messages
|
|
312
320
|
this.worker.on("message", (response: WorkerResponse) => {
|
|
@@ -502,6 +510,10 @@ export class EmbeddingService {
|
|
|
502
510
|
}
|
|
503
511
|
} catch (error) {
|
|
504
512
|
this.initPromise = null;
|
|
513
|
+
const code = (error as { code?: string }).code;
|
|
514
|
+
if (code === "MODULE_NOT_FOUND" || code === "ERR_MODULE_NOT_FOUND") {
|
|
515
|
+
throw new OnnxRuntimeUnavailableError("text embedding");
|
|
516
|
+
}
|
|
505
517
|
const message = error instanceof Error ? error.message : "Unknown error";
|
|
506
518
|
throw new Error(`Failed to load embedding model: ${message}`);
|
|
507
519
|
}
|
|
@@ -1275,6 +1287,92 @@ export class EmbeddingService {
|
|
|
1275
1287
|
this.config = { ...this.config, device: "cpu" };
|
|
1276
1288
|
}
|
|
1277
1289
|
|
|
1290
|
+
/**
|
|
1291
|
+
* Terminate the worker thread and prepare for re-spawn on next use.
|
|
1292
|
+
*
|
|
1293
|
+
* Unlike dispose() which only releases the ONNX pipeline within the
|
|
1294
|
+
* existing worker thread, this method terminates the worker thread
|
|
1295
|
+
* process entirely. This forces the OS to reclaim ALL memory held by
|
|
1296
|
+
* the ONNX Runtime C++ arena allocator, which glibc malloc fragmentation
|
|
1297
|
+
* prevents dispose() from returning.
|
|
1298
|
+
*
|
|
1299
|
+
* After termination, the next generateEmbedding() call will automatically
|
|
1300
|
+
* re-spawn a new worker thread via ensureWorkerReady().
|
|
1301
|
+
*
|
|
1302
|
+
* Key behaviors:
|
|
1303
|
+
* - lastCrashTime is reset to 0 (planned terminate, not a crash)
|
|
1304
|
+
* - workerRestartCount is preserved (not incremented)
|
|
1305
|
+
* - currentProvider is preserved for re-spawn
|
|
1306
|
+
* - Mutual exclusion via workerInitPromise
|
|
1307
|
+
*/
|
|
1308
|
+
async terminateAndRespawn(): Promise<void> {
|
|
1309
|
+
// T-4: Set workerInitPromise to prevent concurrent spawns during terminate
|
|
1310
|
+
const terminatePromise = (async (): Promise<void> => {
|
|
1311
|
+
// T-3: Save workerRestartCount before terminate (crash handler increments it)
|
|
1312
|
+
const savedRestartCount = this.workerRestartCount;
|
|
1313
|
+
|
|
1314
|
+
// Clear idle timer
|
|
1315
|
+
if (this.idleTimer !== null) {
|
|
1316
|
+
clearTimeout(this.idleTimer);
|
|
1317
|
+
this.idleTimer = null;
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
if (this.worker) {
|
|
1321
|
+
try {
|
|
1322
|
+
// Send terminate message so worker can dispose pipeline
|
|
1323
|
+
await this.sendWorkerMessage({
|
|
1324
|
+
type: "terminate",
|
|
1325
|
+
requestId: generateRequestId(),
|
|
1326
|
+
});
|
|
1327
|
+
} catch {
|
|
1328
|
+
// May fail if worker already exited
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
// Force terminate the worker thread
|
|
1332
|
+
try {
|
|
1333
|
+
await this.worker.terminate();
|
|
1334
|
+
} catch {
|
|
1335
|
+
// Already terminated
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
this.worker = null;
|
|
1339
|
+
this.workerReady = false;
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
// Also dispose in-process pipeline if present
|
|
1343
|
+
await this.disposeInProcess();
|
|
1344
|
+
|
|
1345
|
+
// Reject any remaining pending requests
|
|
1346
|
+
for (const [id, pending] of this.pendingRequests) {
|
|
1347
|
+
clearTimeout(pending.timer);
|
|
1348
|
+
pending.reject(new Error("Service terminated for respawn"));
|
|
1349
|
+
this.pendingRequests.delete(id);
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
// T-3: Restore workerRestartCount (planned terminate, not a crash)
|
|
1353
|
+
this.workerRestartCount = savedRestartCount;
|
|
1354
|
+
|
|
1355
|
+
// T-3: Reset lastCrashTime (planned terminate, no cooldown needed)
|
|
1356
|
+
this.lastCrashTime = 0;
|
|
1357
|
+
|
|
1358
|
+
this.inferencesSinceRecycle = 0;
|
|
1359
|
+
|
|
1360
|
+
// T-5: currentProvider is NOT reset — re-spawn will use same provider
|
|
1361
|
+
|
|
1362
|
+
// eslint-disable-next-line no-console
|
|
1363
|
+
console.log("[ML] Worker Thread terminated for respawn (planned)");
|
|
1364
|
+
})();
|
|
1365
|
+
|
|
1366
|
+
// T-4: Hold workerInitPromise to block concurrent generateEmbedding() calls
|
|
1367
|
+
this.workerInitPromise = terminatePromise.then(() => {
|
|
1368
|
+
// Clear workerInitPromise after terminate completes so next
|
|
1369
|
+
// ensureWorkerReady() can spawn a fresh worker
|
|
1370
|
+
this.workerInitPromise = null;
|
|
1371
|
+
});
|
|
1372
|
+
|
|
1373
|
+
await terminatePromise;
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1278
1376
|
/**
|
|
1279
1377
|
* Get the number of worker thread restarts that have occurred.
|
|
1280
1378
|
*/
|
|
@@ -144,7 +144,7 @@ export interface WorkerDisposeResponse {
|
|
|
144
144
|
}
|
|
145
145
|
|
|
146
146
|
/**
|
|
147
|
-
* Successful response from terminate (sent
|
|
147
|
+
* Successful response from terminate (sent before parent calls worker.terminate()).
|
|
148
148
|
*/
|
|
149
149
|
export interface WorkerTerminateResponse {
|
|
150
150
|
type: "terminate";
|
|
@@ -20,10 +20,14 @@
|
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
22
|
import { parentPort } from "node:worker_threads";
|
|
23
|
-
import { createRequire } from "node:module";
|
|
24
|
-
import fs from "node:fs";
|
|
25
|
-
import nodePath from "node:path";
|
|
26
23
|
import type { WorkerMessage, WorkerResponse, WorkerErrorResponse } from "./worker-thread-types.js";
|
|
24
|
+
import { OnnxRuntimeUnavailableError } from "../onnx-availability.js";
|
|
25
|
+
import {
|
|
26
|
+
detectExecutionProvider,
|
|
27
|
+
verifyCudaAvailability,
|
|
28
|
+
isLdLibraryPathSetAtOsLevel,
|
|
29
|
+
} from "../onnx-provider-detect.js";
|
|
30
|
+
import type { ExecutionProvider } from "../onnx-provider-detect.js";
|
|
27
31
|
|
|
28
32
|
// =====================================================
|
|
29
33
|
// Pipeline types (mirrors service.ts DisposablePipeline)
|
|
@@ -38,83 +42,6 @@ interface DisposablePipeline {
|
|
|
38
42
|
dispose?: () => Promise<unknown[]>;
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
// =====================================================
|
|
42
|
-
// GPU execution provider detection
|
|
43
|
-
// =====================================================
|
|
44
|
-
|
|
45
|
-
type ExecutionProvider = "cpu" | "cuda";
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Detect the ONNX execution provider based on environment configuration.
|
|
49
|
-
*
|
|
50
|
-
* Checks ONNX_EXECUTION_PROVIDER env var and verifies that the CUDA
|
|
51
|
-
* provider shared library is available on disk. Falls back to CPU
|
|
52
|
-
* gracefully if the provider is not installed.
|
|
53
|
-
*/
|
|
54
|
-
function detectExecutionProvider(): ExecutionProvider {
|
|
55
|
-
const envProvider = process.env.ONNX_EXECUTION_PROVIDER;
|
|
56
|
-
|
|
57
|
-
if (envProvider === "cuda" || envProvider === "rocm") {
|
|
58
|
-
const cudaAvailable = verifyCudaAvailability();
|
|
59
|
-
if (cudaAvailable) {
|
|
60
|
-
return "cuda";
|
|
61
|
-
}
|
|
62
|
-
console.warn(
|
|
63
|
-
"[EmbeddingWorker] ONNX_EXECUTION_PROVIDER=%s but CUDA provider not available, falling back to CPU",
|
|
64
|
-
envProvider
|
|
65
|
-
);
|
|
66
|
-
return "cpu";
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
return "cpu";
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
/**
|
|
73
|
-
* Verify that CUDA provider (libonnxruntime_providers_cuda.so) is available.
|
|
74
|
-
*
|
|
75
|
-
* onnxruntime-node can download the CUDA provider shared library via
|
|
76
|
-
* `ONNXRUNTIME_NODE_INSTALL_CUDA=v12 node .../install.js`. This function
|
|
77
|
-
* checks for its presence on disk rather than trying to resolve a
|
|
78
|
-
* non-existent npm package (onnxruntime-gpu is Python-only).
|
|
79
|
-
*/
|
|
80
|
-
function verifyCudaAvailability(): boolean {
|
|
81
|
-
try {
|
|
82
|
-
const esmRequire = createRequire(import.meta.url);
|
|
83
|
-
const ortNodePath = esmRequire.resolve("onnxruntime-node");
|
|
84
|
-
// require.resolve returns e.g. .../onnxruntime-node/dist/index.js
|
|
85
|
-
// Walk up to package root by finding the directory containing package.json
|
|
86
|
-
let packageDir = nodePath.dirname(ortNodePath);
|
|
87
|
-
for (let i = 0; i < 5; i++) {
|
|
88
|
-
if (fs.existsSync(nodePath.join(packageDir, "package.json"))) break;
|
|
89
|
-
packageDir = nodePath.dirname(packageDir);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
// Search across napi versions (v3, v6, etc.) for CUDA provider
|
|
93
|
-
const binDir = nodePath.join(packageDir, "bin");
|
|
94
|
-
if (fs.existsSync(binDir)) {
|
|
95
|
-
const napiDirs = fs.readdirSync(binDir).filter((d) => d.startsWith("napi-v"));
|
|
96
|
-
for (const napiDir of napiDirs) {
|
|
97
|
-
const cudaProviderPath = nodePath.join(
|
|
98
|
-
binDir,
|
|
99
|
-
napiDir,
|
|
100
|
-
"linux",
|
|
101
|
-
"x64",
|
|
102
|
-
"libonnxruntime_providers_cuda.so"
|
|
103
|
-
);
|
|
104
|
-
if (fs.existsSync(cudaProviderPath)) {
|
|
105
|
-
return true;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
console.warn("[EmbeddingWorker] CUDA provider not found in: %s", binDir);
|
|
111
|
-
return false;
|
|
112
|
-
} catch {
|
|
113
|
-
console.warn("[EmbeddingWorker] Cannot verify CUDA provider: onnxruntime-node not found");
|
|
114
|
-
return false;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
45
|
// =====================================================
|
|
119
46
|
// Worker state
|
|
120
47
|
// =====================================================
|
|
@@ -146,26 +73,6 @@ function normalizeVector(vector: number[]): number[] {
|
|
|
146
73
|
return vector.map((val) => val / norm);
|
|
147
74
|
}
|
|
148
75
|
|
|
149
|
-
/**
|
|
150
|
-
* Verify that LD_LIBRARY_PATH is set at the OS level (not just process.env).
|
|
151
|
-
*
|
|
152
|
-
* dlopen() reads LD_LIBRARY_PATH from the kernel environment (/proc/self/environ),
|
|
153
|
-
* NOT from process.env modifications made at runtime. If LD_LIBRARY_PATH was set
|
|
154
|
-
* after process startup (e.g. by loadEnvLocal()), dlopen() cannot find CUDA
|
|
155
|
-
* libraries and ONNX Runtime CUDA provider initialization will segfault or throw.
|
|
156
|
-
*
|
|
157
|
-
* Returns true only if LD_LIBRARY_PATH was present in the original process environment.
|
|
158
|
-
*/
|
|
159
|
-
function isLdLibraryPathSetAtOsLevel(): boolean {
|
|
160
|
-
try {
|
|
161
|
-
const procEnv = fs.readFileSync("/proc/self/environ", "utf-8");
|
|
162
|
-
return procEnv.includes("LD_LIBRARY_PATH");
|
|
163
|
-
} catch {
|
|
164
|
-
// /proc/self/environ not available (non-Linux) — assume set
|
|
165
|
-
return true;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
|
|
169
76
|
async function initializePipeline(): Promise<void> {
|
|
170
77
|
if (pipeline) return;
|
|
171
78
|
if (initPromise) return initPromise;
|
|
@@ -175,7 +82,7 @@ async function initializePipeline(): Promise<void> {
|
|
|
175
82
|
// Only auto-detect from env on first init. If provider was explicitly
|
|
176
83
|
// set by a switch-provider message, respect that setting.
|
|
177
84
|
if (!providerExplicitlySet) {
|
|
178
|
-
resolvedProvider = detectExecutionProvider();
|
|
85
|
+
resolvedProvider = detectExecutionProvider("EmbeddingWorker");
|
|
179
86
|
}
|
|
180
87
|
|
|
181
88
|
// Safety check: if CUDA is requested but LD_LIBRARY_PATH was not set at
|
|
@@ -236,6 +143,10 @@ async function initializePipeline(): Promise<void> {
|
|
|
236
143
|
console.log("[EmbeddingWorker] ONNX pipeline ready (provider: %s)", resolvedProvider);
|
|
237
144
|
} catch (error) {
|
|
238
145
|
initPromise = null;
|
|
146
|
+
const code = (error as { code?: string }).code;
|
|
147
|
+
if (code === "MODULE_NOT_FOUND" || code === "ERR_MODULE_NOT_FOUND") {
|
|
148
|
+
throw new OnnxRuntimeUnavailableError("text embedding");
|
|
149
|
+
}
|
|
239
150
|
const message = error instanceof Error ? error.message : "Unknown error";
|
|
240
151
|
throw new Error(`Failed to load embedding model in worker thread: ${message}`);
|
|
241
152
|
}
|
|
@@ -436,8 +347,8 @@ async function handleMessage(message: WorkerMessage): Promise<void> {
|
|
|
436
347
|
requestId: message.requestId,
|
|
437
348
|
success: true,
|
|
438
349
|
});
|
|
439
|
-
//
|
|
440
|
-
|
|
350
|
+
// Parent calls worker.terminate() to stop this thread.
|
|
351
|
+
// No process.exit() — let parent control the lifecycle.
|
|
441
352
|
break;
|
|
442
353
|
}
|
|
443
354
|
|
|
@@ -446,7 +357,7 @@ async function handleMessage(message: WorkerMessage): Promise<void> {
|
|
|
446
357
|
|
|
447
358
|
// If switching to CUDA, verify availability first
|
|
448
359
|
if (targetProvider === "cuda") {
|
|
449
|
-
const canUseCuda = verifyCudaAvailability();
|
|
360
|
+
const canUseCuda = verifyCudaAvailability("EmbeddingWorker");
|
|
450
361
|
if (!canUseCuda) {
|
|
451
362
|
// Cannot switch to CUDA — respond with current (cpu) provider
|
|
452
363
|
sendResponse({
|
package/src/index.ts
CHANGED
|
@@ -49,6 +49,21 @@ export { DINOv2Service, DINOV2_EMBEDDING_DIMENSION, DINOV2_INPUT_SIZE } from "./
|
|
|
49
49
|
export type { DINOv2ServiceConfig } from "./dinov2/index.js";
|
|
50
50
|
export type { DINOv2WorkerMessage, DINOv2WorkerResponse } from "./dinov2/index.js";
|
|
51
51
|
|
|
52
|
+
// ONNX Runtime availability checker
|
|
53
|
+
export {
|
|
54
|
+
isOnnxRuntimeAvailable,
|
|
55
|
+
OnnxRuntimeUnavailableError,
|
|
56
|
+
safeImportOnnx,
|
|
57
|
+
} from "./onnx-availability.js";
|
|
58
|
+
|
|
59
|
+
// ML Worker Thread resource limits (PR7e-β1)
|
|
60
|
+
export {
|
|
61
|
+
loadMLWorkerResourceLimits,
|
|
62
|
+
getMLWorkerThreadOptions,
|
|
63
|
+
MLWorkerResourceLimitsSchema,
|
|
64
|
+
} from "./config/worker-resource-limits.js";
|
|
65
|
+
export type { MLWorkerResourceLimits } from "./config/worker-resource-limits.js";
|
|
66
|
+
|
|
52
67
|
// Search utilities exports (SearchService removed in v0.1.0)
|
|
53
68
|
export { calculateRRF, mergeWithRRF, normalizeRRFScore, toRankedItems } from "./search/index.js";
|
|
54
69
|
export { executeHybridSearch } from "./search/index.js";
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 TKMD and Reftrix Contributors
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Cached availability checker for onnxruntime-node (optionalDependency).
|
|
6
|
+
* When unavailable, ML features degrade gracefully.
|
|
7
|
+
*
|
|
8
|
+
* @module onnx-availability
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
let _available: boolean | null = null;
|
|
12
|
+
let _unavailableReason: string | null = null;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Check if onnxruntime-node is available (async, caches result).
|
|
16
|
+
*/
|
|
17
|
+
export async function isOnnxRuntimeAvailable(): Promise<boolean> {
|
|
18
|
+
if (_available !== null) return _available;
|
|
19
|
+
try {
|
|
20
|
+
await import("onnxruntime-node");
|
|
21
|
+
_available = true;
|
|
22
|
+
_unavailableReason = null;
|
|
23
|
+
} catch {
|
|
24
|
+
_available = false;
|
|
25
|
+
_unavailableReason = "onnxruntime-node is not installed";
|
|
26
|
+
}
|
|
27
|
+
return _available;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Synchronous check — returns null if not yet probed.
|
|
32
|
+
*/
|
|
33
|
+
export function isOnnxRuntimeAvailableSync(): boolean | null {
|
|
34
|
+
return _available;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Returns the reason onnxruntime-node is unavailable, or null if available.
|
|
39
|
+
*/
|
|
40
|
+
export function getOnnxUnavailableReason(): string | null {
|
|
41
|
+
return _unavailableReason;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Error thrown when ML features require onnxruntime-node but it is not installed.
|
|
46
|
+
*/
|
|
47
|
+
export class OnnxRuntimeUnavailableError extends Error {
|
|
48
|
+
constructor(context?: string) {
|
|
49
|
+
super(
|
|
50
|
+
`ML features require onnxruntime-node which is not available. ` +
|
|
51
|
+
`Install with: pnpm add onnxruntime-node` +
|
|
52
|
+
(context ? ` (${context})` : "")
|
|
53
|
+
);
|
|
54
|
+
this.name = "OnnxRuntimeUnavailableError";
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
|
|
59
|
+
type OnnxRuntimeModule = typeof import("onnxruntime-node");
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Safely import onnxruntime-node with a sanitized error on failure.
|
|
63
|
+
*/
|
|
64
|
+
export async function safeImportOnnx(): Promise<OnnxRuntimeModule> {
|
|
65
|
+
try {
|
|
66
|
+
return await import("onnxruntime-node");
|
|
67
|
+
} catch {
|
|
68
|
+
throw new OnnxRuntimeUnavailableError();
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* For testing: reset cached state.
|
|
74
|
+
*/
|
|
75
|
+
export function _resetForTesting(): void {
|
|
76
|
+
_available = null;
|
|
77
|
+
_unavailableReason = null;
|
|
78
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
/**
|
|
3
|
+
* ONNX Execution Provider Detection Utilities
|
|
4
|
+
*
|
|
5
|
+
* Shared logic for detecting the ONNX execution provider (CPU vs CUDA)
|
|
6
|
+
* based on environment configuration and CUDA shared library availability.
|
|
7
|
+
*
|
|
8
|
+
* Used by both the e5-base embedding worker thread and the DINOv2 worker thread
|
|
9
|
+
* to avoid code duplication. Extracted from embeddings/worker-thread.ts in v0.4.0.
|
|
10
|
+
*
|
|
11
|
+
* @module onnx-provider-detect
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { createRequire } from "node:module";
|
|
15
|
+
import fs from "node:fs";
|
|
16
|
+
import nodePath from "node:path";
|
|
17
|
+
|
|
18
|
+
// =====================================================
|
|
19
|
+
// Types
|
|
20
|
+
// =====================================================
|
|
21
|
+
|
|
22
|
+
export type ExecutionProvider = "cpu" | "cuda";
|
|
23
|
+
|
|
24
|
+
// =====================================================
|
|
25
|
+
// Provider Detection
|
|
26
|
+
// =====================================================
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Detect the ONNX execution provider based on environment configuration.
|
|
30
|
+
*
|
|
31
|
+
* Checks ONNX_EXECUTION_PROVIDER env var and verifies that the CUDA
|
|
32
|
+
* provider shared library is available on disk. Falls back to CPU
|
|
33
|
+
* gracefully if the provider is not installed.
|
|
34
|
+
*
|
|
35
|
+
* @param logPrefix - Log prefix for warning messages (e.g. "EmbeddingWorker", "DINOv2Worker")
|
|
36
|
+
*/
|
|
37
|
+
export function detectExecutionProvider(logPrefix?: string): ExecutionProvider {
|
|
38
|
+
const prefix = logPrefix ?? "OnnxProvider";
|
|
39
|
+
const envProvider = process.env.ONNX_EXECUTION_PROVIDER;
|
|
40
|
+
|
|
41
|
+
if (envProvider === "cuda" || envProvider === "rocm") {
|
|
42
|
+
const cudaAvailable = verifyCudaAvailability(logPrefix);
|
|
43
|
+
if (cudaAvailable) {
|
|
44
|
+
return "cuda";
|
|
45
|
+
}
|
|
46
|
+
console.warn(
|
|
47
|
+
"[%s] ONNX_EXECUTION_PROVIDER=%s but CUDA provider not available, falling back to CPU",
|
|
48
|
+
prefix,
|
|
49
|
+
envProvider
|
|
50
|
+
);
|
|
51
|
+
return "cpu";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return "cpu";
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// =====================================================
|
|
58
|
+
// CUDA Availability Verification
|
|
59
|
+
// =====================================================
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Verify that CUDA provider (libonnxruntime_providers_cuda.so) is available.
|
|
63
|
+
*
|
|
64
|
+
* onnxruntime-node can download the CUDA provider shared library via
|
|
65
|
+
* `ONNXRUNTIME_NODE_INSTALL_CUDA=v12 node .../install.js`. This function
|
|
66
|
+
* checks for its presence on disk rather than trying to resolve a
|
|
67
|
+
* non-existent npm package (onnxruntime-gpu is Python-only).
|
|
68
|
+
*
|
|
69
|
+
* @param logPrefix - Log prefix for warning messages
|
|
70
|
+
*/
|
|
71
|
+
export function verifyCudaAvailability(logPrefix?: string): boolean {
|
|
72
|
+
const prefix = logPrefix ?? "OnnxProvider";
|
|
73
|
+
try {
|
|
74
|
+
const esmRequire = createRequire(import.meta.url);
|
|
75
|
+
const ortNodePath = esmRequire.resolve("onnxruntime-node");
|
|
76
|
+
// require.resolve returns e.g. .../onnxruntime-node/dist/index.js
|
|
77
|
+
// Walk up to package root by finding the directory containing package.json
|
|
78
|
+
let packageDir = nodePath.dirname(ortNodePath);
|
|
79
|
+
for (let i = 0; i < 5; i++) {
|
|
80
|
+
if (fs.existsSync(nodePath.join(packageDir, "package.json"))) break;
|
|
81
|
+
packageDir = nodePath.dirname(packageDir);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Search across napi versions (v3, v6, etc.) for CUDA provider
|
|
85
|
+
const binDir = nodePath.join(packageDir, "bin");
|
|
86
|
+
if (fs.existsSync(binDir)) {
|
|
87
|
+
const napiDirs = fs.readdirSync(binDir).filter((d) => d.startsWith("napi-v"));
|
|
88
|
+
for (const napiDir of napiDirs) {
|
|
89
|
+
const cudaProviderPath = nodePath.join(
|
|
90
|
+
binDir,
|
|
91
|
+
napiDir,
|
|
92
|
+
"linux",
|
|
93
|
+
"x64",
|
|
94
|
+
"libonnxruntime_providers_cuda.so"
|
|
95
|
+
);
|
|
96
|
+
if (fs.existsSync(cudaProviderPath)) {
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
console.warn("[%s] CUDA provider not found in: %s", prefix, binDir);
|
|
103
|
+
return false;
|
|
104
|
+
} catch {
|
|
105
|
+
console.warn("[%s] Cannot verify CUDA provider: onnxruntime-node not found", prefix);
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// =====================================================
|
|
111
|
+
// LD_LIBRARY_PATH Check
|
|
112
|
+
// =====================================================
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Verify that LD_LIBRARY_PATH is set at the OS level (not just process.env).
|
|
116
|
+
*
|
|
117
|
+
* dlopen() reads LD_LIBRARY_PATH from the kernel environment (/proc/self/environ),
|
|
118
|
+
* NOT from process.env modifications made at runtime. If LD_LIBRARY_PATH was set
|
|
119
|
+
* after process startup (e.g. by loadEnvLocal()), dlopen() cannot find CUDA
|
|
120
|
+
* libraries and ONNX Runtime CUDA provider initialization will segfault or throw.
|
|
121
|
+
*
|
|
122
|
+
* Returns true only if LD_LIBRARY_PATH was present in the original process environment.
|
|
123
|
+
*
|
|
124
|
+
* @returns true if LD_LIBRARY_PATH is set at OS level, or if /proc/self/environ is unavailable
|
|
125
|
+
*/
|
|
126
|
+
export function isLdLibraryPathSetAtOsLevel(): boolean {
|
|
127
|
+
try {
|
|
128
|
+
const procEnv = fs.readFileSync("/proc/self/environ", "utf-8");
|
|
129
|
+
return procEnv.includes("LD_LIBRARY_PATH");
|
|
130
|
+
} catch {
|
|
131
|
+
// /proc/self/environ not available (non-Linux) — assume set
|
|
132
|
+
return true;
|
|
133
|
+
}
|
|
134
|
+
}
|