@reftrixmcp/ml 0.1.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/config/worker-resource-limits.d.ts +89 -0
  2. package/dist/config/worker-resource-limits.d.ts.map +1 -0
  3. package/dist/config/worker-resource-limits.js +156 -0
  4. package/dist/config/worker-resource-limits.js.map +1 -0
  5. package/dist/dinov2/service.d.ts +13 -0
  6. package/dist/dinov2/service.d.ts.map +1 -1
  7. package/dist/dinov2/service.js +86 -8
  8. package/dist/dinov2/service.js.map +1 -1
  9. package/dist/dinov2/worker-thread.js +72 -7
  10. package/dist/dinov2/worker-thread.js.map +1 -1
  11. package/dist/embeddings/service.d.ts +19 -0
  12. package/dist/embeddings/service.d.ts.map +1 -1
  13. package/dist/embeddings/service.js +87 -1
  14. package/dist/embeddings/service.js.map +1 -1
  15. package/dist/embeddings/worker-thread-types.d.ts +1 -1
  16. package/dist/embeddings/worker-thread.d.ts.map +1 -1
  17. package/dist/embeddings/worker-thread.js +10 -85
  18. package/dist/embeddings/worker-thread.js.map +1 -1
  19. package/dist/index.d.ts +3 -0
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +4 -0
  22. package/dist/index.js.map +1 -1
  23. package/dist/onnx-availability.d.ts +29 -0
  24. package/dist/onnx-availability.d.ts.map +1 -0
  25. package/dist/onnx-availability.js +69 -0
  26. package/dist/onnx-availability.js.map +1 -0
  27. package/dist/onnx-provider-detect.d.ts +47 -0
  28. package/dist/onnx-provider-detect.d.ts.map +1 -0
  29. package/dist/onnx-provider-detect.js +111 -0
  30. package/dist/onnx-provider-detect.js.map +1 -0
  31. package/package.json +8 -3
  32. package/src/config/worker-resource-limits.ts +176 -0
  33. package/src/dinov2/service.ts +105 -8
  34. package/src/dinov2/worker-thread.ts +85 -8
  35. package/src/embeddings/service.ts +99 -1
  36. package/src/embeddings/worker-thread-types.ts +1 -1
  37. package/src/embeddings/worker-thread.ts +15 -104
  38. package/src/index.ts +15 -0
  39. package/src/onnx-availability.ts +78 -0
  40. package/src/onnx-provider-detect.ts +134 -0
@@ -28,16 +28,20 @@ import type {
28
28
  } from "./worker-thread-types.js";
29
29
 
30
30
  // =====================================================
31
- // Dynamic import of onnxruntime-node
31
+ // Dynamic import of onnxruntime-node (optional dependency)
32
32
  // =====================================================
33
33
 
34
+ import { safeImportOnnx } from "../onnx-availability.js";
35
+ import { detectExecutionProvider, isLdLibraryPathSetAtOsLevel } from "../onnx-provider-detect.js";
36
+ import type { ExecutionProvider } from "../onnx-provider-detect.js";
37
+
34
38
  // eslint-disable-next-line @typescript-eslint/consistent-type-imports
35
39
  type OrtModule = typeof import("onnxruntime-node");
36
40
  let ort: OrtModule | null = null;
37
41
 
38
42
  async function getOrt(): Promise<OrtModule> {
39
43
  if (!ort) {
40
- ort = await import("onnxruntime-node");
44
+ ort = await safeImportOnnx();
41
45
  }
42
46
  return ort;
43
47
  }
@@ -137,16 +141,71 @@ function l2Normalize(vec: Float32Array): Float32Array {
137
141
  // Session management
138
142
  // =====================================================
139
143
 
144
+ /** Resolved execution provider for the current session. */
145
+ let resolvedProvider: ExecutionProvider = "cpu";
146
+
140
147
  async function initializeSession(modelPath: string): Promise<void> {
141
148
  if (session) return;
142
149
 
150
+ const provider = detectExecutionProvider("DINOv2Worker");
151
+
152
+ // Safety check: if CUDA is requested but LD_LIBRARY_PATH was not set at
153
+ // the OS level (only set via loadEnvLocal at runtime), CUDA init will crash
154
+ // because dlopen() can't find CUDA shared libraries. Fall back to CPU.
155
+ let effectiveProvider = provider;
156
+ if (effectiveProvider === "cuda" && !isLdLibraryPathSetAtOsLevel()) {
157
+ console.warn(
158
+ "[DINOv2Worker] CUDA requested but LD_LIBRARY_PATH not set at OS level. " +
159
+ "dlopen() cannot find CUDA libraries. Falling back to CPU. " +
160
+ "To use CUDA, set LD_LIBRARY_PATH before starting the Node.js process."
161
+ );
162
+ effectiveProvider = "cpu";
163
+ }
164
+
165
+ const sessionOptions =
166
+ effectiveProvider === "cuda"
167
+ ? {
168
+ executionProviders: ["CUDAExecutionProvider", "CPUExecutionProvider"],
169
+ enableMemPattern: false,
170
+ }
171
+ : {
172
+ executionProviders: ["cpu"],
173
+ enableCpuMemArena: false,
174
+ enableMemPattern: false,
175
+ };
176
+
143
177
  const ortModule = await getOrt();
144
- session = (await ortModule.InferenceSession.create(modelPath, {
145
- executionProviders: ["cpu"],
146
- })) as unknown as OrtSession;
178
+
179
+ // Attempt to create session with the resolved provider.
180
+ // If CUDA init fails (e.g. library not found), catch and retry with CPU.
181
+ try {
182
+ session = (await ortModule.InferenceSession.create(
183
+ modelPath,
184
+ sessionOptions
185
+ )) as unknown as OrtSession;
186
+ resolvedProvider = effectiveProvider;
187
+ } catch (providerError) {
188
+ if (effectiveProvider !== "cpu") {
189
+ // CUDA init failed — fallback to CPU
190
+ const errorMsg =
191
+ providerError instanceof Error ? providerError.message : String(providerError);
192
+ console.warn(
193
+ "[DINOv2Worker] CUDA session creation failed, falling back to CPU: %s",
194
+ errorMsg
195
+ );
196
+ session = (await ortModule.InferenceSession.create(modelPath, {
197
+ executionProviders: ["cpu"],
198
+ enableCpuMemArena: false,
199
+ enableMemPattern: false,
200
+ })) as unknown as OrtSession;
201
+ resolvedProvider = "cpu";
202
+ } else {
203
+ throw providerError;
204
+ }
205
+ }
147
206
 
148
207
  // eslint-disable-next-line no-console
149
- console.log("[DINOv2Worker] Session initialized:", modelPath);
208
+ console.log("[DINOv2Worker] Session initialized (provider: %s):", resolvedProvider, modelPath);
150
209
  }
151
210
 
152
211
  async function disposeSession(): Promise<void> {
@@ -185,8 +244,18 @@ async function runInference(
185
244
  // Create input tensor [1, 3, height, width]
186
245
  const inputTensor = new ortModule.Tensor("float32", inputData, [1, 3, height, width]);
187
246
 
188
- // Run inference
189
- const output = await session.run({ pixel_values: inputTensor });
247
+ // Run inference — use try/finally to ensure tensor disposal (OOM-FIX-1)
248
+ // ONNX Tensor holds native C++ memory that V8 GC cannot reclaim.
249
+ // Without explicit dispose(), ~1.4MB/inference leaks as native memory.
250
+ let output: Record<string, { data: Float32Array; dims: readonly number[] }>;
251
+ try {
252
+ output = await session.run({ pixel_values: inputTensor });
253
+ } finally {
254
+ // Dispose input tensor immediately after inference completes
255
+ if (typeof (inputTensor as unknown as { dispose?: () => void }).dispose === "function") {
256
+ (inputTensor as unknown as { dispose: () => void }).dispose();
257
+ }
258
+ }
190
259
 
191
260
  // Extract CLS token from last_hidden_state [1, 257, 768]
192
261
  const lastHiddenState = output.last_hidden_state;
@@ -210,6 +279,14 @@ async function runInference(
210
279
  clsToken[i] = data[i]!;
211
280
  }
212
281
 
282
+ // Dispose output tensors after data extraction (OOM-FIX-1)
283
+ for (const key of Object.keys(output)) {
284
+ const tensor = output[key] as unknown as { dispose?: () => void };
285
+ if (typeof tensor?.dispose === "function") {
286
+ tensor.dispose();
287
+ }
288
+ }
289
+
213
290
  // L2 normalize with NaN/Infinity defense
214
291
  const normalized = l2Normalize(clsToken);
215
292
 
@@ -21,6 +21,8 @@ import { fileURLToPath } from "node:url";
21
21
  import path from "node:path";
22
22
  import type { EmbeddingTextType, EmbeddingServiceConfig, CacheStats } from "./types.js";
23
23
  import type { WorkerMessage, WorkerResponse } from "./worker-thread-types.js";
24
+ import { OnnxRuntimeUnavailableError } from "../onnx-availability.js";
25
+ import { getMLWorkerThreadOptions } from "../config/worker-resource-limits.js";
24
26
 
25
27
  /**
26
28
  * Disposable pipeline interface (used only in in-process fallback mode).
@@ -306,7 +308,13 @@ export class EmbeddingService {
306
308
  // worker_threads Workers, so we must explicitly pass an empty array
307
309
  // to prevent the parent's flags (set by WorkerSupervisor fork())
308
310
  // from propagating.
309
- this.worker = new Worker(scriptPath, { execArgv: [] });
311
+ //
312
+ // PR7e-β1: Add explicit resourceLimits to prevent OOM when ONNX
313
+ // falls back to CPU (onnxruntime-node). Without resourceLimits,
314
+ // V8 old-space defaults to < 4GB and e5-base CPU inference OOMs.
315
+ // See ADR-0012 §3 BLOCKER 4 and config/worker-resource-limits.ts.
316
+ const workerOptions = { execArgv: [], ...getMLWorkerThreadOptions() };
317
+ this.worker = new Worker(scriptPath, workerOptions);
310
318
 
311
319
  // Handle worker messages
312
320
  this.worker.on("message", (response: WorkerResponse) => {
@@ -502,6 +510,10 @@ export class EmbeddingService {
502
510
  }
503
511
  } catch (error) {
504
512
  this.initPromise = null;
513
+ const code = (error as { code?: string }).code;
514
+ if (code === "MODULE_NOT_FOUND" || code === "ERR_MODULE_NOT_FOUND") {
515
+ throw new OnnxRuntimeUnavailableError("text embedding");
516
+ }
505
517
  const message = error instanceof Error ? error.message : "Unknown error";
506
518
  throw new Error(`Failed to load embedding model: ${message}`);
507
519
  }
@@ -1275,6 +1287,92 @@ export class EmbeddingService {
1275
1287
  this.config = { ...this.config, device: "cpu" };
1276
1288
  }
1277
1289
 
1290
+ /**
1291
+ * Terminate the worker thread and prepare for re-spawn on next use.
1292
+ *
1293
+ * Unlike dispose() which only releases the ONNX pipeline within the
1294
+ * existing worker thread, this method terminates the worker thread
1295
+ * process entirely. This forces the OS to reclaim ALL memory held by
1296
+ * the ONNX Runtime C++ arena allocator, which glibc malloc fragmentation
1297
+ * prevents dispose() from returning.
1298
+ *
1299
+ * After termination, the next generateEmbedding() call will automatically
1300
+ * re-spawn a new worker thread via ensureWorkerReady().
1301
+ *
1302
+ * Key behaviors:
1303
+ * - lastCrashTime is reset to 0 (planned terminate, not a crash)
1304
+ * - workerRestartCount is preserved (not incremented)
1305
+ * - currentProvider is preserved for re-spawn
1306
+ * - Mutual exclusion via workerInitPromise
1307
+ */
1308
+ async terminateAndRespawn(): Promise<void> {
1309
+ // T-4: Set workerInitPromise to prevent concurrent spawns during terminate
1310
+ const terminatePromise = (async (): Promise<void> => {
1311
+ // T-3: Save workerRestartCount before terminate (crash handler increments it)
1312
+ const savedRestartCount = this.workerRestartCount;
1313
+
1314
+ // Clear idle timer
1315
+ if (this.idleTimer !== null) {
1316
+ clearTimeout(this.idleTimer);
1317
+ this.idleTimer = null;
1318
+ }
1319
+
1320
+ if (this.worker) {
1321
+ try {
1322
+ // Send terminate message so worker can dispose pipeline
1323
+ await this.sendWorkerMessage({
1324
+ type: "terminate",
1325
+ requestId: generateRequestId(),
1326
+ });
1327
+ } catch {
1328
+ // May fail if worker already exited
1329
+ }
1330
+
1331
+ // Force terminate the worker thread
1332
+ try {
1333
+ await this.worker.terminate();
1334
+ } catch {
1335
+ // Already terminated
1336
+ }
1337
+
1338
+ this.worker = null;
1339
+ this.workerReady = false;
1340
+ }
1341
+
1342
+ // Also dispose in-process pipeline if present
1343
+ await this.disposeInProcess();
1344
+
1345
+ // Reject any remaining pending requests
1346
+ for (const [id, pending] of this.pendingRequests) {
1347
+ clearTimeout(pending.timer);
1348
+ pending.reject(new Error("Service terminated for respawn"));
1349
+ this.pendingRequests.delete(id);
1350
+ }
1351
+
1352
+ // T-3: Restore workerRestartCount (planned terminate, not a crash)
1353
+ this.workerRestartCount = savedRestartCount;
1354
+
1355
+ // T-3: Reset lastCrashTime (planned terminate, no cooldown needed)
1356
+ this.lastCrashTime = 0;
1357
+
1358
+ this.inferencesSinceRecycle = 0;
1359
+
1360
+ // T-5: currentProvider is NOT reset — re-spawn will use same provider
1361
+
1362
+ // eslint-disable-next-line no-console
1363
+ console.log("[ML] Worker Thread terminated for respawn (planned)");
1364
+ })();
1365
+
1366
+ // T-4: Hold workerInitPromise to block concurrent generateEmbedding() calls
1367
+ this.workerInitPromise = terminatePromise.then(() => {
1368
+ // Clear workerInitPromise after terminate completes so next
1369
+ // ensureWorkerReady() can spawn a fresh worker
1370
+ this.workerInitPromise = null;
1371
+ });
1372
+
1373
+ await terminatePromise;
1374
+ }
1375
+
1278
1376
  /**
1279
1377
  * Get the number of worker thread restarts that have occurred.
1280
1378
  */
@@ -144,7 +144,7 @@ export interface WorkerDisposeResponse {
144
144
  }
145
145
 
146
146
  /**
147
- * Successful response from terminate (sent just before process.exit).
147
+ * Successful response from terminate (sent before parent calls worker.terminate()).
148
148
  */
149
149
  export interface WorkerTerminateResponse {
150
150
  type: "terminate";
@@ -20,10 +20,14 @@
20
20
  */
21
21
 
22
22
  import { parentPort } from "node:worker_threads";
23
- import { createRequire } from "node:module";
24
- import fs from "node:fs";
25
- import nodePath from "node:path";
26
23
  import type { WorkerMessage, WorkerResponse, WorkerErrorResponse } from "./worker-thread-types.js";
24
+ import { OnnxRuntimeUnavailableError } from "../onnx-availability.js";
25
+ import {
26
+ detectExecutionProvider,
27
+ verifyCudaAvailability,
28
+ isLdLibraryPathSetAtOsLevel,
29
+ } from "../onnx-provider-detect.js";
30
+ import type { ExecutionProvider } from "../onnx-provider-detect.js";
27
31
 
28
32
  // =====================================================
29
33
  // Pipeline types (mirrors service.ts DisposablePipeline)
@@ -38,83 +42,6 @@ interface DisposablePipeline {
38
42
  dispose?: () => Promise<unknown[]>;
39
43
  }
40
44
 
41
- // =====================================================
42
- // GPU execution provider detection
43
- // =====================================================
44
-
45
- type ExecutionProvider = "cpu" | "cuda";
46
-
47
- /**
48
- * Detect the ONNX execution provider based on environment configuration.
49
- *
50
- * Checks ONNX_EXECUTION_PROVIDER env var and verifies that the CUDA
51
- * provider shared library is available on disk. Falls back to CPU
52
- * gracefully if the provider is not installed.
53
- */
54
- function detectExecutionProvider(): ExecutionProvider {
55
- const envProvider = process.env.ONNX_EXECUTION_PROVIDER;
56
-
57
- if (envProvider === "cuda" || envProvider === "rocm") {
58
- const cudaAvailable = verifyCudaAvailability();
59
- if (cudaAvailable) {
60
- return "cuda";
61
- }
62
- console.warn(
63
- "[EmbeddingWorker] ONNX_EXECUTION_PROVIDER=%s but CUDA provider not available, falling back to CPU",
64
- envProvider
65
- );
66
- return "cpu";
67
- }
68
-
69
- return "cpu";
70
- }
71
-
72
- /**
73
- * Verify that CUDA provider (libonnxruntime_providers_cuda.so) is available.
74
- *
75
- * onnxruntime-node can download the CUDA provider shared library via
76
- * `ONNXRUNTIME_NODE_INSTALL_CUDA=v12 node .../install.js`. This function
77
- * checks for its presence on disk rather than trying to resolve a
78
- * non-existent npm package (onnxruntime-gpu is Python-only).
79
- */
80
- function verifyCudaAvailability(): boolean {
81
- try {
82
- const esmRequire = createRequire(import.meta.url);
83
- const ortNodePath = esmRequire.resolve("onnxruntime-node");
84
- // require.resolve returns e.g. .../onnxruntime-node/dist/index.js
85
- // Walk up to package root by finding the directory containing package.json
86
- let packageDir = nodePath.dirname(ortNodePath);
87
- for (let i = 0; i < 5; i++) {
88
- if (fs.existsSync(nodePath.join(packageDir, "package.json"))) break;
89
- packageDir = nodePath.dirname(packageDir);
90
- }
91
-
92
- // Search across napi versions (v3, v6, etc.) for CUDA provider
93
- const binDir = nodePath.join(packageDir, "bin");
94
- if (fs.existsSync(binDir)) {
95
- const napiDirs = fs.readdirSync(binDir).filter((d) => d.startsWith("napi-v"));
96
- for (const napiDir of napiDirs) {
97
- const cudaProviderPath = nodePath.join(
98
- binDir,
99
- napiDir,
100
- "linux",
101
- "x64",
102
- "libonnxruntime_providers_cuda.so"
103
- );
104
- if (fs.existsSync(cudaProviderPath)) {
105
- return true;
106
- }
107
- }
108
- }
109
-
110
- console.warn("[EmbeddingWorker] CUDA provider not found in: %s", binDir);
111
- return false;
112
- } catch {
113
- console.warn("[EmbeddingWorker] Cannot verify CUDA provider: onnxruntime-node not found");
114
- return false;
115
- }
116
- }
117
-
118
45
  // =====================================================
119
46
  // Worker state
120
47
  // =====================================================
@@ -146,26 +73,6 @@ function normalizeVector(vector: number[]): number[] {
146
73
  return vector.map((val) => val / norm);
147
74
  }
148
75
 
149
- /**
150
- * Verify that LD_LIBRARY_PATH is set at the OS level (not just process.env).
151
- *
152
- * dlopen() reads LD_LIBRARY_PATH from the kernel environment (/proc/self/environ),
153
- * NOT from process.env modifications made at runtime. If LD_LIBRARY_PATH was set
154
- * after process startup (e.g. by loadEnvLocal()), dlopen() cannot find CUDA
155
- * libraries and ONNX Runtime CUDA provider initialization will segfault or throw.
156
- *
157
- * Returns true only if LD_LIBRARY_PATH was present in the original process environment.
158
- */
159
- function isLdLibraryPathSetAtOsLevel(): boolean {
160
- try {
161
- const procEnv = fs.readFileSync("/proc/self/environ", "utf-8");
162
- return procEnv.includes("LD_LIBRARY_PATH");
163
- } catch {
164
- // /proc/self/environ not available (non-Linux) — assume set
165
- return true;
166
- }
167
- }
168
-
169
76
  async function initializePipeline(): Promise<void> {
170
77
  if (pipeline) return;
171
78
  if (initPromise) return initPromise;
@@ -175,7 +82,7 @@ async function initializePipeline(): Promise<void> {
175
82
  // Only auto-detect from env on first init. If provider was explicitly
176
83
  // set by a switch-provider message, respect that setting.
177
84
  if (!providerExplicitlySet) {
178
- resolvedProvider = detectExecutionProvider();
85
+ resolvedProvider = detectExecutionProvider("EmbeddingWorker");
179
86
  }
180
87
 
181
88
  // Safety check: if CUDA is requested but LD_LIBRARY_PATH was not set at
@@ -236,6 +143,10 @@ async function initializePipeline(): Promise<void> {
236
143
  console.log("[EmbeddingWorker] ONNX pipeline ready (provider: %s)", resolvedProvider);
237
144
  } catch (error) {
238
145
  initPromise = null;
146
+ const code = (error as { code?: string }).code;
147
+ if (code === "MODULE_NOT_FOUND" || code === "ERR_MODULE_NOT_FOUND") {
148
+ throw new OnnxRuntimeUnavailableError("text embedding");
149
+ }
239
150
  const message = error instanceof Error ? error.message : "Unknown error";
240
151
  throw new Error(`Failed to load embedding model in worker thread: ${message}`);
241
152
  }
@@ -436,8 +347,8 @@ async function handleMessage(message: WorkerMessage): Promise<void> {
436
347
  requestId: message.requestId,
437
348
  success: true,
438
349
  });
439
- // Allow the response to be flushed before exiting
440
- setTimeout(() => process.exit(0), 50);
350
+ // Parent calls worker.terminate() to stop this thread.
351
+ // No process.exit() — let parent control the lifecycle.
441
352
  break;
442
353
  }
443
354
 
@@ -446,7 +357,7 @@ async function handleMessage(message: WorkerMessage): Promise<void> {
446
357
 
447
358
  // If switching to CUDA, verify availability first
448
359
  if (targetProvider === "cuda") {
449
- const canUseCuda = verifyCudaAvailability();
360
+ const canUseCuda = verifyCudaAvailability("EmbeddingWorker");
450
361
  if (!canUseCuda) {
451
362
  // Cannot switch to CUDA — respond with current (cpu) provider
452
363
  sendResponse({
package/src/index.ts CHANGED
@@ -49,6 +49,21 @@ export { DINOv2Service, DINOV2_EMBEDDING_DIMENSION, DINOV2_INPUT_SIZE } from "./
49
49
  export type { DINOv2ServiceConfig } from "./dinov2/index.js";
50
50
  export type { DINOv2WorkerMessage, DINOv2WorkerResponse } from "./dinov2/index.js";
51
51
 
52
+ // ONNX Runtime availability checker
53
+ export {
54
+ isOnnxRuntimeAvailable,
55
+ OnnxRuntimeUnavailableError,
56
+ safeImportOnnx,
57
+ } from "./onnx-availability.js";
58
+
59
+ // ML Worker Thread resource limits (PR7e-β1)
60
+ export {
61
+ loadMLWorkerResourceLimits,
62
+ getMLWorkerThreadOptions,
63
+ MLWorkerResourceLimitsSchema,
64
+ } from "./config/worker-resource-limits.js";
65
+ export type { MLWorkerResourceLimits } from "./config/worker-resource-limits.js";
66
+
52
67
  // Search utilities exports (SearchService removed in v0.1.0)
53
68
  export { calculateRRF, mergeWithRRF, normalizeRRFScore, toRankedItems } from "./search/index.js";
54
69
  export { executeHybridSearch } from "./search/index.js";
@@ -0,0 +1,78 @@
1
+ // SPDX-FileCopyrightText: 2026 TKMD and Reftrix Contributors
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+
4
+ /**
5
+ * Cached availability checker for onnxruntime-node (optionalDependency).
6
+ * When unavailable, ML features degrade gracefully.
7
+ *
8
+ * @module onnx-availability
9
+ */
10
+
11
+ let _available: boolean | null = null;
12
+ let _unavailableReason: string | null = null;
13
+
14
+ /**
15
+ * Check if onnxruntime-node is available (async, caches result).
16
+ */
17
+ export async function isOnnxRuntimeAvailable(): Promise<boolean> {
18
+ if (_available !== null) return _available;
19
+ try {
20
+ await import("onnxruntime-node");
21
+ _available = true;
22
+ _unavailableReason = null;
23
+ } catch {
24
+ _available = false;
25
+ _unavailableReason = "onnxruntime-node is not installed";
26
+ }
27
+ return _available;
28
+ }
29
+
30
+ /**
31
+ * Synchronous check — returns null if not yet probed.
32
+ */
33
+ export function isOnnxRuntimeAvailableSync(): boolean | null {
34
+ return _available;
35
+ }
36
+
37
+ /**
38
+ * Returns the reason onnxruntime-node is unavailable, or null if available.
39
+ */
40
+ export function getOnnxUnavailableReason(): string | null {
41
+ return _unavailableReason;
42
+ }
43
+
44
+ /**
45
+ * Error thrown when ML features require onnxruntime-node but it is not installed.
46
+ */
47
+ export class OnnxRuntimeUnavailableError extends Error {
48
+ constructor(context?: string) {
49
+ super(
50
+ `ML features require onnxruntime-node which is not available. ` +
51
+ `Install with: pnpm add onnxruntime-node` +
52
+ (context ? ` (${context})` : "")
53
+ );
54
+ this.name = "OnnxRuntimeUnavailableError";
55
+ }
56
+ }
57
+
58
+ // eslint-disable-next-line @typescript-eslint/consistent-type-imports
59
+ type OnnxRuntimeModule = typeof import("onnxruntime-node");
60
+
61
+ /**
62
+ * Safely import onnxruntime-node with a sanitized error on failure.
63
+ */
64
+ export async function safeImportOnnx(): Promise<OnnxRuntimeModule> {
65
+ try {
66
+ return await import("onnxruntime-node");
67
+ } catch {
68
+ throw new OnnxRuntimeUnavailableError();
69
+ }
70
+ }
71
+
72
+ /**
73
+ * For testing: reset cached state.
74
+ */
75
+ export function _resetForTesting(): void {
76
+ _available = null;
77
+ _unavailableReason = null;
78
+ }
@@ -0,0 +1,134 @@
1
+ // SPDX-License-Identifier: AGPL-3.0-only
2
+ /**
3
+ * ONNX Execution Provider Detection Utilities
4
+ *
5
+ * Shared logic for detecting the ONNX execution provider (CPU vs CUDA)
6
+ * based on environment configuration and CUDA shared library availability.
7
+ *
8
+ * Used by both the e5-base embedding worker thread and the DINOv2 worker thread
9
+ * to avoid code duplication. Extracted from embeddings/worker-thread.ts in v0.4.0.
10
+ *
11
+ * @module onnx-provider-detect
12
+ */
13
+
14
+ import { createRequire } from "node:module";
15
+ import fs from "node:fs";
16
+ import nodePath from "node:path";
17
+
18
+ // =====================================================
19
+ // Types
20
+ // =====================================================
21
+
22
+ export type ExecutionProvider = "cpu" | "cuda";
23
+
24
+ // =====================================================
25
+ // Provider Detection
26
+ // =====================================================
27
+
28
+ /**
29
+ * Detect the ONNX execution provider based on environment configuration.
30
+ *
31
+ * Checks ONNX_EXECUTION_PROVIDER env var and verifies that the CUDA
32
+ * provider shared library is available on disk. Falls back to CPU
33
+ * gracefully if the provider is not installed.
34
+ *
35
+ * @param logPrefix - Log prefix for warning messages (e.g. "EmbeddingWorker", "DINOv2Worker")
36
+ */
37
+ export function detectExecutionProvider(logPrefix?: string): ExecutionProvider {
38
+ const prefix = logPrefix ?? "OnnxProvider";
39
+ const envProvider = process.env.ONNX_EXECUTION_PROVIDER;
40
+
41
+ if (envProvider === "cuda" || envProvider === "rocm") {
42
+ const cudaAvailable = verifyCudaAvailability(logPrefix);
43
+ if (cudaAvailable) {
44
+ return "cuda";
45
+ }
46
+ console.warn(
47
+ "[%s] ONNX_EXECUTION_PROVIDER=%s but CUDA provider not available, falling back to CPU",
48
+ prefix,
49
+ envProvider
50
+ );
51
+ return "cpu";
52
+ }
53
+
54
+ return "cpu";
55
+ }
56
+
57
+ // =====================================================
58
+ // CUDA Availability Verification
59
+ // =====================================================
60
+
61
+ /**
62
+ * Verify that CUDA provider (libonnxruntime_providers_cuda.so) is available.
63
+ *
64
+ * onnxruntime-node can download the CUDA provider shared library via
65
+ * `ONNXRUNTIME_NODE_INSTALL_CUDA=v12 node .../install.js`. This function
66
+ * checks for its presence on disk rather than trying to resolve a
67
+ * non-existent npm package (onnxruntime-gpu is Python-only).
68
+ *
69
+ * @param logPrefix - Log prefix for warning messages
70
+ */
71
+ export function verifyCudaAvailability(logPrefix?: string): boolean {
72
+ const prefix = logPrefix ?? "OnnxProvider";
73
+ try {
74
+ const esmRequire = createRequire(import.meta.url);
75
+ const ortNodePath = esmRequire.resolve("onnxruntime-node");
76
+ // require.resolve returns e.g. .../onnxruntime-node/dist/index.js
77
+ // Walk up to package root by finding the directory containing package.json
78
+ let packageDir = nodePath.dirname(ortNodePath);
79
+ for (let i = 0; i < 5; i++) {
80
+ if (fs.existsSync(nodePath.join(packageDir, "package.json"))) break;
81
+ packageDir = nodePath.dirname(packageDir);
82
+ }
83
+
84
+ // Search across napi versions (v3, v6, etc.) for CUDA provider
85
+ const binDir = nodePath.join(packageDir, "bin");
86
+ if (fs.existsSync(binDir)) {
87
+ const napiDirs = fs.readdirSync(binDir).filter((d) => d.startsWith("napi-v"));
88
+ for (const napiDir of napiDirs) {
89
+ const cudaProviderPath = nodePath.join(
90
+ binDir,
91
+ napiDir,
92
+ "linux",
93
+ "x64",
94
+ "libonnxruntime_providers_cuda.so"
95
+ );
96
+ if (fs.existsSync(cudaProviderPath)) {
97
+ return true;
98
+ }
99
+ }
100
+ }
101
+
102
+ console.warn("[%s] CUDA provider not found in: %s", prefix, binDir);
103
+ return false;
104
+ } catch {
105
+ console.warn("[%s] Cannot verify CUDA provider: onnxruntime-node not found", prefix);
106
+ return false;
107
+ }
108
+ }
109
+
110
+ // =====================================================
111
+ // LD_LIBRARY_PATH Check
112
+ // =====================================================
113
+
114
+ /**
115
+ * Verify that LD_LIBRARY_PATH is set at the OS level (not just process.env).
116
+ *
117
+ * dlopen() reads LD_LIBRARY_PATH from the kernel environment (/proc/self/environ),
118
+ * NOT from process.env modifications made at runtime. If LD_LIBRARY_PATH was set
119
+ * after process startup (e.g. by loadEnvLocal()), dlopen() cannot find CUDA
120
+ * libraries and ONNX Runtime CUDA provider initialization will segfault or throw.
121
+ *
122
+ * Returns true only if LD_LIBRARY_PATH was present in the original process environment.
123
+ *
124
+ * @returns true if LD_LIBRARY_PATH is set at OS level, or if /proc/self/environ is unavailable
125
+ */
126
+ export function isLdLibraryPathSetAtOsLevel(): boolean {
127
+ try {
128
+ const procEnv = fs.readFileSync("/proc/self/environ", "utf-8");
129
+ return procEnv.includes("LD_LIBRARY_PATH");
130
+ } catch {
131
+ // /proc/self/environ not available (non-Linux) — assume set
132
+ return true;
133
+ }
134
+ }