omnius 1.0.640 → 1.0.642
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +370 -54
- package/dist/scripts/audio-speaker-embedding-worker.py +38 -16
- package/dist/scripts/ocr-advanced.py +218 -21
- package/docs/DISCOVERY.json +14 -2
- package/docs/rest/endpoints/chat.md +3 -1
- package/docs/rest/endpoints/voice-vision.md +16 -5
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5859,7 +5859,7 @@ function usesSystemSitePackages() {
|
|
|
5859
5859
|
function loadManifest3() {
|
|
5860
5860
|
try {
|
|
5861
5861
|
const manifest = JSON.parse(readFileSync10(manifestPath3(), "utf8"));
|
|
5862
|
-
if (manifest.runtimeVersion !== RUNTIME_VERSION3 || manifest.model !== MODEL_NAME2 || manifest.modelRevision !== MODEL_REVISION2 || manifest.modelDigest !== `sha256:${MODEL_SHA2562}` || manifest.backend !== "onnxruntime-cpu" || manifest.onnxRuntimeVersion !== ONNXRUNTIME_VERSION || manifest.numpyVersion !== NUMPY_VERSION || manifest.preprocessing?.implementation !== FBANK_IMPLEMENTATION || manifest.preprocessing?.
|
|
5862
|
+
if (manifest.runtimeVersion !== RUNTIME_VERSION3 || manifest.model !== MODEL_NAME2 || manifest.modelRevision !== MODEL_REVISION2 || manifest.modelDigest !== `sha256:${MODEL_SHA2562}` || manifest.backend !== "onnxruntime-cpu" || manifest.onnxRuntimeVersion !== ONNXRUNTIME_VERSION || manifest.numpyVersion !== NUMPY_VERSION || manifest.preprocessing?.implementation !== FBANK_IMPLEMENTATION || manifest.preprocessing?.validation !== FBANK_VALIDATION || !hasExpectedSpeakerRuntimePackages(manifest.packages) || usesSystemSitePackages() || !existsSync11(modelPath2()) || !existsSync11(managedPython())) {
|
|
5863
5863
|
return null;
|
|
5864
5864
|
}
|
|
5865
5865
|
return sha256File4(modelPath2()) === MODEL_SHA2562 ? manifest : null;
|
|
@@ -5914,8 +5914,8 @@ async function preprocessingProbe(python2, workerScript) {
|
|
|
5914
5914
|
try {
|
|
5915
5915
|
const output2 = await execFileText2(python2, [workerScript, "--preprocessing-probe"], { timeout: 3e4, env: managedEnv3() });
|
|
5916
5916
|
const event = JSON.parse(output2.trim().split(/\r?\n/).pop() || "{}");
|
|
5917
|
-
if (event.type !== "preprocessing_probe" || event.implementation !== FBANK_IMPLEMENTATION || event.
|
|
5918
|
-
return { verified: false, error: "managed NumPy Kaldi fbank
|
|
5917
|
+
if (event.type !== "preprocessing_probe" || event.implementation !== FBANK_IMPLEMENTATION || event.validation?.validation !== FBANK_VALIDATION) {
|
|
5918
|
+
return { verified: false, error: "managed NumPy Kaldi fbank did not satisfy the pinned versioned calibration invariants" };
|
|
5919
5919
|
}
|
|
5920
5920
|
return { verified: true };
|
|
5921
5921
|
} catch (error) {
|
|
@@ -6089,7 +6089,7 @@ async function getSpeakerEmbeddingReadiness() {
|
|
|
6089
6089
|
packages: manifest?.packages,
|
|
6090
6090
|
preprocessing: {
|
|
6091
6091
|
implementation: FBANK_IMPLEMENTATION,
|
|
6092
|
-
|
|
6092
|
+
validation: FBANK_VALIDATION,
|
|
6093
6093
|
verified: runtime2.preprocessingVerified
|
|
6094
6094
|
},
|
|
6095
6095
|
providers: runtime2.providers,
|
|
@@ -6120,7 +6120,7 @@ async function ensureSpeakerEmbeddingSetup() {
|
|
|
6120
6120
|
await installPinnedSpeakerRuntimeDependencies(python2);
|
|
6121
6121
|
const runtime2 = await probeRuntime(python2, findWorkerScript4());
|
|
6122
6122
|
if (!runtime2.available || runtime2.onnxruntimeVersion !== ONNXRUNTIME_VERSION || runtime2.numpyVersion !== NUMPY_VERSION || !runtime2.preprocessingVerified || !runtime2.providers.includes("CPUExecutionProvider") || usesSystemSitePackages()) {
|
|
6123
|
-
throw new Error(`Managed CPU NumPy Kaldi-fbank validation failed after speaker setup${runtime2.error ? `: ${runtime2.error}` : ""}. The private speaker venv must prove pinned NumPy ${NUMPY_VERSION}, ONNX Runtime ${ONNXRUNTIME_VERSION}, the bundled preprocessing
|
|
6123
|
+
throw new Error(`Managed CPU NumPy Kaldi-fbank validation failed after speaker setup${runtime2.error ? `: ${runtime2.error}` : ""}. The private speaker venv must prove pinned NumPy ${NUMPY_VERSION}, ONNX Runtime ${ONNXRUNTIME_VERSION}, the bundled versioned preprocessing invariants, and CPUExecutionProvider without importing Torch or Torchaudio.`);
|
|
6124
6124
|
}
|
|
6125
6125
|
await downloadChecked2(MODEL_URL2, modelPath2(), MODEL_SHA2562);
|
|
6126
6126
|
const manifest = {
|
|
@@ -6134,7 +6134,7 @@ async function ensureSpeakerEmbeddingSetup() {
|
|
|
6134
6134
|
packages: speakerRuntimeManifest(),
|
|
6135
6135
|
preprocessing: {
|
|
6136
6136
|
implementation: FBANK_IMPLEMENTATION,
|
|
6137
|
-
|
|
6137
|
+
validation: FBANK_VALIDATION
|
|
6138
6138
|
},
|
|
6139
6139
|
pythonVersion: runtime2.pythonVersion ?? bootstrapRuntime.pythonVersion,
|
|
6140
6140
|
installedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -6234,7 +6234,7 @@ async function activateSpeakerEmbedding() {
|
|
|
6234
6234
|
}
|
|
6235
6235
|
});
|
|
6236
6236
|
});
|
|
6237
|
-
if (ready.pid !== worker2.pid || ready.backend !== "onnxruntime-cpu" || ready.provider !== "CPUExecutionProvider" || ready.model !== MODEL_NAME2 || ready.model_digest !== manifest.modelDigest || ready.dimension !== 512 || !ready.warmed || ready.preprocessing !== FBANK_IMPLEMENTATION || ready.
|
|
6237
|
+
if (ready.pid !== worker2.pid || ready.backend !== "onnxruntime-cpu" || ready.provider !== "CPUExecutionProvider" || ready.model !== MODEL_NAME2 || ready.model_digest !== manifest.modelDigest || ready.dimension !== 512 || !ready.warmed || ready.preprocessing !== FBANK_IMPLEMENTATION || ready.preprocessing_validation?.validation !== FBANK_VALIDATION) {
|
|
6238
6238
|
throw new Error("Speaker embedding worker did not report the pinned CPU CAM++ runtime");
|
|
6239
6239
|
}
|
|
6240
6240
|
state3.ready = ready;
|
|
@@ -6372,7 +6372,7 @@ async function bootstrapSpeakerEmbeddingRuntime() {
|
|
|
6372
6372
|
await ensureSpeakerEmbeddingSetup();
|
|
6373
6373
|
return activateSpeakerEmbedding();
|
|
6374
6374
|
}
|
|
6375
|
-
var RUNTIME_VERSION3, MODEL_NAME2, MODEL_REVISION2, MODEL_FILE, MODEL_URL2, MODEL_SHA2562, ONNXRUNTIME_VERSION, ONNXRUNTIME_WHEEL, ONNXRUNTIME_WHEEL_SHA256, NUMPY_VERSION, NUMPY_WHEEL, NUMPY_WHEEL_SHA256, FBANK_IMPLEMENTATION,
|
|
6375
|
+
var RUNTIME_VERSION3, MODEL_NAME2, MODEL_REVISION2, MODEL_FILE, MODEL_URL2, MODEL_SHA2562, ONNXRUNTIME_VERSION, ONNXRUNTIME_WHEEL, ONNXRUNTIME_WHEEL_SHA256, NUMPY_VERSION, NUMPY_WHEEL, NUMPY_WHEEL_SHA256, FBANK_IMPLEMENTATION, FBANK_VALIDATION, SETUP_TIMEOUT_MS3, START_TIMEOUT_MS3, EMBED_TIMEOUT_MS2, WESPEAKER_CAMPP_ONNX_SHA256, WESPEAKER_CAMPP_MODEL_REVISION, WESPEAKER_CAMPP_ONNXRUNTIME_VERSION, WESPEAKER_CAMPP_NUMPY_VERSION, WESPEAKER_CAMPP_FBANK_IMPLEMENTATION, WESPEAKER_CAMPP_FBANK_VALIDATION, WESPEAKER_CAMPP_EMBEDDING_SET_SLUG, SPEAKER_RUNTIME_WHEELS, state3;
|
|
6376
6376
|
var init_speaker_embedding_runtime = __esm({
|
|
6377
6377
|
"packages/execution/dist/speaker-embedding-runtime.js"() {
|
|
6378
6378
|
"use strict";
|
|
@@ -6380,7 +6380,7 @@ var init_speaker_embedding_runtime = __esm({
|
|
|
6380
6380
|
init_process_async();
|
|
6381
6381
|
init_model_store();
|
|
6382
6382
|
init_venv_paths();
|
|
6383
|
-
RUNTIME_VERSION3 =
|
|
6383
|
+
RUNTIME_VERSION3 = 5;
|
|
6384
6384
|
MODEL_NAME2 = "wespeaker-voxceleb-campplus";
|
|
6385
6385
|
MODEL_REVISION2 = "acf623ad8ca746e50baa432255cf8fc57c669c45";
|
|
6386
6386
|
MODEL_FILE = "voxceleb_CAM++.onnx";
|
|
@@ -6393,7 +6393,7 @@ var init_speaker_embedding_runtime = __esm({
|
|
|
6393
6393
|
NUMPY_WHEEL = "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";
|
|
6394
6394
|
NUMPY_WHEEL_SHA256 = "d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4";
|
|
6395
6395
|
FBANK_IMPLEMENTATION = "kaldi-fbank-numpy-v1";
|
|
6396
|
-
|
|
6396
|
+
FBANK_VALIDATION = "kaldi-fbank-invariants-v2";
|
|
6397
6397
|
SETUP_TIMEOUT_MS3 = 10 * 6e4;
|
|
6398
6398
|
START_TIMEOUT_MS3 = 6e4;
|
|
6399
6399
|
EMBED_TIMEOUT_MS2 = 45e3;
|
|
@@ -6402,6 +6402,7 @@ var init_speaker_embedding_runtime = __esm({
|
|
|
6402
6402
|
WESPEAKER_CAMPP_ONNXRUNTIME_VERSION = ONNXRUNTIME_VERSION;
|
|
6403
6403
|
WESPEAKER_CAMPP_NUMPY_VERSION = NUMPY_VERSION;
|
|
6404
6404
|
WESPEAKER_CAMPP_FBANK_IMPLEMENTATION = FBANK_IMPLEMENTATION;
|
|
6405
|
+
WESPEAKER_CAMPP_FBANK_VALIDATION = FBANK_VALIDATION;
|
|
6405
6406
|
WESPEAKER_CAMPP_EMBEDDING_SET_SLUG = "speaker-identity-wespeaker-voxceleb-campplus-acf623ad8ca746e50baa432255cf8fc57c669c45-fbank80-cmn-fullclip-l2";
|
|
6406
6407
|
SPEAKER_RUNTIME_WHEELS = [
|
|
6407
6408
|
{
|
|
@@ -345916,6 +345917,55 @@ function ocrPythonEnv(extra = {}) {
|
|
|
345916
345917
|
...extra
|
|
345917
345918
|
};
|
|
345918
345919
|
}
|
|
345920
|
+
function assessAdvancedOcrEvidence(input) {
|
|
345921
|
+
const text3 = String(input.text ?? "").trim();
|
|
345922
|
+
const chars = text3.length;
|
|
345923
|
+
const confidence2 = Number.isFinite(input.confidence) ? input.confidence : 0;
|
|
345924
|
+
const lines = Number.isFinite(input.lines) ? input.lines : 0;
|
|
345925
|
+
if (chars === 0) {
|
|
345926
|
+
return { state: "low_information", accepted: false, reason: "no_readable_text", chars, confidence: confidence2, lines };
|
|
345927
|
+
}
|
|
345928
|
+
const alnum = [...text3].filter((character) => /[\p{L}\p{N}]/u.test(character)).length;
|
|
345929
|
+
const alnumRatio2 = alnum / Math.max(1, chars);
|
|
345930
|
+
if (chars < MIN_SUBSTANTIVE_OCR_CHARS && confidence2 < MIN_ACCEPTED_OCR_CONFIDENCE) {
|
|
345931
|
+
return {
|
|
345932
|
+
state: "low_information",
|
|
345933
|
+
accepted: false,
|
|
345934
|
+
reason: "insufficient_low_confidence_text",
|
|
345935
|
+
chars,
|
|
345936
|
+
confidence: confidence2,
|
|
345937
|
+
lines,
|
|
345938
|
+
alnum_ratio: Number(alnumRatio2.toFixed(3))
|
|
345939
|
+
};
|
|
345940
|
+
}
|
|
345941
|
+
if (chars >= HIGH_VOLUME_GARBAGE_CHARS && confidence2 < HIGH_VOLUME_GARBAGE_CONFIDENCE || confidence2 < MIN_ACCEPTED_OCR_CONFIDENCE || alnumRatio2 < 0.45) {
|
|
345942
|
+
return {
|
|
345943
|
+
state: "rejected",
|
|
345944
|
+
accepted: false,
|
|
345945
|
+
reason: chars >= HIGH_VOLUME_GARBAGE_CHARS && confidence2 < HIGH_VOLUME_GARBAGE_CONFIDENCE ? "high_volume_low_confidence_text" : "low_confidence_or_symbol_heavy_text",
|
|
345946
|
+
chars,
|
|
345947
|
+
confidence: confidence2,
|
|
345948
|
+
lines,
|
|
345949
|
+
alnum_ratio: Number(alnumRatio2.toFixed(3))
|
|
345950
|
+
};
|
|
345951
|
+
}
|
|
345952
|
+
return {
|
|
345953
|
+
state: "accepted",
|
|
345954
|
+
accepted: true,
|
|
345955
|
+
reason: "confidence_and_text_quality_met",
|
|
345956
|
+
chars,
|
|
345957
|
+
confidence: confidence2,
|
|
345958
|
+
lines,
|
|
345959
|
+
alnum_ratio: Number(alnumRatio2.toFixed(3))
|
|
345960
|
+
};
|
|
345961
|
+
}
|
|
345962
|
+
function isTerminalSmallCropOcrRejection(evidence) {
|
|
345963
|
+
if (evidence.state !== "rejected")
|
|
345964
|
+
return false;
|
|
345965
|
+
if (evidence.reason === "high_volume_low_confidence_text")
|
|
345966
|
+
return true;
|
|
345967
|
+
return evidence.chars >= TERMINAL_SYMBOL_GARBAGE_CHARS && (evidence.alnum_ratio ?? 1) < TERMINAL_SYMBOL_GARBAGE_ALNUM_RATIO;
|
|
345968
|
+
}
|
|
345919
345969
|
function ocrDiagnostic(code8, message2, extra = {}) {
|
|
345920
345970
|
return {
|
|
345921
345971
|
schema: "omnius.ocr-diagnostic.v1",
|
|
@@ -345934,7 +345984,7 @@ function parsePipelineDiagnostic(error) {
|
|
|
345934
345984
|
return null;
|
|
345935
345985
|
}
|
|
345936
345986
|
}
|
|
345937
|
-
var OCR_PIPELINE_TIMEOUT_MS, OCR_PROCESS_DRAIN_MS, OcrImageAdvancedTool;
|
|
345987
|
+
var OCR_PIPELINE_TIMEOUT_MS, OCR_PROCESS_DRAIN_MS, MIN_ACCEPTED_OCR_CONFIDENCE, MIN_SUBSTANTIVE_OCR_CHARS, HIGH_VOLUME_GARBAGE_CHARS, HIGH_VOLUME_GARBAGE_CONFIDENCE, TERMINAL_SYMBOL_GARBAGE_CHARS, TERMINAL_SYMBOL_GARBAGE_ALNUM_RATIO, OcrImageAdvancedTool;
|
|
345938
345988
|
var init_ocr_image_advanced = __esm({
|
|
345939
345989
|
"packages/execution/dist/tools/ocr-image-advanced.js"() {
|
|
345940
345990
|
"use strict";
|
|
@@ -345942,6 +345992,12 @@ var init_ocr_image_advanced = __esm({
|
|
|
345942
345992
|
init_ocr_advanced_runtime();
|
|
345943
345993
|
OCR_PIPELINE_TIMEOUT_MS = 8e4;
|
|
345944
345994
|
OCR_PROCESS_DRAIN_MS = 3e3;
|
|
345995
|
+
MIN_ACCEPTED_OCR_CONFIDENCE = 50;
|
|
345996
|
+
MIN_SUBSTANTIVE_OCR_CHARS = 12;
|
|
345997
|
+
HIGH_VOLUME_GARBAGE_CHARS = 64;
|
|
345998
|
+
HIGH_VOLUME_GARBAGE_CONFIDENCE = 35;
|
|
345999
|
+
TERMINAL_SYMBOL_GARBAGE_CHARS = 24;
|
|
346000
|
+
TERMINAL_SYMBOL_GARBAGE_ALNUM_RATIO = 0.2;
|
|
345945
346001
|
OcrImageAdvancedTool = class {
|
|
345946
346002
|
workingDir;
|
|
345947
346003
|
name = "ocr_image_advanced";
|
|
@@ -346136,7 +346192,10 @@ var init_ocr_image_advanced = __esm({
|
|
|
346136
346192
|
stage: result.diagnostic?.stage ?? "pipeline",
|
|
346137
346193
|
deadline_ms: result.diagnostic?.deadline_ms ?? OCR_PIPELINE_TIMEOUT_MS,
|
|
346138
346194
|
attempts_completed: result.diagnostic?.attempts_completed ?? null,
|
|
346139
|
-
attempts_planned: result.diagnostic?.attempts_planned ?? null
|
|
346195
|
+
attempts_planned: result.diagnostic?.attempts_planned ?? null,
|
|
346196
|
+
evidence: result.diagnostic?.evidence ?? null,
|
|
346197
|
+
terminal_small_crop_rejection: result.diagnostic?.terminal_small_crop_rejection ?? null,
|
|
346198
|
+
attempt_cap_seconds: result.diagnostic?.attempt_cap_seconds ?? null
|
|
346140
346199
|
})
|
|
346141
346200
|
};
|
|
346142
346201
|
}
|
|
@@ -346160,22 +346219,62 @@ var init_ocr_image_advanced = __esm({
|
|
|
346160
346219
|
data: result
|
|
346161
346220
|
};
|
|
346162
346221
|
}
|
|
346222
|
+
const boundaryEvidence = assessAdvancedOcrEvidence({
|
|
346223
|
+
text: result.text,
|
|
346224
|
+
confidence: result.confidence,
|
|
346225
|
+
lines: result.lines
|
|
346226
|
+
});
|
|
346227
|
+
if (boundaryEvidence.state === "rejected") {
|
|
346228
|
+
return {
|
|
346229
|
+
success: false,
|
|
346230
|
+
output: "",
|
|
346231
|
+
error: "OCR evidence rejected: low-confidence text was suppressed and is not presented as extracted evidence.",
|
|
346232
|
+
durationMs: performance.now() - start2,
|
|
346233
|
+
data: ocrDiagnostic("ocr_evidence_rejected", "OCR output did not meet evidence-quality requirements.", {
|
|
346234
|
+
evidence: boundaryEvidence,
|
|
346235
|
+
terminal_small_crop_rejection: isTerminalSmallCropOcrRejection(boundaryEvidence)
|
|
346236
|
+
})
|
|
346237
|
+
};
|
|
346238
|
+
}
|
|
346239
|
+
const report2 = {
|
|
346240
|
+
...result,
|
|
346241
|
+
...boundaryEvidence.state === "low_information" ? { text: "", chars: 0, lines: 0, score: 0 } : {},
|
|
346242
|
+
quality: {
|
|
346243
|
+
...result.quality ?? {},
|
|
346244
|
+
...boundaryEvidence,
|
|
346245
|
+
schema: "omnius.ocr-evidence.v1"
|
|
346246
|
+
},
|
|
346247
|
+
...boundaryEvidence.state === "low_information" && !result.diagnostic ? {
|
|
346248
|
+
diagnostic: {
|
|
346249
|
+
schema: "omnius.ocr-diagnostic.v1",
|
|
346250
|
+
code: "ocr_low_information",
|
|
346251
|
+
message: "OCR produced no accepted readable text; the result is low-information rather than evidence.",
|
|
346252
|
+
stage: "evidence_quality"
|
|
346253
|
+
}
|
|
346254
|
+
} : {}
|
|
346255
|
+
};
|
|
346163
346256
|
const parts = [];
|
|
346164
|
-
parts.push(`OCR extracted from ${basename11(imagePath)} (${
|
|
346165
|
-
|
|
346166
|
-
|
|
346167
|
-
|
|
346168
|
-
|
|
346257
|
+
parts.push(`OCR extracted from ${basename11(imagePath)} (${report2.image_size})`);
|
|
346258
|
+
if (boundaryEvidence.state === "low_information") {
|
|
346259
|
+
parts.push(`OCR found no accepted readable text (${boundaryEvidence.reason}); low-information result.`);
|
|
346260
|
+
} else {
|
|
346261
|
+
parts.push(`Best variant: ${report2.variant} (confidence: ${report2.confidence}%, ${report2.chars} chars, ${report2.lines} lines, score: ${report2.score})`);
|
|
346262
|
+
}
|
|
346263
|
+
parts.push(`Variants tested: ${report2.variants_tested}`);
|
|
346264
|
+
if (report2.output_files) {
|
|
346265
|
+
const files = report2.output_files;
|
|
346169
346266
|
const saved = [files.txt, files.csv, files.pdf].filter(Boolean);
|
|
346170
346267
|
parts.push(`Output files: ${saved.join(", ")}`);
|
|
346171
346268
|
}
|
|
346172
|
-
|
|
346173
|
-
|
|
346174
|
-
|
|
346175
|
-
|
|
346269
|
+
if (boundaryEvidence.state === "accepted") {
|
|
346270
|
+
parts.push("");
|
|
346271
|
+
parts.push("--- Extracted Text ---");
|
|
346272
|
+
parts.push(report2.text);
|
|
346273
|
+
}
|
|
346274
|
+
if (report2.regions) {
|
|
346176
346275
|
parts.push("");
|
|
346177
346276
|
parts.push("--- Region Extraction ---");
|
|
346178
|
-
for (const [rname, rtext] of Object.entries(
|
|
346277
|
+
for (const [rname, rtext] of Object.entries(report2.regions)) {
|
|
346179
346278
|
if (rtext) {
|
|
346180
346279
|
parts.push(`
|
|
346181
346280
|
[${rname.toUpperCase()}]`);
|
|
@@ -346183,8 +346282,8 @@ var init_ocr_image_advanced = __esm({
|
|
|
346183
346282
|
}
|
|
346184
346283
|
}
|
|
346185
346284
|
}
|
|
346186
|
-
if (
|
|
346187
|
-
const sorted = Object.entries(
|
|
346285
|
+
if (report2.all_variants) {
|
|
346286
|
+
const sorted = Object.entries(report2.all_variants).filter(([_, v]) => v.chars > 0).sort((a2, b) => b[1].score - a2[1].score).slice(0, 5);
|
|
346188
346287
|
if (sorted.length > 1) {
|
|
346189
346288
|
parts.push("");
|
|
346190
346289
|
parts.push("--- Top Variants ---");
|
|
@@ -346201,7 +346300,7 @@ var init_ocr_image_advanced = __esm({
|
|
|
346201
346300
|
success: true,
|
|
346202
346301
|
output: parts.join("\n"),
|
|
346203
346302
|
durationMs: performance.now() - start2,
|
|
346204
|
-
data:
|
|
346303
|
+
data: report2
|
|
346205
346304
|
};
|
|
346206
346305
|
} catch (err) {
|
|
346207
346306
|
const pipelineResult = parsePipelineDiagnostic(err);
|
|
@@ -346215,7 +346314,8 @@ var init_ocr_image_advanced = __esm({
|
|
|
346215
346314
|
stage: pipelineResult.diagnostic?.stage ?? "pipeline",
|
|
346216
346315
|
deadline_ms: pipelineResult.diagnostic?.deadline_ms ?? OCR_PIPELINE_TIMEOUT_MS,
|
|
346217
346316
|
attempts_completed: pipelineResult.diagnostic?.attempts_completed ?? null,
|
|
346218
|
-
attempts_planned: pipelineResult.diagnostic?.attempts_planned ?? null
|
|
346317
|
+
attempts_planned: pipelineResult.diagnostic?.attempts_planned ?? null,
|
|
346318
|
+
evidence: pipelineResult.diagnostic?.evidence ?? null
|
|
346219
346319
|
})
|
|
346220
346320
|
};
|
|
346221
346321
|
}
|
|
@@ -616978,6 +617078,7 @@ __export(dist_exports2, {
|
|
|
616978
617078
|
VisualTriggerTool: () => VisualTriggerTool,
|
|
616979
617079
|
WESPEAKER_CAMPP_EMBEDDING_SET_SLUG: () => WESPEAKER_CAMPP_EMBEDDING_SET_SLUG,
|
|
616980
617080
|
WESPEAKER_CAMPP_FBANK_IMPLEMENTATION: () => WESPEAKER_CAMPP_FBANK_IMPLEMENTATION,
|
|
617081
|
+
WESPEAKER_CAMPP_FBANK_VALIDATION: () => WESPEAKER_CAMPP_FBANK_VALIDATION,
|
|
616981
617082
|
WESPEAKER_CAMPP_MODEL_REVISION: () => WESPEAKER_CAMPP_MODEL_REVISION,
|
|
616982
617083
|
WESPEAKER_CAMPP_NUMPY_VERSION: () => WESPEAKER_CAMPP_NUMPY_VERSION,
|
|
616983
617084
|
WESPEAKER_CAMPP_ONNXRUNTIME_VERSION: () => WESPEAKER_CAMPP_ONNXRUNTIME_VERSION,
|
|
@@ -687469,8 +687570,11 @@ import { existsSync as existsSync128, mkdirSync as mkdirSync79, readFileSync as
|
|
|
687469
687570
|
import { dirname as dirname51, join as join140 } from "node:path";
|
|
687470
687571
|
import { homedir as homedir41 } from "node:os";
|
|
687471
687572
|
import { fileURLToPath as fileURLToPath25 } from "node:url";
|
|
687573
|
+
function managedAsrRuntimeRoot() {
|
|
687574
|
+
return process.env["OMNIUS_ASR_RUNTIME_ROOT"]?.trim() || MANAGED_ASR_ROOT;
|
|
687575
|
+
}
|
|
687472
687576
|
function managedAsrStateFile(engineId) {
|
|
687473
|
-
return join140(
|
|
687577
|
+
return join140(managedAsrRuntimeRoot(), engineId, "models.json");
|
|
687474
687578
|
}
|
|
687475
687579
|
function managedAsrModelDir(engineId) {
|
|
687476
687580
|
return join140(MANAGED_ASR_MODEL_ROOT, engineId);
|
|
@@ -687483,16 +687587,18 @@ function readManagedAsrState(engineId) {
|
|
|
687483
687587
|
}
|
|
687484
687588
|
}
|
|
687485
687589
|
function persistManagedAsrReadiness(readiness) {
|
|
687486
|
-
const root = join140(
|
|
687590
|
+
const root = join140(managedAsrRuntimeRoot(), readiness.engineId);
|
|
687487
687591
|
mkdirSync79(root, { recursive: true });
|
|
687488
687592
|
const state5 = readManagedAsrState(readiness.engineId);
|
|
687489
687593
|
state5.models ??= {};
|
|
687490
687594
|
state5.models[readiness.modelId] = {
|
|
687491
687595
|
installed: readiness.installed,
|
|
687492
687596
|
weightsReady: readiness.weightsReady,
|
|
687597
|
+
active: readiness.active,
|
|
687493
687598
|
device: readiness.device,
|
|
687494
687599
|
lastError: readiness.lastError,
|
|
687495
|
-
pulledAt: readiness.pulledAt
|
|
687600
|
+
pulledAt: readiness.pulledAt,
|
|
687601
|
+
activeAt: readiness.activeAt
|
|
687496
687602
|
};
|
|
687497
687603
|
writeFileSync70(managedAsrStateFile(readiness.engineId), `${JSON.stringify(state5, null, 2)}
|
|
687498
687604
|
`, "utf8");
|
|
@@ -687885,17 +687991,34 @@ ${log22.slice(-2e3)}` };
|
|
|
687885
687991
|
function getManagedAsrReadiness(engineId, modelId) {
|
|
687886
687992
|
assertManagedAsrModel(engineId, modelId);
|
|
687887
687993
|
const saved = readManagedAsrState(engineId).models?.[modelId];
|
|
687994
|
+
const active = saved?.active === true && saved.weightsReady === true && typeof saved.activeAt === "string" && /^cuda(?::\d+)?$/i.test(saved.device ?? "");
|
|
687888
687995
|
return {
|
|
687889
687996
|
engineId,
|
|
687890
687997
|
modelId,
|
|
687891
687998
|
installed: Boolean(saved?.installed && existsSync128(getVenvPython())),
|
|
687892
687999
|
weightsReady: Boolean(saved?.weightsReady),
|
|
687893
|
-
active
|
|
688000
|
+
// Managed direct ASR has no resident process. `active` therefore means
|
|
688001
|
+
// this exact model last completed a CUDA inference successfully, not that
|
|
688002
|
+
// a Python child happens to still be alive after a request returned.
|
|
688003
|
+
active,
|
|
687894
688004
|
device: saved?.device,
|
|
687895
688005
|
lastError: saved?.lastError,
|
|
687896
|
-
pulledAt: saved?.pulledAt
|
|
688006
|
+
pulledAt: saved?.pulledAt,
|
|
688007
|
+
activeAt: saved?.activeAt
|
|
687897
688008
|
};
|
|
687898
688009
|
}
|
|
688010
|
+
function recordManagedAsrInference(input, outcome) {
|
|
688011
|
+
const prior = getManagedAsrReadiness(input.engineId, input.modelId);
|
|
688012
|
+
const cuda = typeof input.device === "string" && /^cuda(?::\d+)?$/i.test(input.device);
|
|
688013
|
+
const activeAt = outcome.ok && cuda ? (/* @__PURE__ */ new Date()).toISOString() : void 0;
|
|
688014
|
+
persistManagedAsrReadiness({
|
|
688015
|
+
...prior,
|
|
688016
|
+
device: input.device || prior.device,
|
|
688017
|
+
active: Boolean(activeAt),
|
|
688018
|
+
activeAt,
|
|
688019
|
+
lastError: outcome.ok ? void 0 : outcome.error
|
|
688020
|
+
});
|
|
688021
|
+
}
|
|
687899
688022
|
function ensureManagedAsrModel(input) {
|
|
687900
688023
|
const engineId = input.engineId.trim().toLowerCase();
|
|
687901
688024
|
const modelId = input.modelId.trim().toLowerCase();
|
|
@@ -688058,12 +688181,16 @@ function transcribeManagedNemotronFile(input) {
|
|
|
688058
688181
|
const ready = events.find((event) => event["type"] === "ready");
|
|
688059
688182
|
const error = events.find((event) => event["type"] === "error");
|
|
688060
688183
|
if (result.status !== 0 || !transcript) {
|
|
688061
|
-
|
|
688184
|
+
const message2 = String(error?.["message"] ?? (`${result.stderr || ""}`.trim() || result.error || "Nemotron file transcription failed"));
|
|
688185
|
+
recordManagedAsrInference({ engineId: "nemotron-streaming", modelId, device: existing.device }, { ok: false, error: message2 });
|
|
688186
|
+
throw new Error(message2);
|
|
688062
688187
|
}
|
|
688188
|
+
const device2 = typeof ready?.["device"] === "string" ? ready["device"] : existing.device;
|
|
688189
|
+
recordManagedAsrInference({ engineId: "nemotron-streaming", modelId, device: device2 }, { ok: true });
|
|
688063
688190
|
return {
|
|
688064
688191
|
text: String(transcript["text"] ?? ""),
|
|
688065
688192
|
duration: Number.isFinite(Number(transcript["audioSeconds"])) ? Number(transcript["audioSeconds"]) : null,
|
|
688066
|
-
device:
|
|
688193
|
+
device: device2,
|
|
688067
688194
|
segments: [],
|
|
688068
688195
|
warnings: []
|
|
688069
688196
|
};
|
|
@@ -688102,6 +688229,7 @@ function transcribeManagedWhisperFile(input) {
|
|
|
688102
688229
|
const payload = [...parseWorkerEvents(`${result.stdout || ""}`)].reverse().find((event) => event["ok"] === true);
|
|
688103
688230
|
if (result.status !== 0 || !payload) {
|
|
688104
688231
|
const detail = `${result.stderr || ""}`.trim() || String(result.error ?? "Whisper file transcription failed");
|
|
688232
|
+
recordManagedAsrInference({ engineId: "openai-whisper", modelId, device: existing.device }, { ok: false, error: detail });
|
|
688105
688233
|
throw new Error(detail);
|
|
688106
688234
|
}
|
|
688107
688235
|
const segments = Array.isArray(payload["segments"]) ? payload["segments"].flatMap((segment) => {
|
|
@@ -688113,10 +688241,12 @@ function transcribeManagedWhisperFile(input) {
|
|
|
688113
688241
|
text: String(value2["text"] ?? "")
|
|
688114
688242
|
}];
|
|
688115
688243
|
}) : [];
|
|
688244
|
+
const device2 = typeof payload["device"] === "string" ? payload["device"] : existing.device;
|
|
688245
|
+
recordManagedAsrInference({ engineId: "openai-whisper", modelId, device: device2 }, { ok: true });
|
|
688116
688246
|
return {
|
|
688117
688247
|
text: String(payload["text"] ?? ""),
|
|
688118
688248
|
duration: Number.isFinite(Number(payload["duration"])) ? Number(payload["duration"]) : null,
|
|
688119
|
-
device:
|
|
688249
|
+
device: device2,
|
|
688120
688250
|
segments,
|
|
688121
688251
|
warnings: []
|
|
688122
688252
|
};
|
|
@@ -688152,17 +688282,21 @@ function transcribeManagedAsrFile(input) {
|
|
|
688152
688282
|
const payload = [...events].reverse().find((event) => event["type"] === "transcript");
|
|
688153
688283
|
if (result.status !== 0 || !payload) {
|
|
688154
688284
|
const workerError = [...events].reverse().find((event) => event["type"] === "error");
|
|
688155
|
-
|
|
688285
|
+
const message2 = String(workerError?.["message"] ?? result.stderr ?? result.error ?? "Voxtral file transcription failed");
|
|
688286
|
+
recordManagedAsrInference({ engineId, modelId, device: existing.device }, { ok: false, error: message2 });
|
|
688287
|
+
throw new Error(message2);
|
|
688156
688288
|
}
|
|
688157
688289
|
const segments = Array.isArray(payload["segments"]) ? payload["segments"].flatMap((segment) => {
|
|
688158
688290
|
if (!segment || typeof segment !== "object") return [];
|
|
688159
688291
|
const value2 = segment;
|
|
688160
688292
|
return [{ start: Number(value2["start"] ?? 0), end: Number(value2["end"] ?? 0), text: String(value2["text"] ?? "") }];
|
|
688161
688293
|
}) : [];
|
|
688294
|
+
const device2 = typeof payload["device"] === "string" ? payload["device"] : existing.device;
|
|
688295
|
+
recordManagedAsrInference({ engineId, modelId, device: device2 }, { ok: true });
|
|
688162
688296
|
return {
|
|
688163
688297
|
text: String(payload["text"] ?? ""),
|
|
688164
688298
|
duration: Number.isFinite(Number(payload["duration"])) ? Number(payload["duration"]) : null,
|
|
688165
|
-
device:
|
|
688299
|
+
device: device2,
|
|
688166
688300
|
segments,
|
|
688167
688301
|
warnings: []
|
|
688168
688302
|
};
|
|
@@ -688625,6 +688759,11 @@ function updateListenLiveState(patch) {
|
|
|
688625
688759
|
function getListenLiveState() {
|
|
688626
688760
|
return listenLiveState;
|
|
688627
688761
|
}
|
|
688762
|
+
function childProcessIsRunning(process4) {
|
|
688763
|
+
return Boolean(
|
|
688764
|
+
process4 && process4.exitCode === null && process4.signalCode === null && !process4.killed
|
|
688765
|
+
);
|
|
688766
|
+
}
|
|
688628
688767
|
function clamp0112(value2) {
|
|
688629
688768
|
if (!Number.isFinite(value2)) return 0;
|
|
688630
688769
|
return Math.max(0, Math.min(1, value2));
|
|
@@ -689286,7 +689425,7 @@ var init_listen = __esm({
|
|
|
689286
689425
|
lastVisibleVadAt = 0;
|
|
689287
689426
|
stderrTail = "";
|
|
689288
689427
|
get ready() {
|
|
689289
|
-
return this._ready;
|
|
689428
|
+
return this._ready && childProcessIsRunning(this.process);
|
|
689290
689429
|
}
|
|
689291
689430
|
async start() {
|
|
689292
689431
|
let pyPath = "python3";
|
|
@@ -689425,17 +689564,32 @@ ${text3}`.slice(-2e3);
|
|
|
689425
689564
|
});
|
|
689426
689565
|
onChildError(this.process, (err) => {
|
|
689427
689566
|
clearTimeout(timeout2);
|
|
689428
|
-
|
|
689567
|
+
const wasReady = this._ready;
|
|
689568
|
+
this._ready = false;
|
|
689569
|
+
this.process = null;
|
|
689570
|
+
if (wasReady) {
|
|
689571
|
+
updateListenLiveState({ phase: "error", lastStatus: err.message.slice(0, 160) });
|
|
689572
|
+
this.emit("error", err);
|
|
689573
|
+
} else {
|
|
689574
|
+
reject(err);
|
|
689575
|
+
}
|
|
689429
689576
|
});
|
|
689430
689577
|
onChildClose(this.process, (code8) => {
|
|
689431
689578
|
this.unregisterBrokerModel();
|
|
689432
|
-
|
|
689579
|
+
const wasReady = this._ready;
|
|
689580
|
+
this._ready = false;
|
|
689581
|
+
this.process = null;
|
|
689582
|
+
if (!wasReady) {
|
|
689433
689583
|
clearTimeout(timeout2);
|
|
689434
689584
|
reject(
|
|
689435
689585
|
new Error(
|
|
689436
689586
|
`${this.engineId} worker exited with code ${code8} before ready` + (this.stderrTail ? `: ${this.stderrTail.trim()}` : "")
|
|
689437
689587
|
)
|
|
689438
689588
|
);
|
|
689589
|
+
} else {
|
|
689590
|
+
const message2 = `${this.engineId} worker exited with code ${code8}` + (this.stderrTail ? `: ${this.stderrTail.trim()}` : "");
|
|
689591
|
+
updateListenLiveState({ phase: "error", lastStatus: message2.slice(0, 160) });
|
|
689592
|
+
this.emit("error", new Error(message2));
|
|
689439
689593
|
}
|
|
689440
689594
|
});
|
|
689441
689595
|
});
|
|
@@ -689500,6 +689654,10 @@ ${text3}`.slice(-2e3);
|
|
|
689500
689654
|
liveTranscriber = null;
|
|
689501
689655
|
// TranscribeLive from transcribe-cli or WhisperFallbackTranscriber
|
|
689502
689656
|
active = false;
|
|
689657
|
+
/** Set only after the selected worker has emitted/finished its ready signal. */
|
|
689658
|
+
liveWorkerReady = false;
|
|
689659
|
+
/** Identity captured when that worker became ready; never infer it from a later selection. */
|
|
689660
|
+
runningSelection = null;
|
|
689503
689661
|
paused = false;
|
|
689504
689662
|
// Pause state for voicechat (stops mic but keeps transcriber)
|
|
689505
689663
|
silenceTimer = null;
|
|
@@ -689542,6 +689700,27 @@ ${text3}`.slice(-2e3);
|
|
|
689542
689700
|
get currentMode() {
|
|
689543
689701
|
return this.config.mode;
|
|
689544
689702
|
}
|
|
689703
|
+
/**
|
|
689704
|
+
* Runtime truth for a resident live ASR pipeline. This does not claim that
|
|
689705
|
+
* a selected-but-unstarted model is ready, and it verifies fallback worker
|
|
689706
|
+
* process liveness rather than trusting an old `ready` event.
|
|
689707
|
+
*/
|
|
689708
|
+
get liveRuntimeReadiness() {
|
|
689709
|
+
const fallbackHealthy = !(this.liveTranscriber instanceof WhisperFallbackTranscriber) || this.liveTranscriber.ready === true;
|
|
689710
|
+
const workerReady = Boolean(
|
|
689711
|
+
this.active && this.liveWorkerReady && this.runningSelection && this.liveTranscriber && fallbackHealthy
|
|
689712
|
+
);
|
|
689713
|
+
return {
|
|
689714
|
+
// Keep the captured identity while reporting a failed worker so the
|
|
689715
|
+
// daemon can surface its error against the exact selection rather than
|
|
689716
|
+
// silently falling back to persisted weights state.
|
|
689717
|
+
engineId: this.runningSelection?.engineId ?? null,
|
|
689718
|
+
modelId: this.runningSelection?.modelId ?? null,
|
|
689719
|
+
workerReady,
|
|
689720
|
+
micActive: workerReady && !this.paused && childProcessIsRunning(this.micProcess),
|
|
689721
|
+
paused: this.paused
|
|
689722
|
+
};
|
|
689723
|
+
}
|
|
689545
689724
|
get pendingTranscript() {
|
|
689546
689725
|
return this.pendingText;
|
|
689547
689726
|
}
|
|
@@ -689812,6 +689991,8 @@ ${text3}`.slice(-2e3);
|
|
|
689812
689991
|
*/
|
|
689813
689992
|
async start() {
|
|
689814
689993
|
if (this.active) return "Already listening.";
|
|
689994
|
+
this.liveWorkerReady = false;
|
|
689995
|
+
this.runningSelection = null;
|
|
689815
689996
|
const unavailableReason = getAsrEngineUnavailableReason(this.config.engineId);
|
|
689816
689997
|
if (unavailableReason) return unavailableReason;
|
|
689817
689998
|
if (this.config.engineId === "vibevoice-transformers") {
|
|
@@ -689941,6 +690122,8 @@ ${text3}`.slice(-2e3);
|
|
|
689941
690122
|
}
|
|
689942
690123
|
);
|
|
689943
690124
|
this.liveTranscriber.on("error", (err) => {
|
|
690125
|
+
this.liveWorkerReady = false;
|
|
690126
|
+
updateListenLiveState({ phase: "error", lastStatus: err.message.slice(0, 160) });
|
|
689944
690127
|
this.emit("error", err);
|
|
689945
690128
|
});
|
|
689946
690129
|
await new Promise((resolve90, reject) => {
|
|
@@ -689999,6 +690182,8 @@ ${text3}`.slice(-2e3);
|
|
|
689999
690182
|
if (this.config.mode === "auto") this.resetSilenceTimer();
|
|
690000
690183
|
});
|
|
690001
690184
|
fallback.on("error", (err) => {
|
|
690185
|
+
this.liveWorkerReady = false;
|
|
690186
|
+
updateListenLiveState({ phase: "error", lastStatus: err.message.slice(0, 160) });
|
|
690002
690187
|
this.emit("error", err);
|
|
690003
690188
|
});
|
|
690004
690189
|
this.liveTranscriber = fallback;
|
|
@@ -690011,6 +690196,11 @@ transcribe-cli error: ${transcribeCliError}` : "";
|
|
|
690011
690196
|
}
|
|
690012
690197
|
}
|
|
690013
690198
|
this.active = true;
|
|
690199
|
+
this.liveWorkerReady = true;
|
|
690200
|
+
this.runningSelection = {
|
|
690201
|
+
engineId: this.config.engineId,
|
|
690202
|
+
modelId: this.config.modelId
|
|
690203
|
+
};
|
|
690014
690204
|
this.paused = false;
|
|
690015
690205
|
this.spawnMicProcess(micCmd);
|
|
690016
690206
|
this.blinkState = true;
|
|
@@ -690037,6 +690227,8 @@ transcribe-cli error: ${transcribeCliError}` : "";
|
|
|
690037
690227
|
async stop() {
|
|
690038
690228
|
if (!this.active) return "Not listening.";
|
|
690039
690229
|
this.active = false;
|
|
690230
|
+
this.liveWorkerReady = false;
|
|
690231
|
+
this.runningSelection = null;
|
|
690040
690232
|
this.owners.clear();
|
|
690041
690233
|
this.blinkState = false;
|
|
690042
690234
|
this.micWaterfall = [];
|
|
@@ -792931,6 +793123,13 @@ function getRuntimeStatus() {
|
|
|
792931
793123
|
const persisted = _listenEngine ? null : loadGlobalSettings();
|
|
792932
793124
|
const asrEngineId = _listenEngine?.currentEngine ?? persisted?.asrEngine ?? "openai-whisper";
|
|
792933
793125
|
const asrModelId = _listenEngine?.currentModel ?? persisted?.asrModel ?? "medium";
|
|
793126
|
+
const liveAsr = _listenEngine?.liveRuntimeReadiness ?? null;
|
|
793127
|
+
const selectedLivePipeline = Boolean(
|
|
793128
|
+
liveAsr?.engineId === asrEngineId && liveAsr?.modelId === asrModelId
|
|
793129
|
+
);
|
|
793130
|
+
const selectedLiveWorker = selectedLivePipeline && liveAsr?.workerReady === true;
|
|
793131
|
+
const selectedLiveMic = selectedLiveWorker && liveAsr?.micActive === true;
|
|
793132
|
+
const managedAsr = asrEngineId === "openai-whisper" || asrEngineId === "nemotron-streaming" || asrEngineId === "voxtral-transformers" ? getManagedAsrReadiness(asrEngineId, asrModelId) : null;
|
|
792934
793133
|
const vibePhase = vibe.active ? "ready" : vibe.lastError ? "error" : vibe.weightsReady ? "weights-ready" : vibe.installed ? "runtime-installed" : "not-installed";
|
|
792935
793134
|
return {
|
|
792936
793135
|
state: _state3,
|
|
@@ -792942,9 +793141,9 @@ function getRuntimeStatus() {
|
|
|
792942
793141
|
listenPaused: _listenEngine?.isPaused ?? false,
|
|
792943
793142
|
asrEngineId,
|
|
792944
793143
|
asrModelId,
|
|
792945
|
-
asrBackend: asrEngineId === "vibevoice-transformers" ? "vibevoice-transformers" : listenState.backend,
|
|
792946
|
-
asrPhase: asrEngineId === "vibevoice-transformers" ? vibePhase : listenState.phase,
|
|
792947
|
-
asrReady: asrEngineId === "vibevoice-transformers" ? vibe.active : Boolean(_listenEngine?.isActive),
|
|
793144
|
+
asrBackend: asrEngineId === "vibevoice-transformers" ? "vibevoice-transformers" : managedAsr ? asrEngineId : listenState.backend,
|
|
793145
|
+
asrPhase: asrEngineId === "vibevoice-transformers" ? vibePhase : selectedLiveMic ? "listening" : selectedLiveWorker && liveAsr?.paused && listenState.phase === "paused" ? "paused" : selectedLivePipeline && listenState.phase === "error" ? "error" : managedAsr?.active ? "ready" : managedAsr?.lastError ? "error" : managedAsr?.weightsReady ? "weights-ready" : "not-started",
|
|
793146
|
+
asrReady: asrEngineId === "vibevoice-transformers" ? vibe.active : managedAsr ? managedAsr.active || selectedLiveMic : Boolean(_listenEngine?.isActive),
|
|
792948
793147
|
micDevice: listenState.micDevice || null,
|
|
792949
793148
|
micSourceKind: listenState.micSourceKind,
|
|
792950
793149
|
micSourceChannels: listenState.micSourceChannels,
|
|
@@ -793173,6 +793372,7 @@ var init_voice_runtime = __esm({
|
|
|
793173
793372
|
init_listen();
|
|
793174
793373
|
init_dist5();
|
|
793175
793374
|
init_voicechat();
|
|
793375
|
+
init_py_embed();
|
|
793176
793376
|
_voiceEngine = null;
|
|
793177
793377
|
_listenEngine = null;
|
|
793178
793378
|
_voiceChatSession = null;
|
|
@@ -828251,8 +828451,10 @@ function getOpenApiSpec() {
|
|
|
828251
828451
|
include_daemon_tools: { type: "array", items: { type: "string", enum: ["read", "run", "admin"] }, description: "Scopes from which Omnius may offer its bounded core daemon-tool catalog. Use daemon_tool_names to request other exact tools." },
|
|
828252
828452
|
daemon_tool_names: { type: "array", items: { type: "string" }, description: "Optional exact allowlist of daemon tool names. Use this to keep local-model tool prompts small." },
|
|
828253
828453
|
agent_timeout_s: { type: "number", minimum: 5, maximum: 600, default: 45, description: "Total server-side agent-loop deadline. Distinct from per-backend timeout_s." },
|
|
828454
|
+
agent_max_tool_rounds: { type: "integer", minimum: 1, maximum: 8, default: 1, description: "Maximum daemon-tool planning rounds. The default executes one tool round, then removes daemon schemas for lower-latency final synthesis." },
|
|
828455
|
+
agent_prefetch_web_search: { type: "boolean", default: false, description: "Explicitly execute authorized web_search with the latest user text before one backend synthesis. factual-first enables this automatically." },
|
|
828254
828456
|
max_turns: { type: "integer", description: "Q2 — agent_loop max iterations (default 8, max 64)." },
|
|
828255
|
-
prompt_template: { type: "string", enum: ["factual-first"], description: "
|
|
828457
|
+
prompt_template: { type: "string", enum: ["factual-first"], description: "Factual-first prefetches authorized web_search from the latest user turn and performs one grounded synthesis generation." }
|
|
828256
828458
|
} } } } },
|
|
828257
828459
|
responses: { 200: { description: "OpenAI chat.completion shape, SSE if stream=true. agent_loop responses include _agent_loop:{turns,log,done,reason,elapsed_ms,backend_transport}." }, 504: { description: "Backend round or total agent-loop deadline expired" }, 508: { description: "The model repeated an identical daemon tool call" }, ...ErrorResponses }
|
|
828258
828460
|
}
|
|
@@ -828873,7 +829075,7 @@ function getOpenApiSpec() {
|
|
|
828873
829075
|
post: {
|
|
828874
829076
|
summary: "Provision and activate one role-typed audio embedding runtime",
|
|
828875
829077
|
tags: ["Audio"],
|
|
828876
|
-
description: "Admin-only. Requires kind=acoustic|speaker|semantic in the query (canonical) or JSON body. acoustic reuses the pinned JetPack YAMNet/TensorRT setup; speaker creates a private CPython 3.10 CPU-only WeSpeaker CAM++ venv, installs only checksum-pinned NumPy, ONNX Runtime, and its locked CPU support wheels with --no-deps/--no-index, then downloads the pinned model. Its 80-bin Kaldi-compatible NumPy fbank is validated with a
|
|
829078
|
+
description: "Admin-only. Requires kind=acoustic|speaker|semantic in the query (canonical) or JSON body. acoustic reuses the pinned JetPack YAMNet/TensorRT setup; speaker creates a private CPython 3.10 CPU-only WeSpeaker CAM++ venv, installs only checksum-pinned NumPy, ONNX Runtime, and its locked CPU support wheels with --no-deps/--no-index, then downloads the pinned model. Its 80-bin Kaldi-compatible NumPy fbank is validated with a versioned cross-platform numeric-invariant probe before readiness. The speaker path never imports, links, replaces, or otherwise depends on JetPack Torch/Torchaudio, so an incompatible generic Torchaudio wheel cannot affect CUDA-enabled Egg Torch. semantic installs the isolated JetPack CUDA CLAP dependencies and pinned model. This is the only REST operation allowed to provision. It activates and warms only the requested role worker; no role is substituted for another.",
|
|
828877
829079
|
parameters: [{ name: "kind", in: "query", required: false, schema: { type: "string", enum: ["acoustic", "speaker", "semantic"] }, description: "Canonical required role selector; JSON body.kind is accepted for compatibility." }],
|
|
828878
829080
|
requestBody: { required: false, content: { "application/json": { schema: { type: "object", properties: { kind: { type: "string", enum: ["acoustic", "speaker", "semantic"] } } } } } },
|
|
828879
829081
|
responses: { 200: { description: "Requested role runtime provisioned and warm." }, 202: { description: "Provisioning finished but requested role did not prove warm; poll its health URL." }, 400: { description: "kind is missing or invalid." }, 403: { description: "Admin scope required." }, 500: { description: "Role-specific setup/preflight failed." } }
|
|
@@ -830485,6 +830687,32 @@ function getAgentLoopTotalTimeoutMs(body) {
|
|
|
830485
830687
|
AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS
|
|
830486
830688
|
);
|
|
830487
830689
|
}
|
|
830690
|
+
function getAgentLoopPlannerMaxTokens() {
|
|
830691
|
+
return boundedModelCheckInteger(
|
|
830692
|
+
Number(process.env["OMNIUS_AGENT_LOOP_PLANNER_MAX_TOKENS"]),
|
|
830693
|
+
AGENT_LOOP_PLANNER_DEFAULT_MAX_TOKENS,
|
|
830694
|
+
32,
|
|
830695
|
+
512
|
|
830696
|
+
);
|
|
830697
|
+
}
|
|
830698
|
+
function getAgentLoopMaxToolRounds(body) {
|
|
830699
|
+
const requested = body["agent_max_tool_rounds"];
|
|
830700
|
+
const configured = Number(process.env["OMNIUS_AGENT_LOOP_MAX_TOOL_ROUNDS"]);
|
|
830701
|
+
return boundedModelCheckInteger(
|
|
830702
|
+
requested ?? configured,
|
|
830703
|
+
AGENT_LOOP_MAX_TOOL_ROUNDS_DEFAULT,
|
|
830704
|
+
1,
|
|
830705
|
+
8
|
|
830706
|
+
);
|
|
830707
|
+
}
|
|
830708
|
+
function getAgentLoopToolResultMaxChars() {
|
|
830709
|
+
return boundedModelCheckInteger(
|
|
830710
|
+
Number(process.env["OMNIUS_AGENT_LOOP_TOOL_RESULT_MAX_CHARS"]),
|
|
830711
|
+
AGENT_LOOP_TOOL_RESULT_DEFAULT_MAX_CHARS,
|
|
830712
|
+
1e3,
|
|
830713
|
+
32e3
|
|
830714
|
+
);
|
|
830715
|
+
}
|
|
830488
830716
|
function compactAgentLoopToolResult(value2) {
|
|
830489
830717
|
let encoded;
|
|
830490
830718
|
try {
|
|
@@ -830492,11 +830720,12 @@ function compactAgentLoopToolResult(value2) {
|
|
|
830492
830720
|
} catch {
|
|
830493
830721
|
encoded = JSON.stringify({ success: false, error: "tool result was not JSON serializable" });
|
|
830494
830722
|
}
|
|
830495
|
-
|
|
830723
|
+
const maxChars = getAgentLoopToolResultMaxChars();
|
|
830724
|
+
if (encoded.length <= maxChars) return encoded;
|
|
830496
830725
|
return JSON.stringify({
|
|
830497
830726
|
truncated: true,
|
|
830498
830727
|
original_chars: encoded.length,
|
|
830499
|
-
preview: encoded.slice(0,
|
|
830728
|
+
preview: encoded.slice(0, maxChars)
|
|
830500
830729
|
});
|
|
830501
830730
|
}
|
|
830502
830731
|
function parseJsonObject8(value2) {
|
|
@@ -832987,6 +833216,7 @@ async function runAgentLoopChatCompletions(opts) {
|
|
|
832987
833216
|
const targetTransport = requestProviderTransport(targetUrl, route?.endpoint);
|
|
832988
833217
|
const useNativeOllamaChat = targetTransport.protocol === "ollama";
|
|
832989
833218
|
const maxTurns = typeof requestBody["max_turns"] === "number" ? Math.max(1, Math.min(64, requestBody["max_turns"])) : 8;
|
|
833219
|
+
const maxDaemonToolRounds = getAgentLoopMaxToolRounds(requestBody);
|
|
832990
833220
|
const remoteIp = (req3.socket?.remoteAddress || "").replace(/^::ffff:/, "");
|
|
832991
833221
|
const origin = /^(127\.\d+\.\d+\.\d+|::1|localhost)$/.test(remoteIp) ? "loopback" : "remote";
|
|
832992
833222
|
const reqAuth = req3;
|
|
@@ -833118,6 +833348,77 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833118
833348
|
const chatId = `chatcmpl-${randomBytes30(12).toString("hex")}`;
|
|
833119
833349
|
const turnsLog = [];
|
|
833120
833350
|
const seenDaemonCalls = /* @__PURE__ */ new Set();
|
|
833351
|
+
let daemonToolRoundsExecuted = 0;
|
|
833352
|
+
const prefetchWebSearch = (promptTemplate === "factual-first" || requestBody["agent_prefetch_web_search"] === true) && advertisedDaemonNames.has("web_search") && callerTools.length === 0 && !messages2.some((message2) => message2.role === "tool" || (message2.tool_calls?.length ?? 0) > 0);
|
|
833353
|
+
const latestUserQuery = [...messages2].reverse().find((message2) => message2.role === "user" && typeof message2.content === "string")?.content?.trim().slice(0, 1e3);
|
|
833354
|
+
if (prefetchWebSearch && latestUserQuery) {
|
|
833355
|
+
const meta = daemonTools.get("web_search");
|
|
833356
|
+
let tool = null;
|
|
833357
|
+
if (meta) {
|
|
833358
|
+
try {
|
|
833359
|
+
tool = new meta.ToolClass(process.cwd());
|
|
833360
|
+
} catch {
|
|
833361
|
+
try {
|
|
833362
|
+
tool = new meta.ToolClass();
|
|
833363
|
+
} catch {
|
|
833364
|
+
tool = null;
|
|
833365
|
+
}
|
|
833366
|
+
}
|
|
833367
|
+
}
|
|
833368
|
+
if (tool) {
|
|
833369
|
+
const callId = `prefetch_web_search_${randomBytes30(6).toString("hex")}`;
|
|
833370
|
+
const args = { query: latestUserQuery };
|
|
833371
|
+
const fingerprint3 = `web_search:${JSON.stringify(args)}`;
|
|
833372
|
+
messages2.push({
|
|
833373
|
+
role: "assistant",
|
|
833374
|
+
content: "",
|
|
833375
|
+
tool_calls: [{
|
|
833376
|
+
id: callId,
|
|
833377
|
+
type: "function",
|
|
833378
|
+
function: { name: "web_search", arguments: JSON.stringify(args) }
|
|
833379
|
+
}]
|
|
833380
|
+
});
|
|
833381
|
+
let toolResult;
|
|
833382
|
+
try {
|
|
833383
|
+
activeTool = tool;
|
|
833384
|
+
const remainingMs = totalDeadline - Date.now();
|
|
833385
|
+
if (remainingMs <= 0) throw new Error("agent loop deadline expired before factual-first search");
|
|
833386
|
+
const execution = await executeAgentLoopToolBounded(
|
|
833387
|
+
tool,
|
|
833388
|
+
args,
|
|
833389
|
+
Math.min(getAgentLoopToolTimeoutMs(), remainingMs),
|
|
833390
|
+
clientDisconnect
|
|
833391
|
+
);
|
|
833392
|
+
if (execution.clientDisconnected || isClientDisconnected()) return;
|
|
833393
|
+
toolResult = execution.result;
|
|
833394
|
+
} catch (error) {
|
|
833395
|
+
if (isClientDisconnected()) return;
|
|
833396
|
+
toolResult = {
|
|
833397
|
+
success: false,
|
|
833398
|
+
output: "",
|
|
833399
|
+
error: error instanceof Error ? error.message : String(error),
|
|
833400
|
+
durationMs: 0
|
|
833401
|
+
};
|
|
833402
|
+
} finally {
|
|
833403
|
+
activeTool = null;
|
|
833404
|
+
}
|
|
833405
|
+
messages2.push({
|
|
833406
|
+
role: "tool",
|
|
833407
|
+
tool_call_id: callId,
|
|
833408
|
+
name: "web_search",
|
|
833409
|
+
content: compactAgentLoopToolResult(toolResult)
|
|
833410
|
+
});
|
|
833411
|
+
seenDaemonCalls.add(fingerprint3);
|
|
833412
|
+
daemonToolRoundsExecuted = 1;
|
|
833413
|
+
turnsLog.push({
|
|
833414
|
+
turn: 0,
|
|
833415
|
+
tool_calls: 1,
|
|
833416
|
+
daemon_executed: 1,
|
|
833417
|
+
client_yielded: 0,
|
|
833418
|
+
prefetched: true
|
|
833419
|
+
});
|
|
833420
|
+
}
|
|
833421
|
+
}
|
|
833121
833422
|
for (let turn = 1; turn <= maxTurns; turn++) {
|
|
833122
833423
|
if (isClientDisconnected()) return;
|
|
833123
833424
|
if (Date.now() > totalDeadline) {
|
|
@@ -833142,15 +833443,19 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833142
833443
|
return;
|
|
833143
833444
|
}
|
|
833144
833445
|
const transportMessages = forceNoThinkForTools ? messages2.map((message3) => typeof message3.content === "string" ? { ...message3, content: stripNoThinkPromptDirectives(message3.content) } : message3) : messages2;
|
|
833446
|
+
const daemonToolsForTurn = daemonToolRoundsExecuted < maxDaemonToolRounds ? daemonToolEntries : [];
|
|
833447
|
+
const toolsForTurn = [...callerTools, ...daemonToolsForTurn];
|
|
833448
|
+
const planningTurn = toolsForTurn.length > 0;
|
|
833145
833449
|
const ollamaOptions = {};
|
|
833146
833450
|
if (typeof requestBody["temperature"] === "number") ollamaOptions["temperature"] = requestBody["temperature"];
|
|
833147
833451
|
if (typeof requestBody["top_p"] === "number") ollamaOptions["top_p"] = requestBody["top_p"];
|
|
833148
|
-
if (
|
|
833452
|
+
if (planningTurn) ollamaOptions["num_predict"] = getAgentLoopPlannerMaxTokens();
|
|
833453
|
+
else if (typeof requestBody["max_tokens"] === "number") ollamaOptions["num_predict"] = requestBody["max_tokens"];
|
|
833149
833454
|
if (typeof requestBody["seed"] === "number") ollamaOptions["seed"] = requestBody["seed"];
|
|
833150
833455
|
const turnBody = useNativeOllamaChat ? {
|
|
833151
833456
|
model: originalModel,
|
|
833152
833457
|
messages: agentLoopMessagesForOllama(transportMessages),
|
|
833153
|
-
tools:
|
|
833458
|
+
tools: toolsForTurn.length > 0 ? toolsForTurn : void 0,
|
|
833154
833459
|
stream: false,
|
|
833155
833460
|
think: false,
|
|
833156
833461
|
options: ollamaOptions
|
|
@@ -833158,7 +833463,8 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833158
833463
|
...requestBody,
|
|
833159
833464
|
model: originalModel,
|
|
833160
833465
|
messages: transportMessages,
|
|
833161
|
-
tools:
|
|
833466
|
+
tools: toolsForTurn.length > 0 ? toolsForTurn : void 0,
|
|
833467
|
+
...planningTurn ? { max_tokens: getAgentLoopPlannerMaxTokens() } : {},
|
|
833162
833468
|
stream: false,
|
|
833163
833469
|
...forceNoThinkForTools ? { think: false } : {},
|
|
833164
833470
|
// Strip our own loop-control fields so they don't confuse the backend
|
|
@@ -833166,6 +833472,8 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833166
833472
|
include_daemon_tools: void 0,
|
|
833167
833473
|
daemon_tool_names: void 0,
|
|
833168
833474
|
agent_timeout_s: void 0,
|
|
833475
|
+
agent_max_tool_rounds: void 0,
|
|
833476
|
+
agent_prefetch_web_search: void 0,
|
|
833169
833477
|
max_turns: void 0,
|
|
833170
833478
|
timeout_s: void 0,
|
|
833171
833479
|
realtime: void 0,
|
|
@@ -833432,6 +833740,7 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833432
833740
|
daemon_executed: executed,
|
|
833433
833741
|
client_yielded: 0
|
|
833434
833742
|
});
|
|
833743
|
+
if (executed > 0) daemonToolRoundsExecuted += 1;
|
|
833435
833744
|
}
|
|
833436
833745
|
jsonResponse(res, 200, {
|
|
833437
833746
|
id: chatId,
|
|
@@ -837627,7 +837936,10 @@ data: ${JSON.stringify(data)}
|
|
|
837627
837936
|
const baseReadiness = engine.id === "vibevoice-transformers" ? vibe : getManagedAsrReadiness(engine.id, selectedModel ?? engine.models[0].id);
|
|
837628
837937
|
const readiness = {
|
|
837629
837938
|
...baseReadiness,
|
|
837630
|
-
|
|
837939
|
+
// Direct managed ASR is request-scoped rather than a resident child
|
|
837940
|
+
// process. Expose its successful CUDA inference only for the exact
|
|
837941
|
+
// selected engine/model; never promote an unselected model to active.
|
|
837942
|
+
active: unavailableReason ? false : selected && baseReadiness.active,
|
|
837631
837943
|
disabled: Boolean(unavailableReason),
|
|
837632
837944
|
...unavailableReason ? { unavailableReason, lastError: unavailableReason } : {}
|
|
837633
837945
|
};
|
|
@@ -837641,7 +837953,7 @@ data: ${JSON.stringify(data)}
|
|
|
837641
837953
|
const modelBaseReadiness = engine.id === "vibevoice-transformers" ? vibe : getManagedAsrReadiness(engine.id, model.id);
|
|
837642
837954
|
const modelReadiness = {
|
|
837643
837955
|
...modelBaseReadiness,
|
|
837644
|
-
active: unavailableReason ? false : modelBaseReadiness.active,
|
|
837956
|
+
active: unavailableReason ? false : selected && model.id === listen.currentModel && modelBaseReadiness.active,
|
|
837645
837957
|
disabled: Boolean(unavailableReason),
|
|
837646
837958
|
...unavailableReason ? { unavailableReason, lastError: unavailableReason } : {}
|
|
837647
837959
|
};
|
|
@@ -837683,9 +837995,11 @@ data: ${JSON.stringify(data)}
|
|
|
837683
837995
|
runtimes: listAsrEngines().filter((engine) => engine.id === "openai-whisper" || engine.id === "nemotron-streaming" || engine.id === "voxtral-transformers").flatMap((engine) => engine.models.map((model) => {
|
|
837684
837996
|
const readiness = getManagedAsrReadiness(engine.id, model.id);
|
|
837685
837997
|
const unavailableReason = getAsrEngineUnavailableReason(engine.id);
|
|
837998
|
+
const selected = engine.id === listen.currentEngine && model.id === listen.currentModel;
|
|
837686
837999
|
return {
|
|
837687
838000
|
...readiness,
|
|
837688
|
-
active: unavailableReason ? false : readiness.active,
|
|
838001
|
+
active: unavailableReason ? false : selected && readiness.active,
|
|
838002
|
+
selected,
|
|
837689
838003
|
disabled: Boolean(unavailableReason),
|
|
837690
838004
|
...unavailableReason ? { unavailableReason, lastError: unavailableReason } : {}
|
|
837691
838005
|
};
|
|
@@ -841937,7 +842251,7 @@ function setTimerEnabled(name10, enabled2) {
|
|
|
841937
842251
|
return false;
|
|
841938
842252
|
}
|
|
841939
842253
|
}
|
|
841940
|
-
var require4, NEXUS_DIRECTORY_ORIGIN3, NEXUS_SPONSORS_URL3, SERVER_BOOT_IDENTITY, voiceTtsTransactionTail, endpointRegistry, modelRouteMap, endpointUsage, _lastEndpointDiagnostics, BACKEND_TIMEOUT_DEFAULT_MS, BACKEND_TIMEOUT_MAX_MS, AGENT_LOOP_TOTAL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOTAL_TIMEOUT_MIN_MS, AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS, AGENT_LOOP_TOOL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOOL_TIMEOUT_MAX_MS,
|
|
842254
|
+
var require4, NEXUS_DIRECTORY_ORIGIN3, NEXUS_SPONSORS_URL3, SERVER_BOOT_IDENTITY, voiceTtsTransactionTail, endpointRegistry, modelRouteMap, endpointUsage, _lastEndpointDiagnostics, BACKEND_TIMEOUT_DEFAULT_MS, BACKEND_TIMEOUT_MAX_MS, AGENT_LOOP_TOTAL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOTAL_TIMEOUT_MIN_MS, AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS, AGENT_LOOP_TOOL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOOL_TIMEOUT_MAX_MS, AGENT_LOOP_TOOL_RESULT_DEFAULT_MAX_CHARS, AGENT_LOOP_PLANNER_DEFAULT_MAX_TOKENS, AGENT_LOOP_MAX_TOOL_ROUNDS_DEFAULT, DEFAULT_AGENT_LOOP_DAEMON_TOOL_NAMES, MODEL_LIST_TIMEOUT_DEFAULT_MS, MODEL_CHECK_TIMEOUT_DEFAULT_MS, MODEL_CHECK_TIMEOUT_MAX_MS, MODEL_CHECK_MAX_TOKENS, cancelledAgentLoopTools, metrics, startedAt, FILE_MIME_BY_EXT, realtimeOllamaFallbackCache, runningProcesses, LOCAL_UI_SESSION_COOKIE, LOCAL_UI_SESSION_TOKEN, perKeyUsage, CRON_MARKER2;
|
|
841941
842255
|
var init_serve = __esm({
|
|
841942
842256
|
"packages/cli/src/api/serve.ts"() {
|
|
841943
842257
|
init_config();
|
|
@@ -841998,7 +842312,9 @@ var init_serve = __esm({
|
|
|
841998
842312
|
AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS = 10 * 60 * 1e3;
|
|
841999
842313
|
AGENT_LOOP_TOOL_TIMEOUT_DEFAULT_MS = 3e4;
|
|
842000
842314
|
AGENT_LOOP_TOOL_TIMEOUT_MAX_MS = 12e4;
|
|
842001
|
-
|
|
842315
|
+
AGENT_LOOP_TOOL_RESULT_DEFAULT_MAX_CHARS = 6e3;
|
|
842316
|
+
AGENT_LOOP_PLANNER_DEFAULT_MAX_TOKENS = 96;
|
|
842317
|
+
AGENT_LOOP_MAX_TOOL_ROUNDS_DEFAULT = 1;
|
|
842002
842318
|
DEFAULT_AGENT_LOOP_DAEMON_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
842003
842319
|
"web_search",
|
|
842004
842320
|
"web_fetch",
|