omnius 1.0.640 → 1.0.641
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +218 -50
- package/dist/scripts/audio-speaker-embedding-worker.py +38 -16
- package/dist/scripts/ocr-advanced.py +137 -8
- package/docs/DISCOVERY.json +8 -1
- package/docs/rest/endpoints/chat.md +2 -1
- package/docs/rest/endpoints/voice-vision.md +10 -2
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5859,7 +5859,7 @@ function usesSystemSitePackages() {
|
|
|
5859
5859
|
function loadManifest3() {
|
|
5860
5860
|
try {
|
|
5861
5861
|
const manifest = JSON.parse(readFileSync10(manifestPath3(), "utf8"));
|
|
5862
|
-
if (manifest.runtimeVersion !== RUNTIME_VERSION3 || manifest.model !== MODEL_NAME2 || manifest.modelRevision !== MODEL_REVISION2 || manifest.modelDigest !== `sha256:${MODEL_SHA2562}` || manifest.backend !== "onnxruntime-cpu" || manifest.onnxRuntimeVersion !== ONNXRUNTIME_VERSION || manifest.numpyVersion !== NUMPY_VERSION || manifest.preprocessing?.implementation !== FBANK_IMPLEMENTATION || manifest.preprocessing?.
|
|
5862
|
+
if (manifest.runtimeVersion !== RUNTIME_VERSION3 || manifest.model !== MODEL_NAME2 || manifest.modelRevision !== MODEL_REVISION2 || manifest.modelDigest !== `sha256:${MODEL_SHA2562}` || manifest.backend !== "onnxruntime-cpu" || manifest.onnxRuntimeVersion !== ONNXRUNTIME_VERSION || manifest.numpyVersion !== NUMPY_VERSION || manifest.preprocessing?.implementation !== FBANK_IMPLEMENTATION || manifest.preprocessing?.validation !== FBANK_VALIDATION || !hasExpectedSpeakerRuntimePackages(manifest.packages) || usesSystemSitePackages() || !existsSync11(modelPath2()) || !existsSync11(managedPython())) {
|
|
5863
5863
|
return null;
|
|
5864
5864
|
}
|
|
5865
5865
|
return sha256File4(modelPath2()) === MODEL_SHA2562 ? manifest : null;
|
|
@@ -5914,8 +5914,8 @@ async function preprocessingProbe(python2, workerScript) {
|
|
|
5914
5914
|
try {
|
|
5915
5915
|
const output2 = await execFileText2(python2, [workerScript, "--preprocessing-probe"], { timeout: 3e4, env: managedEnv3() });
|
|
5916
5916
|
const event = JSON.parse(output2.trim().split(/\r?\n/).pop() || "{}");
|
|
5917
|
-
if (event.type !== "preprocessing_probe" || event.implementation !== FBANK_IMPLEMENTATION || event.
|
|
5918
|
-
return { verified: false, error: "managed NumPy Kaldi fbank
|
|
5917
|
+
if (event.type !== "preprocessing_probe" || event.implementation !== FBANK_IMPLEMENTATION || event.validation?.validation !== FBANK_VALIDATION) {
|
|
5918
|
+
return { verified: false, error: "managed NumPy Kaldi fbank did not satisfy the pinned versioned calibration invariants" };
|
|
5919
5919
|
}
|
|
5920
5920
|
return { verified: true };
|
|
5921
5921
|
} catch (error) {
|
|
@@ -6089,7 +6089,7 @@ async function getSpeakerEmbeddingReadiness() {
|
|
|
6089
6089
|
packages: manifest?.packages,
|
|
6090
6090
|
preprocessing: {
|
|
6091
6091
|
implementation: FBANK_IMPLEMENTATION,
|
|
6092
|
-
|
|
6092
|
+
validation: FBANK_VALIDATION,
|
|
6093
6093
|
verified: runtime2.preprocessingVerified
|
|
6094
6094
|
},
|
|
6095
6095
|
providers: runtime2.providers,
|
|
@@ -6120,7 +6120,7 @@ async function ensureSpeakerEmbeddingSetup() {
|
|
|
6120
6120
|
await installPinnedSpeakerRuntimeDependencies(python2);
|
|
6121
6121
|
const runtime2 = await probeRuntime(python2, findWorkerScript4());
|
|
6122
6122
|
if (!runtime2.available || runtime2.onnxruntimeVersion !== ONNXRUNTIME_VERSION || runtime2.numpyVersion !== NUMPY_VERSION || !runtime2.preprocessingVerified || !runtime2.providers.includes("CPUExecutionProvider") || usesSystemSitePackages()) {
|
|
6123
|
-
throw new Error(`Managed CPU NumPy Kaldi-fbank validation failed after speaker setup${runtime2.error ? `: ${runtime2.error}` : ""}. The private speaker venv must prove pinned NumPy ${NUMPY_VERSION}, ONNX Runtime ${ONNXRUNTIME_VERSION}, the bundled preprocessing
|
|
6123
|
+
throw new Error(`Managed CPU NumPy Kaldi-fbank validation failed after speaker setup${runtime2.error ? `: ${runtime2.error}` : ""}. The private speaker venv must prove pinned NumPy ${NUMPY_VERSION}, ONNX Runtime ${ONNXRUNTIME_VERSION}, the bundled versioned preprocessing invariants, and CPUExecutionProvider without importing Torch or Torchaudio.`);
|
|
6124
6124
|
}
|
|
6125
6125
|
await downloadChecked2(MODEL_URL2, modelPath2(), MODEL_SHA2562);
|
|
6126
6126
|
const manifest = {
|
|
@@ -6134,7 +6134,7 @@ async function ensureSpeakerEmbeddingSetup() {
|
|
|
6134
6134
|
packages: speakerRuntimeManifest(),
|
|
6135
6135
|
preprocessing: {
|
|
6136
6136
|
implementation: FBANK_IMPLEMENTATION,
|
|
6137
|
-
|
|
6137
|
+
validation: FBANK_VALIDATION
|
|
6138
6138
|
},
|
|
6139
6139
|
pythonVersion: runtime2.pythonVersion ?? bootstrapRuntime.pythonVersion,
|
|
6140
6140
|
installedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -6234,7 +6234,7 @@ async function activateSpeakerEmbedding() {
|
|
|
6234
6234
|
}
|
|
6235
6235
|
});
|
|
6236
6236
|
});
|
|
6237
|
-
if (ready.pid !== worker2.pid || ready.backend !== "onnxruntime-cpu" || ready.provider !== "CPUExecutionProvider" || ready.model !== MODEL_NAME2 || ready.model_digest !== manifest.modelDigest || ready.dimension !== 512 || !ready.warmed || ready.preprocessing !== FBANK_IMPLEMENTATION || ready.
|
|
6237
|
+
if (ready.pid !== worker2.pid || ready.backend !== "onnxruntime-cpu" || ready.provider !== "CPUExecutionProvider" || ready.model !== MODEL_NAME2 || ready.model_digest !== manifest.modelDigest || ready.dimension !== 512 || !ready.warmed || ready.preprocessing !== FBANK_IMPLEMENTATION || ready.preprocessing_validation?.validation !== FBANK_VALIDATION) {
|
|
6238
6238
|
throw new Error("Speaker embedding worker did not report the pinned CPU CAM++ runtime");
|
|
6239
6239
|
}
|
|
6240
6240
|
state3.ready = ready;
|
|
@@ -6372,7 +6372,7 @@ async function bootstrapSpeakerEmbeddingRuntime() {
|
|
|
6372
6372
|
await ensureSpeakerEmbeddingSetup();
|
|
6373
6373
|
return activateSpeakerEmbedding();
|
|
6374
6374
|
}
|
|
6375
|
-
var RUNTIME_VERSION3, MODEL_NAME2, MODEL_REVISION2, MODEL_FILE, MODEL_URL2, MODEL_SHA2562, ONNXRUNTIME_VERSION, ONNXRUNTIME_WHEEL, ONNXRUNTIME_WHEEL_SHA256, NUMPY_VERSION, NUMPY_WHEEL, NUMPY_WHEEL_SHA256, FBANK_IMPLEMENTATION,
|
|
6375
|
+
var RUNTIME_VERSION3, MODEL_NAME2, MODEL_REVISION2, MODEL_FILE, MODEL_URL2, MODEL_SHA2562, ONNXRUNTIME_VERSION, ONNXRUNTIME_WHEEL, ONNXRUNTIME_WHEEL_SHA256, NUMPY_VERSION, NUMPY_WHEEL, NUMPY_WHEEL_SHA256, FBANK_IMPLEMENTATION, FBANK_VALIDATION, SETUP_TIMEOUT_MS3, START_TIMEOUT_MS3, EMBED_TIMEOUT_MS2, WESPEAKER_CAMPP_ONNX_SHA256, WESPEAKER_CAMPP_MODEL_REVISION, WESPEAKER_CAMPP_ONNXRUNTIME_VERSION, WESPEAKER_CAMPP_NUMPY_VERSION, WESPEAKER_CAMPP_FBANK_IMPLEMENTATION, WESPEAKER_CAMPP_FBANK_VALIDATION, WESPEAKER_CAMPP_EMBEDDING_SET_SLUG, SPEAKER_RUNTIME_WHEELS, state3;
|
|
6376
6376
|
var init_speaker_embedding_runtime = __esm({
|
|
6377
6377
|
"packages/execution/dist/speaker-embedding-runtime.js"() {
|
|
6378
6378
|
"use strict";
|
|
@@ -6380,7 +6380,7 @@ var init_speaker_embedding_runtime = __esm({
|
|
|
6380
6380
|
init_process_async();
|
|
6381
6381
|
init_model_store();
|
|
6382
6382
|
init_venv_paths();
|
|
6383
|
-
RUNTIME_VERSION3 =
|
|
6383
|
+
RUNTIME_VERSION3 = 5;
|
|
6384
6384
|
MODEL_NAME2 = "wespeaker-voxceleb-campplus";
|
|
6385
6385
|
MODEL_REVISION2 = "acf623ad8ca746e50baa432255cf8fc57c669c45";
|
|
6386
6386
|
MODEL_FILE = "voxceleb_CAM++.onnx";
|
|
@@ -6393,7 +6393,7 @@ var init_speaker_embedding_runtime = __esm({
|
|
|
6393
6393
|
NUMPY_WHEEL = "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl";
|
|
6394
6394
|
NUMPY_WHEEL_SHA256 = "d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4";
|
|
6395
6395
|
FBANK_IMPLEMENTATION = "kaldi-fbank-numpy-v1";
|
|
6396
|
-
|
|
6396
|
+
FBANK_VALIDATION = "kaldi-fbank-invariants-v2";
|
|
6397
6397
|
SETUP_TIMEOUT_MS3 = 10 * 6e4;
|
|
6398
6398
|
START_TIMEOUT_MS3 = 6e4;
|
|
6399
6399
|
EMBED_TIMEOUT_MS2 = 45e3;
|
|
@@ -6402,6 +6402,7 @@ var init_speaker_embedding_runtime = __esm({
|
|
|
6402
6402
|
WESPEAKER_CAMPP_ONNXRUNTIME_VERSION = ONNXRUNTIME_VERSION;
|
|
6403
6403
|
WESPEAKER_CAMPP_NUMPY_VERSION = NUMPY_VERSION;
|
|
6404
6404
|
WESPEAKER_CAMPP_FBANK_IMPLEMENTATION = FBANK_IMPLEMENTATION;
|
|
6405
|
+
WESPEAKER_CAMPP_FBANK_VALIDATION = FBANK_VALIDATION;
|
|
6405
6406
|
WESPEAKER_CAMPP_EMBEDDING_SET_SLUG = "speaker-identity-wespeaker-voxceleb-campplus-acf623ad8ca746e50baa432255cf8fc57c669c45-fbank80-cmn-fullclip-l2";
|
|
6406
6407
|
SPEAKER_RUNTIME_WHEELS = [
|
|
6407
6408
|
{
|
|
@@ -345916,6 +345917,48 @@ function ocrPythonEnv(extra = {}) {
|
|
|
345916
345917
|
...extra
|
|
345917
345918
|
};
|
|
345918
345919
|
}
|
|
345920
|
+
function assessAdvancedOcrEvidence(input) {
|
|
345921
|
+
const text3 = String(input.text ?? "").trim();
|
|
345922
|
+
const chars = text3.length;
|
|
345923
|
+
const confidence2 = Number.isFinite(input.confidence) ? input.confidence : 0;
|
|
345924
|
+
const lines = Number.isFinite(input.lines) ? input.lines : 0;
|
|
345925
|
+
if (chars === 0) {
|
|
345926
|
+
return { state: "low_information", accepted: false, reason: "no_readable_text", chars, confidence: confidence2, lines };
|
|
345927
|
+
}
|
|
345928
|
+
const alnum = [...text3].filter((character) => /[\p{L}\p{N}]/u.test(character)).length;
|
|
345929
|
+
const alnumRatio2 = alnum / Math.max(1, chars);
|
|
345930
|
+
if (chars < MIN_SUBSTANTIVE_OCR_CHARS && confidence2 < MIN_ACCEPTED_OCR_CONFIDENCE) {
|
|
345931
|
+
return {
|
|
345932
|
+
state: "low_information",
|
|
345933
|
+
accepted: false,
|
|
345934
|
+
reason: "insufficient_low_confidence_text",
|
|
345935
|
+
chars,
|
|
345936
|
+
confidence: confidence2,
|
|
345937
|
+
lines,
|
|
345938
|
+
alnum_ratio: Number(alnumRatio2.toFixed(3))
|
|
345939
|
+
};
|
|
345940
|
+
}
|
|
345941
|
+
if (chars >= HIGH_VOLUME_GARBAGE_CHARS && confidence2 < HIGH_VOLUME_GARBAGE_CONFIDENCE || confidence2 < MIN_ACCEPTED_OCR_CONFIDENCE || alnumRatio2 < 0.45) {
|
|
345942
|
+
return {
|
|
345943
|
+
state: "rejected",
|
|
345944
|
+
accepted: false,
|
|
345945
|
+
reason: chars >= HIGH_VOLUME_GARBAGE_CHARS && confidence2 < HIGH_VOLUME_GARBAGE_CONFIDENCE ? "high_volume_low_confidence_text" : "low_confidence_or_symbol_heavy_text",
|
|
345946
|
+
chars,
|
|
345947
|
+
confidence: confidence2,
|
|
345948
|
+
lines,
|
|
345949
|
+
alnum_ratio: Number(alnumRatio2.toFixed(3))
|
|
345950
|
+
};
|
|
345951
|
+
}
|
|
345952
|
+
return {
|
|
345953
|
+
state: "accepted",
|
|
345954
|
+
accepted: true,
|
|
345955
|
+
reason: "confidence_and_text_quality_met",
|
|
345956
|
+
chars,
|
|
345957
|
+
confidence: confidence2,
|
|
345958
|
+
lines,
|
|
345959
|
+
alnum_ratio: Number(alnumRatio2.toFixed(3))
|
|
345960
|
+
};
|
|
345961
|
+
}
|
|
345919
345962
|
function ocrDiagnostic(code8, message2, extra = {}) {
|
|
345920
345963
|
return {
|
|
345921
345964
|
schema: "omnius.ocr-diagnostic.v1",
|
|
@@ -345934,7 +345977,7 @@ function parsePipelineDiagnostic(error) {
|
|
|
345934
345977
|
return null;
|
|
345935
345978
|
}
|
|
345936
345979
|
}
|
|
345937
|
-
var OCR_PIPELINE_TIMEOUT_MS, OCR_PROCESS_DRAIN_MS, OcrImageAdvancedTool;
|
|
345980
|
+
var OCR_PIPELINE_TIMEOUT_MS, OCR_PROCESS_DRAIN_MS, MIN_ACCEPTED_OCR_CONFIDENCE, MIN_SUBSTANTIVE_OCR_CHARS, HIGH_VOLUME_GARBAGE_CHARS, HIGH_VOLUME_GARBAGE_CONFIDENCE, OcrImageAdvancedTool;
|
|
345938
345981
|
var init_ocr_image_advanced = __esm({
|
|
345939
345982
|
"packages/execution/dist/tools/ocr-image-advanced.js"() {
|
|
345940
345983
|
"use strict";
|
|
@@ -345942,6 +345985,10 @@ var init_ocr_image_advanced = __esm({
|
|
|
345942
345985
|
init_ocr_advanced_runtime();
|
|
345943
345986
|
OCR_PIPELINE_TIMEOUT_MS = 8e4;
|
|
345944
345987
|
OCR_PROCESS_DRAIN_MS = 3e3;
|
|
345988
|
+
MIN_ACCEPTED_OCR_CONFIDENCE = 50;
|
|
345989
|
+
MIN_SUBSTANTIVE_OCR_CHARS = 12;
|
|
345990
|
+
HIGH_VOLUME_GARBAGE_CHARS = 64;
|
|
345991
|
+
HIGH_VOLUME_GARBAGE_CONFIDENCE = 35;
|
|
345945
345992
|
OcrImageAdvancedTool = class {
|
|
345946
345993
|
workingDir;
|
|
345947
345994
|
name = "ocr_image_advanced";
|
|
@@ -346136,7 +346183,8 @@ var init_ocr_image_advanced = __esm({
|
|
|
346136
346183
|
stage: result.diagnostic?.stage ?? "pipeline",
|
|
346137
346184
|
deadline_ms: result.diagnostic?.deadline_ms ?? OCR_PIPELINE_TIMEOUT_MS,
|
|
346138
346185
|
attempts_completed: result.diagnostic?.attempts_completed ?? null,
|
|
346139
|
-
attempts_planned: result.diagnostic?.attempts_planned ?? null
|
|
346186
|
+
attempts_planned: result.diagnostic?.attempts_planned ?? null,
|
|
346187
|
+
evidence: result.diagnostic?.evidence ?? null
|
|
346140
346188
|
})
|
|
346141
346189
|
};
|
|
346142
346190
|
}
|
|
@@ -346160,22 +346208,61 @@ var init_ocr_image_advanced = __esm({
|
|
|
346160
346208
|
data: result
|
|
346161
346209
|
};
|
|
346162
346210
|
}
|
|
346211
|
+
const boundaryEvidence = assessAdvancedOcrEvidence({
|
|
346212
|
+
text: result.text,
|
|
346213
|
+
confidence: result.confidence,
|
|
346214
|
+
lines: result.lines
|
|
346215
|
+
});
|
|
346216
|
+
if (boundaryEvidence.state === "rejected") {
|
|
346217
|
+
return {
|
|
346218
|
+
success: false,
|
|
346219
|
+
output: "",
|
|
346220
|
+
error: "OCR evidence rejected: low-confidence text was suppressed and is not presented as extracted evidence.",
|
|
346221
|
+
durationMs: performance.now() - start2,
|
|
346222
|
+
data: ocrDiagnostic("ocr_evidence_rejected", "OCR output did not meet evidence-quality requirements.", {
|
|
346223
|
+
evidence: boundaryEvidence
|
|
346224
|
+
})
|
|
346225
|
+
};
|
|
346226
|
+
}
|
|
346227
|
+
const report2 = {
|
|
346228
|
+
...result,
|
|
346229
|
+
...boundaryEvidence.state === "low_information" ? { text: "", chars: 0, lines: 0, score: 0 } : {},
|
|
346230
|
+
quality: {
|
|
346231
|
+
...result.quality ?? {},
|
|
346232
|
+
...boundaryEvidence,
|
|
346233
|
+
schema: "omnius.ocr-evidence.v1"
|
|
346234
|
+
},
|
|
346235
|
+
...boundaryEvidence.state === "low_information" && !result.diagnostic ? {
|
|
346236
|
+
diagnostic: {
|
|
346237
|
+
schema: "omnius.ocr-diagnostic.v1",
|
|
346238
|
+
code: "ocr_low_information",
|
|
346239
|
+
message: "OCR produced no accepted readable text; the result is low-information rather than evidence.",
|
|
346240
|
+
stage: "evidence_quality"
|
|
346241
|
+
}
|
|
346242
|
+
} : {}
|
|
346243
|
+
};
|
|
346163
346244
|
const parts = [];
|
|
346164
|
-
parts.push(`OCR extracted from ${basename11(imagePath)} (${
|
|
346165
|
-
|
|
346166
|
-
|
|
346167
|
-
|
|
346168
|
-
|
|
346245
|
+
parts.push(`OCR extracted from ${basename11(imagePath)} (${report2.image_size})`);
|
|
346246
|
+
if (boundaryEvidence.state === "low_information") {
|
|
346247
|
+
parts.push(`OCR found no accepted readable text (${boundaryEvidence.reason}); low-information result.`);
|
|
346248
|
+
} else {
|
|
346249
|
+
parts.push(`Best variant: ${report2.variant} (confidence: ${report2.confidence}%, ${report2.chars} chars, ${report2.lines} lines, score: ${report2.score})`);
|
|
346250
|
+
}
|
|
346251
|
+
parts.push(`Variants tested: ${report2.variants_tested}`);
|
|
346252
|
+
if (report2.output_files) {
|
|
346253
|
+
const files = report2.output_files;
|
|
346169
346254
|
const saved = [files.txt, files.csv, files.pdf].filter(Boolean);
|
|
346170
346255
|
parts.push(`Output files: ${saved.join(", ")}`);
|
|
346171
346256
|
}
|
|
346172
|
-
|
|
346173
|
-
|
|
346174
|
-
|
|
346175
|
-
|
|
346257
|
+
if (boundaryEvidence.state === "accepted") {
|
|
346258
|
+
parts.push("");
|
|
346259
|
+
parts.push("--- Extracted Text ---");
|
|
346260
|
+
parts.push(report2.text);
|
|
346261
|
+
}
|
|
346262
|
+
if (report2.regions) {
|
|
346176
346263
|
parts.push("");
|
|
346177
346264
|
parts.push("--- Region Extraction ---");
|
|
346178
|
-
for (const [rname, rtext] of Object.entries(
|
|
346265
|
+
for (const [rname, rtext] of Object.entries(report2.regions)) {
|
|
346179
346266
|
if (rtext) {
|
|
346180
346267
|
parts.push(`
|
|
346181
346268
|
[${rname.toUpperCase()}]`);
|
|
@@ -346183,8 +346270,8 @@ var init_ocr_image_advanced = __esm({
|
|
|
346183
346270
|
}
|
|
346184
346271
|
}
|
|
346185
346272
|
}
|
|
346186
|
-
if (
|
|
346187
|
-
const sorted = Object.entries(
|
|
346273
|
+
if (report2.all_variants) {
|
|
346274
|
+
const sorted = Object.entries(report2.all_variants).filter(([_, v]) => v.chars > 0).sort((a2, b) => b[1].score - a2[1].score).slice(0, 5);
|
|
346188
346275
|
if (sorted.length > 1) {
|
|
346189
346276
|
parts.push("");
|
|
346190
346277
|
parts.push("--- Top Variants ---");
|
|
@@ -346201,7 +346288,7 @@ var init_ocr_image_advanced = __esm({
|
|
|
346201
346288
|
success: true,
|
|
346202
346289
|
output: parts.join("\n"),
|
|
346203
346290
|
durationMs: performance.now() - start2,
|
|
346204
|
-
data:
|
|
346291
|
+
data: report2
|
|
346205
346292
|
};
|
|
346206
346293
|
} catch (err) {
|
|
346207
346294
|
const pipelineResult = parsePipelineDiagnostic(err);
|
|
@@ -346215,7 +346302,8 @@ var init_ocr_image_advanced = __esm({
|
|
|
346215
346302
|
stage: pipelineResult.diagnostic?.stage ?? "pipeline",
|
|
346216
346303
|
deadline_ms: pipelineResult.diagnostic?.deadline_ms ?? OCR_PIPELINE_TIMEOUT_MS,
|
|
346217
346304
|
attempts_completed: pipelineResult.diagnostic?.attempts_completed ?? null,
|
|
346218
|
-
attempts_planned: pipelineResult.diagnostic?.attempts_planned ?? null
|
|
346305
|
+
attempts_planned: pipelineResult.diagnostic?.attempts_planned ?? null,
|
|
346306
|
+
evidence: pipelineResult.diagnostic?.evidence ?? null
|
|
346219
346307
|
})
|
|
346220
346308
|
};
|
|
346221
346309
|
}
|
|
@@ -616978,6 +617066,7 @@ __export(dist_exports2, {
|
|
|
616978
617066
|
VisualTriggerTool: () => VisualTriggerTool,
|
|
616979
617067
|
WESPEAKER_CAMPP_EMBEDDING_SET_SLUG: () => WESPEAKER_CAMPP_EMBEDDING_SET_SLUG,
|
|
616980
617068
|
WESPEAKER_CAMPP_FBANK_IMPLEMENTATION: () => WESPEAKER_CAMPP_FBANK_IMPLEMENTATION,
|
|
617069
|
+
WESPEAKER_CAMPP_FBANK_VALIDATION: () => WESPEAKER_CAMPP_FBANK_VALIDATION,
|
|
616981
617070
|
WESPEAKER_CAMPP_MODEL_REVISION: () => WESPEAKER_CAMPP_MODEL_REVISION,
|
|
616982
617071
|
WESPEAKER_CAMPP_NUMPY_VERSION: () => WESPEAKER_CAMPP_NUMPY_VERSION,
|
|
616983
617072
|
WESPEAKER_CAMPP_ONNXRUNTIME_VERSION: () => WESPEAKER_CAMPP_ONNXRUNTIME_VERSION,
|
|
@@ -687469,8 +687558,11 @@ import { existsSync as existsSync128, mkdirSync as mkdirSync79, readFileSync as
|
|
|
687469
687558
|
import { dirname as dirname51, join as join140 } from "node:path";
|
|
687470
687559
|
import { homedir as homedir41 } from "node:os";
|
|
687471
687560
|
import { fileURLToPath as fileURLToPath25 } from "node:url";
|
|
687561
|
+
function managedAsrRuntimeRoot() {
|
|
687562
|
+
return process.env["OMNIUS_ASR_RUNTIME_ROOT"]?.trim() || MANAGED_ASR_ROOT;
|
|
687563
|
+
}
|
|
687472
687564
|
function managedAsrStateFile(engineId) {
|
|
687473
|
-
return join140(
|
|
687565
|
+
return join140(managedAsrRuntimeRoot(), engineId, "models.json");
|
|
687474
687566
|
}
|
|
687475
687567
|
function managedAsrModelDir(engineId) {
|
|
687476
687568
|
return join140(MANAGED_ASR_MODEL_ROOT, engineId);
|
|
@@ -687483,16 +687575,18 @@ function readManagedAsrState(engineId) {
|
|
|
687483
687575
|
}
|
|
687484
687576
|
}
|
|
687485
687577
|
function persistManagedAsrReadiness(readiness) {
|
|
687486
|
-
const root = join140(
|
|
687578
|
+
const root = join140(managedAsrRuntimeRoot(), readiness.engineId);
|
|
687487
687579
|
mkdirSync79(root, { recursive: true });
|
|
687488
687580
|
const state5 = readManagedAsrState(readiness.engineId);
|
|
687489
687581
|
state5.models ??= {};
|
|
687490
687582
|
state5.models[readiness.modelId] = {
|
|
687491
687583
|
installed: readiness.installed,
|
|
687492
687584
|
weightsReady: readiness.weightsReady,
|
|
687585
|
+
active: readiness.active,
|
|
687493
687586
|
device: readiness.device,
|
|
687494
687587
|
lastError: readiness.lastError,
|
|
687495
|
-
pulledAt: readiness.pulledAt
|
|
687588
|
+
pulledAt: readiness.pulledAt,
|
|
687589
|
+
activeAt: readiness.activeAt
|
|
687496
687590
|
};
|
|
687497
687591
|
writeFileSync70(managedAsrStateFile(readiness.engineId), `${JSON.stringify(state5, null, 2)}
|
|
687498
687592
|
`, "utf8");
|
|
@@ -687885,17 +687979,34 @@ ${log22.slice(-2e3)}` };
|
|
|
687885
687979
|
function getManagedAsrReadiness(engineId, modelId) {
|
|
687886
687980
|
assertManagedAsrModel(engineId, modelId);
|
|
687887
687981
|
const saved = readManagedAsrState(engineId).models?.[modelId];
|
|
687982
|
+
const active = saved?.active === true && saved.weightsReady === true && typeof saved.activeAt === "string" && /^cuda(?::\d+)?$/i.test(saved.device ?? "");
|
|
687888
687983
|
return {
|
|
687889
687984
|
engineId,
|
|
687890
687985
|
modelId,
|
|
687891
687986
|
installed: Boolean(saved?.installed && existsSync128(getVenvPython())),
|
|
687892
687987
|
weightsReady: Boolean(saved?.weightsReady),
|
|
687893
|
-
active
|
|
687988
|
+
// Managed direct ASR has no resident process. `active` therefore means
|
|
687989
|
+
// this exact model last completed a CUDA inference successfully, not that
|
|
687990
|
+
// a Python child happens to still be alive after a request returned.
|
|
687991
|
+
active,
|
|
687894
687992
|
device: saved?.device,
|
|
687895
687993
|
lastError: saved?.lastError,
|
|
687896
|
-
pulledAt: saved?.pulledAt
|
|
687994
|
+
pulledAt: saved?.pulledAt,
|
|
687995
|
+
activeAt: saved?.activeAt
|
|
687897
687996
|
};
|
|
687898
687997
|
}
|
|
687998
|
+
function recordManagedAsrInference(input, outcome) {
|
|
687999
|
+
const prior = getManagedAsrReadiness(input.engineId, input.modelId);
|
|
688000
|
+
const cuda = typeof input.device === "string" && /^cuda(?::\d+)?$/i.test(input.device);
|
|
688001
|
+
const activeAt = outcome.ok && cuda ? (/* @__PURE__ */ new Date()).toISOString() : void 0;
|
|
688002
|
+
persistManagedAsrReadiness({
|
|
688003
|
+
...prior,
|
|
688004
|
+
device: input.device || prior.device,
|
|
688005
|
+
active: Boolean(activeAt),
|
|
688006
|
+
activeAt,
|
|
688007
|
+
lastError: outcome.ok ? void 0 : outcome.error
|
|
688008
|
+
});
|
|
688009
|
+
}
|
|
687899
688010
|
function ensureManagedAsrModel(input) {
|
|
687900
688011
|
const engineId = input.engineId.trim().toLowerCase();
|
|
687901
688012
|
const modelId = input.modelId.trim().toLowerCase();
|
|
@@ -688058,12 +688169,16 @@ function transcribeManagedNemotronFile(input) {
|
|
|
688058
688169
|
const ready = events.find((event) => event["type"] === "ready");
|
|
688059
688170
|
const error = events.find((event) => event["type"] === "error");
|
|
688060
688171
|
if (result.status !== 0 || !transcript) {
|
|
688061
|
-
|
|
688172
|
+
const message2 = String(error?.["message"] ?? (`${result.stderr || ""}`.trim() || result.error || "Nemotron file transcription failed"));
|
|
688173
|
+
recordManagedAsrInference({ engineId: "nemotron-streaming", modelId, device: existing.device }, { ok: false, error: message2 });
|
|
688174
|
+
throw new Error(message2);
|
|
688062
688175
|
}
|
|
688176
|
+
const device2 = typeof ready?.["device"] === "string" ? ready["device"] : existing.device;
|
|
688177
|
+
recordManagedAsrInference({ engineId: "nemotron-streaming", modelId, device: device2 }, { ok: true });
|
|
688063
688178
|
return {
|
|
688064
688179
|
text: String(transcript["text"] ?? ""),
|
|
688065
688180
|
duration: Number.isFinite(Number(transcript["audioSeconds"])) ? Number(transcript["audioSeconds"]) : null,
|
|
688066
|
-
device:
|
|
688181
|
+
device: device2,
|
|
688067
688182
|
segments: [],
|
|
688068
688183
|
warnings: []
|
|
688069
688184
|
};
|
|
@@ -688102,6 +688217,7 @@ function transcribeManagedWhisperFile(input) {
|
|
|
688102
688217
|
const payload = [...parseWorkerEvents(`${result.stdout || ""}`)].reverse().find((event) => event["ok"] === true);
|
|
688103
688218
|
if (result.status !== 0 || !payload) {
|
|
688104
688219
|
const detail = `${result.stderr || ""}`.trim() || String(result.error ?? "Whisper file transcription failed");
|
|
688220
|
+
recordManagedAsrInference({ engineId: "openai-whisper", modelId, device: existing.device }, { ok: false, error: detail });
|
|
688105
688221
|
throw new Error(detail);
|
|
688106
688222
|
}
|
|
688107
688223
|
const segments = Array.isArray(payload["segments"]) ? payload["segments"].flatMap((segment) => {
|
|
@@ -688113,10 +688229,12 @@ function transcribeManagedWhisperFile(input) {
|
|
|
688113
688229
|
text: String(value2["text"] ?? "")
|
|
688114
688230
|
}];
|
|
688115
688231
|
}) : [];
|
|
688232
|
+
const device2 = typeof payload["device"] === "string" ? payload["device"] : existing.device;
|
|
688233
|
+
recordManagedAsrInference({ engineId: "openai-whisper", modelId, device: device2 }, { ok: true });
|
|
688116
688234
|
return {
|
|
688117
688235
|
text: String(payload["text"] ?? ""),
|
|
688118
688236
|
duration: Number.isFinite(Number(payload["duration"])) ? Number(payload["duration"]) : null,
|
|
688119
|
-
device:
|
|
688237
|
+
device: device2,
|
|
688120
688238
|
segments,
|
|
688121
688239
|
warnings: []
|
|
688122
688240
|
};
|
|
@@ -688152,17 +688270,21 @@ function transcribeManagedAsrFile(input) {
|
|
|
688152
688270
|
const payload = [...events].reverse().find((event) => event["type"] === "transcript");
|
|
688153
688271
|
if (result.status !== 0 || !payload) {
|
|
688154
688272
|
const workerError = [...events].reverse().find((event) => event["type"] === "error");
|
|
688155
|
-
|
|
688273
|
+
const message2 = String(workerError?.["message"] ?? result.stderr ?? result.error ?? "Voxtral file transcription failed");
|
|
688274
|
+
recordManagedAsrInference({ engineId, modelId, device: existing.device }, { ok: false, error: message2 });
|
|
688275
|
+
throw new Error(message2);
|
|
688156
688276
|
}
|
|
688157
688277
|
const segments = Array.isArray(payload["segments"]) ? payload["segments"].flatMap((segment) => {
|
|
688158
688278
|
if (!segment || typeof segment !== "object") return [];
|
|
688159
688279
|
const value2 = segment;
|
|
688160
688280
|
return [{ start: Number(value2["start"] ?? 0), end: Number(value2["end"] ?? 0), text: String(value2["text"] ?? "") }];
|
|
688161
688281
|
}) : [];
|
|
688282
|
+
const device2 = typeof payload["device"] === "string" ? payload["device"] : existing.device;
|
|
688283
|
+
recordManagedAsrInference({ engineId, modelId, device: device2 }, { ok: true });
|
|
688162
688284
|
return {
|
|
688163
688285
|
text: String(payload["text"] ?? ""),
|
|
688164
688286
|
duration: Number.isFinite(Number(payload["duration"])) ? Number(payload["duration"]) : null,
|
|
688165
|
-
device:
|
|
688287
|
+
device: device2,
|
|
688166
688288
|
segments,
|
|
688167
688289
|
warnings: []
|
|
688168
688290
|
};
|
|
@@ -792931,6 +793053,7 @@ function getRuntimeStatus() {
|
|
|
792931
793053
|
const persisted = _listenEngine ? null : loadGlobalSettings();
|
|
792932
793054
|
const asrEngineId = _listenEngine?.currentEngine ?? persisted?.asrEngine ?? "openai-whisper";
|
|
792933
793055
|
const asrModelId = _listenEngine?.currentModel ?? persisted?.asrModel ?? "medium";
|
|
793056
|
+
const managedAsr = asrEngineId === "openai-whisper" || asrEngineId === "nemotron-streaming" || asrEngineId === "voxtral-transformers" ? getManagedAsrReadiness(asrEngineId, asrModelId) : null;
|
|
792934
793057
|
const vibePhase = vibe.active ? "ready" : vibe.lastError ? "error" : vibe.weightsReady ? "weights-ready" : vibe.installed ? "runtime-installed" : "not-installed";
|
|
792935
793058
|
return {
|
|
792936
793059
|
state: _state3,
|
|
@@ -792942,9 +793065,9 @@ function getRuntimeStatus() {
|
|
|
792942
793065
|
listenPaused: _listenEngine?.isPaused ?? false,
|
|
792943
793066
|
asrEngineId,
|
|
792944
793067
|
asrModelId,
|
|
792945
|
-
asrBackend: asrEngineId === "vibevoice-transformers" ? "vibevoice-transformers" : listenState.backend,
|
|
792946
|
-
asrPhase: asrEngineId === "vibevoice-transformers" ? vibePhase : listenState.phase,
|
|
792947
|
-
asrReady: asrEngineId === "vibevoice-transformers" ? vibe.active : Boolean(_listenEngine?.isActive),
|
|
793068
|
+
asrBackend: asrEngineId === "vibevoice-transformers" ? "vibevoice-transformers" : managedAsr ? asrEngineId : listenState.backend,
|
|
793069
|
+
asrPhase: asrEngineId === "vibevoice-transformers" ? vibePhase : managedAsr?.active ? "ready" : managedAsr?.lastError ? "error" : managedAsr?.weightsReady ? "weights-ready" : listenState.phase,
|
|
793070
|
+
asrReady: asrEngineId === "vibevoice-transformers" ? vibe.active : managedAsr ? managedAsr.active : Boolean(_listenEngine?.isActive),
|
|
792948
793071
|
micDevice: listenState.micDevice || null,
|
|
792949
793072
|
micSourceKind: listenState.micSourceKind,
|
|
792950
793073
|
micSourceChannels: listenState.micSourceChannels,
|
|
@@ -793173,6 +793296,7 @@ var init_voice_runtime = __esm({
|
|
|
793173
793296
|
init_listen();
|
|
793174
793297
|
init_dist5();
|
|
793175
793298
|
init_voicechat();
|
|
793299
|
+
init_py_embed();
|
|
793176
793300
|
_voiceEngine = null;
|
|
793177
793301
|
_listenEngine = null;
|
|
793178
793302
|
_voiceChatSession = null;
|
|
@@ -828251,6 +828375,7 @@ function getOpenApiSpec() {
|
|
|
828251
828375
|
include_daemon_tools: { type: "array", items: { type: "string", enum: ["read", "run", "admin"] }, description: "Scopes from which Omnius may offer its bounded core daemon-tool catalog. Use daemon_tool_names to request other exact tools." },
|
|
828252
828376
|
daemon_tool_names: { type: "array", items: { type: "string" }, description: "Optional exact allowlist of daemon tool names. Use this to keep local-model tool prompts small." },
|
|
828253
828377
|
agent_timeout_s: { type: "number", minimum: 5, maximum: 600, default: 45, description: "Total server-side agent-loop deadline. Distinct from per-backend timeout_s." },
|
|
828378
|
+
agent_max_tool_rounds: { type: "integer", minimum: 1, maximum: 8, default: 1, description: "Maximum daemon-tool planning rounds. The default executes one tool round, then removes daemon schemas for lower-latency final synthesis." },
|
|
828254
828379
|
max_turns: { type: "integer", description: "Q2 — agent_loop max iterations (default 8, max 64)." },
|
|
828255
828380
|
prompt_template: { type: "string", enum: ["factual-first"], description: "Q8 — prepended system policy template. 'factual-first' instructs model to call web_search FIRST for any factual question." }
|
|
828256
828381
|
} } } } },
|
|
@@ -828873,7 +828998,7 @@ function getOpenApiSpec() {
|
|
|
828873
828998
|
post: {
|
|
828874
828999
|
summary: "Provision and activate one role-typed audio embedding runtime",
|
|
828875
829000
|
tags: ["Audio"],
|
|
828876
|
-
description: "Admin-only. Requires kind=acoustic|speaker|semantic in the query (canonical) or JSON body. acoustic reuses the pinned JetPack YAMNet/TensorRT setup; speaker creates a private CPython 3.10 CPU-only WeSpeaker CAM++ venv, installs only checksum-pinned NumPy, ONNX Runtime, and its locked CPU support wheels with --no-deps/--no-index, then downloads the pinned model. Its 80-bin Kaldi-compatible NumPy fbank is validated with a
|
|
829001
|
+
description: "Admin-only. Requires kind=acoustic|speaker|semantic in the query (canonical) or JSON body. acoustic reuses the pinned JetPack YAMNet/TensorRT setup; speaker creates a private CPython 3.10 CPU-only WeSpeaker CAM++ venv, installs only checksum-pinned NumPy, ONNX Runtime, and its locked CPU support wheels with --no-deps/--no-index, then downloads the pinned model. Its 80-bin Kaldi-compatible NumPy fbank is validated with a versioned cross-platform numeric-invariant probe before readiness. The speaker path never imports, links, replaces, or otherwise depends on JetPack Torch/Torchaudio, so an incompatible generic Torchaudio wheel cannot affect CUDA-enabled Egg Torch. semantic installs the isolated JetPack CUDA CLAP dependencies and pinned model. This is the only REST operation allowed to provision. It activates and warms only the requested role worker; no role is substituted for another.",
|
|
828877
829002
|
parameters: [{ name: "kind", in: "query", required: false, schema: { type: "string", enum: ["acoustic", "speaker", "semantic"] }, description: "Canonical required role selector; JSON body.kind is accepted for compatibility." }],
|
|
828878
829003
|
requestBody: { required: false, content: { "application/json": { schema: { type: "object", properties: { kind: { type: "string", enum: ["acoustic", "speaker", "semantic"] } } } } } },
|
|
828879
829004
|
responses: { 200: { description: "Requested role runtime provisioned and warm." }, 202: { description: "Provisioning finished but requested role did not prove warm; poll its health URL." }, 400: { description: "kind is missing or invalid." }, 403: { description: "Admin scope required." }, 500: { description: "Role-specific setup/preflight failed." } }
|
|
@@ -830485,6 +830610,32 @@ function getAgentLoopTotalTimeoutMs(body) {
|
|
|
830485
830610
|
AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS
|
|
830486
830611
|
);
|
|
830487
830612
|
}
|
|
830613
|
+
function getAgentLoopPlannerMaxTokens() {
|
|
830614
|
+
return boundedModelCheckInteger(
|
|
830615
|
+
Number(process.env["OMNIUS_AGENT_LOOP_PLANNER_MAX_TOKENS"]),
|
|
830616
|
+
AGENT_LOOP_PLANNER_DEFAULT_MAX_TOKENS,
|
|
830617
|
+
32,
|
|
830618
|
+
512
|
|
830619
|
+
);
|
|
830620
|
+
}
|
|
830621
|
+
function getAgentLoopMaxToolRounds(body) {
|
|
830622
|
+
const requested = body["agent_max_tool_rounds"];
|
|
830623
|
+
const configured = Number(process.env["OMNIUS_AGENT_LOOP_MAX_TOOL_ROUNDS"]);
|
|
830624
|
+
return boundedModelCheckInteger(
|
|
830625
|
+
requested ?? configured,
|
|
830626
|
+
AGENT_LOOP_MAX_TOOL_ROUNDS_DEFAULT,
|
|
830627
|
+
1,
|
|
830628
|
+
8
|
|
830629
|
+
);
|
|
830630
|
+
}
|
|
830631
|
+
function getAgentLoopToolResultMaxChars() {
|
|
830632
|
+
return boundedModelCheckInteger(
|
|
830633
|
+
Number(process.env["OMNIUS_AGENT_LOOP_TOOL_RESULT_MAX_CHARS"]),
|
|
830634
|
+
AGENT_LOOP_TOOL_RESULT_DEFAULT_MAX_CHARS,
|
|
830635
|
+
1e3,
|
|
830636
|
+
32e3
|
|
830637
|
+
);
|
|
830638
|
+
}
|
|
830488
830639
|
function compactAgentLoopToolResult(value2) {
|
|
830489
830640
|
let encoded;
|
|
830490
830641
|
try {
|
|
@@ -830492,11 +830643,12 @@ function compactAgentLoopToolResult(value2) {
|
|
|
830492
830643
|
} catch {
|
|
830493
830644
|
encoded = JSON.stringify({ success: false, error: "tool result was not JSON serializable" });
|
|
830494
830645
|
}
|
|
830495
|
-
|
|
830646
|
+
const maxChars = getAgentLoopToolResultMaxChars();
|
|
830647
|
+
if (encoded.length <= maxChars) return encoded;
|
|
830496
830648
|
return JSON.stringify({
|
|
830497
830649
|
truncated: true,
|
|
830498
830650
|
original_chars: encoded.length,
|
|
830499
|
-
preview: encoded.slice(0,
|
|
830651
|
+
preview: encoded.slice(0, maxChars)
|
|
830500
830652
|
});
|
|
830501
830653
|
}
|
|
830502
830654
|
function parseJsonObject8(value2) {
|
|
@@ -832987,6 +833139,7 @@ async function runAgentLoopChatCompletions(opts) {
|
|
|
832987
833139
|
const targetTransport = requestProviderTransport(targetUrl, route?.endpoint);
|
|
832988
833140
|
const useNativeOllamaChat = targetTransport.protocol === "ollama";
|
|
832989
833141
|
const maxTurns = typeof requestBody["max_turns"] === "number" ? Math.max(1, Math.min(64, requestBody["max_turns"])) : 8;
|
|
833142
|
+
const maxDaemonToolRounds = getAgentLoopMaxToolRounds(requestBody);
|
|
832990
833143
|
const remoteIp = (req3.socket?.remoteAddress || "").replace(/^::ffff:/, "");
|
|
832991
833144
|
const origin = /^(127\.\d+\.\d+\.\d+|::1|localhost)$/.test(remoteIp) ? "loopback" : "remote";
|
|
832992
833145
|
const reqAuth = req3;
|
|
@@ -833118,6 +833271,7 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833118
833271
|
const chatId = `chatcmpl-${randomBytes30(12).toString("hex")}`;
|
|
833119
833272
|
const turnsLog = [];
|
|
833120
833273
|
const seenDaemonCalls = /* @__PURE__ */ new Set();
|
|
833274
|
+
let daemonToolRoundsExecuted = 0;
|
|
833121
833275
|
for (let turn = 1; turn <= maxTurns; turn++) {
|
|
833122
833276
|
if (isClientDisconnected()) return;
|
|
833123
833277
|
if (Date.now() > totalDeadline) {
|
|
@@ -833142,15 +833296,19 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833142
833296
|
return;
|
|
833143
833297
|
}
|
|
833144
833298
|
const transportMessages = forceNoThinkForTools ? messages2.map((message3) => typeof message3.content === "string" ? { ...message3, content: stripNoThinkPromptDirectives(message3.content) } : message3) : messages2;
|
|
833299
|
+
const daemonToolsForTurn = daemonToolRoundsExecuted < maxDaemonToolRounds ? daemonToolEntries : [];
|
|
833300
|
+
const toolsForTurn = [...callerTools, ...daemonToolsForTurn];
|
|
833301
|
+
const planningTurn = toolsForTurn.length > 0;
|
|
833145
833302
|
const ollamaOptions = {};
|
|
833146
833303
|
if (typeof requestBody["temperature"] === "number") ollamaOptions["temperature"] = requestBody["temperature"];
|
|
833147
833304
|
if (typeof requestBody["top_p"] === "number") ollamaOptions["top_p"] = requestBody["top_p"];
|
|
833148
|
-
if (
|
|
833305
|
+
if (planningTurn) ollamaOptions["num_predict"] = getAgentLoopPlannerMaxTokens();
|
|
833306
|
+
else if (typeof requestBody["max_tokens"] === "number") ollamaOptions["num_predict"] = requestBody["max_tokens"];
|
|
833149
833307
|
if (typeof requestBody["seed"] === "number") ollamaOptions["seed"] = requestBody["seed"];
|
|
833150
833308
|
const turnBody = useNativeOllamaChat ? {
|
|
833151
833309
|
model: originalModel,
|
|
833152
833310
|
messages: agentLoopMessagesForOllama(transportMessages),
|
|
833153
|
-
tools:
|
|
833311
|
+
tools: toolsForTurn.length > 0 ? toolsForTurn : void 0,
|
|
833154
833312
|
stream: false,
|
|
833155
833313
|
think: false,
|
|
833156
833314
|
options: ollamaOptions
|
|
@@ -833158,7 +833316,8 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833158
833316
|
...requestBody,
|
|
833159
833317
|
model: originalModel,
|
|
833160
833318
|
messages: transportMessages,
|
|
833161
|
-
tools:
|
|
833319
|
+
tools: toolsForTurn.length > 0 ? toolsForTurn : void 0,
|
|
833320
|
+
...planningTurn ? { max_tokens: getAgentLoopPlannerMaxTokens() } : {},
|
|
833162
833321
|
stream: false,
|
|
833163
833322
|
...forceNoThinkForTools ? { think: false } : {},
|
|
833164
833323
|
// Strip our own loop-control fields so they don't confuse the backend
|
|
@@ -833166,6 +833325,7 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833166
833325
|
include_daemon_tools: void 0,
|
|
833167
833326
|
daemon_tool_names: void 0,
|
|
833168
833327
|
agent_timeout_s: void 0,
|
|
833328
|
+
agent_max_tool_rounds: void 0,
|
|
833169
833329
|
max_turns: void 0,
|
|
833170
833330
|
timeout_s: void 0,
|
|
833171
833331
|
realtime: void 0,
|
|
@@ -833432,6 +833592,7 @@ ${messages2[firstSystemIdx].content}`
|
|
|
833432
833592
|
daemon_executed: executed,
|
|
833433
833593
|
client_yielded: 0
|
|
833434
833594
|
});
|
|
833595
|
+
if (executed > 0) daemonToolRoundsExecuted += 1;
|
|
833435
833596
|
}
|
|
833436
833597
|
jsonResponse(res, 200, {
|
|
833437
833598
|
id: chatId,
|
|
@@ -837627,7 +837788,10 @@ data: ${JSON.stringify(data)}
|
|
|
837627
837788
|
const baseReadiness = engine.id === "vibevoice-transformers" ? vibe : getManagedAsrReadiness(engine.id, selectedModel ?? engine.models[0].id);
|
|
837628
837789
|
const readiness = {
|
|
837629
837790
|
...baseReadiness,
|
|
837630
|
-
|
|
837791
|
+
// Direct managed ASR is request-scoped rather than a resident child
|
|
837792
|
+
// process. Expose its successful CUDA inference only for the exact
|
|
837793
|
+
// selected engine/model; never promote an unselected model to active.
|
|
837794
|
+
active: unavailableReason ? false : selected && baseReadiness.active,
|
|
837631
837795
|
disabled: Boolean(unavailableReason),
|
|
837632
837796
|
...unavailableReason ? { unavailableReason, lastError: unavailableReason } : {}
|
|
837633
837797
|
};
|
|
@@ -837641,7 +837805,7 @@ data: ${JSON.stringify(data)}
|
|
|
837641
837805
|
const modelBaseReadiness = engine.id === "vibevoice-transformers" ? vibe : getManagedAsrReadiness(engine.id, model.id);
|
|
837642
837806
|
const modelReadiness = {
|
|
837643
837807
|
...modelBaseReadiness,
|
|
837644
|
-
active: unavailableReason ? false : modelBaseReadiness.active,
|
|
837808
|
+
active: unavailableReason ? false : selected && model.id === listen.currentModel && modelBaseReadiness.active,
|
|
837645
837809
|
disabled: Boolean(unavailableReason),
|
|
837646
837810
|
...unavailableReason ? { unavailableReason, lastError: unavailableReason } : {}
|
|
837647
837811
|
};
|
|
@@ -837683,9 +837847,11 @@ data: ${JSON.stringify(data)}
|
|
|
837683
837847
|
runtimes: listAsrEngines().filter((engine) => engine.id === "openai-whisper" || engine.id === "nemotron-streaming" || engine.id === "voxtral-transformers").flatMap((engine) => engine.models.map((model) => {
|
|
837684
837848
|
const readiness = getManagedAsrReadiness(engine.id, model.id);
|
|
837685
837849
|
const unavailableReason = getAsrEngineUnavailableReason(engine.id);
|
|
837850
|
+
const selected = engine.id === listen.currentEngine && model.id === listen.currentModel;
|
|
837686
837851
|
return {
|
|
837687
837852
|
...readiness,
|
|
837688
|
-
active: unavailableReason ? false : readiness.active,
|
|
837853
|
+
active: unavailableReason ? false : selected && readiness.active,
|
|
837854
|
+
selected,
|
|
837689
837855
|
disabled: Boolean(unavailableReason),
|
|
837690
837856
|
...unavailableReason ? { unavailableReason, lastError: unavailableReason } : {}
|
|
837691
837857
|
};
|
|
@@ -841937,7 +842103,7 @@ function setTimerEnabled(name10, enabled2) {
|
|
|
841937
842103
|
return false;
|
|
841938
842104
|
}
|
|
841939
842105
|
}
|
|
841940
|
-
var require4, NEXUS_DIRECTORY_ORIGIN3, NEXUS_SPONSORS_URL3, SERVER_BOOT_IDENTITY, voiceTtsTransactionTail, endpointRegistry, modelRouteMap, endpointUsage, _lastEndpointDiagnostics, BACKEND_TIMEOUT_DEFAULT_MS, BACKEND_TIMEOUT_MAX_MS, AGENT_LOOP_TOTAL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOTAL_TIMEOUT_MIN_MS, AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS, AGENT_LOOP_TOOL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOOL_TIMEOUT_MAX_MS,
|
|
842106
|
+
var require4, NEXUS_DIRECTORY_ORIGIN3, NEXUS_SPONSORS_URL3, SERVER_BOOT_IDENTITY, voiceTtsTransactionTail, endpointRegistry, modelRouteMap, endpointUsage, _lastEndpointDiagnostics, BACKEND_TIMEOUT_DEFAULT_MS, BACKEND_TIMEOUT_MAX_MS, AGENT_LOOP_TOTAL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOTAL_TIMEOUT_MIN_MS, AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS, AGENT_LOOP_TOOL_TIMEOUT_DEFAULT_MS, AGENT_LOOP_TOOL_TIMEOUT_MAX_MS, AGENT_LOOP_TOOL_RESULT_DEFAULT_MAX_CHARS, AGENT_LOOP_PLANNER_DEFAULT_MAX_TOKENS, AGENT_LOOP_MAX_TOOL_ROUNDS_DEFAULT, DEFAULT_AGENT_LOOP_DAEMON_TOOL_NAMES, MODEL_LIST_TIMEOUT_DEFAULT_MS, MODEL_CHECK_TIMEOUT_DEFAULT_MS, MODEL_CHECK_TIMEOUT_MAX_MS, MODEL_CHECK_MAX_TOKENS, cancelledAgentLoopTools, metrics, startedAt, FILE_MIME_BY_EXT, realtimeOllamaFallbackCache, runningProcesses, LOCAL_UI_SESSION_COOKIE, LOCAL_UI_SESSION_TOKEN, perKeyUsage, CRON_MARKER2;
|
|
841941
842107
|
var init_serve = __esm({
|
|
841942
842108
|
"packages/cli/src/api/serve.ts"() {
|
|
841943
842109
|
init_config();
|
|
@@ -841998,7 +842164,9 @@ var init_serve = __esm({
|
|
|
841998
842164
|
AGENT_LOOP_TOTAL_TIMEOUT_MAX_MS = 10 * 60 * 1e3;
|
|
841999
842165
|
AGENT_LOOP_TOOL_TIMEOUT_DEFAULT_MS = 3e4;
|
|
842000
842166
|
AGENT_LOOP_TOOL_TIMEOUT_MAX_MS = 12e4;
|
|
842001
|
-
|
|
842167
|
+
AGENT_LOOP_TOOL_RESULT_DEFAULT_MAX_CHARS = 6e3;
|
|
842168
|
+
AGENT_LOOP_PLANNER_DEFAULT_MAX_TOKENS = 96;
|
|
842169
|
+
AGENT_LOOP_MAX_TOOL_ROUNDS_DEFAULT = 1;
|
|
842002
842170
|
DEFAULT_AGENT_LOOP_DAEMON_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
842003
842171
|
"web_search",
|
|
842004
842172
|
"web_fetch",
|
|
@@ -37,10 +37,11 @@ MIN_RMS = 0.003
|
|
|
37
37
|
MIN_PEAK = 0.01
|
|
38
38
|
MAX_CLIPPED_FRACTION = 0.01
|
|
39
39
|
FBANK_IMPLEMENTATION = "kaldi-fbank-numpy-v1"
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
|
|
40
|
+
# A CPU FFT may differ in harmless last bits across x86/aarch64 and NumPy
|
|
41
|
+
# builds. Validate the public Kaldi configuration with numeric invariants,
|
|
42
|
+
# not a byte hash of implementation-dependent FFT output.
|
|
43
|
+
FBANK_VALIDATION = "kaldi-fbank-invariants-v2"
|
|
44
|
+
CALIBRATION_BAND_MEANS = (11.1405, 17.5231, 12.8845, 9.2557, 7.7213, 6.7842, 6.6405)
|
|
44
45
|
|
|
45
46
|
|
|
46
47
|
def emit(payload: dict) -> None:
|
|
@@ -163,25 +164,46 @@ def kaldi_fbank80_numpy(waveform, np):
|
|
|
163
164
|
|
|
164
165
|
|
|
165
166
|
def validate_kaldi_fbank_numpy(np):
|
|
166
|
-
"""Validate the no-Torch preprocessing path before readiness is true.
|
|
167
|
+
"""Validate the no-Torch preprocessing path before readiness is true.
|
|
168
|
+
|
|
169
|
+
The calibration intentionally asserts shape, finite/CMN invariants, the
|
|
170
|
+
expected 220/440 Hz Kaldi mel-band profile and energy range with tolerances
|
|
171
|
+
that are stable across supported CPU FFT implementations. It must reject a
|
|
172
|
+
changed window, frame layout, PCM scale, mel bank, pre-emphasis or CMN, but
|
|
173
|
+
must not reject Jetson due to last-bit numerical differences.
|
|
174
|
+
"""
|
|
167
175
|
t = np.arange(SAMPLE_RATE * 2, dtype=np.float32) / np.float32(SAMPLE_RATE)
|
|
168
176
|
samples = (
|
|
169
177
|
np.float32(0.075) * np.sin(np.float32(2.0 * np.pi * 220.0) * t)
|
|
170
178
|
+ np.float32(0.025) * np.sin(np.float32(2.0 * np.pi * 440.0) * t)
|
|
171
179
|
).astype(np.float32)
|
|
172
|
-
|
|
173
|
-
features = (
|
|
180
|
+
raw_features = kaldi_fbank80_numpy(samples * np.float32(1 << 15), np)
|
|
181
|
+
features = (
|
|
182
|
+
raw_features - np.mean(raw_features, axis=0, dtype=np.float32, keepdims=True)
|
|
183
|
+
).astype(np.float32)
|
|
174
184
|
if features.shape != (198, 80) or not np.all(np.isfinite(features)):
|
|
175
185
|
raise RuntimeError("NumPy Kaldi fbank calibration produced an invalid feature matrix")
|
|
176
|
-
|
|
186
|
+
cmn_abs_mean_max = float(np.max(np.abs(np.mean(features, axis=0))))
|
|
187
|
+
if cmn_abs_mean_max > 1e-3:
|
|
177
188
|
raise RuntimeError("NumPy Kaldi fbank calibration failed utterance CMN")
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
189
|
+
raw_min = float(np.min(raw_features))
|
|
190
|
+
raw_max = float(np.max(raw_features))
|
|
191
|
+
cmn_rms = float(np.sqrt(np.mean(np.square(features))))
|
|
192
|
+
band_means = raw_features.mean(axis=0)[[0, 5, 10, 20, 40, 60, 79]]
|
|
193
|
+
if not (4.5 < raw_min < 6.0 and 19.0 < raw_max < 21.5 and 0.55 < cmn_rms < 0.75):
|
|
181
194
|
raise RuntimeError(
|
|
182
|
-
"NumPy Kaldi fbank calibration
|
|
195
|
+
"NumPy Kaldi fbank calibration energy/range invariant failed"
|
|
183
196
|
)
|
|
184
|
-
|
|
197
|
+
if not np.allclose(band_means, np.asarray(CALIBRATION_BAND_MEANS, dtype=np.float32), rtol=0.0, atol=0.25):
|
|
198
|
+
raise RuntimeError("NumPy Kaldi fbank calibration mel-band invariant failed")
|
|
199
|
+
return {
|
|
200
|
+
"validation": FBANK_VALIDATION,
|
|
201
|
+
"frames": int(features.shape[0]),
|
|
202
|
+
"bins": int(features.shape[1]),
|
|
203
|
+
"cmn_abs_mean_max": round(cmn_abs_mean_max, 8),
|
|
204
|
+
"cmn_rms": round(cmn_rms, 6),
|
|
205
|
+
"raw_feature_range": [round(raw_min, 6), round(raw_max, 6)],
|
|
206
|
+
}
|
|
185
207
|
|
|
186
208
|
|
|
187
209
|
class WeSpeakerCamPlus:
|
|
@@ -191,7 +213,7 @@ class WeSpeakerCamPlus:
|
|
|
191
213
|
import onnxruntime as ort
|
|
192
214
|
|
|
193
215
|
self.np = np
|
|
194
|
-
self.
|
|
216
|
+
self.preprocessing_validation = validate_kaldi_fbank_numpy(np)
|
|
195
217
|
options = ort.SessionOptions()
|
|
196
218
|
options.inter_op_num_threads = 1
|
|
197
219
|
options.intra_op_num_threads = 1
|
|
@@ -265,7 +287,7 @@ def main() -> int:
|
|
|
265
287
|
{
|
|
266
288
|
"type": "preprocessing_probe",
|
|
267
289
|
"implementation": FBANK_IMPLEMENTATION,
|
|
268
|
-
"
|
|
290
|
+
"validation": validate_kaldi_fbank_numpy(np),
|
|
269
291
|
}
|
|
270
292
|
)
|
|
271
293
|
return 0
|
|
@@ -285,7 +307,7 @@ def main() -> int:
|
|
|
285
307
|
"model_load_ms": round(worker.model_load_ms, 3),
|
|
286
308
|
"warmed": True,
|
|
287
309
|
"preprocessing": FBANK_IMPLEMENTATION,
|
|
288
|
-
"
|
|
310
|
+
"preprocessing_validation": worker.preprocessing_validation,
|
|
289
311
|
}
|
|
290
312
|
)
|
|
291
313
|
for line in sys.stdin:
|
|
@@ -180,6 +180,10 @@ MEDIUM_IMAGE_AREA_PX = 2_000_000
|
|
|
180
180
|
MAX_PIPELINE_DEADLINE_MS = 80_000
|
|
181
181
|
MAX_TESSERACT_ATTEMPT_SECONDS = 12.0
|
|
182
182
|
MIN_TESSERACT_ATTEMPT_SECONDS = 0.25
|
|
183
|
+
MIN_ACCEPTED_CONFIDENCE = 50.0
|
|
184
|
+
MIN_SUBSTANTIVE_TEXT_CHARS = 12
|
|
185
|
+
HIGH_VOLUME_GARBAGE_CHARS = 64
|
|
186
|
+
HIGH_VOLUME_GARBAGE_CONFIDENCE = 35.0
|
|
183
187
|
ACTIVE_DEADLINE = None
|
|
184
188
|
|
|
185
189
|
|
|
@@ -273,14 +277,69 @@ def run_tesseract(binary_img, deadline, language="eng", psm=6):
|
|
|
273
277
|
return text, avg_conf, line_count
|
|
274
278
|
|
|
275
279
|
|
|
276
|
-
def
|
|
280
|
+
def assess_ocr_evidence(text, confidence, line_count):
|
|
281
|
+
"""Classify OCR output before it can become agent-visible evidence."""
|
|
282
|
+
normalized = str(text or "").strip()
|
|
283
|
+
chars = len(normalized)
|
|
284
|
+
if chars == 0:
|
|
285
|
+
return {
|
|
286
|
+
"state": "low_information",
|
|
287
|
+
"accepted": False,
|
|
288
|
+
"reason": "no_readable_text",
|
|
289
|
+
"chars": 0,
|
|
290
|
+
"confidence": round(float(confidence), 1),
|
|
291
|
+
"lines": int(line_count),
|
|
292
|
+
}
|
|
293
|
+
alnum_ratio = sum(character.isalnum() for character in normalized) / max(1, chars)
|
|
294
|
+
if chars < MIN_SUBSTANTIVE_TEXT_CHARS and confidence < MIN_ACCEPTED_CONFIDENCE:
|
|
295
|
+
return {
|
|
296
|
+
"state": "low_information",
|
|
297
|
+
"accepted": False,
|
|
298
|
+
"reason": "insufficient_low_confidence_text",
|
|
299
|
+
"chars": chars,
|
|
300
|
+
"confidence": round(float(confidence), 1),
|
|
301
|
+
"lines": int(line_count),
|
|
302
|
+
"alnum_ratio": round(alnum_ratio, 3),
|
|
303
|
+
}
|
|
304
|
+
if (
|
|
305
|
+
(chars >= HIGH_VOLUME_GARBAGE_CHARS and confidence < HIGH_VOLUME_GARBAGE_CONFIDENCE)
|
|
306
|
+
or confidence < MIN_ACCEPTED_CONFIDENCE
|
|
307
|
+
or alnum_ratio < 0.45
|
|
308
|
+
):
|
|
309
|
+
reason = (
|
|
310
|
+
"high_volume_low_confidence_text"
|
|
311
|
+
if chars >= HIGH_VOLUME_GARBAGE_CHARS and confidence < HIGH_VOLUME_GARBAGE_CONFIDENCE
|
|
312
|
+
else "low_confidence_or_symbol_heavy_text"
|
|
313
|
+
)
|
|
314
|
+
return {
|
|
315
|
+
"state": "rejected",
|
|
316
|
+
"accepted": False,
|
|
317
|
+
"reason": reason,
|
|
318
|
+
"chars": chars,
|
|
319
|
+
"confidence": round(float(confidence), 1),
|
|
320
|
+
"lines": int(line_count),
|
|
321
|
+
"alnum_ratio": round(alnum_ratio, 3),
|
|
322
|
+
}
|
|
323
|
+
return {
|
|
324
|
+
"state": "accepted",
|
|
325
|
+
"accepted": True,
|
|
326
|
+
"reason": "confidence_and_text_quality_met",
|
|
327
|
+
"chars": chars,
|
|
328
|
+
"confidence": round(float(confidence), 1),
|
|
329
|
+
"lines": int(line_count),
|
|
330
|
+
"alnum_ratio": round(alnum_ratio, 3),
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def compute_score(text, confidence, line_count, evidence=None):
|
|
277
335
|
"""Combined scoring heuristic:
|
|
278
336
|
- confidence * sqrt(char_count) — rewards quality and coverage
|
|
279
337
|
- + line_count * 10 — bonus for structured output (more lines = better parse)
|
|
280
338
|
The agent discovered that line-count is a strong proxy for successful parsing
|
|
281
339
|
on structured documents like invoices and forms."""
|
|
340
|
+
quality = evidence or assess_ocr_evidence(text, confidence, line_count)
|
|
282
341
|
char_count = len(text)
|
|
283
|
-
if char_count == 0:
|
|
342
|
+
if not quality["accepted"] or char_count == 0:
|
|
284
343
|
return 0
|
|
285
344
|
return confidence * (char_count ** 0.5) + line_count * 10
|
|
286
345
|
|
|
@@ -319,7 +378,8 @@ def build_ocr_plan(image_area_px, single_psm=None):
|
|
|
319
378
|
|
|
320
379
|
|
|
321
380
|
def has_sufficient_evidence(text, confidence, line_count):
|
|
322
|
-
|
|
381
|
+
quality = assess_ocr_evidence(text, confidence, line_count)
|
|
382
|
+
return quality["accepted"] and confidence >= 70.0 and line_count >= 1
|
|
323
383
|
|
|
324
384
|
|
|
325
385
|
# ---------------------------------------------------------------------------
|
|
@@ -420,15 +480,17 @@ def run_variant_plan(gray, plan, deadline, language, debug_dir=None, debug_prefi
|
|
|
420
480
|
ocr_errors.append(f"{key}: {error}")
|
|
421
481
|
continue
|
|
422
482
|
char_count = len(text)
|
|
423
|
-
|
|
483
|
+
evidence = assess_ocr_evidence(text, confidence, line_count)
|
|
484
|
+
score = compute_score(text, confidence, line_count, evidence)
|
|
424
485
|
all_results[key] = {
|
|
425
486
|
"text": text,
|
|
426
487
|
"chars": char_count,
|
|
427
488
|
"lines": line_count,
|
|
428
489
|
"confidence": round(confidence, 1),
|
|
429
490
|
"score": round(score, 1),
|
|
491
|
+
"evidence": evidence,
|
|
430
492
|
}
|
|
431
|
-
if score > best_score:
|
|
493
|
+
if evidence["accepted"] and score > best_score:
|
|
432
494
|
best_score = score
|
|
433
495
|
best_key = key
|
|
434
496
|
if has_sufficient_evidence(text, confidence, line_count):
|
|
@@ -467,10 +529,66 @@ def run_pipeline(image_path, deadline, language="eng", do_regions=False, debug_d
|
|
|
467
529
|
)
|
|
468
530
|
attempts_completed = len(all_results)
|
|
469
531
|
if not best_key:
|
|
470
|
-
|
|
471
|
-
|
|
532
|
+
if not all_results:
|
|
533
|
+
detail = ocr_errors[-1] if ocr_errors else "no preprocessing variant completed"
|
|
534
|
+
return {"error": f"Tesseract failed for every bounded OCR attempt: {detail}"}
|
|
535
|
+
rejected = [item for item in all_results.values() if item["evidence"]["state"] == "rejected"]
|
|
536
|
+
if rejected:
|
|
537
|
+
worst = max(rejected, key=lambda item: (item["chars"], -item["confidence"]))
|
|
538
|
+
evidence = worst["evidence"]
|
|
539
|
+
message = (
|
|
540
|
+
"OCR text was suppressed because it did not meet evidence-quality requirements "
|
|
541
|
+
f"({evidence['reason']}; {evidence['chars']} chars at {evidence['confidence']}% confidence)."
|
|
542
|
+
)
|
|
543
|
+
return {
|
|
544
|
+
"error": message,
|
|
545
|
+
"diagnostic": {
|
|
546
|
+
**diagnostic("ocr_evidence_rejected", message, deadline, "evidence_quality", attempts_completed, len(plan)),
|
|
547
|
+
"evidence": evidence,
|
|
548
|
+
},
|
|
549
|
+
}
|
|
550
|
+
# Empty or tiny non-substantive detections are valid observations:
|
|
551
|
+
# do not invent text and do not report them as a pipeline error.
|
|
552
|
+
result = {
|
|
553
|
+
"text": "",
|
|
554
|
+
"confidence": 0.0,
|
|
555
|
+
"variant": "none",
|
|
556
|
+
"chars": 0,
|
|
557
|
+
"lines": 0,
|
|
558
|
+
"score": 0.0,
|
|
559
|
+
"image_size": f"{w_orig}x{h_orig}",
|
|
560
|
+
"variants_tested": len(all_results),
|
|
561
|
+
"all_variants": {},
|
|
562
|
+
"quality": {
|
|
563
|
+
"schema": "omnius.ocr-evidence.v1",
|
|
564
|
+
"state": "low_information",
|
|
565
|
+
"accepted": False,
|
|
566
|
+
"reason": "no_accepted_readable_text",
|
|
567
|
+
"low_information_variants": len(all_results),
|
|
568
|
+
},
|
|
569
|
+
"diagnostic": diagnostic(
|
|
570
|
+
"ocr_low_information",
|
|
571
|
+
"OCR produced no accepted readable text; the result is low-information rather than evidence.",
|
|
572
|
+
deadline,
|
|
573
|
+
"evidence_quality",
|
|
574
|
+
attempts_completed,
|
|
575
|
+
len(plan),
|
|
576
|
+
),
|
|
577
|
+
"strategy": {
|
|
578
|
+
"effective_area_px": effective_area,
|
|
579
|
+
"attempts_planned": len(plan),
|
|
580
|
+
"attempts_completed": attempts_completed,
|
|
581
|
+
"early_exit": False,
|
|
582
|
+
"deadline_ms": deadline.deadline_ms,
|
|
583
|
+
},
|
|
584
|
+
}
|
|
585
|
+
return result
|
|
472
586
|
|
|
473
587
|
best = all_results[best_key]
|
|
588
|
+
accepted_results = {
|
|
589
|
+
key: value for key, value in all_results.items()
|
|
590
|
+
if value["evidence"]["state"] == "accepted"
|
|
591
|
+
}
|
|
474
592
|
result = {
|
|
475
593
|
"text": best["text"],
|
|
476
594
|
"confidence": best["confidence"],
|
|
@@ -480,7 +598,18 @@ def run_pipeline(image_path, deadline, language="eng", do_regions=False, debug_d
|
|
|
480
598
|
"score": best["score"],
|
|
481
599
|
"image_size": f"{w_orig}x{h_orig}",
|
|
482
600
|
"variants_tested": len(all_results),
|
|
483
|
-
|
|
601
|
+
# Never leak rejected raw OCR as alternate evidence to agents.
|
|
602
|
+
"all_variants": accepted_results,
|
|
603
|
+
"quality": {
|
|
604
|
+
"schema": "omnius.ocr-evidence.v1",
|
|
605
|
+
**best["evidence"],
|
|
606
|
+
"rejected_variants": sum(
|
|
607
|
+
1 for item in all_results.values() if item["evidence"]["state"] == "rejected"
|
|
608
|
+
),
|
|
609
|
+
"low_information_variants": sum(
|
|
610
|
+
1 for item in all_results.values() if item["evidence"]["state"] == "low_information"
|
|
611
|
+
),
|
|
612
|
+
},
|
|
484
613
|
"strategy": {
|
|
485
614
|
"effective_area_px": effective_area,
|
|
486
615
|
"attempts_planned": len(plan),
|
package/docs/DISCOVERY.json
CHANGED
|
@@ -5399,7 +5399,7 @@
|
|
|
5399
5399
|
"tags": [
|
|
5400
5400
|
"Audio"
|
|
5401
5401
|
],
|
|
5402
|
-
"description": "Admin-only. Requires kind=acoustic|speaker|semantic in the query (canonical) or JSON body. acoustic reuses the pinned JetPack YAMNet/TensorRT setup; speaker creates a private CPython 3.10 CPU-only WeSpeaker CAM++ venv, installs only checksum-pinned NumPy, ONNX Runtime, and its locked CPU support wheels with --no-deps/--no-index, then downloads the pinned model. Its 80-bin Kaldi-compatible NumPy fbank is validated with a
|
|
5402
|
+
"description": "Admin-only. Requires kind=acoustic|speaker|semantic in the query (canonical) or JSON body. acoustic reuses the pinned JetPack YAMNet/TensorRT setup; speaker creates a private CPython 3.10 CPU-only WeSpeaker CAM++ venv, installs only checksum-pinned NumPy, ONNX Runtime, and its locked CPU support wheels with --no-deps/--no-index, then downloads the pinned model. Its 80-bin Kaldi-compatible NumPy fbank is validated with a versioned cross-platform numeric-invariant probe before readiness. The speaker path never imports, links, replaces, or otherwise depends on JetPack Torch/Torchaudio, so an incompatible generic Torchaudio wheel cannot affect CUDA-enabled Egg Torch. semantic installs the isolated JetPack CUDA CLAP dependencies and pinned model. This is the only REST operation allowed to provision. It activates and warms only the requested role worker; no role is substituted for another.",
|
|
5403
5403
|
"parameters": [
|
|
5404
5404
|
{
|
|
5405
5405
|
"name": "kind",
|
|
@@ -6156,6 +6156,13 @@
|
|
|
6156
6156
|
"default": 45,
|
|
6157
6157
|
"description": "Total server-side agent-loop deadline. Distinct from per-backend timeout_s."
|
|
6158
6158
|
},
|
|
6159
|
+
"agent_max_tool_rounds": {
|
|
6160
|
+
"type": "integer",
|
|
6161
|
+
"minimum": 1,
|
|
6162
|
+
"maximum": 8,
|
|
6163
|
+
"default": 1,
|
|
6164
|
+
"description": "Maximum daemon-tool planning rounds. The default executes one tool round, then removes daemon schemas for lower-latency final synthesis."
|
|
6165
|
+
},
|
|
6159
6166
|
"max_turns": {
|
|
6160
6167
|
"type": "integer",
|
|
6161
6168
|
"description": "Q2 — agent_loop max iterations (default 8, max 64)."
|
|
@@ -46,6 +46,7 @@ Important body fields:
|
|
|
46
46
|
| `include_daemon_tools` | array | Permit the bounded core daemon-tool catalog by scope: `read`, `run`, `admin` |
|
|
47
47
|
| `daemon_tool_names` | array | Exact daemon-tool allowlist; recommended for local models |
|
|
48
48
|
| `agent_timeout_s` | number | Whole-loop deadline, default 45 seconds and maximum 600 |
|
|
49
|
+
| `agent_max_tool_rounds` | integer | Daemon tool rounds before forced final synthesis; default 1 |
|
|
49
50
|
| `max_turns` | integer | Server-side agent loop turn cap |
|
|
50
51
|
| `prompt_template` | string | Optional template such as `factual-first` |
|
|
51
52
|
|
|
@@ -129,4 +130,4 @@ For ASR/TTS systems that only need the text brain, use `/realtime` or `/v1/realt
|
|
|
129
130
|
|
|
130
131
|
`/v1/chat/completions` can run an internal tool loop when `agent_loop: true`. This lets clients collapse multiple model/tool round trips into one daemon request. Daemon tool calls execute inline; client-owned tool calls can still be yielded in OpenAI-compatible shape.
|
|
131
132
|
|
|
132
|
-
Ollama-backed loops use its native `/api/chat` tool protocol. `timeout_s` applies to each backend round, while `agent_timeout_s` bounds the complete loop and defaults to 45 seconds. Omnius returns a typed HTTP 504 when
|
|
133
|
+
Ollama-backed loops use its native `/api/chat` tool protocol. `timeout_s` applies to each backend round, while `agent_timeout_s` bounds the complete loop and defaults to 45 seconds. The planning turn is capped at 96 output tokens. By default Omnius executes one daemon-tool round, then removes daemon schemas for the final synthesis; set `agent_max_tool_rounds` only when a workflow genuinely needs deeper tool chaining. Omnius returns a typed HTTP 504 when the total budget expires and HTTP 508 when a model repeats the same daemon tool with identical arguments. Tool results are capped at 6,000 characters before the next prompt. Without `daemon_tool_names`, Omnius offers only a compact core catalog permitted by `include_daemon_tools`; `prompt_template: "factual-first"` narrows it further to `web_search` and `web_fetch`.
|
|
@@ -135,6 +135,13 @@ return diagnostics. A timeout or cancellation returns
|
|
|
135
135
|
`ocr_cancelled`; cancellation terminates the Python/Tesseract process group
|
|
136
136
|
with TERM followed by KILL.
|
|
137
137
|
|
|
138
|
+
OCR text is evidence-gated before it is returned. Strong text with adequate
|
|
139
|
+
confidence is accepted. An empty or tiny non-substantive result is a successful
|
|
140
|
+
`omnius.ocr-evidence.v1` `low_information` observation with diagnostic code
|
|
141
|
+
`ocr_low_information`, not a fabricated transcript. High-volume low-confidence
|
|
142
|
+
or symbol-heavy output is suppressed and returned as
|
|
143
|
+
`ocr_evidence_rejected`; its raw text is not exposed as alternate OCR evidence.
|
|
144
|
+
|
|
138
145
|
## TTS
|
|
139
146
|
|
|
140
147
|
`POST /v1/voice/tts` returns audio bytes. `format` can be `wav` or `pcm`. `X-Sample-Rate` reports the sample rate.
|
|
@@ -270,8 +277,9 @@ managed venv installs it with `--no-deps --no-index`.
|
|
|
270
277
|
The worker implements the WeSpeaker CAM++ 80-bin Kaldi configuration in pure
|
|
271
278
|
NumPy: 25 ms / 10 ms Hamming frames, dither disabled, Kaldi pre-emphasis and
|
|
272
279
|
mel bank behavior, then full-clip CMN without CVN. Readiness invokes a
|
|
273
|
-
|
|
274
|
-
|
|
280
|
+
versioned CPU preprocessing probe. It checks the fixed Kaldi configuration,
|
|
281
|
+
feature shape, CMN, energy range, and expected mel-band profile with explicit
|
|
282
|
+
cross-platform numeric tolerances before it can report ready. The speaker path neither imports
|
|
275
283
|
nor links Torch or Torchaudio, so the generic `torchaudio-2.2.0` ABI mismatch
|
|
276
284
|
cannot bind against, replace, or otherwise affect JetPack's CUDA-enabled Egg
|
|
277
285
|
Torch. Inference remains install-free and network-free; failed package imports
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omnius",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.641",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "omnius",
|
|
9
|
-
"version": "1.0.
|
|
9
|
+
"version": "1.0.641",
|
|
10
10
|
"bundleDependencies": [
|
|
11
11
|
"image-to-ascii"
|
|
12
12
|
],
|
package/package.json
CHANGED