omnius 1.0.413 → 1.0.414
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +232 -71
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -41700,28 +41700,26 @@ var init_transcribe_tool = __esm({
|
|
|
41700
41700
|
}
|
|
41701
41701
|
if (effectiveModel !== askedModel)
|
|
41702
41702
|
model = effectiveModel;
|
|
41703
|
-
const failures = [];
|
|
41704
|
-
let managedPython = "";
|
|
41705
|
-
try {
|
|
41706
|
-
managedPython = await this.ensureManagedWhisperRuntime();
|
|
41707
|
-
process.env.TRANSCRIBE_PYTHON = managedPython;
|
|
41708
|
-
await ensureTranscribeCliPythonModule(managedPython);
|
|
41709
|
-
} catch (err) {
|
|
41710
|
-
failures.push(`managed ASR runtime bootstrap: ${err instanceof Error ? err.message : String(err)}`);
|
|
41711
|
-
}
|
|
41712
41703
|
if (backend === "managed-whisper") {
|
|
41713
41704
|
const managed2 = await this.execViaManagedWhisper(filePath, model, start2);
|
|
41714
41705
|
if (managed2.success)
|
|
41715
41706
|
return managed2;
|
|
41716
|
-
if (managed2.error)
|
|
41717
|
-
failures.push(managed2.error);
|
|
41718
41707
|
return {
|
|
41719
41708
|
success: false,
|
|
41720
41709
|
output: "",
|
|
41721
|
-
error: `Managed Whisper ASR failed. ${
|
|
41710
|
+
error: `Managed Whisper ASR failed. ${String(managed2.error || managed2.output || "unknown error").slice(0, 1200)}`,
|
|
41722
41711
|
durationMs: performance.now() - start2
|
|
41723
41712
|
};
|
|
41724
41713
|
}
|
|
41714
|
+
const failures = [];
|
|
41715
|
+
let managedPython = "";
|
|
41716
|
+
try {
|
|
41717
|
+
managedPython = await this.ensureManagedWhisperRuntime();
|
|
41718
|
+
process.env.TRANSCRIBE_PYTHON = managedPython;
|
|
41719
|
+
await ensureTranscribeCliPythonModule(managedPython);
|
|
41720
|
+
} catch (err) {
|
|
41721
|
+
failures.push(`managed ASR runtime bootstrap: ${err instanceof Error ? err.message : String(err)}`);
|
|
41722
|
+
}
|
|
41725
41723
|
const tc = await loadTranscribeCli() ?? await ensureManagedTranscribeCliRuntime();
|
|
41726
41724
|
if (tc && backend !== "managed-whisper") {
|
|
41727
41725
|
try {
|
|
@@ -603158,6 +603156,16 @@ function isAudioFailureText(value2) {
|
|
|
603158
603156
|
if (!text2) return false;
|
|
603159
603157
|
return /^(?:ASR|VAD|Transcription)\s+(?:failed|unavailable)|No module named ['"]?transcribe_cli|torchcodec|torchaudio version/i.test(text2);
|
|
603160
603158
|
}
|
|
603159
|
+
function sanitizeLiveAudioError(value2) {
|
|
603160
|
+
const text2 = String(value2 ?? "").trim();
|
|
603161
|
+
if (!text2) return "";
|
|
603162
|
+
if (/transcribe-cli module:\s*No module named ['"]?transcribe_cli/i.test(text2) || /No module named ['"]?transcribe_cli/i.test(text2)) {
|
|
603163
|
+
const parts = text2.split(/\s*\|\s*/g).map((part) => part.trim()).filter((part) => part && !/transcribe-cli module:\s*No module named ['"]?transcribe_cli/i.test(part) && !/No module named ['"]?transcribe_cli/i.test(part));
|
|
603164
|
+
const preserved = parts.join(" | ").trim();
|
|
603165
|
+
return preserved || "ASR unavailable: managed Whisper runtime is still bootstrapping or failed; transcribe-cli is not required for live ASR.";
|
|
603166
|
+
}
|
|
603167
|
+
return text2;
|
|
603168
|
+
}
|
|
603161
603169
|
function readPcm16WavActivity(filePath, bins = 36) {
|
|
603162
603170
|
try {
|
|
603163
603171
|
const buf = readFileSync88(filePath);
|
|
@@ -603272,39 +603280,12 @@ function parseLiveSpeechDecision(value2) {
|
|
|
603272
603280
|
};
|
|
603273
603281
|
}
|
|
603274
603282
|
function fallbackLiveSpeechDecision(proposal) {
|
|
603275
|
-
if (proposal.kind === "unknown_person") {
|
|
603276
|
-
return {
|
|
603277
|
-
speak: true,
|
|
603278
|
-
text: proposal.defaultText,
|
|
603279
|
-
confidence: 0.72,
|
|
603280
|
-
action: "speak",
|
|
603281
|
-
reason: "unknown person visible and identity clarification is useful"
|
|
603282
|
-
};
|
|
603283
|
-
}
|
|
603284
|
-
if (proposal.kind === "audio_intent") {
|
|
603285
|
-
return {
|
|
603286
|
-
speak: true,
|
|
603287
|
-
text: proposal.defaultText,
|
|
603288
|
-
confidence: 0.68,
|
|
603289
|
-
action: "speak",
|
|
603290
|
-
reason: "audio intake classified speech as addressed to Omnius"
|
|
603291
|
-
};
|
|
603292
|
-
}
|
|
603293
|
-
if (proposal.urgency === "high") {
|
|
603294
|
-
return {
|
|
603295
|
-
speak: true,
|
|
603296
|
-
text: proposal.defaultText,
|
|
603297
|
-
confidence: 0.6,
|
|
603298
|
-
action: "speak",
|
|
603299
|
-
reason: "high-urgency live trigger"
|
|
603300
|
-
};
|
|
603301
|
-
}
|
|
603302
603283
|
return {
|
|
603303
603284
|
speak: false,
|
|
603304
603285
|
text: "",
|
|
603305
603286
|
confidence: 0.55,
|
|
603306
|
-
action: "silent",
|
|
603307
|
-
reason: "
|
|
603287
|
+
action: proposal.urgency === "high" ? "queue_review" : "silent",
|
|
603288
|
+
reason: "live speech arbiter unavailable; refusing canned vocal fallback"
|
|
603308
603289
|
};
|
|
603309
603290
|
}
|
|
603310
603291
|
async function classifyLiveSpeech(proposal, config) {
|
|
@@ -603337,7 +603318,7 @@ async function classifyLiveSpeech(proposal, config) {
|
|
|
603337
603318
|
`urgency=${proposal.urgency}`,
|
|
603338
603319
|
`summary=${proposal.summary}`,
|
|
603339
603320
|
`evidence=${proposal.evidence}`,
|
|
603340
|
-
`
|
|
603321
|
+
proposal.speechGoal ? `speech_goal=${proposal.speechGoal}` : ""
|
|
603341
603322
|
].join("\n").slice(0, 1600)
|
|
603342
603323
|
}
|
|
603343
603324
|
]
|
|
@@ -603637,7 +603618,7 @@ function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
|
|
|
603637
603618
|
lines.push(`├${horizontal}┤`);
|
|
603638
603619
|
const transcriptFailure = isAudioFailureText(snapshot.audio.transcript) ? snapshot.audio.transcript : "";
|
|
603639
603620
|
const transcript = transcriptFailure ? "" : snapshot.audio.transcript;
|
|
603640
|
-
const audioError = [snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | ");
|
|
603621
|
+
const audioError = sanitizeLiveAudioError([snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | "));
|
|
603641
603622
|
const audioBits = [
|
|
603642
603623
|
`audio input=${snapshot.audio.input || snapshot.config.selectedAudioInput || "none"}`,
|
|
603643
603624
|
snapshot.audio.activity ? `activity ${snapshot.audio.activity.waveform} rms=${snapshot.audio.activity.rmsDb.toFixed(1)}dB active=${Math.round(snapshot.audio.activity.activeRatio * 100)}%` : "",
|
|
@@ -604020,7 +604001,7 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
604020
604001
|
if (snapshot.audio) {
|
|
604021
604002
|
const transcriptFailure = isAudioFailureText(snapshot.audio.transcript) ? snapshot.audio.transcript : "";
|
|
604022
604003
|
const transcript = transcriptFailure ? "" : snapshot.audio.transcript;
|
|
604023
|
-
const audioError = [snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | ");
|
|
604004
|
+
const audioError = sanitizeLiveAudioError([snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | "));
|
|
604024
604005
|
const audioEvents = (snapshot.events ?? []).filter((event) => event.kind === "audio_transcript" || event.kind === "audio_sound").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 8);
|
|
604025
604006
|
if (audioEvents.length > 0) {
|
|
604026
604007
|
lines.push("");
|
|
@@ -604464,6 +604445,10 @@ var init_live_sensors = __esm({
|
|
|
604464
604445
|
setCameraRotation(device, rotation, source, evidence, confidence2) {
|
|
604465
604446
|
const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
|
|
604466
604447
|
if (!target) return null;
|
|
604448
|
+
const current = this.config.cameraOrientation?.[target];
|
|
604449
|
+
if (source === "auto" && current?.source === "manual") {
|
|
604450
|
+
return current;
|
|
604451
|
+
}
|
|
604467
604452
|
const orientation = {
|
|
604468
604453
|
rotation,
|
|
604469
604454
|
source,
|
|
@@ -604929,7 +604914,7 @@ var init_live_sensors = __esm({
|
|
|
604929
604914
|
source,
|
|
604930
604915
|
summary: `${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
|
|
604931
604916
|
evidence: unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; "),
|
|
604932
|
-
|
|
604917
|
+
speechGoal: "If appropriate, ask the visible unknown person for the name Omnius should associate with the face crop. Do not claim an identity.",
|
|
604933
604918
|
urgency: "high",
|
|
604934
604919
|
observedAt: sampledAt
|
|
604935
604920
|
}, 9e4);
|
|
@@ -604946,7 +604931,7 @@ var init_live_sensors = __esm({
|
|
|
604946
604931
|
source,
|
|
604947
604932
|
summary: `Visual trigger hit in live camera stream ${source}`,
|
|
604948
604933
|
evidence: triggerEvents.map((entry) => entry.summary).join("; "),
|
|
604949
|
-
|
|
604934
|
+
speechGoal: "If immediate clarification is warranted, ask whether Omnius should inspect the visible trigger.",
|
|
604950
604935
|
urgency: "high",
|
|
604951
604936
|
observedAt: sampledAt
|
|
604952
604937
|
}, 45e3);
|
|
@@ -605143,7 +605128,7 @@ ${output}`).join("\n\n");
|
|
|
605143
605128
|
source: capture.device,
|
|
605144
605129
|
summary: "Live audio intake classified speech as intended for Omnius",
|
|
605145
605130
|
evidence: `transcript=${transcript}; confidence=${intake.confidence.toFixed(2)}; reason=${intake.reason}`,
|
|
605146
|
-
|
|
605131
|
+
speechGoal: "Respond naturally to the addressed utterance using live context; keep it brief.",
|
|
605147
605132
|
urgency: "high",
|
|
605148
605133
|
observedAt: this.snapshot.audio.updatedAt
|
|
605149
605134
|
}, 2e4);
|
|
@@ -695869,7 +695854,10 @@ var init_task_manager_singleton = __esm({
|
|
|
695869
695854
|
// packages/cli/src/tui/voicechat.ts
|
|
695870
695855
|
var voicechat_exports = {};
|
|
695871
695856
|
__export(voicechat_exports, {
|
|
695872
|
-
VoiceChatSession: () => VoiceChatSession
|
|
695857
|
+
VoiceChatSession: () => VoiceChatSession,
|
|
695858
|
+
buildRealtimeVoiceMessages: () => buildRealtimeVoiceMessages,
|
|
695859
|
+
extractVoiceModelReply: () => extractVoiceModelReply,
|
|
695860
|
+
isLikelyAgentSpeechEcho: () => isLikelyAgentSpeechEcho
|
|
695873
695861
|
});
|
|
695874
695862
|
import { EventEmitter as EventEmitter12 } from "node:events";
|
|
695875
695863
|
function clamp0116(x) {
|
|
@@ -695966,19 +695954,138 @@ function stripToolJsonLines(text2) {
|
|
|
695966
695954
|
});
|
|
695967
695955
|
return kept.join("\n").trim();
|
|
695968
695956
|
}
|
|
695969
|
-
|
|
695957
|
+
function compactSpeechText(value2, maxLength = 8e3) {
|
|
695958
|
+
const text2 = String(value2 ?? "").replace(/\s+/g, " ").replace(/\s+([,.!?;:])/g, "$1").trim();
|
|
695959
|
+
return text2 ? text2.slice(0, maxLength) : "";
|
|
695960
|
+
}
|
|
695961
|
+
function comparableSpeechText(value2) {
|
|
695962
|
+
return String(value2 ?? "").toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
|
695963
|
+
}
|
|
695964
|
+
function comparableTokens(value2) {
|
|
695965
|
+
return comparableSpeechText(value2).split(/\s+/).filter(Boolean);
|
|
695966
|
+
}
|
|
695967
|
+
function jaccardSimilarity(aTokens, bTokens) {
|
|
695968
|
+
const a2 = new Set(aTokens);
|
|
695969
|
+
const b = new Set(bTokens);
|
|
695970
|
+
if (!a2.size || !b.size) return 0;
|
|
695971
|
+
let overlap = 0;
|
|
695972
|
+
for (const token of a2) if (b.has(token)) overlap++;
|
|
695973
|
+
return overlap / Math.max(1, Math.min(a2.size, b.size));
|
|
695974
|
+
}
|
|
695975
|
+
function isMostlyContainedInOrder(needleTokens, haystackTokens) {
|
|
695976
|
+
if (!needleTokens.length || !haystackTokens.length) return false;
|
|
695977
|
+
let index = 0;
|
|
695978
|
+
for (const token of haystackTokens) {
|
|
695979
|
+
if (token === needleTokens[index]) index++;
|
|
695980
|
+
if (index >= needleTokens.length) return true;
|
|
695981
|
+
}
|
|
695982
|
+
return false;
|
|
695983
|
+
}
|
|
695984
|
+
function isLikelyAgentSpeechEcho({
|
|
695985
|
+
heardText,
|
|
695986
|
+
agentText,
|
|
695987
|
+
elapsedMs: elapsedMs2
|
|
695988
|
+
}) {
|
|
695989
|
+
const heard = compactSpeechText(heardText, 1200);
|
|
695990
|
+
if (!heard) return { likelyEcho: false, echoSimilarity: 0, reason: "empty" };
|
|
695991
|
+
if (!Number.isFinite(elapsedMs2) || elapsedMs2 > AGENT_ECHO_WINDOW_MS) {
|
|
695992
|
+
return { likelyEcho: false, echoSimilarity: 0, reason: "outside_window" };
|
|
695993
|
+
}
|
|
695994
|
+
const heardTokens = comparableTokens(heard);
|
|
695995
|
+
const agentTokens = comparableTokens(agentText);
|
|
695996
|
+
if (!agentTokens.length) return { likelyEcho: false, echoSimilarity: 0, reason: "no_agent_speech" };
|
|
695997
|
+
const agentComparable = agentTokens.join(" ");
|
|
695998
|
+
const heardComparable = heardTokens.join(" ");
|
|
695999
|
+
const echoSimilarity = jaccardSimilarity(heardTokens, agentTokens);
|
|
696000
|
+
const exactEcho = Boolean(heardComparable && agentComparable && (agentComparable.includes(heardComparable) || heardComparable.includes(agentComparable)));
|
|
696001
|
+
const orderedEcho = heardTokens.length >= 3 && isMostlyContainedInOrder(heardTokens, agentTokens);
|
|
696002
|
+
const likelyEcho = echoSimilarity >= AGENT_ECHO_SIMILARITY || exactEcho || orderedEcho;
|
|
696003
|
+
return {
|
|
696004
|
+
likelyEcho,
|
|
696005
|
+
echoSimilarity,
|
|
696006
|
+
reason: likelyEcho ? "speaker_echo" : "pass"
|
|
696007
|
+
};
|
|
696008
|
+
}
|
|
696009
|
+
function clipVoiceReply(text2, maxLength = MAX_VOICE_REPLY_CHARS) {
|
|
696010
|
+
const cleaned = String(text2 || "").replace(/\s+/g, " ").replace(/^[`"']+|[`"']+$/g, "").trim();
|
|
696011
|
+
return cleaned ? cleaned.slice(0, maxLength) : "";
|
|
696012
|
+
}
|
|
696013
|
+
function tryParseJsonObject(value2) {
|
|
696014
|
+
const text2 = value2.trim();
|
|
696015
|
+
if (!text2.startsWith("{") || !text2.endsWith("}")) return null;
|
|
696016
|
+
try {
|
|
696017
|
+
const parsed = JSON.parse(text2);
|
|
696018
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
696019
|
+
} catch {
|
|
696020
|
+
return null;
|
|
696021
|
+
}
|
|
696022
|
+
}
|
|
696023
|
+
function extractVoiceModelReply(value2) {
|
|
696024
|
+
const cleaned = String(value2 ?? "").replace(/^\s*```(?:json)?/i, "").replace(/```\s*$/i, "").trim();
|
|
696025
|
+
const start2 = cleaned.indexOf("{");
|
|
696026
|
+
const end = cleaned.lastIndexOf("}");
|
|
696027
|
+
const embedded = tryParseJsonObject(cleaned) || (start2 >= 0 && end > start2 ? tryParseJsonObject(cleaned.slice(start2, end + 1)) : null);
|
|
696028
|
+
if (embedded && typeof embedded["text"] === "string") {
|
|
696029
|
+
const actionRaw = String(embedded["action"] ?? "speak").toLowerCase();
|
|
696030
|
+
const action = actionRaw === "hangup" ? "hangup" : actionRaw === "silent" || actionRaw === "none" ? "silent" : "speak";
|
|
696031
|
+
return {
|
|
696032
|
+
action,
|
|
696033
|
+
text: clipVoiceReply(embedded["text"]),
|
|
696034
|
+
structured: true,
|
|
696035
|
+
internalReason: typeof embedded["internal_reason"] === "string" ? clipVoiceReply(embedded["internal_reason"], 500) : typeof embedded["internalReason"] === "string" ? clipVoiceReply(embedded["internalReason"], 500) : null
|
|
696036
|
+
};
|
|
696037
|
+
}
|
|
696038
|
+
const text2 = cleaned.replace(/^\s*```(?:[a-z]+)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").replace(/^(?:agent|assistant|ai|voice|speaker|reply)\s*:\s*/i, "").trim();
|
|
696039
|
+
return {
|
|
696040
|
+
action: text2 ? "speak" : "silent",
|
|
696041
|
+
text: clipVoiceReply(text2),
|
|
696042
|
+
structured: false,
|
|
696043
|
+
internalReason: null
|
|
696044
|
+
};
|
|
696045
|
+
}
|
|
696046
|
+
function isContextSnapshotTurn(turn) {
|
|
696047
|
+
return turn.role === "system" && turn.content.startsWith(CONTEXT_SNAPSHOT_PREFIX);
|
|
696048
|
+
}
|
|
696049
|
+
function isPinnedSystemTurn(turn) {
|
|
696050
|
+
return turn.role === "system" && (turn.content === SYSTEM_PROMPT2 || turn.content.startsWith("Available tools (emit one-line JSON:"));
|
|
696051
|
+
}
|
|
696052
|
+
function buildRealtimeVoiceMessages(turns, maxMessages = MAX_REALTIME_MODEL_MESSAGES) {
|
|
696053
|
+
const pinned = [];
|
|
696054
|
+
const dynamic = [];
|
|
696055
|
+
let latestSnapshot = null;
|
|
696056
|
+
for (const turn of turns) {
|
|
696057
|
+
if (isPinnedSystemTurn(turn) && !pinned.some((p2) => p2.content === turn.content)) {
|
|
696058
|
+
pinned.push(turn);
|
|
696059
|
+
} else if (isContextSnapshotTurn(turn)) {
|
|
696060
|
+
latestSnapshot = turn;
|
|
696061
|
+
} else {
|
|
696062
|
+
dynamic.push(turn);
|
|
696063
|
+
}
|
|
696064
|
+
}
|
|
696065
|
+
if (latestSnapshot) dynamic.push(latestSnapshot);
|
|
696066
|
+
const available = Math.max(0, maxMessages - pinned.length);
|
|
696067
|
+
return [...pinned, ...dynamic.slice(-available)];
|
|
696068
|
+
}
|
|
696069
|
+
var VAD_SILENCE_MS, MAX_SEGMENT_MS, SUMMARY_INJECTION_INTERVAL, MAX_CONTEXT_TURNS, MAX_REALTIME_MODEL_MESSAGES, CONTEXT_SNAPSHOT_PREFIX, MAX_VOICE_REPLY_CHARS, AGENT_ECHO_WINDOW_MS, AGENT_ECHO_SIMILARITY, SYSTEM_PROMPT2, MIN_SIGNAL_SCORE, NOISE_ONLY_RE, VoiceChatSession;
|
|
695970
696070
|
var init_voicechat = __esm({
|
|
695971
696071
|
"packages/cli/src/tui/voicechat.ts"() {
|
|
695972
696072
|
"use strict";
|
|
695973
696073
|
VAD_SILENCE_MS = 3e3;
|
|
695974
696074
|
MAX_SEGMENT_MS = 6500;
|
|
696075
|
+
SUMMARY_INJECTION_INTERVAL = 4;
|
|
695975
696076
|
MAX_CONTEXT_TURNS = 20;
|
|
696077
|
+
MAX_REALTIME_MODEL_MESSAGES = 16;
|
|
696078
|
+
CONTEXT_SNAPSHOT_PREFIX = "Context snapshot (read-only):";
|
|
696079
|
+
MAX_VOICE_REPLY_CHARS = 700;
|
|
696080
|
+
AGENT_ECHO_WINDOW_MS = 12e3;
|
|
696081
|
+
AGENT_ECHO_SIMILARITY = 0.72;
|
|
695976
696082
|
SYSTEM_PROMPT2 = `You are a voice assistant having a live spoken conversation. Keep responses extremely brief — 1-2 sentences max. You're speaking aloud, not writing. Be conversational, direct, and helpful. Don't use markdown or formatting — just natural speech.
|
|
695977
696083
|
|
|
695978
696084
|
Rules:
|
|
695979
696085
|
- Never invent environment facts (cwd, OS, specs, repo state). If you need a precise fact from the main agent, request a tool by outputting on a single line EXACTLY one JSON object: {"tool": string, "args": object} and nothing else. Then wait for the tool result before answering.
|
|
695980
696086
|
- You may also request to relay a user task to the main agent by emitting {"tool":"voice_to_main","args":{"message":"...","start":true}}.
|
|
695981
|
-
- Prefer tools for factual queries; otherwise, answer directly with a short reply
|
|
696087
|
+
- Prefer tools for factual queries; otherwise, answer directly with a short reply.
|
|
696088
|
+
- Internal notes must never be spoken. If you need an internal reason, return compact JSON: {"action":"speak","text":"exact words to speak aloud","internal_reason":"brief nonspoken reason"}.`;
|
|
695982
696089
|
MIN_SIGNAL_SCORE = 0.4;
|
|
695983
696090
|
NOISE_ONLY_RE = /^(?:[.·…\s,;:!?\-–—_()\[\]{}"'`]+|(?:uh|um|erm|hmm|mm+|uhh+|umm+)[\s.!?]*)+$/i;
|
|
695984
696091
|
VoiceChatSession = class extends EventEmitter12 {
|
|
@@ -696008,6 +696115,8 @@ Rules:
|
|
|
696008
696115
|
silenceTimer = null;
|
|
696009
696116
|
maxSegmentTimer = null;
|
|
696010
696117
|
lastSignalScore = null;
|
|
696118
|
+
listenPausedForResponse = false;
|
|
696119
|
+
lastAgentSpeech = null;
|
|
696011
696120
|
// Abort control for inference
|
|
696012
696121
|
abortController = null;
|
|
696013
696122
|
// Callbacks
|
|
@@ -696083,6 +696192,10 @@ Rules:
|
|
|
696083
696192
|
this.context.push({ role: "system", content: this.toolCatalogNote });
|
|
696084
696193
|
}
|
|
696085
696194
|
this.turnCount = 0;
|
|
696195
|
+
this.voiceTranscript = [];
|
|
696196
|
+
this.relayTranscript = [];
|
|
696197
|
+
this.lastAgentSpeech = null;
|
|
696198
|
+
this.listenPausedForResponse = false;
|
|
696086
696199
|
if (this.verbose) this.onStatus("VoiceChat active — LISTENING");
|
|
696087
696200
|
this._onTranscript = (...args) => {
|
|
696088
696201
|
let text2;
|
|
@@ -696172,6 +696285,12 @@ Rules:
|
|
|
696172
696285
|
if (this._state !== "LISTENING" && this._state !== "CAPTURING") {
|
|
696173
696286
|
return;
|
|
696174
696287
|
}
|
|
696288
|
+
const echo = this.classifyAgentEcho(text2);
|
|
696289
|
+
if (echo.likelyEcho) {
|
|
696290
|
+
if (this.debugSnr) this.onStatus(`Ignoring likely agent-speaker echo (${echo.echoSimilarity.toFixed(2)}): ${truncateForLog(text2, 48)}`);
|
|
696291
|
+
this.emit("voiceEchoFiltered", echo);
|
|
696292
|
+
return;
|
|
696293
|
+
}
|
|
696175
696294
|
if (this._state === "LISTENING") {
|
|
696176
696295
|
this.setState("CAPTURING");
|
|
696177
696296
|
this.captureBuffer = "";
|
|
@@ -696222,6 +696341,7 @@ Rules:
|
|
|
696222
696341
|
return;
|
|
696223
696342
|
}
|
|
696224
696343
|
this.setState("TRANSCRIBING");
|
|
696344
|
+
this.pauseListenForResponse();
|
|
696225
696345
|
this.onUserSpeech(text2);
|
|
696226
696346
|
this.voiceTranscript.push({ role: "user", content: text2, ts: Date.now() });
|
|
696227
696347
|
this.context.push({ role: "user", content: text2 });
|
|
@@ -696232,9 +696352,7 @@ Rules:
|
|
|
696232
696352
|
} catch {
|
|
696233
696353
|
}
|
|
696234
696354
|
}
|
|
696235
|
-
|
|
696236
|
-
this.context.splice(1, 1);
|
|
696237
|
-
}
|
|
696355
|
+
this.trimContext();
|
|
696238
696356
|
this.think();
|
|
696239
696357
|
}
|
|
696240
696358
|
// ---------------------------------------------------------------------------
|
|
@@ -696245,18 +696363,20 @@ Rules:
|
|
|
696245
696363
|
this.setState("THINKING");
|
|
696246
696364
|
if (this.verbose) this.onStatus("Thinking...");
|
|
696247
696365
|
this.abortController = new AbortController();
|
|
696366
|
+
const lastUser = this.latestUserText();
|
|
696248
696367
|
try {
|
|
696249
696368
|
if (this.toolRelay?.contextSnapshot) {
|
|
696250
696369
|
try {
|
|
696251
696370
|
const snap = await Promise.resolve(this.toolRelay.contextSnapshot());
|
|
696252
696371
|
if (snap && snap.trim()) {
|
|
696372
|
+
this.dropTransientSystemTurns();
|
|
696253
696373
|
this.context.push({ role: "system", content: `Context snapshot (read-only):
|
|
696254
696374
|
${snap.trim()}` });
|
|
696375
|
+
this.trimContext();
|
|
696255
696376
|
}
|
|
696256
696377
|
} catch {
|
|
696257
696378
|
}
|
|
696258
696379
|
}
|
|
696259
|
-
const lastUser = [...this.context].reverse().find((m2) => m2.role === "user")?.content || "";
|
|
696260
696380
|
let preAnswered = false;
|
|
696261
696381
|
if (this.heuristicsEnabled && this.toolRelay && lastUser) {
|
|
696262
696382
|
const lower = lastUser.toLowerCase();
|
|
@@ -696310,6 +696430,7 @@ ${out}` });
|
|
|
696310
696430
|
}
|
|
696311
696431
|
this.context.push({ role: "system", content: `Tool ${name10} result (authoritative):
|
|
696312
696432
|
${toolOutput}` });
|
|
696433
|
+
this.trimContext();
|
|
696313
696434
|
}
|
|
696314
696435
|
if (!this.active) return;
|
|
696315
696436
|
if (this.heuristicsEnabled && this.toolRelay && /\b(can't|cannot)\b/i.test(response) && this.toolCatalogNote) {
|
|
@@ -696317,18 +696438,25 @@ ${toolOutput}` });
|
|
|
696317
696438
|
response = await this.streamOllamaInference(this.abortController.signal);
|
|
696318
696439
|
}
|
|
696319
696440
|
if (response.trim()) {
|
|
696320
|
-
const
|
|
696441
|
+
const reply = extractVoiceModelReply(stripToolJsonLines(response.trim()));
|
|
696442
|
+
const finalSpoken = reply.text;
|
|
696443
|
+
if (!finalSpoken || reply.action === "silent") {
|
|
696444
|
+
return;
|
|
696445
|
+
}
|
|
696446
|
+
if (typeof this.voice.waitUntilIdle === "function") {
|
|
696447
|
+
try {
|
|
696448
|
+
await this.voice.waitUntilIdle();
|
|
696449
|
+
} catch {
|
|
696450
|
+
}
|
|
696451
|
+
}
|
|
696321
696452
|
this.context.push({ role: "assistant", content: finalSpoken });
|
|
696453
|
+
this.trimContext();
|
|
696322
696454
|
this.setState("SPEAKING");
|
|
696323
696455
|
this.onAgentSpeech(finalSpoken);
|
|
696324
|
-
|
|
696325
|
-
this.listen.pause();
|
|
696326
|
-
} catch {
|
|
696327
|
-
}
|
|
696456
|
+
this.lastAgentSpeech = { text: finalSpoken, startedAt: Date.now() };
|
|
696328
696457
|
this.voice.speak(finalSpoken);
|
|
696329
696458
|
this.voiceTranscript.push({ role: "assistant", content: finalSpoken, ts: Date.now() });
|
|
696330
|
-
this.
|
|
696331
|
-
if (this.runner) {
|
|
696459
|
+
if (this.runner && this.turnCount % SUMMARY_INJECTION_INTERVAL === 0) {
|
|
696332
696460
|
this.injectSummary();
|
|
696333
696461
|
}
|
|
696334
696462
|
if (typeof this.voice.waitUntilIdle === "function") {
|
|
@@ -696337,9 +696465,12 @@ ${toolOutput}` });
|
|
|
696337
696465
|
} catch {
|
|
696338
696466
|
}
|
|
696339
696467
|
} else {
|
|
696340
|
-
const estimatedMs = Math.max(1500,
|
|
696468
|
+
const estimatedMs = Math.max(1500, finalSpoken.length / 5 * (6e4 / 150));
|
|
696341
696469
|
await new Promise((r2) => setTimeout(r2, estimatedMs));
|
|
696342
696470
|
}
|
|
696471
|
+
if (reply.action === "hangup") {
|
|
696472
|
+
this.active = false;
|
|
696473
|
+
}
|
|
696343
696474
|
}
|
|
696344
696475
|
} catch (err) {
|
|
696345
696476
|
if (!this.active) return;
|
|
@@ -696351,14 +696482,44 @@ ${toolOutput}` });
|
|
|
696351
696482
|
this.abortController = null;
|
|
696352
696483
|
}
|
|
696353
696484
|
if (this.active) {
|
|
696354
|
-
|
|
696355
|
-
await this.listen.resume();
|
|
696356
|
-
} catch {
|
|
696357
|
-
}
|
|
696485
|
+
await this.resumeListenAfterResponse();
|
|
696358
696486
|
this.setState("LISTENING");
|
|
696359
696487
|
if (this.verbose) this.onStatus("LISTENING...");
|
|
696360
696488
|
}
|
|
696361
696489
|
}
|
|
696490
|
+
latestUserText() {
|
|
696491
|
+
return [...this.context].reverse().find((m2) => m2.role === "user")?.content || "";
|
|
696492
|
+
}
|
|
696493
|
+
pauseListenForResponse() {
|
|
696494
|
+
if (this.listenPausedForResponse) return;
|
|
696495
|
+
try {
|
|
696496
|
+
this.listen.pause();
|
|
696497
|
+
} catch {
|
|
696498
|
+
}
|
|
696499
|
+
this.listenPausedForResponse = true;
|
|
696500
|
+
}
|
|
696501
|
+
async resumeListenAfterResponse() {
|
|
696502
|
+
if (!this.listenPausedForResponse) return;
|
|
696503
|
+
try {
|
|
696504
|
+
await this.listen.resume();
|
|
696505
|
+
} catch {
|
|
696506
|
+
}
|
|
696507
|
+
this.listenPausedForResponse = false;
|
|
696508
|
+
}
|
|
696509
|
+
classifyAgentEcho(text2) {
|
|
696510
|
+
const agentText = this.lastAgentSpeech?.text || "";
|
|
696511
|
+
const elapsedMs2 = this.lastAgentSpeech ? Date.now() - this.lastAgentSpeech.startedAt : Number.POSITIVE_INFINITY;
|
|
696512
|
+
return isLikelyAgentSpeechEcho({ heardText: text2, agentText, elapsedMs: elapsedMs2 });
|
|
696513
|
+
}
|
|
696514
|
+
dropTransientSystemTurns() {
|
|
696515
|
+
this.context = this.context.filter((turn) => !isContextSnapshotTurn(turn));
|
|
696516
|
+
}
|
|
696517
|
+
trimContext() {
|
|
696518
|
+
this.context = buildRealtimeVoiceMessages(this.context, MAX_CONTEXT_TURNS + 2);
|
|
696519
|
+
}
|
|
696520
|
+
buildRealtimeMessagesForRequest() {
|
|
696521
|
+
return buildRealtimeVoiceMessages(this.context, MAX_REALTIME_MODEL_MESSAGES);
|
|
696522
|
+
}
|
|
696362
696523
|
/**
|
|
696363
696524
|
* Stream inference. Tries native Ollama /api/chat first (supports think:false
|
|
696364
696525
|
* for reasoning models), falls back to OpenAI-compat /v1/chat/completions.
|
|
@@ -696370,11 +696531,11 @@ ${toolOutput}` });
|
|
|
696370
696531
|
try {
|
|
696371
696532
|
const nativeBody = JSON.stringify({
|
|
696372
696533
|
model: this.model,
|
|
696373
|
-
messages: this.
|
|
696534
|
+
messages: this.buildRealtimeMessagesForRequest(),
|
|
696374
696535
|
stream: true,
|
|
696375
696536
|
think: false,
|
|
696376
696537
|
// Disable reasoning — voice chat needs fast, direct responses
|
|
696377
|
-
options: { temperature: 0.
|
|
696538
|
+
options: { temperature: 0.5, num_predict: 192 }
|
|
696378
696539
|
});
|
|
696379
696540
|
const res2 = await fetch(`${baseUrl2}/api/chat`, {
|
|
696380
696541
|
method: "POST",
|
|
@@ -696391,10 +696552,10 @@ ${toolOutput}` });
|
|
|
696391
696552
|
}
|
|
696392
696553
|
const openaiBody = JSON.stringify({
|
|
696393
696554
|
model: this.model,
|
|
696394
|
-
messages: this.
|
|
696555
|
+
messages: this.buildRealtimeMessagesForRequest(),
|
|
696395
696556
|
stream: true,
|
|
696396
|
-
temperature: 0.
|
|
696397
|
-
max_tokens:
|
|
696557
|
+
temperature: 0.5,
|
|
696558
|
+
max_tokens: 384
|
|
696398
696559
|
});
|
|
696399
696560
|
const endpoint = baseUrl2.includes("/v1") ? `${baseUrl2}/chat/completions` : `${baseUrl2}/v1/chat/completions`;
|
|
696400
696561
|
const res = await fetch(endpoint, { method: "POST", headers, body: openaiBody, signal });
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omnius",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.414",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "omnius",
|
|
9
|
-
"version": "1.0.
|
|
9
|
+
"version": "1.0.414",
|
|
10
10
|
"bundleDependencies": [
|
|
11
11
|
"image-to-ascii"
|
|
12
12
|
],
|
package/package.json
CHANGED