@elizaos/shared 2.0.3-beta.2 → 2.0.3-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app-hero-art.d.ts.map +1 -1
- package/app-hero-art.js +1 -7
- package/app-hero-art.js.map +1 -1
- package/config/app-config.d.ts +1 -1
- package/config/boot-config-store.d.ts +1 -4
- package/config/boot-config-store.d.ts.map +1 -1
- package/config/boot-config-store.js +1 -45
- package/config/boot-config-store.js.map +1 -1
- package/contracts/apps.d.ts +9 -2
- package/contracts/apps.d.ts.map +1 -1
- package/contracts/apps.js +10 -67
- package/contracts/apps.js.map +1 -1
- package/contracts/conversation-routes.d.ts +0 -4
- package/contracts/conversation-routes.d.ts.map +1 -1
- package/contracts/conversation-routes.js +0 -1
- package/contracts/conversation-routes.js.map +1 -1
- package/env-utils.d.ts +10 -1
- package/env-utils.d.ts.map +1 -1
- package/env-utils.js +7 -9
- package/env-utils.js.map +1 -1
- package/events.d.ts +2 -0
- package/events.d.ts.map +1 -0
- package/events.js +2 -0
- package/events.js.map +1 -0
- package/format-error.d.ts +5 -4
- package/format-error.d.ts.map +1 -1
- package/format-error.js +5 -6
- package/format-error.js.map +1 -1
- package/hardware-catalog/index.d.ts +3 -3
- package/hardware-catalog/index.d.ts.map +1 -1
- package/hardware-catalog/index.js +1 -1
- package/index.d.ts +4 -1
- package/index.d.ts.map +1 -1
- package/index.js +4 -1
- package/index.js.map +1 -1
- package/local-inference/catalog.d.ts +29 -4
- package/local-inference/catalog.d.ts.map +1 -1
- package/local-inference/catalog.js +126 -102
- package/local-inference/catalog.js.map +1 -1
- package/local-inference/device-fit.d.ts +73 -0
- package/local-inference/device-fit.d.ts.map +1 -0
- package/local-inference/device-fit.js +155 -0
- package/local-inference/device-fit.js.map +1 -0
- package/local-inference/hf-proxy.d.ts +43 -0
- package/local-inference/hf-proxy.d.ts.map +1 -0
- package/local-inference/hf-proxy.js +50 -0
- package/local-inference/hf-proxy.js.map +1 -0
- package/local-inference/hub-auth.d.ts +26 -0
- package/local-inference/hub-auth.d.ts.map +1 -0
- package/local-inference/hub-auth.js +48 -0
- package/local-inference/hub-auth.js.map +1 -0
- package/local-inference/index.d.ts +6 -1
- package/local-inference/index.d.ts.map +1 -1
- package/local-inference/index.js +6 -1
- package/local-inference/index.js.map +1 -1
- package/local-inference/routing-preferences.d.ts +5 -1
- package/local-inference/routing-preferences.d.ts.map +1 -1
- package/local-inference/routing-preferences.js +6 -0
- package/local-inference/routing-preferences.js.map +1 -1
- package/local-inference/runtime-class.d.ts +29 -0
- package/local-inference/runtime-class.d.ts.map +1 -0
- package/local-inference/runtime-class.js +30 -0
- package/local-inference/runtime-class.js.map +1 -0
- package/local-inference/types.d.ts +45 -7
- package/local-inference/types.d.ts.map +1 -1
- package/local-inference/voice-models.d.ts +4 -4
- package/local-inference/voice-models.d.ts.map +1 -1
- package/local-inference/voice-models.js +42 -19
- package/local-inference/voice-models.js.map +1 -1
- package/local-inference-gpu/gpu-profile-schema.d.ts +5 -5
- package/loopback-trust.d.ts +73 -0
- package/loopback-trust.d.ts.map +1 -0
- package/loopback-trust.js +241 -0
- package/loopback-trust.js.map +1 -0
- package/package.json +85 -4
- package/settings-debug.d.ts +5 -16
- package/settings-debug.d.ts.map +1 -1
- package/settings-debug.js +5 -118
- package/settings-debug.js.map +1 -1
- package/steward-session-client/index.d.ts +5 -27
- package/steward-session-client/index.d.ts.map +1 -1
- package/steward-session-client/index.js +5 -61
- package/steward-session-client/index.js.map +1 -1
- package/test-env-config.d.ts +81 -0
- package/test-env-config.d.ts.map +1 -0
- package/test-env-config.js +125 -0
- package/test-env-config.js.map +1 -0
- package/transcripts.d.ts +2 -0
- package/transcripts.d.ts.map +1 -1
- package/transcripts.js.map +1 -1
- package/utils/env.d.ts +1 -0
- package/utils/env.d.ts.map +1 -1
- package/utils/env.js +1 -10
- package/utils/env.js.map +1 -1
- package/utils/errors.d.ts +2 -1
- package/utils/errors.d.ts.map +1 -1
- package/utils/errors.js +2 -5
- package/utils/errors.js.map +1 -1
- package/utils/string-hash.d.ts +11 -0
- package/utils/string-hash.d.ts.map +1 -0
- package/utils/string-hash.js +17 -0
- package/utils/string-hash.js.map +1 -0
- package/view-hero-art.d.ts.map +1 -1
- package/view-hero-art.js +1 -7
- package/view-hero-art.js.map +1 -1
- package/voice/owner-inference.d.ts +57 -0
- package/voice/owner-inference.d.ts.map +1 -0
- package/voice/owner-inference.js +73 -0
- package/voice/owner-inference.js.map +1 -0
- package/voice/respond-gate.d.ts +88 -0
- package/voice/respond-gate.d.ts.map +1 -0
- package/voice/respond-gate.js +139 -0
- package/voice/respond-gate.js.map +1 -0
- package/voice-eot.d.ts +42 -0
- package/voice-eot.d.ts.map +1 -0
- package/voice-eot.js +158 -0
- package/voice-eot.js.map +1 -0
- package/voice.d.ts +2 -2
- package/voice.js +3 -3
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Owner-candidate inference for voice (#8785).
|
|
3
|
+
*
|
|
4
|
+
* "How does the agent know which speaker is the device OWNER?" Today the owner
|
|
5
|
+
* entity must be enrolled explicitly (the first-run voice flow). But when no
|
|
6
|
+
* owner is set, the agent should still be able to FORM a hypothesis from what it
|
|
7
|
+
* hears — the person who speaks to it most, most confidently, and clearly more
|
|
8
|
+
* than anyone else is the likely owner. This is the pure decision logic a
|
|
9
|
+
* provider/evaluator runs when the owner is unknown: it accumulates recognized
|
|
10
|
+
* voice turns and proposes a candidate only when the evidence is both sufficient
|
|
11
|
+
* (enough confident observations) and unambiguous (a clear lead over the
|
|
12
|
+
* runner-up). Otherwise it stays UNDECIDED — it never guesses an owner from one
|
|
13
|
+
* stray turn, because a wrong owner is a security and personalization hazard.
|
|
14
|
+
*
|
|
15
|
+
* Pure (no I/O, no models); the runtime feeds it the diarized/recognized
|
|
16
|
+
* observations and acts on a decided candidate (e.g. prompt to confirm, or set
|
|
17
|
+
* the owner setting). The Voice Workbench exercises the SAME function so the
|
|
18
|
+
* inference is benchmarked, not just shipped.
|
|
19
|
+
*/
|
|
20
|
+
/** One recognized voice turn: which enrolled/clustered speaker, how confident. */
|
|
21
|
+
export interface OwnerObservation {
|
|
22
|
+
/** The entity/cluster the recognized voice resolved to (null = unrecognized). */
|
|
23
|
+
entityId: string | null;
|
|
24
|
+
/** Recognition confidence 0..1 (cosine-rescaled by the attribution pipeline). */
|
|
25
|
+
confidence: number;
|
|
26
|
+
}
|
|
27
|
+
export interface OwnerInferenceOptions {
|
|
28
|
+
/** Minimum qualifying observations before any candidate is proposed. */
|
|
29
|
+
minObservations?: number;
|
|
30
|
+
/** Confidence floor; observations below it don't count toward a candidate. */
|
|
31
|
+
minConfidence?: number;
|
|
32
|
+
/**
|
|
33
|
+
* Minimum lead (in confidence-weighted score) the top speaker must hold over
|
|
34
|
+
* the runner-up to be unambiguous. Prevents naming an owner in a two-equals
|
|
35
|
+
* household.
|
|
36
|
+
*/
|
|
37
|
+
minMargin?: number;
|
|
38
|
+
}
|
|
39
|
+
export interface OwnerInferenceResult {
|
|
40
|
+
/** The proposed owner entity, or null when the evidence is insufficient. */
|
|
41
|
+
ownerEntityId: string | null;
|
|
42
|
+
/** Confidence-weighted share of the proposed owner (0..1), 0 when undecided. */
|
|
43
|
+
share: number;
|
|
44
|
+
/** Number of qualifying (confident, recognized) observations considered. */
|
|
45
|
+
qualifyingObservations: number;
|
|
46
|
+
/** Why the function decided / declined — surfaced for the provider's logs. */
|
|
47
|
+
reason: string;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Propose the most likely owner from recognized voice observations, or stay
|
|
51
|
+
* undecided. A candidate is returned only when there are at least
|
|
52
|
+
* `minObservations` confident, recognized turns AND the top speaker leads the
|
|
53
|
+
* runner-up by at least `minMargin` (confidence-weighted). Ties and thin
|
|
54
|
+
* evidence yield `ownerEntityId: null`.
|
|
55
|
+
*/
|
|
56
|
+
export declare function resolveOwnerCandidate(observations: ReadonlyArray<OwnerObservation>, options?: OwnerInferenceOptions): OwnerInferenceResult;
|
|
57
|
+
//# sourceMappingURL=owner-inference.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"owner-inference.d.ts","sourceRoot":"","sources":["../../src/voice/owner-inference.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,kFAAkF;AAClF,MAAM,WAAW,gBAAgB;IAC/B,iFAAiF;IACjF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,iFAAiF;IACjF,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,wEAAwE;IACxE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,4EAA4E;IAC5E,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gFAAgF;IAChF,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,sBAAsB,EAAE,MAAM,CAAC;IAC/B,8EAA8E;IAC9E,MAAM,EAAE,MAAM,CAAC;CAChB;AAMD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,YAAY,EAAE,aAAa,CAAC,gBAAgB,CAAC,EAC7C,OAAO,GAAE,qBAA0B,GAClC,oBAAoB,CA4CtB"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Owner-candidate inference for voice (#8785).
|
|
3
|
+
*
|
|
4
|
+
* "How does the agent know which speaker is the device OWNER?" Today the owner
|
|
5
|
+
* entity must be enrolled explicitly (the first-run voice flow). But when no
|
|
6
|
+
* owner is set, the agent should still be able to FORM a hypothesis from what it
|
|
7
|
+
* hears — the person who speaks to it most, most confidently, and clearly more
|
|
8
|
+
* than anyone else is the likely owner. This is the pure decision logic a
|
|
9
|
+
* provider/evaluator runs when the owner is unknown: it accumulates recognized
|
|
10
|
+
* voice turns and proposes a candidate only when the evidence is both sufficient
|
|
11
|
+
* (enough confident observations) and unambiguous (a clear lead over the
|
|
12
|
+
* runner-up). Otherwise it stays UNDECIDED — it never guesses an owner from one
|
|
13
|
+
* stray turn, because a wrong owner is a security and personalization hazard.
|
|
14
|
+
*
|
|
15
|
+
* Pure (no I/O, no models); the runtime feeds it the diarized/recognized
|
|
16
|
+
* observations and acts on a decided candidate (e.g. prompt to confirm, or set
|
|
17
|
+
* the owner setting). The Voice Workbench exercises the SAME function so the
|
|
18
|
+
* inference is benchmarked, not just shipped.
|
|
19
|
+
*/
|
|
20
|
+
const DEFAULT_MIN_OBSERVATIONS = 3;
|
|
21
|
+
const DEFAULT_MIN_CONFIDENCE = 0.7;
|
|
22
|
+
const DEFAULT_MIN_MARGIN = 1;
|
|
23
|
+
/**
|
|
24
|
+
* Propose the most likely owner from recognized voice observations, or stay
|
|
25
|
+
* undecided. A candidate is returned only when there are at least
|
|
26
|
+
* `minObservations` confident, recognized turns AND the top speaker leads the
|
|
27
|
+
* runner-up by at least `minMargin` (confidence-weighted). Ties and thin
|
|
28
|
+
* evidence yield `ownerEntityId: null`.
|
|
29
|
+
*/
|
|
30
|
+
export function resolveOwnerCandidate(observations, options = {}) {
|
|
31
|
+
const minObservations = options.minObservations ?? DEFAULT_MIN_OBSERVATIONS;
|
|
32
|
+
const minConfidence = options.minConfidence ?? DEFAULT_MIN_CONFIDENCE;
|
|
33
|
+
const minMargin = options.minMargin ?? DEFAULT_MIN_MARGIN;
|
|
34
|
+
const scores = new Map();
|
|
35
|
+
let qualifying = 0;
|
|
36
|
+
let totalScore = 0;
|
|
37
|
+
for (const obs of observations) {
|
|
38
|
+
if (obs.entityId === null)
|
|
39
|
+
continue;
|
|
40
|
+
if (!(obs.confidence >= minConfidence))
|
|
41
|
+
continue;
|
|
42
|
+
qualifying += 1;
|
|
43
|
+
const next = (scores.get(obs.entityId) ?? 0) + obs.confidence;
|
|
44
|
+
scores.set(obs.entityId, next);
|
|
45
|
+
totalScore += obs.confidence;
|
|
46
|
+
}
|
|
47
|
+
if (qualifying < minObservations) {
|
|
48
|
+
return {
|
|
49
|
+
ownerEntityId: null,
|
|
50
|
+
share: 0,
|
|
51
|
+
qualifyingObservations: qualifying,
|
|
52
|
+
reason: `insufficient evidence (${qualifying}/${minObservations} confident observations)`,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
const ranked = [...scores.entries()].sort((a, b) => b[1] - a[1]);
|
|
56
|
+
const [topId, topScore] = ranked[0];
|
|
57
|
+
const runnerUpScore = ranked[1]?.[1] ?? 0;
|
|
58
|
+
if (topScore - runnerUpScore < minMargin) {
|
|
59
|
+
return {
|
|
60
|
+
ownerEntityId: null,
|
|
61
|
+
share: totalScore > 0 ? topScore / totalScore : 0,
|
|
62
|
+
qualifyingObservations: qualifying,
|
|
63
|
+
reason: `ambiguous lead (top ${topScore.toFixed(2)} vs runner-up ${runnerUpScore.toFixed(2)}, margin < ${minMargin})`,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
ownerEntityId: topId,
|
|
68
|
+
share: totalScore > 0 ? topScore / totalScore : 0,
|
|
69
|
+
qualifyingObservations: qualifying,
|
|
70
|
+
reason: `dominant speaker (${topScore.toFixed(2)} of ${totalScore.toFixed(2)} over ${qualifying} observations)`,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=owner-inference.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"owner-inference.js","sourceRoot":"","sources":["../../src/voice/owner-inference.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAkCH,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,YAA6C,EAC7C,UAAiC,EAAE;IAEnC,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,wBAAwB,CAAC;IAC5E,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,sBAAsB,CAAC;IACtE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAE1D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI;YAAE,SAAS;QACpC,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,IAAI,aAAa,CAAC;YAAE,SAAS;QACjD,UAAU,IAAI,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC;QAC9D,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/B,UAAU,IAAI,GAAG,CAAC,UAAU,CAAC;IAC/B,CAAC;IAED,IAAI,UAAU,GAAG,eAAe,EAAE,CAAC;QACjC,OAAO;YACL,aAAa,EAAE,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,sBAAsB,EAAE,UAAU;YAClC,MAAM,EAAE,0BAA0B,UAAU,IAAI,eAAe,0BAA0B;SAC1F,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,QAAQ,GAAG,aAAa,GAAG,SAAS,EAAE,CAAC;QACzC,OAAO;YACL,aAAa,EAAE,IAAI;YACnB,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YACjD,sBAAsB,EAAE,UAAU;YAClC,MAAM,EAAE,uBAAuB,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,SAAS,GAAG;SACtH,CAAC;IACJ,CAAC;IAED,OAAO;QACL,aAAa,EAAE,KAAK;QACpB,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACjD,sBAAsB,EAAE,UAAU;QAClC,MAAM,EAAE,qBAAqB,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,gBAAgB;KAChH,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical client-side shouldRespond / echo-rejection gate for always-on voice.
|
|
3
|
+
*
|
|
4
|
+
* In always-on ("hands-free") mode the mic is open continuously, so the
|
|
5
|
+
* recognizer transcribes EVERYTHING it hears — the agent's own text-to-speech
|
|
6
|
+
* bleeding back into the mic, the speaker's disfluent thinking noises ("um…"),
|
|
7
|
+
* and other people in the room. Sending those as turns makes the agent reply to
|
|
8
|
+
* itself, to filler, or to a bystander — the "responding when it's not
|
|
9
|
+
* appropriate" problem.
|
|
10
|
+
*
|
|
11
|
+
* This is the SINGLE definition of that gate, shared by:
|
|
12
|
+
* - the UI shell capture loop (`@elizaos/ui` `voice/should-respond` +
|
|
13
|
+
* `voice/voice-turn-signal` re-export from here), and
|
|
14
|
+
* - the Voice Workbench headless runner (the plugin's real-decision-logic
|
|
15
|
+
* services adapter calls these so the benchmark exercises the SAME code the
|
|
16
|
+
* client runs — no drift between what we test and what we ship).
|
|
17
|
+
*
|
|
18
|
+
* Pure (no DOM, no I/O); the only dependency is the canonical syntactic EOT
|
|
19
|
+
* heuristic in `@elizaos/shared/voice-eot`.
|
|
20
|
+
*/
|
|
21
|
+
/** How recent the agent's reply must be for the echo guard to apply. */
|
|
22
|
+
export declare const ECHO_WINDOW_MS = 9000;
|
|
23
|
+
/** Word-overlap fraction above which a turn is treated as TTS echo. */
|
|
24
|
+
export declare const ECHO_OVERLAP_THRESHOLD = 0.7;
|
|
25
|
+
export interface ShouldRespondContext {
|
|
26
|
+
/** The agent's most recent spoken reply, for the echo guard. */
|
|
27
|
+
recentAgentReply?: string;
|
|
28
|
+
/** Age of that reply in ms; the echo guard applies while it's recent. */
|
|
29
|
+
replyAgeMs?: number;
|
|
30
|
+
/**
|
|
31
|
+
* True while the agent is CURRENTLY speaking. Forces the echo guard on
|
|
32
|
+
* regardless of `replyAgeMs`, because a long reply's TTS is actively bleeding
|
|
33
|
+
* into the open mic even though its message was created many seconds ago (the
|
|
34
|
+
* age-only window would have already expired mid-speech).
|
|
35
|
+
*/
|
|
36
|
+
agentSpeaking?: boolean;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Whether a transcribed voice turn should be sent to the agent (i.e. warrants a
|
|
40
|
+
* response). Returns false for pure disfluency and for near-verbatim echoes of
|
|
41
|
+
* the agent's recent speech.
|
|
42
|
+
*/
|
|
43
|
+
export declare function shouldRespondToVoiceTurn(transcript: string, context?: ShouldRespondContext): boolean;
|
|
44
|
+
/** Mirrors the server-side VoiceTurnSignalMetadata shape the gate parses. */
|
|
45
|
+
export interface VoiceTurnSignal {
|
|
46
|
+
endOfTurnProbability: number;
|
|
47
|
+
nextSpeaker: "agent" | "user" | "unknown";
|
|
48
|
+
agentShouldSpeak: boolean;
|
|
49
|
+
source: string;
|
|
50
|
+
}
|
|
51
|
+
/** Live speaker attribution from diarization (only where audio frames exist). */
|
|
52
|
+
export interface VoiceTurnSpeakerAttribution {
|
|
53
|
+
/** Enrolled entity this turn was attributed to, or null when unknown. */
|
|
54
|
+
entityId: string | null;
|
|
55
|
+
/** Match confidence 0..1 (cosine-rescaled by the attribution pipeline). */
|
|
56
|
+
confidence: number;
|
|
57
|
+
/** True when attributed to the device owner / primary enrolled speaker. */
|
|
58
|
+
isOwner?: boolean;
|
|
59
|
+
}
|
|
60
|
+
export interface BuildVoiceTurnSignalContext extends ShouldRespondContext {
|
|
61
|
+
/** Speaker attribution for this turn (diarization; desktop/server only). */
|
|
62
|
+
speaker?: VoiceTurnSpeakerAttribution;
|
|
63
|
+
/** True when a wake word ("hey eliza") fired within the recent listen window. */
|
|
64
|
+
wakeWordActive?: boolean;
|
|
65
|
+
/** Entity ids the agent answers to without a wake word (owner + enrolled). */
|
|
66
|
+
knownSpeakerEntityIds?: readonly string[];
|
|
67
|
+
/**
|
|
68
|
+
* Cosine similarity (0..1) of THIS turn's speaker embedding against the
|
|
69
|
+
* agent's own TTS-voice imprint, when the speaker encoder is available. High =
|
|
70
|
+
* the agent is hearing ITSELF (its TTS bled back into the mic). This is the
|
|
71
|
+
* ACOUSTIC self-echo signal — it catches an echo the transcript word-overlap
|
|
72
|
+
* guard misses (a mis-transcribed echo whose words don't match the reply).
|
|
73
|
+
*/
|
|
74
|
+
selfVoiceSimilarity?: number;
|
|
75
|
+
}
|
|
76
|
+
/** Server SUPPRESS threshold for EOT — below this reads as "user still talking". */
|
|
77
|
+
export declare const SERVER_EOT_SUPPRESS_THRESHOLD = 0.4;
|
|
78
|
+
/** Only a CONFIDENT bystander attribution is allowed to silence a turn. */
|
|
79
|
+
export declare const BYSTANDER_SUPPRESS_CONFIDENCE = 0.7;
|
|
80
|
+
/**
|
|
81
|
+
* Cosine at/above which an incoming turn is treated as the agent's OWN voice and
|
|
82
|
+
* hard-suppressed. WeSpeaker enrolls the device owner at ~0.78; the agent's own
|
|
83
|
+
* TTS imprint is a tighter, single-source match, so a slightly lower bar is
|
|
84
|
+
* safe and catches echo the transcript guard cannot.
|
|
85
|
+
*/
|
|
86
|
+
export declare const AGENT_SELF_VOICE_THRESHOLD = 0.7;
|
|
87
|
+
export declare function buildVoiceTurnSignal(transcript: string, context?: BuildVoiceTurnSignalContext): VoiceTurnSignal;
|
|
88
|
+
//# sourceMappingURL=respond-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"respond-gate.d.ts","sourceRoot":"","sources":["../../src/voice/respond-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAqBH,wEAAwE;AACxE,eAAO,MAAM,cAAc,OAAO,CAAC;AACnC,uEAAuE;AACvE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAU1C,MAAM,WAAW,oBAAoB;IACnC,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAuBT;AAED,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;IAC1C,gBAAgB,EAAE,OAAO,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,iFAAiF;AACjF,MAAM,WAAW,2BAA2B;IAC1C,yEAAyE;IACzE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;IACnB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,2BAA4B,SAAQ,oBAAoB;IACvE,4EAA4E;IAC5E,OAAO,CAAC,EAAE,2BAA2B,CAAC;IACtC,iFAAiF;IACjF,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,8EAA8E;IAC9E,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,oFAAoF;AACpF,eAAO,MAAM,6BAA6B,MAAM,CAAC;AACjD,2EAA2E;AAC3E,eAAO,MAAM,6BAA6B,MAAM,CAAC;AACjD;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,2BAAgC,GACxC,eAAe,CA0DjB"}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical client-side shouldRespond / echo-rejection gate for always-on voice.
|
|
3
|
+
*
|
|
4
|
+
* In always-on ("hands-free") mode the mic is open continuously, so the
|
|
5
|
+
* recognizer transcribes EVERYTHING it hears — the agent's own text-to-speech
|
|
6
|
+
* bleeding back into the mic, the speaker's disfluent thinking noises ("um…"),
|
|
7
|
+
* and other people in the room. Sending those as turns makes the agent reply to
|
|
8
|
+
* itself, to filler, or to a bystander — the "responding when it's not
|
|
9
|
+
* appropriate" problem.
|
|
10
|
+
*
|
|
11
|
+
* This is the SINGLE definition of that gate, shared by:
|
|
12
|
+
* - the UI shell capture loop (`@elizaos/ui` `voice/should-respond` +
|
|
13
|
+
* `voice/voice-turn-signal` re-export from here), and
|
|
14
|
+
* - the Voice Workbench headless runner (the plugin's real-decision-logic
|
|
15
|
+
* services adapter calls these so the benchmark exercises the SAME code the
|
|
16
|
+
* client runs — no drift between what we test and what we ship).
|
|
17
|
+
*
|
|
18
|
+
* Pure (no DOM, no I/O); the only dependency is the canonical syntactic EOT
|
|
19
|
+
* heuristic in `@elizaos/shared/voice-eot`.
|
|
20
|
+
*/
|
|
21
|
+
import { scoreEndOfTurnHeuristic } from "../voice-eot.js";
|
|
22
|
+
/** Pure disfluencies — never a meaningful turn on their own. NOT answers. */
|
|
23
|
+
const DISFLUENCIES = new Set([
|
|
24
|
+
"um",
|
|
25
|
+
"uh",
|
|
26
|
+
"uhh",
|
|
27
|
+
"uhm",
|
|
28
|
+
"umm",
|
|
29
|
+
"hmm",
|
|
30
|
+
"hm",
|
|
31
|
+
"mm",
|
|
32
|
+
"mmm",
|
|
33
|
+
"er",
|
|
34
|
+
"erm",
|
|
35
|
+
"ah",
|
|
36
|
+
"eh",
|
|
37
|
+
]);
|
|
38
|
+
/** How recent the agent's reply must be for the echo guard to apply. */
|
|
39
|
+
export const ECHO_WINDOW_MS = 9000;
|
|
40
|
+
/** Word-overlap fraction above which a turn is treated as TTS echo. */
|
|
41
|
+
export const ECHO_OVERLAP_THRESHOLD = 0.7;
|
|
42
|
+
function words(text) {
|
|
43
|
+
return text
|
|
44
|
+
.toLowerCase()
|
|
45
|
+
.replace(/[^a-z0-9'\s-]/gi, "")
|
|
46
|
+
.split(/\s+/)
|
|
47
|
+
.filter(Boolean);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Whether a transcribed voice turn should be sent to the agent (i.e. warrants a
|
|
51
|
+
* response). Returns false for pure disfluency and for near-verbatim echoes of
|
|
52
|
+
* the agent's recent speech.
|
|
53
|
+
*/
|
|
54
|
+
export function shouldRespondToVoiceTurn(transcript, context = {}) {
|
|
55
|
+
const w = words(transcript);
|
|
56
|
+
if (w.length === 0)
|
|
57
|
+
return false;
|
|
58
|
+
// Pure disfluency ("um", "uh huh"… with nothing substantive) → ignore.
|
|
59
|
+
if (w.every((word) => DISFLUENCIES.has(word)))
|
|
60
|
+
return false;
|
|
61
|
+
// Self-echo: the agent's own TTS heard back through the mic. Only consider it
|
|
62
|
+
// while the reply is recent, and only for multi-word turns (a one-word answer
|
|
63
|
+
// shouldn't be suppressed just because the word also appears in the reply).
|
|
64
|
+
const reply = context.recentAgentReply?.trim();
|
|
65
|
+
const age = context.replyAgeMs ?? Number.POSITIVE_INFINITY;
|
|
66
|
+
const echoActive = context.agentSpeaking === true || age <= ECHO_WINDOW_MS;
|
|
67
|
+
if (reply && echoActive && w.length >= 2) {
|
|
68
|
+
const replyWords = new Set(words(reply));
|
|
69
|
+
if (replyWords.size > 0) {
|
|
70
|
+
const overlap = w.filter((word) => replyWords.has(word)).length / w.length;
|
|
71
|
+
if (overlap >= ECHO_OVERLAP_THRESHOLD)
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
/** Server SUPPRESS threshold for EOT — below this reads as "user still talking". */
|
|
78
|
+
export const SERVER_EOT_SUPPRESS_THRESHOLD = 0.4;
|
|
79
|
+
/** Only a CONFIDENT bystander attribution is allowed to silence a turn. */
|
|
80
|
+
export const BYSTANDER_SUPPRESS_CONFIDENCE = 0.7;
|
|
81
|
+
/**
|
|
82
|
+
* Cosine at/above which an incoming turn is treated as the agent's OWN voice and
|
|
83
|
+
* hard-suppressed. WeSpeaker enrolls the device owner at ~0.78; the agent's own
|
|
84
|
+
* TTS imprint is a tighter, single-source match, so a slightly lower bar is
|
|
85
|
+
* safe and catches echo the transcript guard cannot.
|
|
86
|
+
*/
|
|
87
|
+
export const AGENT_SELF_VOICE_THRESHOLD = 0.7;
|
|
88
|
+
export function buildVoiceTurnSignal(transcript, context = {}) {
|
|
89
|
+
const endOfTurnProbability = scoreEndOfTurnHeuristic(transcript);
|
|
90
|
+
// Transcript-level gate: the agent's own TTS echoed back through the mic, or
|
|
91
|
+
// pure thinking-noise ("um", "uh").
|
|
92
|
+
let agentShouldSpeak = shouldRespondToVoiceTurn(transcript, context);
|
|
93
|
+
// Audio-frame gate (only when diarization attributed the turn): a CONFIDENT
|
|
94
|
+
// bystander — someone who is neither the owner nor an enrolled speaker — who
|
|
95
|
+
// did NOT say the wake word is cross-talk, not a turn addressed to the agent.
|
|
96
|
+
// An uncertain attribution must never silence a real turn (fail open).
|
|
97
|
+
const speaker = context.speaker;
|
|
98
|
+
if (agentShouldSpeak && speaker && context.wakeWordActive !== true) {
|
|
99
|
+
const known = new Set(context.knownSpeakerEntityIds ?? []);
|
|
100
|
+
const enrolled = speaker.isOwner === true ||
|
|
101
|
+
(speaker.entityId !== null && known.has(speaker.entityId));
|
|
102
|
+
const confidentBystander = !enrolled &&
|
|
103
|
+
speaker.entityId !== null &&
|
|
104
|
+
speaker.confidence >= BYSTANDER_SUPPRESS_CONFIDENCE;
|
|
105
|
+
if (confidentBystander)
|
|
106
|
+
agentShouldSpeak = false;
|
|
107
|
+
}
|
|
108
|
+
// The wake word is an explicit address: it overrides bystander doubt and a
|
|
109
|
+
// soft echo/disfluency miss, because the user deliberately summoned the agent.
|
|
110
|
+
if (context.wakeWordActive === true)
|
|
111
|
+
agentShouldSpeak = true;
|
|
112
|
+
// Acoustic self-voice rejection: if the incoming voice MATCHES the agent's own
|
|
113
|
+
// TTS imprint while it is (or just was) speaking, it is the agent hearing
|
|
114
|
+
// itself — hard-suppress, even past the wake word. This is definitive (the
|
|
115
|
+
// owner cannot sound like the agent's synthetic voice) and catches the echo
|
|
116
|
+
// the transcript word-overlap guard misses (e.g. a mis-transcribed echo, or a
|
|
117
|
+
// "hey eliza" that the agent itself spoke).
|
|
118
|
+
const replyRecent = context.agentSpeaking === true ||
|
|
119
|
+
(context.replyAgeMs ?? Number.POSITIVE_INFINITY) <= ECHO_WINDOW_MS;
|
|
120
|
+
const isSelfVoice = context.selfVoiceSimilarity !== undefined &&
|
|
121
|
+
context.selfVoiceSimilarity >= AGENT_SELF_VOICE_THRESHOLD &&
|
|
122
|
+
replyRecent;
|
|
123
|
+
if (isSelfVoice)
|
|
124
|
+
agentShouldSpeak = false;
|
|
125
|
+
const nextSpeaker = !agentShouldSpeak
|
|
126
|
+
? "user"
|
|
127
|
+
: endOfTurnProbability < SERVER_EOT_SUPPRESS_THRESHOLD
|
|
128
|
+
? "user"
|
|
129
|
+
: "agent";
|
|
130
|
+
const source = isSelfVoice
|
|
131
|
+
? "client-ambient+self-voice"
|
|
132
|
+
: context.wakeWordActive
|
|
133
|
+
? "client-ambient+wakeword"
|
|
134
|
+
: speaker
|
|
135
|
+
? "client-ambient+diarization"
|
|
136
|
+
: "client-ambient";
|
|
137
|
+
return { endOfTurnProbability, nextSpeaker, agentShouldSpeak, source };
|
|
138
|
+
}
|
|
139
|
+
//# sourceMappingURL=respond-gate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"respond-gate.js","sourceRoot":"","sources":["../../src/voice/respond-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEvD,6EAA6E;AAC7E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;CACL,CAAC,CAAC;AAEH,wEAAwE;AACxE,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC;AACnC,uEAAuE;AACvE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAgBD;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CACtC,UAAkB,EAClB,UAAgC,EAAE;IAElC,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5B,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEjC,uEAAuE;IACvE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAE5D,8EAA8E;IAC9E,8EAA8E;IAC9E,4EAA4E;IAC5E,MAAM,KAAK,GAAG,OAAO,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,iBAAiB,CAAC;IAC3D,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,GAAG,IAAI,cAAc,CAAC;IAC3E,IAAI,KAAK,IAAI,UAAU,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACzC,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,OAAO,GACX,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;YAC7D,IAAI,OAAO,IAAI,sBAAsB;gBAAE,OAAO,KAAK,CAAC;QACtD,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAqCD,oFAAoF;AACpF,MAAM,CAAC,MAAM,6BAA6B,GAAG,GAAG,CAAC;AACjD,2EAA2E;AAC3E,MAAM,CAAC,MAAM,6BAA6B,GAAG,GAAG,CAAC;AACjD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAE9C,MAAM,UAAU,oBAAoB,CAClC,UAAkB,EAClB,UAAuC,EAAE;IAEzC,MAAM,oBAAoB,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAAC;IAEjE,6EAA6E;IAC7E,oCAAoC;IACpC,IAAI,gBAAgB,GAAG,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAErE,4EAA4E;IAC5E,6EAA6E;IAC7E,8EAA8E;IAC9E,uEAAuE;IACvE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,IAAI,gBAAgB,IAAI,OAAO,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,QAAQ,GACZ,OAAO,CAAC,OAAO,KAAK,IAAI;YACxB,CAAC,OAAO,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7D,MAAM,kBAAkB,GACtB,CAAC,QAAQ;YACT,OAAO,CAAC,QAAQ,KAAK,IAAI;YACzB,OAAO,CAAC,UAAU,IAAI,6BAA6B,CAAC;QACtD,IAAI,kBAAkB;YAAE,gBAAgB,GAAG,KAAK,CAAC;IACnD,CAAC;IAED,2EAA2E;IAC3E,+EAA+E;IAC/E,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI;QAAE,gBAAgB,GAAG,IAAI,CAAC;IAE7D,+EAA+E;IAC/E,0EAA0E;IAC1E,2EAA2E;IAC3E,4EAA4E;IAC5E,8EAA8E;IAC9E,4CAA4C;IAC5C,MAAM,WAAW,GACf,OAAO,CAAC,aAAa,KAAK,IAAI;QAC9B,CAAC,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,cAAc,CAAC;IACrE,MAAM,WAAW,GACf,OAAO,CAAC,mBAAmB,KAAK,SAAS;QACzC,OAAO,CAAC,mBAAmB,IAAI,0BAA0B;QACzD,WAAW,CAAC;IACd,IAAI,WAAW;QAAE,gBAAgB,GAAG,KAAK,CAAC;IAE1C,MAAM,WAAW,GAAmC,CAAC,gBAAgB;QACnE,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,oBAAoB,GAAG,6BAA6B;YACpD,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,OAAO,CAAC;IAEd,MAAM,MAAM,GAAG,WAAW;QACxB,CAAC,CAAC,2BAA2B;QAC7B,CAAC,CAAC,OAAO,CAAC,cAAc;YACtB,CAAC,CAAC,yBAAyB;YAC3B,CAAC,CAAC,OAAO;gBACP,CAAC,CAAC,4BAA4B;gBAC9B,CAAC,CAAC,gBAAgB,CAAC;IAEzB,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC;AACzE,CAAC"}
|
package/voice-eot.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Heuristic end-of-turn (EOT) scoring — the single source of truth (#8786).
|
|
3
|
+
*
|
|
4
|
+
* The semantic "is the speaker done?" syntactic heuristic used to be implemented
|
|
5
|
+
* twice with drifted behavior: the UI shell capture path
|
|
6
|
+
* (`packages/ui/src/voice/end-of-turn.ts:scoreEndOfTurn`) and the plugin's
|
|
7
|
+
* Tier-3 classifier (`plugin-local-inference .../voice/eot-classifier.ts:
|
|
8
|
+
* HeuristicEotClassifier`). The two had diverged — different rule ORDERING
|
|
9
|
+
* (the plugin scored a 2-word trail-off like "and so" as a complete short
|
|
10
|
+
* command; the UI correctly held it), a missing ellipsis rule on the plugin
|
|
11
|
+
* side, and a different question-tag set. This module is the one canonical
|
|
12
|
+
* implementation both surfaces consume.
|
|
13
|
+
*
|
|
14
|
+
* It lives in `@elizaos/shared` (which both already depend on), is pure +
|
|
15
|
+
* browser-safe (no Node deps), and ships via the `@elizaos/shared/voice-eot`
|
|
16
|
+
* subpath without pulling the whole barrel — mirroring `voice-wer`.
|
|
17
|
+
*
|
|
18
|
+
* The fused composite EOT (ABI v11, `CompositeEotClassifier`) is preferred when
|
|
19
|
+
* the loaded native build wires the semantic model; it blends THIS heuristic as
|
|
20
|
+
* its high-precision syntactic co-signal, so consolidating here also feeds the
|
|
21
|
+
* model path one definition.
|
|
22
|
+
*/
|
|
23
|
+
/**
|
|
24
|
+
* Probability in [0,1] that `transcript` is a COMPLETE turn (the speaker is
|
|
25
|
+
* done). High → commit; low → the utterance trails off, keep listening.
|
|
26
|
+
*
|
|
27
|
+
* Rules fire in priority order; the first match wins:
|
|
28
|
+
*
|
|
29
|
+
* 1 Trailing ellipsis ("…" / "..") 0.20 (trail-off)
|
|
30
|
+
* 2 Sentence-final punctuation (. ! ?) 0.95
|
|
31
|
+
* 3 Question-tag suffix ("right?", "yeah", "correct") 0.85
|
|
32
|
+
* 4 Trailing conjunction (and / but / because / …) 0.15 (mid-clause)
|
|
33
|
+
* 5 Trailing preposition / article (to / the / with …) 0.20 (incomplete NP)
|
|
34
|
+
* 6 Short utterance (< 3 words, no trail-off) 0.70 (command/ack)
|
|
35
|
+
* 7 No signal 0.50
|
|
36
|
+
*
|
|
37
|
+
* Note the conjunction/preposition checks precede the short-utterance rule so a
|
|
38
|
+
* 2-word trail-off ("and so", "going to") is NOT misread as a complete short
|
|
39
|
+
* command.
|
|
40
|
+
*/
|
|
41
|
+
export declare function scoreEndOfTurnHeuristic(transcript: string): number;
|
|
42
|
+
//# sourceMappingURL=voice-eot.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-eot.d.ts","sourceRoot":"","sources":["../src/voice-eot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAqFH;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAkClE"}
|
package/voice-eot.js
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Heuristic end-of-turn (EOT) scoring — the single source of truth (#8786).
|
|
3
|
+
*
|
|
4
|
+
* The semantic "is the speaker done?" syntactic heuristic used to be implemented
|
|
5
|
+
* twice with drifted behavior: the UI shell capture path
|
|
6
|
+
* (`packages/ui/src/voice/end-of-turn.ts:scoreEndOfTurn`) and the plugin's
|
|
7
|
+
* Tier-3 classifier (`plugin-local-inference .../voice/eot-classifier.ts:
|
|
8
|
+
* HeuristicEotClassifier`). The two had diverged — different rule ORDERING
|
|
9
|
+
* (the plugin scored a 2-word trail-off like "and so" as a complete short
|
|
10
|
+
* command; the UI correctly held it), a missing ellipsis rule on the plugin
|
|
11
|
+
* side, and a different question-tag set. This module is the one canonical
|
|
12
|
+
* implementation both surfaces consume.
|
|
13
|
+
*
|
|
14
|
+
* It lives in `@elizaos/shared` (which both already depend on), is pure +
|
|
15
|
+
* browser-safe (no Node deps), and ships via the `@elizaos/shared/voice-eot`
|
|
16
|
+
* subpath without pulling the whole barrel — mirroring `voice-wer`.
|
|
17
|
+
*
|
|
18
|
+
* The fused composite EOT (ABI v11, `CompositeEotClassifier`) is preferred when
|
|
19
|
+
* the loaded native build wires the semantic model; it blends THIS heuristic as
|
|
20
|
+
* its high-precision syntactic co-signal, so consolidating here also feeds the
|
|
21
|
+
* model path one definition.
|
|
22
|
+
*/
|
|
23
|
+
/** Conjunctions that strongly suggest the speaker is mid-clause. */
|
|
24
|
+
const TRAILING_CONJUNCTIONS = new Set([
|
|
25
|
+
"and",
|
|
26
|
+
"but",
|
|
27
|
+
"or",
|
|
28
|
+
"nor",
|
|
29
|
+
"yet",
|
|
30
|
+
"so",
|
|
31
|
+
"because",
|
|
32
|
+
"although",
|
|
33
|
+
"though",
|
|
34
|
+
"while",
|
|
35
|
+
"whereas",
|
|
36
|
+
"if",
|
|
37
|
+
"unless",
|
|
38
|
+
"until",
|
|
39
|
+
"since",
|
|
40
|
+
"when",
|
|
41
|
+
"where",
|
|
42
|
+
"which",
|
|
43
|
+
"that",
|
|
44
|
+
"who",
|
|
45
|
+
"whom",
|
|
46
|
+
"whose",
|
|
47
|
+
]);
|
|
48
|
+
/** Prepositions / articles that imply an incomplete noun phrase follows. */
|
|
49
|
+
const TRAILING_INCOMPLETE = new Set([
|
|
50
|
+
"a",
|
|
51
|
+
"an",
|
|
52
|
+
"the",
|
|
53
|
+
"to",
|
|
54
|
+
"of",
|
|
55
|
+
"in",
|
|
56
|
+
"on",
|
|
57
|
+
"at",
|
|
58
|
+
"by",
|
|
59
|
+
"for",
|
|
60
|
+
"with",
|
|
61
|
+
"from",
|
|
62
|
+
"into",
|
|
63
|
+
"about",
|
|
64
|
+
"through",
|
|
65
|
+
"between",
|
|
66
|
+
"against",
|
|
67
|
+
"during",
|
|
68
|
+
"before",
|
|
69
|
+
"after",
|
|
70
|
+
"without",
|
|
71
|
+
"under",
|
|
72
|
+
"over",
|
|
73
|
+
"above",
|
|
74
|
+
"below",
|
|
75
|
+
"around",
|
|
76
|
+
"beside",
|
|
77
|
+
"beyond",
|
|
78
|
+
"like",
|
|
79
|
+
"near",
|
|
80
|
+
"past",
|
|
81
|
+
"via",
|
|
82
|
+
]);
|
|
83
|
+
/**
|
|
84
|
+
* Question-tag suffixes that end an utterance (matched case-insensitively).
|
|
85
|
+
* The union of both prior surfaces: punctuated forms (the UI set) plus the
|
|
86
|
+
* bare forms (`right` / `yeah` / `correct`) the plugin relied on for the
|
|
87
|
+
* no-trailing-`?` case. A punctuated tag is also caught by the sentence-final
|
|
88
|
+
* punctuation rule, which fires first.
|
|
89
|
+
*/
|
|
90
|
+
const QUESTION_TAGS = [
|
|
91
|
+
"right?",
|
|
92
|
+
"yeah?",
|
|
93
|
+
"ok?",
|
|
94
|
+
"okay?",
|
|
95
|
+
"correct?",
|
|
96
|
+
"hm?",
|
|
97
|
+
"huh?",
|
|
98
|
+
"eh?",
|
|
99
|
+
"right",
|
|
100
|
+
"yeah",
|
|
101
|
+
"correct",
|
|
102
|
+
];
|
|
103
|
+
/**
|
|
104
|
+
* Probability in [0,1] that `transcript` is a COMPLETE turn (the speaker is
|
|
105
|
+
* done). High → commit; low → the utterance trails off, keep listening.
|
|
106
|
+
*
|
|
107
|
+
* Rules fire in priority order; the first match wins:
|
|
108
|
+
*
|
|
109
|
+
* 1 Trailing ellipsis ("…" / "..") 0.20 (trail-off)
|
|
110
|
+
* 2 Sentence-final punctuation (. ! ?) 0.95
|
|
111
|
+
* 3 Question-tag suffix ("right?", "yeah", "correct") 0.85
|
|
112
|
+
* 4 Trailing conjunction (and / but / because / …) 0.15 (mid-clause)
|
|
113
|
+
* 5 Trailing preposition / article (to / the / with …) 0.20 (incomplete NP)
|
|
114
|
+
* 6 Short utterance (< 3 words, no trail-off) 0.70 (command/ack)
|
|
115
|
+
* 7 No signal 0.50
|
|
116
|
+
*
|
|
117
|
+
* Note the conjunction/preposition checks precede the short-utterance rule so a
|
|
118
|
+
* 2-word trail-off ("and so", "going to") is NOT misread as a complete short
|
|
119
|
+
* command.
|
|
120
|
+
*/
|
|
121
|
+
export function scoreEndOfTurnHeuristic(transcript) {
|
|
122
|
+
const text = transcript.trim();
|
|
123
|
+
if (text.length === 0)
|
|
124
|
+
return 0.5;
|
|
125
|
+
// A trailing ellipsis is the strongest trail-off signal — the speaker paused
|
|
126
|
+
// mid-thought. Checked BEFORE sentence-final punctuation, since "..." ends in ".".
|
|
127
|
+
if (/(\.{2,}|…)$/.test(text))
|
|
128
|
+
return 0.2;
|
|
129
|
+
// Sentence-final punctuation → almost certainly done.
|
|
130
|
+
if (/[.!?]$/.test(text))
|
|
131
|
+
return 0.95;
|
|
132
|
+
const lower = text.toLowerCase();
|
|
133
|
+
for (const tag of QUESTION_TAGS) {
|
|
134
|
+
if (lower.endsWith(tag))
|
|
135
|
+
return 0.85;
|
|
136
|
+
}
|
|
137
|
+
const words = lower
|
|
138
|
+
.replace(/[^a-z0-9'\s-]/gi, "")
|
|
139
|
+
.split(/\s+/)
|
|
140
|
+
.filter(Boolean);
|
|
141
|
+
if (words.length === 0)
|
|
142
|
+
return 0.5;
|
|
143
|
+
const lastWord = words[words.length - 1].replace(/[',;:-]+$/, "");
|
|
144
|
+
// Trailing conjunction / preposition / article → mid-clause, the speaker is
|
|
145
|
+
// continuing. Checked BEFORE the short-utterance rule so a 2-word trail-off
|
|
146
|
+
// ("going to", "and so") is NOT misread as a complete short command.
|
|
147
|
+
if (TRAILING_CONJUNCTIONS.has(lastWord))
|
|
148
|
+
return 0.15;
|
|
149
|
+
if (TRAILING_INCOMPLETE.has(lastWord))
|
|
150
|
+
return 0.2;
|
|
151
|
+
// Short utterance that doesn't trail off (a command / acknowledgement) →
|
|
152
|
+
// likely complete ("go home", "yes", "stop").
|
|
153
|
+
if (words.length < 3)
|
|
154
|
+
return 0.7;
|
|
155
|
+
// No strong signal either way — the recognizer's silence is enough.
|
|
156
|
+
return 0.5;
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=voice-eot.js.map
|
package/voice-eot.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-eot.js","sourceRoot":"","sources":["../src/voice-eot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,oEAAoE;AACpE,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC;IACpC,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,SAAS;IACT,UAAU;IACV,QAAQ;IACR,OAAO;IACP,SAAS;IACT,IAAI;IACJ,QAAQ;IACR,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,GAAG;IACH,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,SAAS;IACT,SAAS;IACT,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,SAAS;IACT,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;CACN,CAAC,CAAC;AAEH;;;;;;GAMG;AACH,MAAM,aAAa,GAAG;IACpB,QAAQ;IACR,OAAO;IACP,KAAK;IACL,OAAO;IACP,UAAU;IACV,KAAK;IACL,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,SAAS;CACV,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,uBAAuB,CAAC,UAAkB;IACxD,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAElC,6EAA6E;IAC7E,mFAAmF;IACnF,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IACzC,sDAAsD;IACtD,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;IACvC,CAAC;IAED,MAAM,KAAK,GAAG,KAAK;SAChB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAEnC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAClE,4EAA4E;IAC5E,4EAA4E;IAC5E,qEAAqE;IACrE,IAAI,qBAAqB,CAAC,GAAG,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACrD,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC;QAAE,OAAO,GAAG,CAAC;IAElD,yEAAyE;IACzE,8CAA8C;IAC9C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IAEjC,oEAAoE;IACpE,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/voice.d.ts
CHANGED
|
@@ -35,8 +35,8 @@ export declare const VOICE_PROVIDERS: Array<{
|
|
|
35
35
|
* ASR (automatic speech recognition) provider catalogue. Mirrors
|
|
36
36
|
* `VOICE_PROVIDERS` so the settings UI can render a provider picker for
|
|
37
37
|
* speech-to-text. The legacy whisper.cpp pipeline has been retired —
|
|
38
|
-
* on-device transcription now runs through the local-inference
|
|
39
|
-
* bundle
|
|
38
|
+
* on-device transcription now runs through the local-inference ASR bundle.
|
|
39
|
+
* Release catalog checks gate the bundle lineage before it can be selected.
|
|
40
40
|
*/
|
|
41
41
|
export declare const ASR_PROVIDERS: Array<{
|
|
42
42
|
id: "local-inference" | "eliza-cloud" | "openai";
|
package/voice.js
CHANGED
|
@@ -278,15 +278,15 @@ export const VOICE_PROVIDERS = [
|
|
|
278
278
|
* ASR (automatic speech recognition) provider catalogue. Mirrors
|
|
279
279
|
* `VOICE_PROVIDERS` so the settings UI can render a provider picker for
|
|
280
280
|
* speech-to-text. The legacy whisper.cpp pipeline has been retired —
|
|
281
|
-
* on-device transcription now runs through the local-inference
|
|
282
|
-
* bundle
|
|
281
|
+
* on-device transcription now runs through the local-inference ASR bundle.
|
|
282
|
+
* Release catalog checks gate the bundle lineage before it can be selected.
|
|
283
283
|
*/
|
|
284
284
|
export const ASR_PROVIDERS = [
|
|
285
285
|
{
|
|
286
286
|
id: "local-inference",
|
|
287
287
|
label: "Local inference",
|
|
288
288
|
labelKey: "asr.provider.localInference",
|
|
289
|
-
hint: "On-device
|
|
289
|
+
hint: "On-device local ASR",
|
|
290
290
|
hintKey: "asr.provider.localInference.hint",
|
|
291
291
|
needsKey: false,
|
|
292
292
|
},
|