@wix/legends-platform-sdk 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/hooks/index.js +6 -0
- package/dist/cjs/hooks/index.js.map +1 -1
- package/dist/cjs/hooks/useConversation/livekit/constants.js +4 -1
- package/dist/cjs/hooks/useConversation/livekit/constants.js.map +1 -1
- package/dist/cjs/hooks/useConversation/livekit/useConversation.js +87 -35
- package/dist/cjs/hooks/useConversation/livekit/useConversation.js.map +1 -1
- package/dist/cjs/hooks/useSpeechActivityDetector/helpers.js +87 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/helpers.js.map +1 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/index.js +16 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/index.js.map +1 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/speechActivityProcessorSource.js +55 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/speechActivityProcessorSource.js.map +1 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/types.js +4 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/types.js.map +1 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/useSpeechActivityDetector.js +52 -0
- package/dist/cjs/hooks/useSpeechActivityDetector/useSpeechActivityDetector.js.map +1 -0
- package/dist/cjs/sdk.js +6 -0
- package/dist/cjs/sdk.js.map +1 -1
- package/dist/esm/hooks/index.js +1 -0
- package/dist/esm/hooks/index.js.map +1 -1
- package/dist/esm/hooks/useConversation/livekit/constants.js +3 -0
- package/dist/esm/hooks/useConversation/livekit/constants.js.map +1 -1
- package/dist/esm/hooks/useConversation/livekit/useConversation.js +87 -35
- package/dist/esm/hooks/useConversation/livekit/useConversation.js.map +1 -1
- package/dist/esm/hooks/useSpeechActivityDetector/helpers.js +83 -0
- package/dist/esm/hooks/useSpeechActivityDetector/helpers.js.map +1 -0
- package/dist/esm/hooks/useSpeechActivityDetector/index.js +3 -0
- package/dist/esm/hooks/useSpeechActivityDetector/index.js.map +1 -0
- package/dist/esm/hooks/useSpeechActivityDetector/speechActivityProcessorSource.js +51 -0
- package/dist/esm/hooks/useSpeechActivityDetector/speechActivityProcessorSource.js.map +1 -0
- package/dist/esm/hooks/useSpeechActivityDetector/types.js +2 -0
- package/dist/esm/hooks/useSpeechActivityDetector/types.js.map +1 -0
- package/dist/esm/hooks/useSpeechActivityDetector/useSpeechActivityDetector.js +44 -0
- package/dist/esm/hooks/useSpeechActivityDetector/useSpeechActivityDetector.js.map +1 -0
- package/dist/esm/sdk.js +1 -0
- package/dist/esm/sdk.js.map +1 -1
- package/dist/types/hooks/index.d.ts +1 -0
- package/dist/types/hooks/index.d.ts.map +1 -1
- package/dist/types/hooks/useConversation/livekit/constants.d.ts +3 -0
- package/dist/types/hooks/useConversation/livekit/constants.d.ts.map +1 -1
- package/dist/types/hooks/useConversation/livekit/useConversation.d.ts.map +1 -1
- package/dist/types/hooks/useSpeechActivityDetector/helpers.d.ts +19 -0
- package/dist/types/hooks/useSpeechActivityDetector/helpers.d.ts.map +1 -0
- package/dist/types/hooks/useSpeechActivityDetector/index.d.ts +3 -0
- package/dist/types/hooks/useSpeechActivityDetector/index.d.ts.map +1 -0
- package/dist/types/hooks/useSpeechActivityDetector/speechActivityProcessorSource.d.ts +3 -0
- package/dist/types/hooks/useSpeechActivityDetector/speechActivityProcessorSource.d.ts.map +1 -0
- package/dist/types/hooks/useSpeechActivityDetector/types.d.ts +14 -0
- package/dist/types/hooks/useSpeechActivityDetector/types.d.ts.map +1 -0
- package/dist/types/hooks/useSpeechActivityDetector/useSpeechActivityDetector.d.ts +10 -0
- package/dist/types/hooks/useSpeechActivityDetector/useSpeechActivityDetector.d.ts.map +1 -0
- package/dist/types/sdk.d.ts +1 -0
- package/dist/types/sdk.d.ts.map +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { SPEECH_ACTIVITY_PROCESSOR_NAME, SPEECH_ACTIVITY_PROCESSOR_SOURCE } from './speechActivityProcessorSource';
|
|
2
|
+
const DEFAULT_THRESHOLD_RMS = 0.015;
|
|
3
|
+
const DEFAULT_HANGOVER_MS = 300;
|
|
4
|
+
const WORKLET_BLOCK_SIZE = 128;
|
|
5
|
+
/**
|
|
6
|
+
* Attaches an AudioWorkletNode-based RMS threshold+hangover detector to a
|
|
7
|
+
* LiveKit track's underlying MediaStreamTrack. Runs on the dedicated audio
|
|
8
|
+
* rendering thread — unlike requestAnimationFrame+AnalyserNode, it keeps
|
|
9
|
+
* running when the tab is backgrounded.
|
|
10
|
+
*/
|
|
11
|
+
export function setupSpeechActivityDetector(track, options) {
|
|
12
|
+
if (!track) {
|
|
13
|
+
return {
|
|
14
|
+
cleanup: () => {}
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
let cancelled = false;
|
|
18
|
+
let audioContext = null;
|
|
19
|
+
let mediaStreamSource = null;
|
|
20
|
+
let node = null;
|
|
21
|
+
const thresholdRms = options.thresholdRms ?? DEFAULT_THRESHOLD_RMS;
|
|
22
|
+
const hangoverMs = options.hangoverMs ?? DEFAULT_HANGOVER_MS;
|
|
23
|
+
const setup = async () => {
|
|
24
|
+
const ctx = new AudioContext();
|
|
25
|
+
audioContext = ctx;
|
|
26
|
+
if (ctx.state === 'suspended') {
|
|
27
|
+
await ctx.resume();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// Hangover is expressed in ms; convert to a block count using this
|
|
31
|
+
// context's actual sample rate (block size is fixed at 128 samples by
|
|
32
|
+
// the Web Audio spec, but wall-clock duration per block varies with
|
|
33
|
+
// sample rate across devices/browsers).
|
|
34
|
+
const hangoverBlocks = Math.max(1, Math.round(hangoverMs / 1000 * (ctx.sampleRate / WORKLET_BLOCK_SIZE)));
|
|
35
|
+
const blob = new Blob([SPEECH_ACTIVITY_PROCESSOR_SOURCE], {
|
|
36
|
+
type: 'application/javascript'
|
|
37
|
+
});
|
|
38
|
+
const blobUrl = URL.createObjectURL(blob);
|
|
39
|
+
try {
|
|
40
|
+
await ctx.audioWorklet.addModule(blobUrl);
|
|
41
|
+
} finally {
|
|
42
|
+
URL.revokeObjectURL(blobUrl);
|
|
43
|
+
}
|
|
44
|
+
if (cancelled) {
|
|
45
|
+
await ctx.close();
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
mediaStreamSource = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack]));
|
|
49
|
+
node = new AudioWorkletNode(ctx, SPEECH_ACTIVITY_PROCESSOR_NAME, {
|
|
50
|
+
processorOptions: {
|
|
51
|
+
thresholdRms,
|
|
52
|
+
hangoverBlocks
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
node.port.onmessage = event => {
|
|
56
|
+
var _event$data, _event$data2;
|
|
57
|
+
const timestampMs = performance.now();
|
|
58
|
+
if (((_event$data = event.data) == null ? void 0 : _event$data.type) === 'onset') {
|
|
59
|
+
options.onSpeechStart == null || options.onSpeechStart(timestampMs);
|
|
60
|
+
} else if (((_event$data2 = event.data) == null ? void 0 : _event$data2.type) === 'offset') {
|
|
61
|
+
options.onSpeechEnd == null || options.onSpeechEnd(timestampMs);
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
mediaStreamSource.connect(node);
|
|
65
|
+
};
|
|
66
|
+
setup().catch(error => {
|
|
67
|
+
// eslint-disable-next-line no-console
|
|
68
|
+
console.error('[useSpeechActivityDetector] failed to set up AudioWorklet', error);
|
|
69
|
+
});
|
|
70
|
+
return {
|
|
71
|
+
cleanup: () => {
|
|
72
|
+
var _mediaStreamSource, _audioContext;
|
|
73
|
+
cancelled = true;
|
|
74
|
+
if (node) {
|
|
75
|
+
node.port.onmessage = null;
|
|
76
|
+
node.disconnect();
|
|
77
|
+
}
|
|
78
|
+
(_mediaStreamSource = mediaStreamSource) == null || _mediaStreamSource.disconnect();
|
|
79
|
+
void ((_audioContext = audioContext) == null ? void 0 : _audioContext.close());
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["SPEECH_ACTIVITY_PROCESSOR_NAME","SPEECH_ACTIVITY_PROCESSOR_SOURCE","DEFAULT_THRESHOLD_RMS","DEFAULT_HANGOVER_MS","WORKLET_BLOCK_SIZE","setupSpeechActivityDetector","track","options","cleanup","cancelled","audioContext","mediaStreamSource","node","thresholdRms","hangoverMs","setup","ctx","AudioContext","state","resume","hangoverBlocks","Math","max","round","sampleRate","blob","Blob","type","blobUrl","URL","createObjectURL","audioWorklet","addModule","revokeObjectURL","close","createMediaStreamSource","MediaStream","mediaStreamTrack","AudioWorkletNode","processorOptions","port","onmessage","event","_event$data","_event$data2","timestampMs","performance","now","data","onSpeechStart","onSpeechEnd","connect","catch","error","console","_mediaStreamSource","_audioContext","disconnect"],"sources":["../../../../src/hooks/useSpeechActivityDetector/helpers.ts"],"sourcesContent":["import { Track } from 'livekit-client';\nimport {\n SPEECH_ACTIVITY_PROCESSOR_NAME,\n SPEECH_ACTIVITY_PROCESSOR_SOURCE,\n} from './speechActivityProcessorSource';\n\nconst DEFAULT_THRESHOLD_RMS = 0.015;\nconst DEFAULT_HANGOVER_MS = 300;\nconst WORKLET_BLOCK_SIZE = 128;\n\nexport type SetupSpeechActivityDetectorOptions = {\n thresholdRms?: number;\n hangoverMs?: number;\n minSpeechMs?: number;\n onSpeechStart?: (timestampMs: number) => void;\n onSpeechEnd?: (timestampMs: number) => void;\n onMisfire?: () => void;\n};\n\n/**\n * Attaches an AudioWorkletNode-based RMS threshold+hangover detector to a\n * LiveKit track's underlying MediaStreamTrack. Runs on the dedicated audio\n * rendering thread — unlike requestAnimationFrame+AnalyserNode, it keeps\n * running when the tab is backgrounded.\n */\nexport function setupSpeechActivityDetector(\n track: Track | undefined,\n options: SetupSpeechActivityDetectorOptions,\n): { cleanup: () => void } {\n if (!track) {\n return { cleanup: () => {} };\n }\n\n let cancelled = false;\n let audioContext: AudioContext | null = null;\n let mediaStreamSource: MediaStreamAudioSourceNode | null = null;\n let node: AudioWorkletNode | null = null;\n\n const thresholdRms = options.thresholdRms ?? DEFAULT_THRESHOLD_RMS;\n const hangoverMs = options.hangoverMs ?? DEFAULT_HANGOVER_MS;\n\n const setup = async () => {\n const ctx = new AudioContext();\n audioContext = ctx;\n if (ctx.state === 'suspended') {\n await ctx.resume();\n }\n\n // Hangover is expressed in ms; convert to a block count using this\n // context's actual sample rate (block size is fixed at 128 samples by\n // the Web Audio spec, but wall-clock duration per block varies with\n // sample rate across devices/browsers).\n const hangoverBlocks = Math.max(\n 1,\n Math.round((hangoverMs / 1000) * (ctx.sampleRate / WORKLET_BLOCK_SIZE)),\n );\n\n const blob = new Blob([SPEECH_ACTIVITY_PROCESSOR_SOURCE], {\n type: 'application/javascript',\n });\n const blobUrl = URL.createObjectURL(blob);\n try {\n await ctx.audioWorklet.addModule(blobUrl);\n } finally {\n URL.revokeObjectURL(blobUrl);\n }\n\n if (cancelled) {\n await ctx.close();\n return;\n }\n\n mediaStreamSource = ctx.createMediaStreamSource(\n new MediaStream([track.mediaStreamTrack]),\n );\n node = new AudioWorkletNode(ctx, SPEECH_ACTIVITY_PROCESSOR_NAME, {\n processorOptions: { thresholdRms, hangoverBlocks },\n });\n node.port.onmessage = (event: MessageEvent) => {\n const timestampMs = performance.now();\n if (event.data?.type === 'onset') {\n options.onSpeechStart?.(timestampMs);\n } else if (event.data?.type === 'offset') {\n options.onSpeechEnd?.(timestampMs);\n }\n };\n mediaStreamSource.connect(node);\n };\n\n setup().catch((error) => {\n // eslint-disable-next-line no-console\n console.error(\n '[useSpeechActivityDetector] failed to set up AudioWorklet',\n error,\n );\n });\n\n return {\n cleanup: () => {\n cancelled = true;\n if (node) {\n node.port.onmessage = null;\n node.disconnect();\n }\n mediaStreamSource?.disconnect();\n void audioContext?.close();\n },\n };\n}\n"],"mappings":"AACA,SACEA,8BAA8B,EAC9BC,gCAAgC,QAC3B,iCAAiC;AAExC,MAAMC,qBAAqB,GAAG,KAAK;AACnC,MAAMC,mBAAmB,GAAG,GAAG;AAC/B,MAAMC,kBAAkB,GAAG,GAAG;AAW9B;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASC,2BAA2BA,CACzCC,KAAwB,EACxBC,OAA2C,EAClB;EACzB,IAAI,CAACD,KAAK,EAAE;IACV,OAAO;MAAEE,OAAO,EAAEA,CAAA,KAAM,CAAC;IAAE,CAAC;EAC9B;EAEA,IAAIC,SAAS,GAAG,KAAK;EACrB,IAAIC,YAAiC,GAAG,IAAI;EAC5C,IAAIC,iBAAoD,GAAG,IAAI;EAC/D,IAAIC,IAA6B,GAAG,IAAI;EAExC,MAAMC,YAAY,GAAGN,OAAO,CAACM,YAAY,IAAIX,qBAAqB;EAClE,MAAMY,UAAU,GAAGP,OAAO,CAACO,UAAU,IAAIX,mBAAmB;EAE5D,MAAMY,KAAK,GAAG,MAAAA,CAAA,KAAY;IACxB,MAAMC,GAAG,GAAG,IAAIC,YAAY,CAAC,CAAC;IAC9BP,YAAY,GAAGM,GAAG;IAClB,IAAIA,GAAG,CAACE,KAAK,KAAK,WAAW,EAAE;MAC7B,MAAMF,GAAG,CAACG,MAAM,CAAC,CAAC;IACpB;;IAEA;IACA;IACA;IACA;IACA,MAAMC,cAAc,GAAGC,IAAI,CAACC,GAAG,CAC7B,CAAC,EACDD,IAAI,CAACE,KAAK,CAAET,UAAU,GAAG,IAAI,IAAKE,GAAG,CAACQ,UAAU,GAAGpB,kBAAkB,CAAC,CACxE,CAAC;IAED,MAAMqB,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACzB,gCAAgC,CAAC,EAAE;MACxD0B,IAAI,EAAE;IACR,CAAC,CAAC;IACF,MAAMC,OAAO,GAAGC,GAAG,CAACC,eAAe,CAACL,IAAI,CAAC;IACzC,IAAI;MACF,MAAMT,GAAG,CAACe,YAAY,CAACC,SAAS,CAACJ,OAAO,CAAC;IAC3C,CAAC,SAAS;MACRC,GAAG,CAACI,eAAe,CAACL,OAAO,CAAC;IAC9B;IAEA,IAAInB,SAAS,EAAE;MACb,MAAMO,GAAG,CAACkB,KAAK,CAAC,CAAC;MACjB;IACF;IAEAvB,iBAAiB,GAAGK,GAAG,CAACmB,uBAAuB,CAC7C,IAAIC,WAAW,CAAC,CAAC9B,KAAK,CAAC+B,gBAAgB,CAAC,CAC1C,CAAC;IACDzB,IAAI,GAAG,IAAI0B,gBAAgB,CAACtB,GAAG,EAAEhB,8BAA8B,EAAE;MAC/DuC,gBAAgB,EAAE;QAAE1B,YAAY;QAAEO;MAAe;IACnD,CAAC,CAAC;IACFR,IAAI,CAAC4B,IAAI,CAACC,SAAS,GAAIC,KAAmB,IAAK;MAAA,IAAAC,WAAA,EAAAC,YAAA;MAC7C,MAAMC,WAAW,GAAGC,WAAW,CAACC,GAAG,CAAC,CAAC;MACrC,IAAI,EAAAJ,WAAA,GAAAD,KAAK,CAACM,IAAI,qBAAVL,WAAA,CAAYhB,IAAI,MAAK,OAAO,EAAE;QAChCpB,OAAO,CAAC0C,aAAa,YAArB1C,OAAO,CAAC0C,aAAa,CAAGJ,WAAW,CAAC;MACtC,CAAC,MAAM,IAAI,EAAAD,YAAA,GAAAF,KAAK,CAACM,IAAI,qBAAVJ,YAAA,CAAYjB,IAAI,MAAK,QAAQ,EAAE;QACxCpB,OAAO,CAAC2C,WAAW,YAAnB3C,OAAO,CAAC2C,WAAW,CAAGL,WAAW,CAAC;MACpC;IACF,CAAC;IACDlC,iBAAiB,CAACwC,OAAO,CAACvC,IAAI,CAAC;EACjC,CAAC;EAEDG,KAAK,CAAC,CAAC,CAACqC,KAAK,CAAEC,KAAK,IAAK;IACvB;IACAC,OAAO,CAACD,KAAK,CACX,2DAA2D,EAC3DA,KACF,CAAC;EACH,CAAC,CAAC;EAEF,OAAO;IACL7C,OAAO,EAAEA,CAAA,KAAM;MAAA,IAAA+C,kBAAA,EAAAC,aAAA;MACb/C,SAAS,GAAG,IAAI;MAChB,IAAIG,IAAI,EAAE;QACRA,IAAI,CAAC4B,IAAI,CAACC,SAAS,GAAG,IAAI;QAC1B7B,IAAI,CAAC6C,UAAU,CAAC,CAAC;MACnB;MACA,CAAAF,kBAAA,GAAA5C,iBAAiB,aAAjB4C,kBAAA,CAAmBE,UAAU,CAAC,CAAC;MAC/B,OAAAD,aAAA,GAAK9C,YAAY,qBAAZ8C,aAAA,CAActB,KAAK,CAAC,CAAC;IAC5B;EACF,CAAC;AACH","ignoreList":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sources":["../../../../src/hooks/useSpeechActivityDetector/index.ts"],"sourcesContent":["export * from './useSpeechActivityDetector';\nexport * from './types';\n"],"mappings":"AAAA,cAAc,6BAA6B;AAC3C,cAAc,SAAS","ignoreList":[]}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// AudioWorkletProcessor source, kept as a plain string and loaded via a Blob
|
|
2
|
+
// URL (see helpers.ts). AudioWorkletGlobalScope runs isolated code on the
|
|
3
|
+
// dedicated audio rendering thread — it is not throttled when the tab is
|
|
4
|
+
// backgrounded, unlike requestAnimationFrame-driven analysis. Threshold
|
|
5
|
+
// crossings are posted back to the main thread via `port.postMessage`.
|
|
6
|
+
export const SPEECH_ACTIVITY_PROCESSOR_NAME = 'speech-activity-processor';
|
|
7
|
+
export const SPEECH_ACTIVITY_PROCESSOR_SOURCE = `
|
|
8
|
+
class SpeechActivityProcessor extends AudioWorkletProcessor {
|
|
9
|
+
constructor(options) {
|
|
10
|
+
super();
|
|
11
|
+
const opts = (options && options.processorOptions) || {};
|
|
12
|
+
this.threshold = typeof opts.thresholdRms === 'number' ? opts.thresholdRms : 0.015;
|
|
13
|
+
this.hangoverBlocks = typeof opts.hangoverBlocks === 'number' ? opts.hangoverBlocks : 8;
|
|
14
|
+
this.speaking = false;
|
|
15
|
+
this.silentBlockCount = 0;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
process(inputs) {
|
|
19
|
+
const input = inputs[0];
|
|
20
|
+
const channel = input && input[0];
|
|
21
|
+
if (!channel || channel.length === 0) {
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
let sumSquares = 0;
|
|
26
|
+
for (let i = 0; i < channel.length; i++) {
|
|
27
|
+
sumSquares += channel[i] * channel[i];
|
|
28
|
+
}
|
|
29
|
+
const rms = Math.sqrt(sumSquares / channel.length);
|
|
30
|
+
const isLoud = rms > this.threshold;
|
|
31
|
+
|
|
32
|
+
if (isLoud) {
|
|
33
|
+
this.silentBlockCount = 0;
|
|
34
|
+
if (!this.speaking) {
|
|
35
|
+
this.speaking = true;
|
|
36
|
+
this.port.postMessage({ type: 'onset', rms: rms });
|
|
37
|
+
}
|
|
38
|
+
} else if (this.speaking) {
|
|
39
|
+
this.silentBlockCount += 1;
|
|
40
|
+
if (this.silentBlockCount >= this.hangoverBlocks) {
|
|
41
|
+
this.speaking = false;
|
|
42
|
+
this.port.postMessage({ type: 'offset', rms: rms });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
registerProcessor('${SPEECH_ACTIVITY_PROCESSOR_NAME}', SpeechActivityProcessor);
|
|
50
|
+
`;
|
|
51
|
+
//# sourceMappingURL=speechActivityProcessorSource.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["SPEECH_ACTIVITY_PROCESSOR_NAME","SPEECH_ACTIVITY_PROCESSOR_SOURCE"],"sources":["../../../../src/hooks/useSpeechActivityDetector/speechActivityProcessorSource.ts"],"sourcesContent":["// AudioWorkletProcessor source, kept as a plain string and loaded via a Blob\n// URL (see helpers.ts). AudioWorkletGlobalScope runs isolated code on the\n// dedicated audio rendering thread — it is not throttled when the tab is\n// backgrounded, unlike requestAnimationFrame-driven analysis. Threshold\n// crossings are posted back to the main thread via `port.postMessage`.\nexport const SPEECH_ACTIVITY_PROCESSOR_NAME = 'speech-activity-processor';\n\nexport const SPEECH_ACTIVITY_PROCESSOR_SOURCE = `\nclass SpeechActivityProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n const opts = (options && options.processorOptions) || {};\n this.threshold = typeof opts.thresholdRms === 'number' ? opts.thresholdRms : 0.015;\n this.hangoverBlocks = typeof opts.hangoverBlocks === 'number' ? opts.hangoverBlocks : 8;\n this.speaking = false;\n this.silentBlockCount = 0;\n }\n\n process(inputs) {\n const input = inputs[0];\n const channel = input && input[0];\n if (!channel || channel.length === 0) {\n return true;\n }\n\n let sumSquares = 0;\n for (let i = 0; i < channel.length; i++) {\n sumSquares += channel[i] * channel[i];\n }\n const rms = Math.sqrt(sumSquares / channel.length);\n const isLoud = rms > this.threshold;\n\n if (isLoud) {\n this.silentBlockCount = 0;\n if (!this.speaking) {\n this.speaking = true;\n this.port.postMessage({ type: 'onset', rms: rms });\n }\n } else if (this.speaking) {\n this.silentBlockCount += 1;\n if (this.silentBlockCount >= this.hangoverBlocks) {\n this.speaking = false;\n this.port.postMessage({ type: 'offset', rms: rms });\n }\n }\n return true;\n }\n}\n\nregisterProcessor('${SPEECH_ACTIVITY_PROCESSOR_NAME}', SpeechActivityProcessor);\n`;\n"],"mappings":"AAAA;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMA,8BAA8B,GAAG,2BAA2B;AAEzE,OAAO,MAAMC,gCAAgC,GAAG;AAChD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,qBAAqBD,8BAA8B;AACnD,CAAC","ignoreList":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sources":["../../../../src/hooks/useSpeechActivityDetector/types.ts"],"sourcesContent":["export type UseSpeechActivityDetectorOptions = {\n source: 'user' | 'agent';\n onSpeechStart?: (timestampMs: number) => void;\n onSpeechEnd?: (timestampMs: number) => void;\n /** Fired instead of onSpeechEnd when a segment was shorter than minSpeechMs. */\n onMisfire?: () => void;\n /** RMS threshold (0-1) above which a block counts as \"loud\". Default 0.015. */\n thresholdRms?: number;\n /** Consecutive silence duration before an \"offset\" fires. Default 300ms. */\n hangoverMs?: number;\n /** Minimum onset-to-offset duration to count as real speech, not a blip/echo. Default 0 (no filtering). */\n minSpeechMs?: number;\n};\n"],"mappings":"","ignoreList":[]}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { useEffect, useRef } from 'react';
|
|
2
|
+
import { useParticipantTrack, Source } from '../useParticipantTrack';
|
|
3
|
+
import { setupSpeechActivityDetector } from './helpers';
|
|
4
|
+
/**
|
|
5
|
+
* Acoustic speech onset/offset detector for a participant's microphone
|
|
6
|
+
* track, driven by AudioWorkletNode rather than LiveKit's speaking-state
|
|
7
|
+
* events. Useful when the caller needs a timestamp tied to the actual audio
|
|
8
|
+
* signal (e.g. perceived end-to-end latency measurement) rather than a
|
|
9
|
+
* VAD/turn-detection-derived event.
|
|
10
|
+
*/
|
|
11
|
+
export function useSpeechActivityDetector(options) {
|
|
12
|
+
const source = options.source,
|
|
13
|
+
onSpeechStart = options.onSpeechStart,
|
|
14
|
+
onSpeechEnd = options.onSpeechEnd,
|
|
15
|
+
thresholdRms = options.thresholdRms,
|
|
16
|
+
hangoverMs = options.hangoverMs;
|
|
17
|
+
const _useParticipantTrack = useParticipantTrack({
|
|
18
|
+
type: source,
|
|
19
|
+
source: Source.Microphone
|
|
20
|
+
}),
|
|
21
|
+
track = _useParticipantTrack.track;
|
|
22
|
+
|
|
23
|
+
// Keep latest callbacks available to the effect without retriggering
|
|
24
|
+
// AudioContext/AudioWorkletNode setup on every render.
|
|
25
|
+
const callbacksRef = useRef({
|
|
26
|
+
onSpeechStart,
|
|
27
|
+
onSpeechEnd
|
|
28
|
+
});
|
|
29
|
+
callbacksRef.current = {
|
|
30
|
+
onSpeechStart,
|
|
31
|
+
onSpeechEnd
|
|
32
|
+
};
|
|
33
|
+
useEffect(() => {
|
|
34
|
+
const _setupSpeechActivityD = setupSpeechActivityDetector(track, {
|
|
35
|
+
thresholdRms,
|
|
36
|
+
hangoverMs,
|
|
37
|
+
onSpeechStart: timestampMs => callbacksRef.current.onSpeechStart == null ? void 0 : callbacksRef.current.onSpeechStart(timestampMs),
|
|
38
|
+
onSpeechEnd: timestampMs => callbacksRef.current.onSpeechEnd == null ? void 0 : callbacksRef.current.onSpeechEnd(timestampMs)
|
|
39
|
+
}),
|
|
40
|
+
cleanup = _setupSpeechActivityD.cleanup;
|
|
41
|
+
return cleanup;
|
|
42
|
+
}, [track == null ? void 0 : track.mediaStreamTrack, thresholdRms, hangoverMs]);
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=useSpeechActivityDetector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["useEffect","useRef","useParticipantTrack","Source","setupSpeechActivityDetector","useSpeechActivityDetector","options","source","onSpeechStart","onSpeechEnd","thresholdRms","hangoverMs","_useParticipantTrack","type","Microphone","track","callbacksRef","current","_setupSpeechActivityD","timestampMs","cleanup","mediaStreamTrack"],"sources":["../../../../src/hooks/useSpeechActivityDetector/useSpeechActivityDetector.ts"],"sourcesContent":["import { useEffect, useRef } from 'react';\nimport { useParticipantTrack, Source } from '../useParticipantTrack';\nimport { setupSpeechActivityDetector } from './helpers';\nimport { type UseSpeechActivityDetectorOptions } from './types';\n\n/**\n * Acoustic speech onset/offset detector for a participant's microphone\n * track, driven by AudioWorkletNode rather than LiveKit's speaking-state\n * events. Useful when the caller needs a timestamp tied to the actual audio\n * signal (e.g. perceived end-to-end latency measurement) rather than a\n * VAD/turn-detection-derived event.\n */\nexport function useSpeechActivityDetector(\n options: UseSpeechActivityDetectorOptions,\n): void {\n const { source, onSpeechStart, onSpeechEnd, thresholdRms, hangoverMs } =\n options;\n const { track } = useParticipantTrack({\n type: source,\n source: Source.Microphone,\n });\n\n // Keep latest callbacks available to the effect without retriggering\n // AudioContext/AudioWorkletNode setup on every render.\n const callbacksRef = useRef({ onSpeechStart, onSpeechEnd });\n callbacksRef.current = { onSpeechStart, onSpeechEnd };\n\n useEffect(() => {\n const { cleanup } = setupSpeechActivityDetector(track, {\n thresholdRms,\n hangoverMs,\n onSpeechStart: (timestampMs) =>\n callbacksRef.current.onSpeechStart?.(timestampMs),\n onSpeechEnd: (timestampMs) =>\n callbacksRef.current.onSpeechEnd?.(timestampMs),\n });\n return cleanup;\n }, [track?.mediaStreamTrack, thresholdRms, hangoverMs]);\n}\n"],"mappings":"AAAA,SAASA,SAAS,EAAEC,MAAM,QAAQ,OAAO;AACzC,SAASC,mBAAmB,EAAEC,MAAM,QAAQ,wBAAwB;AACpE,SAASC,2BAA2B,QAAQ,WAAW;AAGvD;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASC,yBAAyBA,CACvCC,OAAyC,EACnC;EACN,MAAQC,MAAM,GACZD,OAAO,CADDC,MAAM;IAAEC,aAAa,GAC3BF,OAAO,CADOE,aAAa;IAAEC,WAAW,GACxCH,OAAO,CADsBG,WAAW;IAAEC,YAAY,GACtDJ,OAAO,CADmCI,YAAY;IAAEC,UAAU,GAClEL,OAAO,CADiDK,UAAU;EAEpE,MAAAC,oBAAA,GAAkBV,mBAAmB,CAAC;MACpCW,IAAI,EAAEN,MAAM;MACZA,MAAM,EAAEJ,MAAM,CAACW;IACjB,CAAC,CAAC;IAHMC,KAAK,GAAAH,oBAAA,CAALG,KAAK;;EAKb;EACA;EACA,MAAMC,YAAY,GAAGf,MAAM,CAAC;IAAEO,aAAa;IAAEC;EAAY,CAAC,CAAC;EAC3DO,YAAY,CAACC,OAAO,GAAG;IAAET,aAAa;IAAEC;EAAY,CAAC;EAErDT,SAAS,CAAC,MAAM;IACd,MAAAkB,qBAAA,GAAoBd,2BAA2B,CAACW,KAAK,EAAE;QACrDL,YAAY;QACZC,UAAU;QACVH,aAAa,EAAGW,WAAW,IACzBH,YAAY,CAACC,OAAO,CAACT,aAAa,oBAAlCQ,YAAY,CAACC,OAAO,CAACT,aAAa,CAAGW,WAAW,CAAC;QACnDV,WAAW,EAAGU,WAAW,IACvBH,YAAY,CAACC,OAAO,CAACR,WAAW,oBAAhCO,YAAY,CAACC,OAAO,CAACR,WAAW,CAAGU,WAAW;MAClD,CAAC,CAAC;MAPMC,OAAO,GAAAF,qBAAA,CAAPE,OAAO;IAQf,OAAOA,OAAO;EAChB,CAAC,EAAE,CAACL,KAAK,oBAALA,KAAK,CAAEM,gBAAgB,EAAEX,YAAY,EAAEC,UAAU,CAAC,CAAC;AACzD","ignoreList":[]}
|
package/dist/esm/sdk.js
CHANGED
|
@@ -21,5 +21,6 @@ export * from './hooks/usePictureInPicture';
|
|
|
21
21
|
export * from './hooks/useMediaPermissions';
|
|
22
22
|
export * from './hooks/useTools';
|
|
23
23
|
export * from './hooks/useParticipantTrack';
|
|
24
|
+
export * from './hooks/useSpeechActivityDetector';
|
|
24
25
|
export * from './conversation/types';
|
|
25
26
|
//# sourceMappingURL=sdk.js.map
|
package/dist/esm/sdk.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["../../src/sdk.ts"],"sourcesContent":["export * from './types';\nexport * from './components/LegendsPlatform';\nexport * from './components/LegendsPlatformQLess';\n\nexport * from './hooks/useTextChat';\nexport * from './hooks/useConversation';\nexport * from './hooks/useTranscriptions';\nexport * from './hooks/useConversationStatus';\nexport * from './hooks/useMicrophone';\nexport * from './hooks/useSpeaker';\nexport * from './hooks/useCamera';\nexport * from './hooks/useMediaRecorder';\nexport * from './hooks/useConversationTimer';\nexport * from './hooks/useAudioLevel';\nexport * from './hooks/useConversationEvents';\nexport * from './hooks/useUserActivity';\nexport * from './hooks/useAudio';\nexport * from './hooks/useScreenShare';\nexport * from './hooks/useConnectionState';\nexport * from './hooks/useActivityState';\nexport * from './hooks/usePictureInPicture';\nexport * from './hooks/useMediaPermissions';\nexport * from './hooks/useTools';\nexport * from './hooks/useParticipantTrack';\n\nexport * from './conversation/types';\n"],"mappings":"AAAA,cAAc,SAAS;AACvB,cAAc,8BAA8B;AAC5C,cAAc,mCAAmC;AAEjD,cAAc,qBAAqB;AACnC,cAAc,yBAAyB;AACvC,cAAc,2BAA2B;AACzC,cAAc,+BAA+B;AAC7C,cAAc,uBAAuB;AACrC,cAAc,oBAAoB;AAClC,cAAc,mBAAmB;AACjC,cAAc,0BAA0B;AACxC,cAAc,8BAA8B;AAC5C,cAAc,uBAAuB;AACrC,cAAc,+BAA+B;AAC7C,cAAc,yBAAyB;AACvC,cAAc,kBAAkB;AAChC,cAAc,wBAAwB;AACtC,cAAc,4BAA4B;AAC1C,cAAc,0BAA0B;AACxC,cAAc,6BAA6B;AAC3C,cAAc,6BAA6B;AAC3C,cAAc,kBAAkB;AAChC,cAAc,6BAA6B;
|
|
1
|
+
{"version":3,"names":[],"sources":["../../src/sdk.ts"],"sourcesContent":["export * from './types';\nexport * from './components/LegendsPlatform';\nexport * from './components/LegendsPlatformQLess';\n\nexport * from './hooks/useTextChat';\nexport * from './hooks/useConversation';\nexport * from './hooks/useTranscriptions';\nexport * from './hooks/useConversationStatus';\nexport * from './hooks/useMicrophone';\nexport * from './hooks/useSpeaker';\nexport * from './hooks/useCamera';\nexport * from './hooks/useMediaRecorder';\nexport * from './hooks/useConversationTimer';\nexport * from './hooks/useAudioLevel';\nexport * from './hooks/useConversationEvents';\nexport * from './hooks/useUserActivity';\nexport * from './hooks/useAudio';\nexport * from './hooks/useScreenShare';\nexport * from './hooks/useConnectionState';\nexport * from './hooks/useActivityState';\nexport * from './hooks/usePictureInPicture';\nexport * from './hooks/useMediaPermissions';\nexport * from './hooks/useTools';\nexport * from './hooks/useParticipantTrack';\nexport * from './hooks/useSpeechActivityDetector';\n\nexport * from './conversation/types';\n"],"mappings":"AAAA,cAAc,SAAS;AACvB,cAAc,8BAA8B;AAC5C,cAAc,mCAAmC;AAEjD,cAAc,qBAAqB;AACnC,cAAc,yBAAyB;AACvC,cAAc,2BAA2B;AACzC,cAAc,+BAA+B;AAC7C,cAAc,uBAAuB;AACrC,cAAc,oBAAoB;AAClC,cAAc,mBAAmB;AACjC,cAAc,0BAA0B;AACxC,cAAc,8BAA8B;AAC5C,cAAc,uBAAuB;AACrC,cAAc,+BAA+B;AAC7C,cAAc,yBAAyB;AACvC,cAAc,kBAAkB;AAChC,cAAc,wBAAwB;AACtC,cAAc,4BAA4B;AAC1C,cAAc,0BAA0B;AACxC,cAAc,6BAA6B;AAC3C,cAAc,6BAA6B;AAC3C,cAAc,kBAAkB;AAChC,cAAc,6BAA6B;AAC3C,cAAc,mCAAmC;AAEjD,cAAc,sBAAsB","ignoreList":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/hooks/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/hooks/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,YAAY,CAAC;AAC3B,cAAc,6BAA6B,CAAC"}
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import type { TurnDetectionData } from './types';
|
|
2
2
|
import type { ConversationEvent } from '../../useConversationEvents';
|
|
3
3
|
export declare const SPEAKING_STATE_EVENT_MAP: Record<TurnDetectionData['participantKind'], Record<TurnDetectionData['state'], ConversationEvent>>;
|
|
4
|
+
export declare const ACOUSTIC_USER_THRESHOLD_RMS = 0.025;
|
|
5
|
+
export declare const ACOUSTIC_USER_HANGOVER_MS = 400;
|
|
6
|
+
export declare const ACOUSTIC_AGENT_HANGOVER_MS = 900;
|
|
4
7
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../../../src/hooks/useConversation/livekit/constants.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAErE,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAC3C,iBAAiB,CAAC,iBAAiB,CAAC,EACpC,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAC,CAUtD,CAAC"}
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../../../src/hooks/useConversation/livekit/constants.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAErE,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAC3C,iBAAiB,CAAC,iBAAiB,CAAC,EACpC,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAC,CAUtD,CAAC;AAEF,eAAO,MAAM,2BAA2B,QAAQ,CAAC;AACjD,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAC7C,eAAO,MAAM,0BAA0B,MAAM,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useConversation.d.ts","sourceRoot":"","sources":["../../../../../src/hooks/useConversation/livekit/useConversation.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EAEvB,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"useConversation.d.ts","sourceRoot":"","sources":["../../../../../src/hooks/useConversation/livekit/useConversation.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EAEvB,MAAM,UAAU,CAAC;AA+BlB,wBAAgB,eAAe,CAAC,EAC9B,QAAe,EACf,IAAI,EACJ,MAAM,EACN,mBAA2B,EAC3B,eAAuB,GACxB,EAAE,sBAAsB,GAAG,qBAAqB,CAmdhD"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { Track } from 'livekit-client';
|
|
2
|
+
export type SetupSpeechActivityDetectorOptions = {
|
|
3
|
+
thresholdRms?: number;
|
|
4
|
+
hangoverMs?: number;
|
|
5
|
+
minSpeechMs?: number;
|
|
6
|
+
onSpeechStart?: (timestampMs: number) => void;
|
|
7
|
+
onSpeechEnd?: (timestampMs: number) => void;
|
|
8
|
+
onMisfire?: () => void;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Attaches an AudioWorkletNode-based RMS threshold+hangover detector to a
|
|
12
|
+
* LiveKit track's underlying MediaStreamTrack. Runs on the dedicated audio
|
|
13
|
+
* rendering thread — unlike requestAnimationFrame+AnalyserNode, it keeps
|
|
14
|
+
* running when the tab is backgrounded.
|
|
15
|
+
*/
|
|
16
|
+
export declare function setupSpeechActivityDetector(track: Track | undefined, options: SetupSpeechActivityDetectorOptions): {
|
|
17
|
+
cleanup: () => void;
|
|
18
|
+
};
|
|
19
|
+
//# sourceMappingURL=helpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useSpeechActivityDetector/helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAUvC,MAAM,MAAM,kCAAkC,GAAG;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9C,WAAW,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5C,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;CACxB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,KAAK,GAAG,SAAS,EACxB,OAAO,EAAE,kCAAkC,GAC1C;IAAE,OAAO,EAAE,MAAM,IAAI,CAAA;CAAE,CAgFzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useSpeechActivityDetector/index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"speechActivityProcessorSource.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useSpeechActivityDetector/speechActivityProcessorSource.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,8BAA8B,8BAA8B,CAAC;AAE1E,eAAO,MAAM,gCAAgC,QA2C5C,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type UseSpeechActivityDetectorOptions = {
|
|
2
|
+
source: 'user' | 'agent';
|
|
3
|
+
onSpeechStart?: (timestampMs: number) => void;
|
|
4
|
+
onSpeechEnd?: (timestampMs: number) => void;
|
|
5
|
+
/** Fired instead of onSpeechEnd when a segment was shorter than minSpeechMs. */
|
|
6
|
+
onMisfire?: () => void;
|
|
7
|
+
/** RMS threshold (0-1) above which a block counts as "loud". Default 0.015. */
|
|
8
|
+
thresholdRms?: number;
|
|
9
|
+
/** Consecutive silence duration before an "offset" fires. Default 300ms. */
|
|
10
|
+
hangoverMs?: number;
|
|
11
|
+
/** Minimum onset-to-offset duration to count as real speech, not a blip/echo. Default 0 (no filtering). */
|
|
12
|
+
minSpeechMs?: number;
|
|
13
|
+
};
|
|
14
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useSpeechActivityDetector/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,gCAAgC,GAAG;IAC7C,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9C,WAAW,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5C,gFAAgF;IAChF,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;IACvB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2GAA2G;IAC3G,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type UseSpeechActivityDetectorOptions } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* Acoustic speech onset/offset detector for a participant's microphone
|
|
4
|
+
* track, driven by AudioWorkletNode rather than LiveKit's speaking-state
|
|
5
|
+
* events. Useful when the caller needs a timestamp tied to the actual audio
|
|
6
|
+
* signal (e.g. perceived end-to-end latency measurement) rather than a
|
|
7
|
+
* VAD/turn-detection-derived event.
|
|
8
|
+
*/
|
|
9
|
+
export declare function useSpeechActivityDetector(options: UseSpeechActivityDetectorOptions): void;
|
|
10
|
+
//# sourceMappingURL=useSpeechActivityDetector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useSpeechActivityDetector.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useSpeechActivityDetector/useSpeechActivityDetector.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,gCAAgC,EAAE,MAAM,SAAS,CAAC;AAEhE;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,gCAAgC,GACxC,IAAI,CAwBN"}
|
package/dist/types/sdk.d.ts
CHANGED
|
@@ -21,5 +21,6 @@ export * from './hooks/usePictureInPicture';
|
|
|
21
21
|
export * from './hooks/useMediaPermissions';
|
|
22
22
|
export * from './hooks/useTools';
|
|
23
23
|
export * from './hooks/useParticipantTrack';
|
|
24
|
+
export * from './hooks/useSpeechActivityDetector';
|
|
24
25
|
export * from './conversation/types';
|
|
25
26
|
//# sourceMappingURL=sdk.d.ts.map
|
package/dist/types/sdk.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sdk.d.ts","sourceRoot":"","sources":["../../src/sdk.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mCAAmC,CAAC;AAElD,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,kBAAkB,CAAC;AACjC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"sdk.d.ts","sourceRoot":"","sources":["../../src/sdk.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,8BAA8B,CAAC;AAC7C,cAAc,mCAAmC,CAAC;AAElD,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,kBAAkB,CAAC;AACjC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,mCAAmC,CAAC;AAElD,cAAc,sBAAsB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "1.
|
|
2
|
+
"version": "1.6.0",
|
|
3
3
|
"name": "@wix/legends-platform-sdk",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"author": {
|
|
@@ -66,7 +66,7 @@
|
|
|
66
66
|
"@wix/yoshi-flow-library": "^6.0.0",
|
|
67
67
|
"@wix/yoshi-style-dependencies": "^6.0.0",
|
|
68
68
|
"express": "^4.17.2",
|
|
69
|
-
"lodash": "^4.
|
|
69
|
+
"lodash": "^4.18.1",
|
|
70
70
|
"react": "^19.0.0",
|
|
71
71
|
"react-dom": "^19.0.0",
|
|
72
72
|
"typescript": "~5.6.2"
|
|
@@ -103,5 +103,5 @@
|
|
|
103
103
|
"wallaby": {
|
|
104
104
|
"autoDetect": true
|
|
105
105
|
},
|
|
106
|
-
"falconPackageHash": "
|
|
106
|
+
"falconPackageHash": "dd4e950c67189a47f9d5faa19a5ec9d0f502ef0718b392f6bfb81656"
|
|
107
107
|
}
|