@absolutejs/voice 0.0.22-beta.312 → 0.0.22-beta.313
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/index.js +104 -1
- package/dist/mediaPipeline.d.ts +50 -0
- package/dist/mediaPipelineRoutes.d.ts +6 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -15,8 +15,8 @@ export type { VoiceRealtimeChannelAssertionInput, VoiceRealtimeChannelAssertionR
|
|
|
15
15
|
export { assertVoiceRealtimeProviderContractEvidence, buildVoiceRealtimeProviderContractMatrix, createVoiceRealtimeProviderContractMatrixPreset, createVoiceRealtimeProviderContractRoutes, evaluateVoiceRealtimeProviderContractEvidence, renderVoiceRealtimeProviderContractHTML } from './realtimeProviderContracts';
|
|
16
16
|
export type { VoiceRealtimeProviderContractAssertionInput, VoiceRealtimeProviderContractAssertionReport, VoiceRealtimeProviderContractCapability, VoiceRealtimeProviderContractCheck, VoiceRealtimeProviderContractDefinition, VoiceRealtimeProviderContractMatrixPresetOptions, VoiceRealtimeProviderContractMatrixInput, VoiceRealtimeProviderContractMatrixReport, VoiceRealtimeProviderContractRoutesOptions, VoiceRealtimeProviderContractRow, VoiceRealtimeProviderPresetProvider, VoiceRealtimeProviderContractStatus } from './realtimeProviderContracts';
|
|
17
17
|
export { buildVoiceDiagnosticsMarkdown, createVoiceDiagnosticsRoutes, resolveVoiceDiagnosticsTraceFilter } from './diagnosticsRoutes';
|
|
18
|
-
export { buildVoiceMediaTransportReport, buildVoiceMediaInterruptionReport, buildVoiceMediaPipelineCalibrationReport, buildVoiceMediaResamplingPlan, buildVoiceMediaVadReport, createVoiceMediaFrame, createVoiceMediaFrameTransformPipeline, createVoiceMediaTransport } from './mediaPipeline';
|
|
19
|
-
export type { VoiceMediaFrame, VoiceMediaFrameKind, VoiceMediaFrameSource, VoiceMediaFrameTransform, VoiceMediaFrameTransformPipeline, VoiceMediaInterruptionInput, VoiceMediaInterruptionReport, VoiceMediaPipelineCalibrationInput, VoiceMediaPipelineCalibrationIssue, VoiceMediaPipelineCalibrationReport, VoiceMediaPipelineStatus, VoiceMediaResamplingPlan, VoiceMediaTransport, VoiceMediaTransportAdapter, VoiceMediaTransportEvent, VoiceMediaTransportEventKind, VoiceMediaTransportOptions, VoiceMediaTransportReport, VoiceMediaTransportState, VoiceMediaVadInput, VoiceMediaVadReport, VoiceMediaVadSegment } from './mediaPipeline';
|
|
18
|
+
export { buildVoiceMediaTransportReport, buildVoiceMediaInterruptionReport, buildVoiceMediaPipelineCalibrationReport, buildVoiceMediaResamplingPlan, buildVoiceMediaProcessorGraphReport, buildVoiceMediaVadReport, createVoiceMediaFrame, createVoiceMediaFrameTransformPipeline, createVoiceMediaProcessorGraph, createVoiceMediaTransport } from './mediaPipeline';
|
|
19
|
+
export type { VoiceMediaFrame, VoiceMediaFrameKind, VoiceMediaFrameSource, VoiceMediaFrameTransform, VoiceMediaFrameTransformPipeline, VoiceMediaInterruptionInput, VoiceMediaInterruptionReport, VoiceMediaPipelineCalibrationInput, VoiceMediaPipelineCalibrationIssue, VoiceMediaPipelineCalibrationReport, VoiceMediaPipelineStatus, VoiceMediaProcessorGraph, VoiceMediaProcessorGraphReport, VoiceMediaProcessorNode, VoiceMediaProcessorNodeEvent, VoiceMediaProcessorNodeKind, VoiceMediaProcessorNodeReport, VoiceMediaResamplingPlan, VoiceMediaTransport, VoiceMediaTransportAdapter, VoiceMediaTransportEvent, VoiceMediaTransportEventKind, VoiceMediaTransportOptions, VoiceMediaTransportReport, VoiceMediaTransportState, VoiceMediaVadInput, VoiceMediaVadReport, VoiceMediaVadSegment } from './mediaPipeline';
|
|
20
20
|
export { assertVoiceMediaPipelineEvidence, buildVoiceMediaPipelineReport, createVoiceMediaPipelineRoutes, evaluateVoiceMediaPipelineEvidence, renderVoiceMediaPipelineHTML, renderVoiceMediaPipelineMarkdown } from './mediaPipelineRoutes';
|
|
21
21
|
export type { VoiceMediaPipelineAssertionInput, VoiceMediaPipelineAssertionReport, VoiceMediaPipelineReport, VoiceMediaPipelineReportOptions, VoiceMediaPipelineRoutesOptions } from './mediaPipelineRoutes';
|
|
22
22
|
export { buildVoiceDemoReadyReport, createVoiceDemoReadyRoutes, renderVoiceDemoReadyHTML } from './demoReadyRoutes';
|
package/dist/index.js
CHANGED
|
@@ -11752,6 +11752,93 @@ var createVoiceMediaFrameTransformPipeline = (input = {}) => {
|
|
|
11752
11752
|
transforms
|
|
11753
11753
|
};
|
|
11754
11754
|
};
|
|
11755
|
+
var normalizeProcessorResult = (frame, result) => {
|
|
11756
|
+
if (result === false || result === undefined) {
|
|
11757
|
+
return [];
|
|
11758
|
+
}
|
|
11759
|
+
if (result === true) {
|
|
11760
|
+
return [frame];
|
|
11761
|
+
}
|
|
11762
|
+
if (Array.isArray(result)) {
|
|
11763
|
+
return result;
|
|
11764
|
+
}
|
|
11765
|
+
return [result];
|
|
11766
|
+
};
|
|
11767
|
+
var buildVoiceMediaProcessorGraphReport = (input) => {
|
|
11768
|
+
const events = input.events ?? [];
|
|
11769
|
+
const nodes = input.nodes.map((node) => {
|
|
11770
|
+
const nodeEvents = events.filter((event) => event.node === node.name);
|
|
11771
|
+
const droppedFrames2 = nodeEvents.reduce((total, event) => total + event.dropped, 0);
|
|
11772
|
+
const emittedFrames2 = nodeEvents.reduce((total, event) => total + event.emitted, 0);
|
|
11773
|
+
const inputFrames2 = nodeEvents.reduce((total, event) => total + event.inputs, 0);
|
|
11774
|
+
return {
|
|
11775
|
+
droppedFrames: droppedFrames2,
|
|
11776
|
+
emittedFrames: emittedFrames2,
|
|
11777
|
+
events: nodeEvents,
|
|
11778
|
+
inputFrames: inputFrames2,
|
|
11779
|
+
kind: node.kind ?? "processor",
|
|
11780
|
+
name: node.name,
|
|
11781
|
+
status: inputFrames2 > 0 && emittedFrames2 === 0 && node.kind !== "sink" ? "warn" : "pass"
|
|
11782
|
+
};
|
|
11783
|
+
});
|
|
11784
|
+
const inputFrames = events.filter((event) => event.node === input.nodes[0]?.name).length;
|
|
11785
|
+
const droppedFrames = events.reduce((total, event) => total + event.dropped, 0);
|
|
11786
|
+
const emittedFrames = input.nodes.at(-1) ? events.filter((event) => event.node === input.nodes.at(-1)?.name).reduce((total, event) => total + event.emitted, 0) : 0;
|
|
11787
|
+
const status = nodes.some((node) => node.status === "warn") ? "warn" : "pass";
|
|
11788
|
+
return {
|
|
11789
|
+
checkedAt: Date.now(),
|
|
11790
|
+
droppedFrames,
|
|
11791
|
+
emittedFrames,
|
|
11792
|
+
events,
|
|
11793
|
+
inputFrames,
|
|
11794
|
+
name: input.name,
|
|
11795
|
+
nodes,
|
|
11796
|
+
status
|
|
11797
|
+
};
|
|
11798
|
+
};
|
|
11799
|
+
var createVoiceMediaProcessorGraph = (input = {}) => {
|
|
11800
|
+
const nodes = input.nodes ?? [];
|
|
11801
|
+
const events = [];
|
|
11802
|
+
const process = async (frame) => {
|
|
11803
|
+
let frames = [frame];
|
|
11804
|
+
for (const node of nodes) {
|
|
11805
|
+
const nextFrames = [];
|
|
11806
|
+
for (const current of frames) {
|
|
11807
|
+
const output = normalizeProcessorResult(current, await node.process(current));
|
|
11808
|
+
events.push({
|
|
11809
|
+
at: Date.now(),
|
|
11810
|
+
dropped: output.length === 0 ? 1 : 0,
|
|
11811
|
+
emitted: output.length,
|
|
11812
|
+
frameId: current.id,
|
|
11813
|
+
inputs: 1,
|
|
11814
|
+
node: node.name
|
|
11815
|
+
});
|
|
11816
|
+
nextFrames.push(...output);
|
|
11817
|
+
}
|
|
11818
|
+
frames = nextFrames;
|
|
11819
|
+
if (frames.length === 0) {
|
|
11820
|
+
break;
|
|
11821
|
+
}
|
|
11822
|
+
}
|
|
11823
|
+
return frames;
|
|
11824
|
+
};
|
|
11825
|
+
return {
|
|
11826
|
+
nodes,
|
|
11827
|
+
process,
|
|
11828
|
+
processMany: async (frames) => {
|
|
11829
|
+
const output = [];
|
|
11830
|
+
for (const frame of frames) {
|
|
11831
|
+
output.push(...await process(frame));
|
|
11832
|
+
}
|
|
11833
|
+
return output;
|
|
11834
|
+
},
|
|
11835
|
+
report: () => buildVoiceMediaProcessorGraphReport({
|
|
11836
|
+
events,
|
|
11837
|
+
name: input.name ?? "voice-media-processor-graph",
|
|
11838
|
+
nodes
|
|
11839
|
+
})
|
|
11840
|
+
};
|
|
11841
|
+
};
|
|
11755
11842
|
var speechProbability = (frame) => {
|
|
11756
11843
|
if (frame.metadata?.isSpeech === true) {
|
|
11757
11844
|
return 1;
|
|
@@ -11939,6 +12026,7 @@ var buildVoiceMediaPipelineReport = (options = {}) => {
|
|
|
11939
12026
|
vad.status,
|
|
11940
12027
|
interruption.status,
|
|
11941
12028
|
resampling?.status ?? "pass",
|
|
12029
|
+
options.processorGraph?.status ?? "pass",
|
|
11942
12030
|
options.transport?.status ?? "pass"
|
|
11943
12031
|
]);
|
|
11944
12032
|
return {
|
|
@@ -11947,6 +12035,7 @@ var buildVoiceMediaPipelineReport = (options = {}) => {
|
|
|
11947
12035
|
frames: frames.length,
|
|
11948
12036
|
interruption,
|
|
11949
12037
|
ok: status === "pass",
|
|
12038
|
+
processorGraph: options.processorGraph,
|
|
11950
12039
|
resampling,
|
|
11951
12040
|
status,
|
|
11952
12041
|
surface: options.surface ?? "voice-media-pipeline",
|
|
@@ -11983,6 +12072,15 @@ var evaluateVoiceMediaPipelineEvidence = (report, input = {}) => {
|
|
|
11983
12072
|
if (input.requireResamplingReady && report.calibration.resamplingRequired && !report.resampling) {
|
|
11984
12073
|
issues.push("Expected resampling plan when calibration requires resampling.");
|
|
11985
12074
|
}
|
|
12075
|
+
if (input.requireProcessorGraph && !report.processorGraph) {
|
|
12076
|
+
issues.push("Expected media processor graph evidence.");
|
|
12077
|
+
}
|
|
12078
|
+
if (input.minProcessorGraphNodes !== undefined && (report.processorGraph?.nodes.length ?? 0) < input.minProcessorGraphNodes) {
|
|
12079
|
+
issues.push(`Expected at least ${String(input.minProcessorGraphNodes)} media processor node(s), found ${String(report.processorGraph?.nodes.length ?? 0)}.`);
|
|
12080
|
+
}
|
|
12081
|
+
if (input.minProcessorGraphEmittedFrames !== undefined && (report.processorGraph?.emittedFrames ?? 0) < input.minProcessorGraphEmittedFrames) {
|
|
12082
|
+
issues.push(`Expected at least ${String(input.minProcessorGraphEmittedFrames)} processor graph output frame(s), found ${String(report.processorGraph?.emittedFrames ?? 0)}.`);
|
|
12083
|
+
}
|
|
11986
12084
|
if (input.requireTransportConnected && !report.transport?.connected) {
|
|
11987
12085
|
issues.push("Expected connected media transport evidence.");
|
|
11988
12086
|
}
|
|
@@ -12022,6 +12120,9 @@ var renderVoiceMediaPipelineMarkdown = (report) => [
|
|
|
12022
12120
|
`- Resampling required: ${report.calibration.resamplingRequired ? "yes" : "no"}`,
|
|
12023
12121
|
`- VAD segments: ${String(report.vad.segments.length)}`,
|
|
12024
12122
|
`- Interruption frames: ${String(report.interruption.interruptionFrames)}`,
|
|
12123
|
+
`- Processor graph: ${report.processorGraph ? `${report.processorGraph.name} (${String(report.processorGraph.nodes.length)} nodes)` : "n/a"}`,
|
|
12124
|
+
`- Processor graph emitted frames: ${String(report.processorGraph?.emittedFrames ?? 0)}`,
|
|
12125
|
+
`- Processor graph dropped frames: ${String(report.processorGraph?.droppedFrames ?? 0)}`,
|
|
12025
12126
|
`- Transport: ${report.transport ? `${report.transport.name} (${report.transport.state})` : "n/a"}`,
|
|
12026
12127
|
`- Transport input frames: ${String(report.transport?.inputFrames ?? 0)}`,
|
|
12027
12128
|
`- Transport output frames: ${String(report.transport?.outputFrames ?? 0)}`,
|
|
@@ -12039,7 +12140,7 @@ var renderVoiceMediaPipelineMarkdown = (report) => [
|
|
|
12039
12140
|
var renderVoiceMediaPipelineHTML = (report, title = "Voice Media Pipeline Proof") => {
|
|
12040
12141
|
const issues = [...report.calibration.issues, ...report.interruption.issues].map((issue) => `<li class="${escapeHtml15(issue.severity)}"><strong>${escapeHtml15(issue.code)}</strong>: ${escapeHtml15(issue.message)}</li>`).join("");
|
|
12041
12142
|
const segments = report.vad.segments.map((segment) => `<tr><td>${escapeHtml15(segment.segmentId)}</td><td>${escapeHtml15(segment.frameCount)}</td><td>${escapeHtml15(segment.durationMs ?? "n/a")}</td><td>${escapeHtml15(segment.turnId ?? "n/a")}</td></tr>`).join("");
|
|
12042
|
-
return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1" /><title>${escapeHtml15(title)}</title><style>body{background:#101418;color:#f7f3e8;font-family:ui-sans-serif,system-ui,sans-serif;margin:0}main{margin:auto;max-width:1100px;padding:32px}.hero,.card{background:#17201d;border:1px solid #2e3d36;border-radius:24px;margin-bottom:16px;padding:22px}.hero{background:linear-gradient(135deg,rgba(20,184,166,.18),rgba(245,158,11,.12))}.eyebrow{color:#5eead4;font-weight:900;letter-spacing:.1em;text-transform:uppercase}h1{font-size:clamp(2.3rem,6vw,4.8rem);letter-spacing:-.06em;line-height:.9;margin:.2rem 0 1rem}.summary{display:grid;gap:12px;grid-template-columns:repeat(auto-fit,minmax(170px,1fr))}.metric{background:#101814;border:1px solid #2e3d36;border-radius:18px;padding:14px}.metric span{color:#a8b5ad;display:block;font-size:.78rem;text-transform:uppercase}.metric strong{display:block;font-size:1.65rem;margin-top:5px}.status{border:1px solid #64748b;border-radius:999px;display:inline-flex;font-weight:900;padding:7px 11px}.pass{color:#86efac}.warn,.warning{color:#fde68a}.fail,.error{color:#fecaca}table{border-collapse:collapse;width:100%}td,th{border-bottom:1px solid #2e3d36;padding:10px;text-align:left}</style></head><body><main><section class="hero"><p class="eyebrow">Native media pipeline</p><h1>${escapeHtml15(title)}</h1><p class="status ${escapeHtml15(report.status)}">${escapeHtml15(report.status)}</p><p>${escapeHtml15(report.surface)}</p><section class="summary"><div class="metric"><span>Frames</span><strong>${String(report.frames)}</strong></div><div class="metric"><span>Input audio</span><strong>${String(report.calibration.inputAudioFrames)}</strong></div><div class="metric"><span>Assistant audio</span><strong>${String(report.calibration.assistantAudioFrames)}</strong></div><div class="metric"><span>Trace linked</span><strong>${String(report.calibration.traceLinkedFrames)}</strong></div><div class="metric"><span>First audio</span><strong>${escapeHtml15(report.calibration.firstAudioLatencyMs ?? "n/a")}ms</strong></div><div class="metric"><span>VAD segments</span><strong>${String(report.vad.segments.length)}</strong></div><div class="metric"><span>Interruptions</span><strong>${String(report.interruption.interruptionFrames)}</strong></div><div class="metric"><span>Resampling</span><strong>${report.calibration.resamplingRequired ? "required" : "not required"}</strong></div><div class="metric"><span>Transport</span><strong>${escapeHtml15(report.transport?.state ?? "n/a")}</strong></div><div class="metric"><span>Transport in/out</span><strong>${String(report.transport?.inputFrames ?? 0)}/${String(report.transport?.outputFrames ?? 0)}</strong></div><div class="metric"><span>Backpressure</span><strong>${String(report.transport?.backpressureEvents ?? 0)}</strong></div></section></section><section class="card"><h2>Issues</h2><ul>${issues || '<li class="pass">No media pipeline issues.</li>'}</ul></section><section class="card"><h2>VAD Segments</h2><table><thead><tr><th>Segment</th><th>Frames</th><th>Duration ms</th><th>Turn</th></tr></thead><tbody>${segments || '<tr><td colspan="4">No VAD segments.</td></tr>'}</tbody></table></section></main></body></html>`;
|
|
12143
|
+
return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1" /><title>${escapeHtml15(title)}</title><style>body{background:#101418;color:#f7f3e8;font-family:ui-sans-serif,system-ui,sans-serif;margin:0}main{margin:auto;max-width:1100px;padding:32px}.hero,.card{background:#17201d;border:1px solid #2e3d36;border-radius:24px;margin-bottom:16px;padding:22px}.hero{background:linear-gradient(135deg,rgba(20,184,166,.18),rgba(245,158,11,.12))}.eyebrow{color:#5eead4;font-weight:900;letter-spacing:.1em;text-transform:uppercase}h1{font-size:clamp(2.3rem,6vw,4.8rem);letter-spacing:-.06em;line-height:.9;margin:.2rem 0 1rem}.summary{display:grid;gap:12px;grid-template-columns:repeat(auto-fit,minmax(170px,1fr))}.metric{background:#101814;border:1px solid #2e3d36;border-radius:18px;padding:14px}.metric span{color:#a8b5ad;display:block;font-size:.78rem;text-transform:uppercase}.metric strong{display:block;font-size:1.65rem;margin-top:5px}.status{border:1px solid #64748b;border-radius:999px;display:inline-flex;font-weight:900;padding:7px 11px}.pass{color:#86efac}.warn,.warning{color:#fde68a}.fail,.error{color:#fecaca}table{border-collapse:collapse;width:100%}td,th{border-bottom:1px solid #2e3d36;padding:10px;text-align:left}</style></head><body><main><section class="hero"><p class="eyebrow">Native media pipeline</p><h1>${escapeHtml15(title)}</h1><p class="status ${escapeHtml15(report.status)}">${escapeHtml15(report.status)}</p><p>${escapeHtml15(report.surface)}</p><section class="summary"><div class="metric"><span>Frames</span><strong>${String(report.frames)}</strong></div><div class="metric"><span>Input audio</span><strong>${String(report.calibration.inputAudioFrames)}</strong></div><div class="metric"><span>Assistant audio</span><strong>${String(report.calibration.assistantAudioFrames)}</strong></div><div class="metric"><span>Trace linked</span><strong>${String(report.calibration.traceLinkedFrames)}</strong></div><div class="metric"><span>First audio</span><strong>${escapeHtml15(report.calibration.firstAudioLatencyMs ?? "n/a")}ms</strong></div><div class="metric"><span>VAD segments</span><strong>${String(report.vad.segments.length)}</strong></div><div class="metric"><span>Interruptions</span><strong>${String(report.interruption.interruptionFrames)}</strong></div><div class="metric"><span>Processor graph</span><strong>${String(report.processorGraph?.nodes.length ?? 0)} nodes</strong></div><div class="metric"><span>Graph out/drop</span><strong>${String(report.processorGraph?.emittedFrames ?? 0)}/${String(report.processorGraph?.droppedFrames ?? 0)}</strong></div><div class="metric"><span>Resampling</span><strong>${report.calibration.resamplingRequired ? "required" : "not required"}</strong></div><div class="metric"><span>Transport</span><strong>${escapeHtml15(report.transport?.state ?? "n/a")}</strong></div><div class="metric"><span>Transport in/out</span><strong>${String(report.transport?.inputFrames ?? 0)}/${String(report.transport?.outputFrames ?? 0)}</strong></div><div class="metric"><span>Backpressure</span><strong>${String(report.transport?.backpressureEvents ?? 0)}</strong></div></section></section><section class="card"><h2>Issues</h2><ul>${issues || '<li class="pass">No media pipeline issues.</li>'}</ul></section><section class="card"><h2>VAD Segments</h2><table><thead><tr><th>Segment</th><th>Frames</th><th>Duration ms</th><th>Turn</th></tr></thead><tbody>${segments || '<tr><td colspan="4">No VAD segments.</td></tr>'}</tbody></table></section></main></body></html>`;
|
|
12043
12144
|
};
|
|
12044
12145
|
var createVoiceMediaPipelineRoutes = (options = {}) => {
|
|
12045
12146
|
const path = options.path ?? "/api/voice/media-pipeline";
|
|
@@ -34871,6 +34972,7 @@ export {
|
|
|
34871
34972
|
createVoiceMemoryAuditEventStore,
|
|
34872
34973
|
createVoiceMemoryAssistantMemoryStore,
|
|
34873
34974
|
createVoiceMediaTransport,
|
|
34975
|
+
createVoiceMediaProcessorGraph,
|
|
34874
34976
|
createVoiceMediaPipelineRoutes,
|
|
34875
34977
|
createVoiceMediaFrameTransformPipeline,
|
|
34876
34978
|
createVoiceMediaFrame,
|
|
@@ -35025,6 +35127,7 @@ export {
|
|
|
35025
35127
|
buildVoiceMediaVadReport,
|
|
35026
35128
|
buildVoiceMediaTransportReport,
|
|
35027
35129
|
buildVoiceMediaResamplingPlan,
|
|
35130
|
+
buildVoiceMediaProcessorGraphReport,
|
|
35028
35131
|
buildVoiceMediaPipelineReport,
|
|
35029
35132
|
buildVoiceMediaPipelineCalibrationReport,
|
|
35030
35133
|
buildVoiceMediaInterruptionReport,
|
package/dist/mediaPipeline.d.ts
CHANGED
|
@@ -34,6 +34,47 @@ export type VoiceMediaFrameTransformPipeline = {
|
|
|
34
34
|
pushMany: (frames: readonly VoiceMediaFrame[]) => Promise<readonly VoiceMediaFrame[]>;
|
|
35
35
|
transforms: readonly VoiceMediaFrameTransform[];
|
|
36
36
|
};
|
|
37
|
+
export type VoiceMediaProcessorNodeKind = 'branch' | 'filter' | 'processor' | 'sink';
|
|
38
|
+
export type VoiceMediaProcessorNodeEvent = {
|
|
39
|
+
at: number;
|
|
40
|
+
dropped: number;
|
|
41
|
+
emitted: number;
|
|
42
|
+
frameId: string;
|
|
43
|
+
inputs: number;
|
|
44
|
+
node: string;
|
|
45
|
+
};
|
|
46
|
+
export type VoiceMediaProcessorNodeReport = {
|
|
47
|
+
droppedFrames: number;
|
|
48
|
+
emittedFrames: number;
|
|
49
|
+
events: readonly VoiceMediaProcessorNodeEvent[];
|
|
50
|
+
inputFrames: number;
|
|
51
|
+
kind: VoiceMediaProcessorNodeKind;
|
|
52
|
+
name: string;
|
|
53
|
+
status: VoiceMediaPipelineStatus;
|
|
54
|
+
};
|
|
55
|
+
export type VoiceMediaProcessorGraphReport = {
|
|
56
|
+
checkedAt: number;
|
|
57
|
+
droppedFrames: number;
|
|
58
|
+
emittedFrames: number;
|
|
59
|
+
events: readonly VoiceMediaProcessorNodeEvent[];
|
|
60
|
+
inputFrames: number;
|
|
61
|
+
name: string;
|
|
62
|
+
nodes: readonly VoiceMediaProcessorNodeReport[];
|
|
63
|
+
status: VoiceMediaPipelineStatus;
|
|
64
|
+
};
|
|
65
|
+
export type VoiceMediaProcessorNode = {
|
|
66
|
+
inputFormat?: AudioFormat;
|
|
67
|
+
kind?: VoiceMediaProcessorNodeKind;
|
|
68
|
+
name: string;
|
|
69
|
+
outputFormat?: AudioFormat;
|
|
70
|
+
process: (frame: VoiceMediaFrame) => boolean | VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined | Promise<boolean | VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined>;
|
|
71
|
+
};
|
|
72
|
+
export type VoiceMediaProcessorGraph = {
|
|
73
|
+
nodes: readonly VoiceMediaProcessorNode[];
|
|
74
|
+
process: (frame: VoiceMediaFrame) => Promise<readonly VoiceMediaFrame[]>;
|
|
75
|
+
processMany: (frames: readonly VoiceMediaFrame[]) => Promise<readonly VoiceMediaFrame[]>;
|
|
76
|
+
report: () => VoiceMediaProcessorGraphReport;
|
|
77
|
+
};
|
|
37
78
|
export type VoiceMediaTransportAdapter = {
|
|
38
79
|
close?: () => Promise<void> | void;
|
|
39
80
|
connect?: () => Promise<void> | void;
|
|
@@ -164,6 +205,15 @@ export declare const buildVoiceMediaResamplingPlan: (input: {
|
|
|
164
205
|
export declare const createVoiceMediaFrameTransformPipeline: (input?: {
|
|
165
206
|
transforms?: readonly VoiceMediaFrameTransform[];
|
|
166
207
|
}) => VoiceMediaFrameTransformPipeline;
|
|
208
|
+
export declare const buildVoiceMediaProcessorGraphReport: (input: {
|
|
209
|
+
events?: readonly VoiceMediaProcessorNodeEvent[];
|
|
210
|
+
name: string;
|
|
211
|
+
nodes: readonly VoiceMediaProcessorNode[];
|
|
212
|
+
}) => VoiceMediaProcessorGraphReport;
|
|
213
|
+
export declare const createVoiceMediaProcessorGraph: (input?: {
|
|
214
|
+
name?: string;
|
|
215
|
+
nodes?: readonly VoiceMediaProcessorNode[];
|
|
216
|
+
}) => VoiceMediaProcessorGraph;
|
|
167
217
|
export declare const buildVoiceMediaVadReport: (input?: VoiceMediaVadInput) => VoiceMediaVadReport;
|
|
168
218
|
export declare const buildVoiceMediaInterruptionReport: (input?: VoiceMediaInterruptionInput) => VoiceMediaInterruptionReport;
|
|
169
219
|
export declare const buildVoiceMediaPipelineCalibrationReport: (input?: VoiceMediaPipelineCalibrationInput) => VoiceMediaPipelineCalibrationReport;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Elysia } from 'elysia';
|
|
2
|
-
import { type VoiceMediaFrame, type VoiceMediaInterruptionReport, type VoiceMediaPipelineCalibrationInput, type VoiceMediaPipelineCalibrationReport, type VoiceMediaPipelineStatus, type VoiceMediaResamplingPlan, type VoiceMediaTransportReport, type VoiceMediaVadReport } from './mediaPipeline';
|
|
2
|
+
import { type VoiceMediaFrame, type VoiceMediaInterruptionReport, type VoiceMediaPipelineCalibrationInput, type VoiceMediaPipelineCalibrationReport, type VoiceMediaPipelineStatus, type VoiceMediaProcessorGraphReport, type VoiceMediaResamplingPlan, type VoiceMediaTransportReport, type VoiceMediaVadReport } from './mediaPipeline';
|
|
3
3
|
export type VoiceMediaPipelineReportOptions = VoiceMediaPipelineCalibrationInput & {
|
|
4
4
|
frames?: readonly VoiceMediaFrame[];
|
|
5
5
|
maxInterruptionLatencyMs?: number;
|
|
6
6
|
maxSilenceFrames?: number;
|
|
7
7
|
minSpeechFrames?: number;
|
|
8
|
+
processorGraph?: VoiceMediaProcessorGraphReport;
|
|
8
9
|
speechEndThreshold?: number;
|
|
9
10
|
speechStartThreshold?: number;
|
|
10
11
|
transport?: VoiceMediaTransportReport;
|
|
@@ -16,6 +17,7 @@ export type VoiceMediaPipelineReport = {
|
|
|
16
17
|
interruption: VoiceMediaInterruptionReport;
|
|
17
18
|
ok: boolean;
|
|
18
19
|
resampling?: VoiceMediaResamplingPlan;
|
|
20
|
+
processorGraph?: VoiceMediaProcessorGraphReport;
|
|
19
21
|
status: VoiceMediaPipelineStatus;
|
|
20
22
|
surface: string;
|
|
21
23
|
transport?: VoiceMediaTransportReport;
|
|
@@ -26,6 +28,8 @@ export type VoiceMediaPipelineAssertionInput = {
|
|
|
26
28
|
maxInterruptionLatencyMs?: number;
|
|
27
29
|
minAssistantAudioFrames?: number;
|
|
28
30
|
minInputAudioFrames?: number;
|
|
31
|
+
minProcessorGraphEmittedFrames?: number;
|
|
32
|
+
minProcessorGraphNodes?: number;
|
|
29
33
|
minTransportInputFrames?: number;
|
|
30
34
|
minTransportOutputFrames?: number;
|
|
31
35
|
minTraceLinkedFrames?: number;
|
|
@@ -33,6 +37,7 @@ export type VoiceMediaPipelineAssertionInput = {
|
|
|
33
37
|
maxTransportBackpressureEvents?: number;
|
|
34
38
|
requireInterruptionFrame?: boolean;
|
|
35
39
|
requirePass?: boolean;
|
|
40
|
+
requireProcessorGraph?: boolean;
|
|
36
41
|
requireResamplingReady?: boolean;
|
|
37
42
|
requireTransportConnected?: boolean;
|
|
38
43
|
};
|