@absolutejs/voice 0.0.22-beta.313 → 0.0.22-beta.314
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +0 -2
- package/dist/index.js +14 -231
- package/dist/mediaPipelineRoutes.d.ts +13 -13
- package/package.json +4 -1
- package/dist/mediaPipeline.d.ts +0 -219
package/dist/index.d.ts
CHANGED
|
@@ -15,8 +15,6 @@ export type { VoiceRealtimeChannelAssertionInput, VoiceRealtimeChannelAssertionR
|
|
|
15
15
|
export { assertVoiceRealtimeProviderContractEvidence, buildVoiceRealtimeProviderContractMatrix, createVoiceRealtimeProviderContractMatrixPreset, createVoiceRealtimeProviderContractRoutes, evaluateVoiceRealtimeProviderContractEvidence, renderVoiceRealtimeProviderContractHTML } from './realtimeProviderContracts';
|
|
16
16
|
export type { VoiceRealtimeProviderContractAssertionInput, VoiceRealtimeProviderContractAssertionReport, VoiceRealtimeProviderContractCapability, VoiceRealtimeProviderContractCheck, VoiceRealtimeProviderContractDefinition, VoiceRealtimeProviderContractMatrixPresetOptions, VoiceRealtimeProviderContractMatrixInput, VoiceRealtimeProviderContractMatrixReport, VoiceRealtimeProviderContractRoutesOptions, VoiceRealtimeProviderContractRow, VoiceRealtimeProviderPresetProvider, VoiceRealtimeProviderContractStatus } from './realtimeProviderContracts';
|
|
17
17
|
export { buildVoiceDiagnosticsMarkdown, createVoiceDiagnosticsRoutes, resolveVoiceDiagnosticsTraceFilter } from './diagnosticsRoutes';
|
|
18
|
-
export { buildVoiceMediaTransportReport, buildVoiceMediaInterruptionReport, buildVoiceMediaPipelineCalibrationReport, buildVoiceMediaResamplingPlan, buildVoiceMediaProcessorGraphReport, buildVoiceMediaVadReport, createVoiceMediaFrame, createVoiceMediaFrameTransformPipeline, createVoiceMediaProcessorGraph, createVoiceMediaTransport } from './mediaPipeline';
|
|
19
|
-
export type { VoiceMediaFrame, VoiceMediaFrameKind, VoiceMediaFrameSource, VoiceMediaFrameTransform, VoiceMediaFrameTransformPipeline, VoiceMediaInterruptionInput, VoiceMediaInterruptionReport, VoiceMediaPipelineCalibrationInput, VoiceMediaPipelineCalibrationIssue, VoiceMediaPipelineCalibrationReport, VoiceMediaPipelineStatus, VoiceMediaProcessorGraph, VoiceMediaProcessorGraphReport, VoiceMediaProcessorNode, VoiceMediaProcessorNodeEvent, VoiceMediaProcessorNodeKind, VoiceMediaProcessorNodeReport, VoiceMediaResamplingPlan, VoiceMediaTransport, VoiceMediaTransportAdapter, VoiceMediaTransportEvent, VoiceMediaTransportEventKind, VoiceMediaTransportOptions, VoiceMediaTransportReport, VoiceMediaTransportState, VoiceMediaVadInput, VoiceMediaVadReport, VoiceMediaVadSegment } from './mediaPipeline';
|
|
20
18
|
export { assertVoiceMediaPipelineEvidence, buildVoiceMediaPipelineReport, createVoiceMediaPipelineRoutes, evaluateVoiceMediaPipelineEvidence, renderVoiceMediaPipelineHTML, renderVoiceMediaPipelineMarkdown } from './mediaPipelineRoutes';
|
|
21
19
|
export type { VoiceMediaPipelineAssertionInput, VoiceMediaPipelineAssertionReport, VoiceMediaPipelineReport, VoiceMediaPipelineReportOptions, VoiceMediaPipelineRoutesOptions } from './mediaPipelineRoutes';
|
|
22
20
|
export { buildVoiceDemoReadyReport, createVoiceDemoReadyRoutes, renderVoiceDemoReadyHTML } from './demoReadyRoutes';
|
package/dist/index.js
CHANGED
|
@@ -11609,7 +11609,10 @@ var createVoiceDiagnosticsRoutes = (options) => {
|
|
|
11609
11609
|
});
|
|
11610
11610
|
return routes;
|
|
11611
11611
|
};
|
|
11612
|
-
// src/
|
|
11612
|
+
// src/mediaPipelineRoutes.ts
|
|
11613
|
+
import { Elysia as Elysia12 } from "elysia";
|
|
11614
|
+
|
|
11615
|
+
// node_modules/@absolutejs/media/dist/index.js
|
|
11613
11616
|
var formatLabel2 = (format) => `${format.container}/${format.encoding}/${String(format.sampleRateHz)}hz/${String(format.channels)}ch`;
|
|
11614
11617
|
var formatMatches2 = (actual, expected) => actual.container === expected.container && actual.encoding === expected.encoding && actual.sampleRateHz === expected.sampleRateHz && actual.channels === expected.channels;
|
|
11615
11618
|
var pushIssue = (issues, severity, code, message) => {
|
|
@@ -11619,97 +11622,7 @@ var numericMetadata = (frame, key) => {
|
|
|
11619
11622
|
const value = frame.metadata?.[key];
|
|
11620
11623
|
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
11621
11624
|
};
|
|
11622
|
-
var
|
|
11623
|
-
var buildVoiceMediaTransportReport = (input) => {
|
|
11624
|
-
const events = input.events ?? [];
|
|
11625
|
-
const state = input.state ?? events.at(-1)?.state ?? "idle";
|
|
11626
|
-
const backpressureEvents = events.filter((event) => event.kind === "backpressure").length;
|
|
11627
|
-
const failed = state === "failed" || events.some((event) => event.kind === "error");
|
|
11628
|
-
return {
|
|
11629
|
-
backpressureEvents,
|
|
11630
|
-
checkedAt: Date.now(),
|
|
11631
|
-
closed: state === "closed",
|
|
11632
|
-
connected: state === "open",
|
|
11633
|
-
events,
|
|
11634
|
-
failed,
|
|
11635
|
-
inputFrames: events.filter((event) => event.kind === "frame-in").length,
|
|
11636
|
-
name: input.name,
|
|
11637
|
-
outputFrames: events.filter((event) => event.kind === "frame-out").length,
|
|
11638
|
-
state,
|
|
11639
|
-
status: failed ? "fail" : backpressureEvents > 0 ? "warn" : "pass"
|
|
11640
|
-
};
|
|
11641
|
-
};
|
|
11642
|
-
var createVoiceMediaTransport = (options) => {
|
|
11643
|
-
let state = "idle";
|
|
11644
|
-
const events = [];
|
|
11645
|
-
const frameHandlers = new Set;
|
|
11646
|
-
const record = (event) => {
|
|
11647
|
-
events.push({ ...event, at: Date.now(), state });
|
|
11648
|
-
};
|
|
11649
|
-
return {
|
|
11650
|
-
close: async () => {
|
|
11651
|
-
state = "closing";
|
|
11652
|
-
await options.onClose?.();
|
|
11653
|
-
state = "closed";
|
|
11654
|
-
record({ kind: "close" });
|
|
11655
|
-
},
|
|
11656
|
-
connect: async () => {
|
|
11657
|
-
try {
|
|
11658
|
-
await options.onConnect?.();
|
|
11659
|
-
state = "open";
|
|
11660
|
-
record({ kind: "connect" });
|
|
11661
|
-
} catch (error) {
|
|
11662
|
-
state = "failed";
|
|
11663
|
-
record({
|
|
11664
|
-
error: error instanceof Error ? error.message : String(error),
|
|
11665
|
-
kind: "error"
|
|
11666
|
-
});
|
|
11667
|
-
throw error;
|
|
11668
|
-
}
|
|
11669
|
-
},
|
|
11670
|
-
events: () => [...events],
|
|
11671
|
-
inputFormat: options.inputFormat,
|
|
11672
|
-
name: options.name,
|
|
11673
|
-
onFrame: (handler) => {
|
|
11674
|
-
frameHandlers.add(handler);
|
|
11675
|
-
return () => frameHandlers.delete(handler);
|
|
11676
|
-
},
|
|
11677
|
-
outputFormat: options.outputFormat,
|
|
11678
|
-
receive: async (frame) => {
|
|
11679
|
-
record({ frameId: frame.id, kind: "frame-in" });
|
|
11680
|
-
if (options.maxBufferedFrames !== undefined && events.filter((event) => event.kind === "frame-in").length > options.maxBufferedFrames) {
|
|
11681
|
-
record({
|
|
11682
|
-
bufferedFrames: events.filter((event) => event.kind === "frame-in").length,
|
|
11683
|
-
kind: "backpressure"
|
|
11684
|
-
});
|
|
11685
|
-
}
|
|
11686
|
-
for (const handler of frameHandlers) {
|
|
11687
|
-
await handler(frame);
|
|
11688
|
-
}
|
|
11689
|
-
},
|
|
11690
|
-
report: () => buildVoiceMediaTransportReport({
|
|
11691
|
-
events,
|
|
11692
|
-
name: options.name,
|
|
11693
|
-
state
|
|
11694
|
-
}),
|
|
11695
|
-
send: async (frame) => {
|
|
11696
|
-
try {
|
|
11697
|
-
await options.onSend?.(frame);
|
|
11698
|
-
record({ frameId: frame.id, kind: "frame-out" });
|
|
11699
|
-
} catch (error) {
|
|
11700
|
-
state = "failed";
|
|
11701
|
-
record({
|
|
11702
|
-
error: error instanceof Error ? error.message : String(error),
|
|
11703
|
-
frameId: frame.id,
|
|
11704
|
-
kind: "error"
|
|
11705
|
-
});
|
|
11706
|
-
throw error;
|
|
11707
|
-
}
|
|
11708
|
-
},
|
|
11709
|
-
state: () => state
|
|
11710
|
-
};
|
|
11711
|
-
};
|
|
11712
|
-
var buildVoiceMediaResamplingPlan = (input) => {
|
|
11625
|
+
var buildMediaResamplingPlan = (input) => {
|
|
11713
11626
|
const required = !formatMatches2(input.inputFormat, input.outputFormat);
|
|
11714
11627
|
return {
|
|
11715
11628
|
inputFormat: input.inputFormat,
|
|
@@ -11719,126 +11632,6 @@ var buildVoiceMediaResamplingPlan = (input) => {
|
|
|
11719
11632
|
status: input.inputFormat.container === input.outputFormat.container && input.inputFormat.encoding === input.outputFormat.encoding && input.inputFormat.channels === input.outputFormat.channels ? "pass" : "warn"
|
|
11720
11633
|
};
|
|
11721
11634
|
};
|
|
11722
|
-
var createVoiceMediaFrameTransformPipeline = (input = {}) => {
|
|
11723
|
-
const transforms = input.transforms ?? [];
|
|
11724
|
-
const push = async (frame) => {
|
|
11725
|
-
let frames = [frame];
|
|
11726
|
-
for (const transform of transforms) {
|
|
11727
|
-
const nextFrames = [];
|
|
11728
|
-
for (const current of frames) {
|
|
11729
|
-
const transformed = await transform.transform(current);
|
|
11730
|
-
if (transformed === undefined) {
|
|
11731
|
-
continue;
|
|
11732
|
-
}
|
|
11733
|
-
if (Array.isArray(transformed)) {
|
|
11734
|
-
nextFrames.push(...transformed);
|
|
11735
|
-
} else {
|
|
11736
|
-
nextFrames.push(transformed);
|
|
11737
|
-
}
|
|
11738
|
-
}
|
|
11739
|
-
frames = nextFrames;
|
|
11740
|
-
}
|
|
11741
|
-
return frames;
|
|
11742
|
-
};
|
|
11743
|
-
return {
|
|
11744
|
-
push,
|
|
11745
|
-
pushMany: async (frames) => {
|
|
11746
|
-
const output = [];
|
|
11747
|
-
for (const frame of frames) {
|
|
11748
|
-
output.push(...await push(frame));
|
|
11749
|
-
}
|
|
11750
|
-
return output;
|
|
11751
|
-
},
|
|
11752
|
-
transforms
|
|
11753
|
-
};
|
|
11754
|
-
};
|
|
11755
|
-
var normalizeProcessorResult = (frame, result) => {
|
|
11756
|
-
if (result === false || result === undefined) {
|
|
11757
|
-
return [];
|
|
11758
|
-
}
|
|
11759
|
-
if (result === true) {
|
|
11760
|
-
return [frame];
|
|
11761
|
-
}
|
|
11762
|
-
if (Array.isArray(result)) {
|
|
11763
|
-
return result;
|
|
11764
|
-
}
|
|
11765
|
-
return [result];
|
|
11766
|
-
};
|
|
11767
|
-
var buildVoiceMediaProcessorGraphReport = (input) => {
|
|
11768
|
-
const events = input.events ?? [];
|
|
11769
|
-
const nodes = input.nodes.map((node) => {
|
|
11770
|
-
const nodeEvents = events.filter((event) => event.node === node.name);
|
|
11771
|
-
const droppedFrames2 = nodeEvents.reduce((total, event) => total + event.dropped, 0);
|
|
11772
|
-
const emittedFrames2 = nodeEvents.reduce((total, event) => total + event.emitted, 0);
|
|
11773
|
-
const inputFrames2 = nodeEvents.reduce((total, event) => total + event.inputs, 0);
|
|
11774
|
-
return {
|
|
11775
|
-
droppedFrames: droppedFrames2,
|
|
11776
|
-
emittedFrames: emittedFrames2,
|
|
11777
|
-
events: nodeEvents,
|
|
11778
|
-
inputFrames: inputFrames2,
|
|
11779
|
-
kind: node.kind ?? "processor",
|
|
11780
|
-
name: node.name,
|
|
11781
|
-
status: inputFrames2 > 0 && emittedFrames2 === 0 && node.kind !== "sink" ? "warn" : "pass"
|
|
11782
|
-
};
|
|
11783
|
-
});
|
|
11784
|
-
const inputFrames = events.filter((event) => event.node === input.nodes[0]?.name).length;
|
|
11785
|
-
const droppedFrames = events.reduce((total, event) => total + event.dropped, 0);
|
|
11786
|
-
const emittedFrames = input.nodes.at(-1) ? events.filter((event) => event.node === input.nodes.at(-1)?.name).reduce((total, event) => total + event.emitted, 0) : 0;
|
|
11787
|
-
const status = nodes.some((node) => node.status === "warn") ? "warn" : "pass";
|
|
11788
|
-
return {
|
|
11789
|
-
checkedAt: Date.now(),
|
|
11790
|
-
droppedFrames,
|
|
11791
|
-
emittedFrames,
|
|
11792
|
-
events,
|
|
11793
|
-
inputFrames,
|
|
11794
|
-
name: input.name,
|
|
11795
|
-
nodes,
|
|
11796
|
-
status
|
|
11797
|
-
};
|
|
11798
|
-
};
|
|
11799
|
-
var createVoiceMediaProcessorGraph = (input = {}) => {
|
|
11800
|
-
const nodes = input.nodes ?? [];
|
|
11801
|
-
const events = [];
|
|
11802
|
-
const process = async (frame) => {
|
|
11803
|
-
let frames = [frame];
|
|
11804
|
-
for (const node of nodes) {
|
|
11805
|
-
const nextFrames = [];
|
|
11806
|
-
for (const current of frames) {
|
|
11807
|
-
const output = normalizeProcessorResult(current, await node.process(current));
|
|
11808
|
-
events.push({
|
|
11809
|
-
at: Date.now(),
|
|
11810
|
-
dropped: output.length === 0 ? 1 : 0,
|
|
11811
|
-
emitted: output.length,
|
|
11812
|
-
frameId: current.id,
|
|
11813
|
-
inputs: 1,
|
|
11814
|
-
node: node.name
|
|
11815
|
-
});
|
|
11816
|
-
nextFrames.push(...output);
|
|
11817
|
-
}
|
|
11818
|
-
frames = nextFrames;
|
|
11819
|
-
if (frames.length === 0) {
|
|
11820
|
-
break;
|
|
11821
|
-
}
|
|
11822
|
-
}
|
|
11823
|
-
return frames;
|
|
11824
|
-
};
|
|
11825
|
-
return {
|
|
11826
|
-
nodes,
|
|
11827
|
-
process,
|
|
11828
|
-
processMany: async (frames) => {
|
|
11829
|
-
const output = [];
|
|
11830
|
-
for (const frame of frames) {
|
|
11831
|
-
output.push(...await process(frame));
|
|
11832
|
-
}
|
|
11833
|
-
return output;
|
|
11834
|
-
},
|
|
11835
|
-
report: () => buildVoiceMediaProcessorGraphReport({
|
|
11836
|
-
events,
|
|
11837
|
-
name: input.name ?? "voice-media-processor-graph",
|
|
11838
|
-
nodes
|
|
11839
|
-
})
|
|
11840
|
-
};
|
|
11841
|
-
};
|
|
11842
11635
|
var speechProbability = (frame) => {
|
|
11843
11636
|
if (frame.metadata?.isSpeech === true) {
|
|
11844
11637
|
return 1;
|
|
@@ -11854,7 +11647,7 @@ var speechProbability = (frame) => {
|
|
|
11854
11647
|
}
|
|
11855
11648
|
return 0;
|
|
11856
11649
|
};
|
|
11857
|
-
var
|
|
11650
|
+
var buildMediaVadReport = (input = {}) => {
|
|
11858
11651
|
const frames = (input.frames ?? []).filter((frame) => frame.kind === "input-audio");
|
|
11859
11652
|
const speechStartThreshold = input.speechStartThreshold ?? 0.6;
|
|
11860
11653
|
const speechEndThreshold = input.speechEndThreshold ?? 0.35;
|
|
@@ -11912,7 +11705,7 @@ var buildVoiceMediaVadReport = (input = {}) => {
|
|
|
11912
11705
|
status: frames.length === 0 ? "warn" : "pass"
|
|
11913
11706
|
};
|
|
11914
11707
|
};
|
|
11915
|
-
var
|
|
11708
|
+
var buildMediaInterruptionReport = (input = {}) => {
|
|
11916
11709
|
const issues = [];
|
|
11917
11710
|
const interruptionFrames = (input.frames ?? []).filter((frame) => frame.kind === "interruption");
|
|
11918
11711
|
const latenciesMs = interruptionFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
|
|
@@ -11931,7 +11724,7 @@ var buildVoiceMediaInterruptionReport = (input = {}) => {
|
|
|
11931
11724
|
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass"
|
|
11932
11725
|
};
|
|
11933
11726
|
};
|
|
11934
|
-
var
|
|
11727
|
+
var buildMediaPipelineCalibrationReport = (input = {}) => {
|
|
11935
11728
|
const frames = input.frames ?? [];
|
|
11936
11729
|
const issues = [];
|
|
11937
11730
|
const inputFrames = frames.filter((frame) => frame.kind === "input-audio");
|
|
@@ -11989,13 +11782,13 @@ var buildVoiceMediaPipelineCalibrationReport = (input = {}) => {
|
|
|
11989
11782
|
resamplingRequired,
|
|
11990
11783
|
resamplingTargetHz,
|
|
11991
11784
|
status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass",
|
|
11992
|
-
surface: input.surface ?? "
|
|
11785
|
+
surface: input.surface ?? "media-pipeline",
|
|
11993
11786
|
traceLinkedFrames,
|
|
11994
11787
|
turnCommitFrames: turnCommitFrames.length
|
|
11995
11788
|
};
|
|
11996
11789
|
};
|
|
11790
|
+
|
|
11997
11791
|
// src/mediaPipelineRoutes.ts
|
|
11998
|
-
import { Elysia as Elysia12 } from "elysia";
|
|
11999
11792
|
var escapeHtml15 = (value) => String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
12000
11793
|
var statusRank2 = {
|
|
12001
11794
|
pass: 0,
|
|
@@ -12005,19 +11798,19 @@ var statusRank2 = {
|
|
|
12005
11798
|
var worstStatus = (statuses) => statuses.reduce((worst, status) => statusRank2[status] > statusRank2[worst] ? status : worst, "pass");
|
|
12006
11799
|
var buildVoiceMediaPipelineReport = (options = {}) => {
|
|
12007
11800
|
const frames = options.frames ?? [];
|
|
12008
|
-
const calibration =
|
|
12009
|
-
const vad =
|
|
11801
|
+
const calibration = buildMediaPipelineCalibrationReport(options);
|
|
11802
|
+
const vad = buildMediaVadReport({
|
|
12010
11803
|
frames,
|
|
12011
11804
|
maxSilenceFrames: options.maxSilenceFrames,
|
|
12012
11805
|
minSpeechFrames: options.minSpeechFrames,
|
|
12013
11806
|
speechEndThreshold: options.speechEndThreshold,
|
|
12014
11807
|
speechStartThreshold: options.speechStartThreshold
|
|
12015
11808
|
});
|
|
12016
|
-
const interruption =
|
|
11809
|
+
const interruption = buildMediaInterruptionReport({
|
|
12017
11810
|
frames,
|
|
12018
11811
|
maxInterruptionLatencyMs: options.maxInterruptionLatencyMs
|
|
12019
11812
|
});
|
|
12020
|
-
const resampling = calibration.inputFormat && calibration.outputFormat ?
|
|
11813
|
+
const resampling = calibration.inputFormat && calibration.outputFormat ? buildMediaResamplingPlan({
|
|
12021
11814
|
inputFormat: calibration.inputFormat,
|
|
12022
11815
|
outputFormat: calibration.outputFormat
|
|
12023
11816
|
}) : undefined;
|
|
@@ -34971,11 +34764,7 @@ export {
|
|
|
34971
34764
|
createVoiceMemoryAuditSinkDeliveryStore,
|
|
34972
34765
|
createVoiceMemoryAuditEventStore,
|
|
34973
34766
|
createVoiceMemoryAssistantMemoryStore,
|
|
34974
|
-
createVoiceMediaTransport,
|
|
34975
|
-
createVoiceMediaProcessorGraph,
|
|
34976
34767
|
createVoiceMediaPipelineRoutes,
|
|
34977
|
-
createVoiceMediaFrameTransformPipeline,
|
|
34978
|
-
createVoiceMediaFrame,
|
|
34979
34768
|
createVoiceLiveOpsRoutes,
|
|
34980
34769
|
createVoiceLiveOpsController,
|
|
34981
34770
|
createVoiceLiveLatencyRoutes,
|
|
@@ -35124,13 +34913,7 @@ export {
|
|
|
35124
34913
|
buildVoiceObservabilityExport,
|
|
35125
34914
|
buildVoiceObservabilityArtifactIndex,
|
|
35126
34915
|
buildVoiceMonitorRunReport,
|
|
35127
|
-
buildVoiceMediaVadReport,
|
|
35128
|
-
buildVoiceMediaTransportReport,
|
|
35129
|
-
buildVoiceMediaResamplingPlan,
|
|
35130
|
-
buildVoiceMediaProcessorGraphReport,
|
|
35131
34916
|
buildVoiceMediaPipelineReport,
|
|
35132
|
-
buildVoiceMediaPipelineCalibrationReport,
|
|
35133
|
-
buildVoiceMediaInterruptionReport,
|
|
35134
34917
|
buildVoiceLiveOpsControlState,
|
|
35135
34918
|
buildVoiceLatencySLOGate,
|
|
35136
34919
|
buildVoiceIncidentBundle,
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
import { Elysia } from 'elysia';
|
|
2
|
-
import { type
|
|
3
|
-
export type VoiceMediaPipelineReportOptions =
|
|
4
|
-
frames?: readonly
|
|
2
|
+
import { type MediaFrame, type MediaInterruptionReport, type MediaPipelineCalibrationInput, type MediaPipelineCalibrationReport, type MediaPipelineStatus, type MediaProcessorGraphReport, type MediaResamplingPlan, type MediaTransportReport, type MediaVadReport } from '@absolutejs/media';
|
|
3
|
+
export type VoiceMediaPipelineReportOptions = MediaPipelineCalibrationInput & {
|
|
4
|
+
frames?: readonly MediaFrame[];
|
|
5
5
|
maxInterruptionLatencyMs?: number;
|
|
6
6
|
maxSilenceFrames?: number;
|
|
7
7
|
minSpeechFrames?: number;
|
|
8
|
-
processorGraph?:
|
|
8
|
+
processorGraph?: MediaProcessorGraphReport;
|
|
9
9
|
speechEndThreshold?: number;
|
|
10
10
|
speechStartThreshold?: number;
|
|
11
|
-
transport?:
|
|
11
|
+
transport?: MediaTransportReport;
|
|
12
12
|
};
|
|
13
13
|
export type VoiceMediaPipelineReport = {
|
|
14
|
-
calibration:
|
|
14
|
+
calibration: MediaPipelineCalibrationReport;
|
|
15
15
|
checkedAt: number;
|
|
16
16
|
frames: number;
|
|
17
|
-
interruption:
|
|
17
|
+
interruption: MediaInterruptionReport;
|
|
18
18
|
ok: boolean;
|
|
19
|
-
resampling?:
|
|
20
|
-
processorGraph?:
|
|
21
|
-
status:
|
|
19
|
+
resampling?: MediaResamplingPlan;
|
|
20
|
+
processorGraph?: MediaProcessorGraphReport;
|
|
21
|
+
status: MediaPipelineStatus;
|
|
22
22
|
surface: string;
|
|
23
|
-
transport?:
|
|
24
|
-
vad:
|
|
23
|
+
transport?: MediaTransportReport;
|
|
24
|
+
vad: MediaVadReport;
|
|
25
25
|
};
|
|
26
26
|
export type VoiceMediaPipelineAssertionInput = {
|
|
27
27
|
maxFirstAudioLatencyMs?: number;
|
|
@@ -44,7 +44,7 @@ export type VoiceMediaPipelineAssertionInput = {
|
|
|
44
44
|
export type VoiceMediaPipelineAssertionReport = {
|
|
45
45
|
issues: string[];
|
|
46
46
|
ok: boolean;
|
|
47
|
-
status:
|
|
47
|
+
status: MediaPipelineStatus;
|
|
48
48
|
surface: string;
|
|
49
49
|
};
|
|
50
50
|
export type VoiceMediaPipelineRoutesOptions = VoiceMediaPipelineReportOptions & {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@absolutejs/voice",
|
|
3
|
-
"version": "0.0.22-beta.
|
|
3
|
+
"version": "0.0.22-beta.314",
|
|
4
4
|
"description": "Voice primitives and Elysia plugin for AbsoluteJS",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -245,6 +245,9 @@
|
|
|
245
245
|
"optional": true
|
|
246
246
|
}
|
|
247
247
|
},
|
|
248
|
+
"dependencies": {
|
|
249
|
+
"@absolutejs/media": "0.0.1-beta.1"
|
|
250
|
+
},
|
|
248
251
|
"devDependencies": {
|
|
249
252
|
"@absolutejs/absolute": "0.19.0-beta.646",
|
|
250
253
|
"@angular/core": "^21.0.0",
|
package/dist/mediaPipeline.d.ts
DELETED
|
@@ -1,219 +0,0 @@
|
|
|
1
|
-
import type { AudioFormat } from './types';
|
|
2
|
-
export type VoiceMediaFrameKind = 'assistant-audio' | 'input-audio' | 'interruption' | 'metadata' | 'transcript' | 'turn-commit';
|
|
3
|
-
export type VoiceMediaFrameSource = 'browser' | 'provider' | 'telephony' | 'voice-runtime';
|
|
4
|
-
export type VoiceMediaPipelineStatus = 'fail' | 'pass' | 'warn';
|
|
5
|
-
export type VoiceMediaResamplingPlan = {
|
|
6
|
-
inputFormat: AudioFormat;
|
|
7
|
-
outputFormat: AudioFormat;
|
|
8
|
-
ratio: number;
|
|
9
|
-
required: boolean;
|
|
10
|
-
status: VoiceMediaPipelineStatus;
|
|
11
|
-
};
|
|
12
|
-
export type VoiceMediaFrame = {
|
|
13
|
-
at?: number;
|
|
14
|
-
audio?: ArrayBuffer | ArrayBufferView;
|
|
15
|
-
durationMs?: number;
|
|
16
|
-
format?: AudioFormat;
|
|
17
|
-
id: string;
|
|
18
|
-
kind: VoiceMediaFrameKind;
|
|
19
|
-
latencyMs?: number;
|
|
20
|
-
metadata?: Record<string, unknown>;
|
|
21
|
-
sessionId?: string;
|
|
22
|
-
source: VoiceMediaFrameSource | (string & {});
|
|
23
|
-
traceEventId?: string;
|
|
24
|
-
turnId?: string;
|
|
25
|
-
};
|
|
26
|
-
export type VoiceMediaFrameTransform = {
|
|
27
|
-
inputFormat?: AudioFormat;
|
|
28
|
-
name: string;
|
|
29
|
-
outputFormat?: AudioFormat;
|
|
30
|
-
transform: (frame: VoiceMediaFrame) => VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined | Promise<VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined>;
|
|
31
|
-
};
|
|
32
|
-
export type VoiceMediaFrameTransformPipeline = {
|
|
33
|
-
push: (frame: VoiceMediaFrame) => Promise<readonly VoiceMediaFrame[]>;
|
|
34
|
-
pushMany: (frames: readonly VoiceMediaFrame[]) => Promise<readonly VoiceMediaFrame[]>;
|
|
35
|
-
transforms: readonly VoiceMediaFrameTransform[];
|
|
36
|
-
};
|
|
37
|
-
export type VoiceMediaProcessorNodeKind = 'branch' | 'filter' | 'processor' | 'sink';
|
|
38
|
-
export type VoiceMediaProcessorNodeEvent = {
|
|
39
|
-
at: number;
|
|
40
|
-
dropped: number;
|
|
41
|
-
emitted: number;
|
|
42
|
-
frameId: string;
|
|
43
|
-
inputs: number;
|
|
44
|
-
node: string;
|
|
45
|
-
};
|
|
46
|
-
export type VoiceMediaProcessorNodeReport = {
|
|
47
|
-
droppedFrames: number;
|
|
48
|
-
emittedFrames: number;
|
|
49
|
-
events: readonly VoiceMediaProcessorNodeEvent[];
|
|
50
|
-
inputFrames: number;
|
|
51
|
-
kind: VoiceMediaProcessorNodeKind;
|
|
52
|
-
name: string;
|
|
53
|
-
status: VoiceMediaPipelineStatus;
|
|
54
|
-
};
|
|
55
|
-
export type VoiceMediaProcessorGraphReport = {
|
|
56
|
-
checkedAt: number;
|
|
57
|
-
droppedFrames: number;
|
|
58
|
-
emittedFrames: number;
|
|
59
|
-
events: readonly VoiceMediaProcessorNodeEvent[];
|
|
60
|
-
inputFrames: number;
|
|
61
|
-
name: string;
|
|
62
|
-
nodes: readonly VoiceMediaProcessorNodeReport[];
|
|
63
|
-
status: VoiceMediaPipelineStatus;
|
|
64
|
-
};
|
|
65
|
-
export type VoiceMediaProcessorNode = {
|
|
66
|
-
inputFormat?: AudioFormat;
|
|
67
|
-
kind?: VoiceMediaProcessorNodeKind;
|
|
68
|
-
name: string;
|
|
69
|
-
outputFormat?: AudioFormat;
|
|
70
|
-
process: (frame: VoiceMediaFrame) => boolean | VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined | Promise<boolean | VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined>;
|
|
71
|
-
};
|
|
72
|
-
export type VoiceMediaProcessorGraph = {
|
|
73
|
-
nodes: readonly VoiceMediaProcessorNode[];
|
|
74
|
-
process: (frame: VoiceMediaFrame) => Promise<readonly VoiceMediaFrame[]>;
|
|
75
|
-
processMany: (frames: readonly VoiceMediaFrame[]) => Promise<readonly VoiceMediaFrame[]>;
|
|
76
|
-
report: () => VoiceMediaProcessorGraphReport;
|
|
77
|
-
};
|
|
78
|
-
export type VoiceMediaTransportAdapter = {
|
|
79
|
-
close?: () => Promise<void> | void;
|
|
80
|
-
connect?: () => Promise<void> | void;
|
|
81
|
-
inputFormat?: AudioFormat;
|
|
82
|
-
name: string;
|
|
83
|
-
onFrame?: (handler: (frame: VoiceMediaFrame) => Promise<void> | void) => () => void;
|
|
84
|
-
outputFormat?: AudioFormat;
|
|
85
|
-
send: (frame: VoiceMediaFrame) => Promise<void> | void;
|
|
86
|
-
};
|
|
87
|
-
export type VoiceMediaTransportState = 'closed' | 'closing' | 'failed' | 'idle' | 'open';
|
|
88
|
-
export type VoiceMediaTransportEventKind = 'backpressure' | 'close' | 'connect' | 'error' | 'frame-in' | 'frame-out';
|
|
89
|
-
export type VoiceMediaTransportEvent = {
|
|
90
|
-
at: number;
|
|
91
|
-
bufferedFrames?: number;
|
|
92
|
-
error?: string;
|
|
93
|
-
frameId?: string;
|
|
94
|
-
kind: VoiceMediaTransportEventKind;
|
|
95
|
-
state: VoiceMediaTransportState;
|
|
96
|
-
};
|
|
97
|
-
export type VoiceMediaTransportReport = {
|
|
98
|
-
backpressureEvents: number;
|
|
99
|
-
checkedAt: number;
|
|
100
|
-
closed: boolean;
|
|
101
|
-
connected: boolean;
|
|
102
|
-
events: readonly VoiceMediaTransportEvent[];
|
|
103
|
-
failed: boolean;
|
|
104
|
-
inputFrames: number;
|
|
105
|
-
name: string;
|
|
106
|
-
outputFrames: number;
|
|
107
|
-
state: VoiceMediaTransportState;
|
|
108
|
-
status: VoiceMediaPipelineStatus;
|
|
109
|
-
};
|
|
110
|
-
export type VoiceMediaTransport = VoiceMediaTransportAdapter & {
|
|
111
|
-
events: () => readonly VoiceMediaTransportEvent[];
|
|
112
|
-
receive: (frame: VoiceMediaFrame) => Promise<void>;
|
|
113
|
-
report: () => VoiceMediaTransportReport;
|
|
114
|
-
state: () => VoiceMediaTransportState;
|
|
115
|
-
};
|
|
116
|
-
export type VoiceMediaTransportOptions = {
|
|
117
|
-
inputFormat?: AudioFormat;
|
|
118
|
-
maxBufferedFrames?: number;
|
|
119
|
-
name: string;
|
|
120
|
-
onClose?: () => Promise<void> | void;
|
|
121
|
-
onConnect?: () => Promise<void> | void;
|
|
122
|
-
onSend?: (frame: VoiceMediaFrame) => Promise<void> | void;
|
|
123
|
-
outputFormat?: AudioFormat;
|
|
124
|
-
};
|
|
125
|
-
export type VoiceMediaPipelineCalibrationInput = {
|
|
126
|
-
expectedInputFormat?: AudioFormat;
|
|
127
|
-
expectedOutputFormat?: AudioFormat;
|
|
128
|
-
frames?: readonly VoiceMediaFrame[];
|
|
129
|
-
inputFormat?: AudioFormat;
|
|
130
|
-
maxBackpressureFrames?: number;
|
|
131
|
-
maxFirstAudioLatencyMs?: number;
|
|
132
|
-
maxJitterMs?: number;
|
|
133
|
-
outputFormat?: AudioFormat;
|
|
134
|
-
requireInterruptionFrame?: boolean;
|
|
135
|
-
requireTraceEvidence?: boolean;
|
|
136
|
-
surface?: string;
|
|
137
|
-
};
|
|
138
|
-
export type VoiceMediaPipelineCalibrationIssue = {
|
|
139
|
-
code: string;
|
|
140
|
-
message: string;
|
|
141
|
-
severity: 'error' | 'warning';
|
|
142
|
-
};
|
|
143
|
-
export type VoiceMediaPipelineCalibrationReport = {
|
|
144
|
-
assistantAudioFrames: number;
|
|
145
|
-
backpressureFrames: number;
|
|
146
|
-
checkedAt: number;
|
|
147
|
-
firstAudioLatencyMs?: number;
|
|
148
|
-
inputAudioFrames: number;
|
|
149
|
-
inputFormat?: AudioFormat;
|
|
150
|
-
interruptionFrames: number;
|
|
151
|
-
issues: VoiceMediaPipelineCalibrationIssue[];
|
|
152
|
-
jitterMs?: number;
|
|
153
|
-
outputFormat?: AudioFormat;
|
|
154
|
-
resamplingRequired: boolean;
|
|
155
|
-
resamplingTargetHz?: number;
|
|
156
|
-
status: VoiceMediaPipelineStatus;
|
|
157
|
-
surface: string;
|
|
158
|
-
traceLinkedFrames: number;
|
|
159
|
-
turnCommitFrames: number;
|
|
160
|
-
};
|
|
161
|
-
export type VoiceMediaVadInput = {
|
|
162
|
-
frames?: readonly VoiceMediaFrame[];
|
|
163
|
-
maxSilenceFrames?: number;
|
|
164
|
-
minSpeechFrames?: number;
|
|
165
|
-
speechEndThreshold?: number;
|
|
166
|
-
speechStartThreshold?: number;
|
|
167
|
-
};
|
|
168
|
-
export type VoiceMediaVadSegment = {
|
|
169
|
-
durationMs?: number;
|
|
170
|
-
endAt?: number;
|
|
171
|
-
frameCount: number;
|
|
172
|
-
segmentId: string;
|
|
173
|
-
sessionId?: string;
|
|
174
|
-
startAt?: number;
|
|
175
|
-
turnId?: string;
|
|
176
|
-
};
|
|
177
|
-
export type VoiceMediaVadReport = {
|
|
178
|
-
checkedAt: number;
|
|
179
|
-
inputAudioFrames: number;
|
|
180
|
-
segments: VoiceMediaVadSegment[];
|
|
181
|
-
status: VoiceMediaPipelineStatus;
|
|
182
|
-
};
|
|
183
|
-
export type VoiceMediaInterruptionInput = {
|
|
184
|
-
frames?: readonly VoiceMediaFrame[];
|
|
185
|
-
maxInterruptionLatencyMs?: number;
|
|
186
|
-
};
|
|
187
|
-
export type VoiceMediaInterruptionReport = {
|
|
188
|
-
checkedAt: number;
|
|
189
|
-
interruptionFrames: number;
|
|
190
|
-
issues: VoiceMediaPipelineCalibrationIssue[];
|
|
191
|
-
latenciesMs: number[];
|
|
192
|
-
status: VoiceMediaPipelineStatus;
|
|
193
|
-
};
|
|
194
|
-
export declare const createVoiceMediaFrame: (frame: VoiceMediaFrame) => VoiceMediaFrame;
|
|
195
|
-
export declare const buildVoiceMediaTransportReport: (input: {
|
|
196
|
-
events?: readonly VoiceMediaTransportEvent[];
|
|
197
|
-
name: string;
|
|
198
|
-
state?: VoiceMediaTransportState;
|
|
199
|
-
}) => VoiceMediaTransportReport;
|
|
200
|
-
export declare const createVoiceMediaTransport: (options: VoiceMediaTransportOptions) => VoiceMediaTransport;
|
|
201
|
-
export declare const buildVoiceMediaResamplingPlan: (input: {
|
|
202
|
-
inputFormat: AudioFormat;
|
|
203
|
-
outputFormat: AudioFormat;
|
|
204
|
-
}) => VoiceMediaResamplingPlan;
|
|
205
|
-
export declare const createVoiceMediaFrameTransformPipeline: (input?: {
|
|
206
|
-
transforms?: readonly VoiceMediaFrameTransform[];
|
|
207
|
-
}) => VoiceMediaFrameTransformPipeline;
|
|
208
|
-
export declare const buildVoiceMediaProcessorGraphReport: (input: {
|
|
209
|
-
events?: readonly VoiceMediaProcessorNodeEvent[];
|
|
210
|
-
name: string;
|
|
211
|
-
nodes: readonly VoiceMediaProcessorNode[];
|
|
212
|
-
}) => VoiceMediaProcessorGraphReport;
|
|
213
|
-
export declare const createVoiceMediaProcessorGraph: (input?: {
|
|
214
|
-
name?: string;
|
|
215
|
-
nodes?: readonly VoiceMediaProcessorNode[];
|
|
216
|
-
}) => VoiceMediaProcessorGraph;
|
|
217
|
-
export declare const buildVoiceMediaVadReport: (input?: VoiceMediaVadInput) => VoiceMediaVadReport;
|
|
218
|
-
export declare const buildVoiceMediaInterruptionReport: (input?: VoiceMediaInterruptionInput) => VoiceMediaInterruptionReport;
|
|
219
|
-
export declare const buildVoiceMediaPipelineCalibrationReport: (input?: VoiceMediaPipelineCalibrationInput) => VoiceMediaPipelineCalibrationReport;
|