@absolutejs/voice 0.0.22-beta.309 → 0.0.22-beta.310

package/dist/index.d.ts

@@ -15,8 +15,8 @@ export type { VoiceRealtimeChannelAssertionInput, VoiceRealtimeChannelAssertionR
 export { assertVoiceRealtimeProviderContractEvidence, buildVoiceRealtimeProviderContractMatrix, createVoiceRealtimeProviderContractMatrixPreset, createVoiceRealtimeProviderContractRoutes, evaluateVoiceRealtimeProviderContractEvidence, renderVoiceRealtimeProviderContractHTML } from './realtimeProviderContracts';
 export type { VoiceRealtimeProviderContractAssertionInput, VoiceRealtimeProviderContractAssertionReport, VoiceRealtimeProviderContractCapability, VoiceRealtimeProviderContractCheck, VoiceRealtimeProviderContractDefinition, VoiceRealtimeProviderContractMatrixPresetOptions, VoiceRealtimeProviderContractMatrixInput, VoiceRealtimeProviderContractMatrixReport, VoiceRealtimeProviderContractRoutesOptions, VoiceRealtimeProviderContractRow, VoiceRealtimeProviderPresetProvider, VoiceRealtimeProviderContractStatus } from './realtimeProviderContracts';
 export { buildVoiceDiagnosticsMarkdown, createVoiceDiagnosticsRoutes, resolveVoiceDiagnosticsTraceFilter } from './diagnosticsRoutes';
-export { buildVoiceMediaPipelineCalibrationReport, createVoiceMediaFrame } from './mediaPipeline';
-export type { VoiceMediaFrame, VoiceMediaFrameKind, VoiceMediaFrameSource, VoiceMediaPipelineCalibrationInput, VoiceMediaPipelineCalibrationIssue, VoiceMediaPipelineCalibrationReport, VoiceMediaPipelineStatus } from './mediaPipeline';
+export { buildVoiceMediaInterruptionReport, buildVoiceMediaPipelineCalibrationReport, buildVoiceMediaResamplingPlan, buildVoiceMediaVadReport, createVoiceMediaFrame, createVoiceMediaFrameTransformPipeline } from './mediaPipeline';
+export type { VoiceMediaFrame, VoiceMediaFrameKind, VoiceMediaFrameSource, VoiceMediaFrameTransform, VoiceMediaFrameTransformPipeline, VoiceMediaInterruptionInput, VoiceMediaInterruptionReport, VoiceMediaPipelineCalibrationInput, VoiceMediaPipelineCalibrationIssue, VoiceMediaPipelineCalibrationReport, VoiceMediaPipelineStatus, VoiceMediaResamplingPlan, VoiceMediaTransportAdapter, VoiceMediaVadInput, VoiceMediaVadReport, VoiceMediaVadSegment } from './mediaPipeline';
 export { buildVoiceDemoReadyReport, createVoiceDemoReadyRoutes, renderVoiceDemoReadyHTML } from './demoReadyRoutes';
 export { buildVoiceDeliverySinkReport, createVoiceDeliverySinkDescriptor, createVoiceDeliverySinkPair, createVoiceDeliverySinkRoutes, createVoiceFileDeliverySink, createVoicePostgresDeliverySink, createVoiceS3DeliverySink, createVoiceSQLiteDeliverySink, createVoiceWebhookDeliverySink, renderVoiceDeliverySinkHTML } from './deliverySinkRoutes';
 export { buildVoiceOpsActionHistoryReport, createVoiceOpsActionAuditRoutes, recordVoiceOpsActionAudit, renderVoiceOpsActionHistoryHTML } from './opsActionAuditRoutes';

package/dist/index.js

@@ -11620,6 +11620,141 @@ var numericMetadata = (frame, key) => {
   return typeof value === "number" && Number.isFinite(value) ? value : undefined;
 };
 var createVoiceMediaFrame = (frame) => frame;
+var buildVoiceMediaResamplingPlan = (input) => {
+  const required = !formatMatches2(input.inputFormat, input.outputFormat);
+  return {
+    inputFormat: input.inputFormat,
+    outputFormat: input.outputFormat,
+    ratio: input.outputFormat.sampleRateHz / input.inputFormat.sampleRateHz,
+    required,
+    status: input.inputFormat.container === input.outputFormat.container && input.inputFormat.encoding === input.outputFormat.encoding && input.inputFormat.channels === input.outputFormat.channels ? "pass" : "warn"
+  };
+};
+var createVoiceMediaFrameTransformPipeline = (input = {}) => {
+  const transforms = input.transforms ?? [];
+  const push = async (frame) => {
+    let frames = [frame];
+    for (const transform of transforms) {
+      const nextFrames = [];
+      for (const current of frames) {
+        const transformed = await transform.transform(current);
+        if (transformed === undefined) {
+          continue;
+        }
+        if (Array.isArray(transformed)) {
+          nextFrames.push(...transformed);
+        } else {
+          nextFrames.push(transformed);
+        }
+      }
+      frames = nextFrames;
+    }
+    return frames;
+  };
+  return {
+    push,
+    pushMany: async (frames) => {
+      const output = [];
+      for (const frame of frames) {
+        output.push(...await push(frame));
+      }
+      return output;
+    },
+    transforms
+  };
+};
+var speechProbability = (frame) => {
+  if (frame.metadata?.isSpeech === true) {
+    return 1;
+  }
+  if (frame.metadata?.isSpeech === false) {
+    return 0;
+  }
+  for (const key of ["speechProbability", "voiceProbability", "rms", "energy"]) {
+    const value = numericMetadata(frame, key);
+    if (value !== undefined) {
+      return value;
+    }
+  }
+  return 0;
+};
+var buildVoiceMediaVadReport = (input = {}) => {
+  const frames = (input.frames ?? []).filter((frame) => frame.kind === "input-audio");
+  const speechStartThreshold = input.speechStartThreshold ?? 0.6;
+  const speechEndThreshold = input.speechEndThreshold ?? 0.35;
+  const minSpeechFrames = input.minSpeechFrames ?? 1;
+  const maxSilenceFrames = input.maxSilenceFrames ?? 1;
+  const segments = [];
+  let activeFrames = [];
+  let silenceFrames = 0;
+  const closeSegment = () => {
+    if (activeFrames.length < minSpeechFrames) {
+      activeFrames = [];
+      silenceFrames = 0;
+      return;
+    }
+    const first = activeFrames[0];
+    const last = activeFrames.at(-1);
+    if (!first) {
+      return;
+    }
+    segments.push({
+      durationMs: first.at !== undefined && last?.at !== undefined ? last.at - first.at + (last.durationMs ?? 0) : undefined,
+      endAt: last?.at !== undefined ? last.at + (last.durationMs ?? 0) : undefined,
+      frameCount: activeFrames.length,
+      segmentId: `vad:${String(segments.length + 1)}`,
+      sessionId: first.sessionId,
+      startAt: first.at,
+      turnId: first.turnId
+    });
+    activeFrames = [];
+    silenceFrames = 0;
+  };
+  for (const frame of frames) {
+    const probability = speechProbability(frame);
+    if (activeFrames.length === 0) {
+      if (probability >= speechStartThreshold) {
+        activeFrames.push(frame);
+      }
+      continue;
+    }
+    activeFrames.push(frame);
+    if (probability <= speechEndThreshold) {
+      silenceFrames += 1;
+    } else {
+      silenceFrames = 0;
+    }
+    if (silenceFrames > maxSilenceFrames) {
+      closeSegment();
+    }
+  }
+  closeSegment();
+  return {
+    checkedAt: Date.now(),
+    inputAudioFrames: frames.length,
+    segments,
+    status: frames.length === 0 ? "warn" : "pass"
+  };
+};
+var buildVoiceMediaInterruptionReport = (input = {}) => {
+  const issues = [];
+  const interruptionFrames = (input.frames ?? []).filter((frame) => frame.kind === "interruption");
+  const latenciesMs = interruptionFrames.map((frame) => frame.latencyMs).filter((latency) => typeof latency === "number");
+  const maxInterruptionLatencyMs = input.maxInterruptionLatencyMs;
+  if (interruptionFrames.length === 0) {
+    pushIssue(issues, "warning", "media.interruption_missing", "No interruption frame was observed.");
+  }
+  if (maxInterruptionLatencyMs !== undefined && latenciesMs.some((latency) => latency > maxInterruptionLatencyMs)) {
+    pushIssue(issues, "error", "media.interruption_latency", `Interruption latency exceeded ${String(maxInterruptionLatencyMs)}ms.`);
+  }
+  return {
+    checkedAt: Date.now(),
+    interruptionFrames: interruptionFrames.length,
+    issues,
+    latenciesMs,
+    status: issues.some((issue) => issue.severity === "error") ? "fail" : issues.length > 0 ? "warn" : "pass"
+  };
+};
 var buildVoiceMediaPipelineCalibrationReport = (input = {}) => {
   const frames = input.frames ?? [];
   const issues = [];
@@ -34456,6 +34591,7 @@ export {
   createVoiceMemoryAuditSinkDeliveryStore,
   createVoiceMemoryAuditEventStore,
   createVoiceMemoryAssistantMemoryStore,
+  createVoiceMediaFrameTransformPipeline,
   createVoiceMediaFrame,
   createVoiceLiveOpsRoutes,
   createVoiceLiveOpsController,
@@ -34605,7 +34741,10 @@ export {
   buildVoiceObservabilityExport,
   buildVoiceObservabilityArtifactIndex,
   buildVoiceMonitorRunReport,
+  buildVoiceMediaVadReport,
+  buildVoiceMediaResamplingPlan,
   buildVoiceMediaPipelineCalibrationReport,
+  buildVoiceMediaInterruptionReport,
   buildVoiceLiveOpsControlState,
   buildVoiceLatencySLOGate,
   buildVoiceIncidentBundle,

package/dist/mediaPipeline.d.ts

@@ -2,6 +2,13 @@ import type { AudioFormat } from './types';
 export type VoiceMediaFrameKind = 'assistant-audio' | 'input-audio' | 'interruption' | 'metadata' | 'transcript' | 'turn-commit';
 export type VoiceMediaFrameSource = 'browser' | 'provider' | 'telephony' | 'voice-runtime';
 export type VoiceMediaPipelineStatus = 'fail' | 'pass' | 'warn';
+export type VoiceMediaResamplingPlan = {
+    inputFormat: AudioFormat;
+    outputFormat: AudioFormat;
+    ratio: number;
+    required: boolean;
+    status: VoiceMediaPipelineStatus;
+};
 export type VoiceMediaFrame = {
     at?: number;
     audio?: ArrayBuffer | ArrayBufferView;
@@ -16,6 +23,26 @@ export type VoiceMediaFrame = {
     traceEventId?: string;
     turnId?: string;
 };
+export type VoiceMediaFrameTransform = {
+    inputFormat?: AudioFormat;
+    name: string;
+    outputFormat?: AudioFormat;
+    transform: (frame: VoiceMediaFrame) => VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined | Promise<VoiceMediaFrame | readonly VoiceMediaFrame[] | undefined>;
+};
+export type VoiceMediaFrameTransformPipeline = {
+    push: (frame: VoiceMediaFrame) => Promise<readonly VoiceMediaFrame[]>;
+    pushMany: (frames: readonly VoiceMediaFrame[]) => Promise<readonly VoiceMediaFrame[]>;
+    transforms: readonly VoiceMediaFrameTransform[];
+};
+export type VoiceMediaTransportAdapter = {
+    close?: () => Promise<void> | void;
+    connect?: () => Promise<void> | void;
+    inputFormat?: AudioFormat;
+    name: string;
+    onFrame?: (handler: (frame: VoiceMediaFrame) => Promise<void> | void) => () => void;
+    outputFormat?: AudioFormat;
+    send: (frame: VoiceMediaFrame) => Promise<void> | void;
+};
 export type VoiceMediaPipelineCalibrationInput = {
     expectedInputFormat?: AudioFormat;
     expectedOutputFormat?: AudioFormat;
@@ -52,5 +79,47 @@ export type VoiceMediaPipelineCalibrationReport = {
     traceLinkedFrames: number;
     turnCommitFrames: number;
 };
+export type VoiceMediaVadInput = {
+    frames?: readonly VoiceMediaFrame[];
+    maxSilenceFrames?: number;
+    minSpeechFrames?: number;
+    speechEndThreshold?: number;
+    speechStartThreshold?: number;
+};
+export type VoiceMediaVadSegment = {
+    durationMs?: number;
+    endAt?: number;
+    frameCount: number;
+    segmentId: string;
+    sessionId?: string;
+    startAt?: number;
+    turnId?: string;
+};
+export type VoiceMediaVadReport = {
+    checkedAt: number;
+    inputAudioFrames: number;
+    segments: VoiceMediaVadSegment[];
+    status: VoiceMediaPipelineStatus;
+};
+export type VoiceMediaInterruptionInput = {
+    frames?: readonly VoiceMediaFrame[];
+    maxInterruptionLatencyMs?: number;
+};
+export type VoiceMediaInterruptionReport = {
+    checkedAt: number;
+    interruptionFrames: number;
+    issues: VoiceMediaPipelineCalibrationIssue[];
+    latenciesMs: number[];
+    status: VoiceMediaPipelineStatus;
+};
 export declare const createVoiceMediaFrame: (frame: VoiceMediaFrame) => VoiceMediaFrame;
+export declare const buildVoiceMediaResamplingPlan: (input: {
+    inputFormat: AudioFormat;
+    outputFormat: AudioFormat;
+}) => VoiceMediaResamplingPlan;
+export declare const createVoiceMediaFrameTransformPipeline: (input?: {
+    transforms?: readonly VoiceMediaFrameTransform[];
+}) => VoiceMediaFrameTransformPipeline;
+export declare const buildVoiceMediaVadReport: (input?: VoiceMediaVadInput) => VoiceMediaVadReport;
+export declare const buildVoiceMediaInterruptionReport: (input?: VoiceMediaInterruptionInput) => VoiceMediaInterruptionReport;
 export declare const buildVoiceMediaPipelineCalibrationReport: (input?: VoiceMediaPipelineCalibrationInput) => VoiceMediaPipelineCalibrationReport;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@absolutejs/voice",
-	"version": "0.0.22-beta.309",
+	"version": "0.0.22-beta.310",
 	"description": "Voice primitives and Elysia plugin for AbsoluteJS",
 	"repository": {
 		"type": "git",