@contractspec/lib.voice 0.2.0

package/dist/browser/stt/diarization-mapper.js

@@ -0,0 +1,42 @@
+// src/stt/diarization-mapper.ts
+class DiarizationMapper {
+  map(segments, labelPrefix = "Speaker") {
+    const speakerOrder = [];
+    const speakerStats = new Map;
+    for (const seg of segments) {
+      if (seg.speakerId && !speakerOrder.includes(seg.speakerId)) {
+        speakerOrder.push(seg.speakerId);
+        speakerStats.set(seg.speakerId, {
+          segmentCount: 0,
+          totalSpeakingMs: 0
+        });
+      }
+    }
+    const labeledSegments = segments.map((seg) => {
+      if (!seg.speakerId)
+        return seg;
+      const index = speakerOrder.indexOf(seg.speakerId);
+      const label = `${labelPrefix} ${index + 1}`;
+      const stats = speakerStats.get(seg.speakerId);
+      if (!stats) {
+        return { ...seg, speakerLabel: label };
+      }
+      stats.segmentCount += 1;
+      stats.totalSpeakingMs += seg.endMs - seg.startMs;
+      return { ...seg, speakerLabel: label };
+    });
+    const speakers = speakerOrder.map((id, index) => {
+      const stats = speakerStats.get(id);
+      return {
+        id,
+        label: `${labelPrefix} ${index + 1}`,
+        segmentCount: stats?.segmentCount ?? 0,
+        totalSpeakingMs: stats?.totalSpeakingMs ?? 0
+      };
+    });
+    return { segments: labeledSegments, speakers };
+  }
+}
+export {
+  DiarizationMapper
+};

package/dist/browser/stt/index.js

@@ -0,0 +1,222 @@
+// src/stt/segment-splitter.ts
+class SegmentSplitter {
+  static DEFAULT_MAX_CHUNK_MS = 5 * 60 * 1000;
+  split(audio, maxChunkMs = SegmentSplitter.DEFAULT_MAX_CHUNK_MS) {
+    const totalDurationMs = audio.durationMs ?? this.estimateDurationMs(audio);
+    if (totalDurationMs <= maxChunkMs) {
+      return [audio];
+    }
+    const chunks = [];
+    const bytesPerMs = audio.data.length / Math.max(totalDurationMs, 1);
+    let offsetMs = 0;
+    while (offsetMs < totalDurationMs) {
+      const chunkDurationMs = Math.min(maxChunkMs, totalDurationMs - offsetMs);
+      const startByte = Math.floor(offsetMs * bytesPerMs);
+      const endByte = Math.floor((offsetMs + chunkDurationMs) * bytesPerMs);
+      chunks.push({
+        data: audio.data.slice(startByte, endByte),
+        format: audio.format,
+        sampleRateHz: audio.sampleRateHz,
+        durationMs: chunkDurationMs,
+        channels: audio.channels
+      });
+      offsetMs += chunkDurationMs;
+    }
+    return chunks;
+  }
+  estimateDurationMs(audio) {
+    const bytesPerSample = 2;
+    const channels = audio.channels ?? 1;
+    const totalSamples = audio.data.length / (bytesPerSample * channels);
+    return Math.ceil(totalSamples / audio.sampleRateHz * 1000);
+  }
+}
+
+// src/stt/diarization-mapper.ts
+class DiarizationMapper {
+  map(segments, labelPrefix = "Speaker") {
+    const speakerOrder = [];
+    const speakerStats = new Map;
+    for (const seg of segments) {
+      if (seg.speakerId && !speakerOrder.includes(seg.speakerId)) {
+        speakerOrder.push(seg.speakerId);
+        speakerStats.set(seg.speakerId, {
+          segmentCount: 0,
+          totalSpeakingMs: 0
+        });
+      }
+    }
+    const labeledSegments = segments.map((seg) => {
+      if (!seg.speakerId)
+        return seg;
+      const index = speakerOrder.indexOf(seg.speakerId);
+      const label = `${labelPrefix} ${index + 1}`;
+      const stats = speakerStats.get(seg.speakerId);
+      if (!stats) {
+        return { ...seg, speakerLabel: label };
+      }
+      stats.segmentCount += 1;
+      stats.totalSpeakingMs += seg.endMs - seg.startMs;
+      return { ...seg, speakerLabel: label };
+    });
+    const speakers = speakerOrder.map((id, index) => {
+      const stats = speakerStats.get(id);
+      return {
+        id,
+        label: `${labelPrefix} ${index + 1}`,
+        segmentCount: stats?.segmentCount ?? 0,
+        totalSpeakingMs: stats?.totalSpeakingMs ?? 0
+      };
+    });
+    return { segments: labeledSegments, speakers };
+  }
+}
+
+// src/stt/subtitle-formatter.ts
+class SubtitleFormatter {
+  toSRT(segments) {
+    return segments.map((seg, i) => {
+      const start = this.formatTimeSRT(seg.startMs);
+      const end = this.formatTimeSRT(seg.endMs);
+      const label = seg.speakerLabel ? `[${seg.speakerLabel}] ` : "";
+      return `${i + 1}
+${start} --> ${end}
+${label}${seg.text}`;
+    }).join(`
+
+`);
+  }
+  toVTT(segments) {
+    const header = `WEBVTT
+
+`;
+    const cues = segments.map((seg, i) => {
+      const start = this.formatTimeVTT(seg.startMs);
+      const end = this.formatTimeVTT(seg.endMs);
+      const label = seg.speakerLabel ? `<v ${seg.speakerLabel}>` : "";
+      return `${i + 1}
+${start} --> ${end}
+${label}${seg.text}`;
+    }).join(`
+
+`);
+    return header + cues;
+  }
+  formatTimeSRT(ms) {
+    const hours = Math.floor(ms / 3600000);
+    const minutes = Math.floor(ms % 3600000 / 60000);
+    const seconds = Math.floor(ms % 60000 / 1000);
+    const millis = ms % 1000;
+    return `${this.pad(hours, 2)}:${this.pad(minutes, 2)}:${this.pad(seconds, 2)},${this.pad(millis, 3)}`;
+  }
+  formatTimeVTT(ms) {
+    const hours = Math.floor(ms / 3600000);
+    const minutes = Math.floor(ms % 3600000 / 60000);
+    const seconds = Math.floor(ms % 60000 / 1000);
+    const millis = ms % 1000;
+    return `${this.pad(hours, 2)}:${this.pad(minutes, 2)}:${this.pad(seconds, 2)}.${this.pad(millis, 3)}`;
+  }
+  pad(value, length) {
+    return value.toString().padStart(length, "0");
+  }
+}
+
+// src/stt/transcriber.ts
+class Transcriber {
+  stt;
+  segmentSplitter = new SegmentSplitter;
+  diarizationMapper = new DiarizationMapper;
+  subtitleFormatter = new SubtitleFormatter;
+  constructor(options) {
+    this.stt = options.stt;
+  }
+  async transcribe(brief) {
+    const projectId = generateProjectId();
+    const chunks = this.segmentSplitter.split(brief.audio);
+    const allSegments = [];
+    let fullText = "";
+    let totalDurationMs = 0;
+    let offsetMs = 0;
+    for (const chunk of chunks) {
+      const result = await this.stt.transcribe({
+        audio: chunk,
+        language: brief.language,
+        diarize: brief.diarize,
+        speakerCount: brief.speakerCount,
+        wordTimestamps: true,
+        vocabularyHints: brief.vocabularyHints
+      });
+      const offsetSegments = result.segments.map((seg) => ({
+        text: seg.text,
+        startMs: seg.startMs + offsetMs,
+        endMs: seg.endMs + offsetMs,
+        speakerId: seg.speakerId,
+        speakerName: seg.speakerName,
+        confidence: seg.confidence
+      }));
+      allSegments.push(...offsetSegments);
+      fullText += (fullText ? " " : "") + result.text;
+      totalDurationMs += result.durationMs;
+      offsetMs += chunk.durationMs ?? 0;
+    }
+    let mappedSegments = allSegments;
+    let speakers;
+    if (brief.diarize) {
+      const mapping = this.diarizationMapper.map(allSegments);
+      mappedSegments = mapping.segments;
+      speakers = mapping.speakers;
+    }
+    const transcript = {
+      text: fullText,
+      segments: mappedSegments,
+      language: brief.language ?? "en",
+      durationMs: totalDurationMs
+    };
+    let subtitles;
+    const format = brief.subtitleFormat ?? "none";
+    if (format === "srt") {
+      subtitles = this.subtitleFormatter.toSRT(mappedSegments);
+    } else if (format === "vtt") {
+      subtitles = this.subtitleFormatter.toVTT(mappedSegments);
+    }
+    return {
+      id: projectId,
+      transcript,
+      subtitles,
+      speakers
+    };
+  }
+  async* transcribeStream(audio, options) {
+    if (!this.stt.transcribeStream) {
+      throw new Error("Streaming transcription not supported by the current STT provider");
+    }
+    const stream = this.stt.transcribeStream(audio, {
+      language: options?.language,
+      diarize: options?.diarize,
+      speakerCount: options?.speakerCount,
+      wordTimestamps: true,
+      vocabularyHints: options?.vocabularyHints
+    });
+    for await (const segment of stream) {
+      yield {
+        text: segment.text,
+        startMs: segment.startMs,
+        endMs: segment.endMs,
+        speakerId: segment.speakerId,
+        speakerLabel: segment.speakerName,
+        confidence: segment.confidence
+      };
+    }
+  }
+}
+function generateProjectId() {
+  const timestamp = Date.now().toString(36);
+  const random = Math.random().toString(36).slice(2, 8);
+  return `stt_${timestamp}_${random}`;
+}
+export {
+  Transcriber,
+  SubtitleFormatter,
+  SegmentSplitter,
+  DiarizationMapper
+};

package/dist/browser/stt/segment-splitter.js

@@ -0,0 +1,36 @@
+// src/stt/segment-splitter.ts
+class SegmentSplitter {
+  static DEFAULT_MAX_CHUNK_MS = 5 * 60 * 1000;
+  split(audio, maxChunkMs = SegmentSplitter.DEFAULT_MAX_CHUNK_MS) {
+    const totalDurationMs = audio.durationMs ?? this.estimateDurationMs(audio);
+    if (totalDurationMs <= maxChunkMs) {
+      return [audio];
+    }
+    const chunks = [];
+    const bytesPerMs = audio.data.length / Math.max(totalDurationMs, 1);
+    let offsetMs = 0;
+    while (offsetMs < totalDurationMs) {
+      const chunkDurationMs = Math.min(maxChunkMs, totalDurationMs - offsetMs);
+      const startByte = Math.floor(offsetMs * bytesPerMs);
+      const endByte = Math.floor((offsetMs + chunkDurationMs) * bytesPerMs);
+      chunks.push({
+        data: audio.data.slice(startByte, endByte),
+        format: audio.format,
+        sampleRateHz: audio.sampleRateHz,
+        durationMs: chunkDurationMs,
+        channels: audio.channels
+      });
+      offsetMs += chunkDurationMs;
+    }
+    return chunks;
+  }
+  estimateDurationMs(audio) {
+    const bytesPerSample = 2;
+    const channels = audio.channels ?? 1;
+    const totalSamples = audio.data.length / (bytesPerSample * channels);
+    return Math.ceil(totalSamples / audio.sampleRateHz * 1000);
+  }
+}
+export {
+  SegmentSplitter
+};

package/dist/browser/stt/subtitle-formatter.js

@@ -0,0 +1,51 @@
+// src/stt/subtitle-formatter.ts
+class SubtitleFormatter {
+  toSRT(segments) {
+    return segments.map((seg, i) => {
+      const start = this.formatTimeSRT(seg.startMs);
+      const end = this.formatTimeSRT(seg.endMs);
+      const label = seg.speakerLabel ? `[${seg.speakerLabel}] ` : "";
+      return `${i + 1}
+${start} --> ${end}
+${label}${seg.text}`;
+    }).join(`
+
+`);
+  }
+  toVTT(segments) {
+    const header = `WEBVTT
+
+`;
+    const cues = segments.map((seg, i) => {
+      const start = this.formatTimeVTT(seg.startMs);
+      const end = this.formatTimeVTT(seg.endMs);
+      const label = seg.speakerLabel ? `<v ${seg.speakerLabel}>` : "";
+      return `${i + 1}
+${start} --> ${end}
+${label}${seg.text}`;
+    }).join(`
+
+`);
+    return header + cues;
+  }
+  formatTimeSRT(ms) {
+    const hours = Math.floor(ms / 3600000);
+    const minutes = Math.floor(ms % 3600000 / 60000);
+    const seconds = Math.floor(ms % 60000 / 1000);
+    const millis = ms % 1000;
+    return `${this.pad(hours, 2)}:${this.pad(minutes, 2)}:${this.pad(seconds, 2)},${this.pad(millis, 3)}`;
+  }
+  formatTimeVTT(ms) {
+    const hours = Math.floor(ms / 3600000);
+    const minutes = Math.floor(ms % 3600000 / 60000);
+    const seconds = Math.floor(ms % 60000 / 1000);
+    const millis = ms % 1000;
+    return `${this.pad(hours, 2)}:${this.pad(minutes, 2)}:${this.pad(seconds, 2)}.${this.pad(millis, 3)}`;
+  }
+  pad(value, length) {
+    return value.toString().padStart(length, "0");
+  }
+}
+export {
+  SubtitleFormatter
+};

package/dist/browser/stt/transcriber.js

@@ -0,0 +1,219 @@
+// src/stt/segment-splitter.ts
+class SegmentSplitter {
+  static DEFAULT_MAX_CHUNK_MS = 5 * 60 * 1000;
+  split(audio, maxChunkMs = SegmentSplitter.DEFAULT_MAX_CHUNK_MS) {
+    const totalDurationMs = audio.durationMs ?? this.estimateDurationMs(audio);
+    if (totalDurationMs <= maxChunkMs) {
+      return [audio];
+    }
+    const chunks = [];
+    const bytesPerMs = audio.data.length / Math.max(totalDurationMs, 1);
+    let offsetMs = 0;
+    while (offsetMs < totalDurationMs) {
+      const chunkDurationMs = Math.min(maxChunkMs, totalDurationMs - offsetMs);
+      const startByte = Math.floor(offsetMs * bytesPerMs);
+      const endByte = Math.floor((offsetMs + chunkDurationMs) * bytesPerMs);
+      chunks.push({
+        data: audio.data.slice(startByte, endByte),
+        format: audio.format,
+        sampleRateHz: audio.sampleRateHz,
+        durationMs: chunkDurationMs,
+        channels: audio.channels
+      });
+      offsetMs += chunkDurationMs;
+    }
+    return chunks;
+  }
+  estimateDurationMs(audio) {
+    const bytesPerSample = 2;
+    const channels = audio.channels ?? 1;
+    const totalSamples = audio.data.length / (bytesPerSample * channels);
+    return Math.ceil(totalSamples / audio.sampleRateHz * 1000);
+  }
+}
+
+// src/stt/diarization-mapper.ts
+class DiarizationMapper {
+  map(segments, labelPrefix = "Speaker") {
+    const speakerOrder = [];
+    const speakerStats = new Map;
+    for (const seg of segments) {
+      if (seg.speakerId && !speakerOrder.includes(seg.speakerId)) {
+        speakerOrder.push(seg.speakerId);
+        speakerStats.set(seg.speakerId, {
+          segmentCount: 0,
+          totalSpeakingMs: 0
+        });
+      }
+    }
+    const labeledSegments = segments.map((seg) => {
+      if (!seg.speakerId)
+        return seg;
+      const index = speakerOrder.indexOf(seg.speakerId);
+      const label = `${labelPrefix} ${index + 1}`;
+      const stats = speakerStats.get(seg.speakerId);
+      if (!stats) {
+        return { ...seg, speakerLabel: label };
+      }
+      stats.segmentCount += 1;
+      stats.totalSpeakingMs += seg.endMs - seg.startMs;
+      return { ...seg, speakerLabel: label };
+    });
+    const speakers = speakerOrder.map((id, index) => {
+      const stats = speakerStats.get(id);
+      return {
+        id,
+        label: `${labelPrefix} ${index + 1}`,
+        segmentCount: stats?.segmentCount ?? 0,
+        totalSpeakingMs: stats?.totalSpeakingMs ?? 0
+      };
+    });
+    return { segments: labeledSegments, speakers };
+  }
+}
+
+// src/stt/subtitle-formatter.ts
+class SubtitleFormatter {
+  toSRT(segments) {
+    return segments.map((seg, i) => {
+      const start = this.formatTimeSRT(seg.startMs);
+      const end = this.formatTimeSRT(seg.endMs);
+      const label = seg.speakerLabel ? `[${seg.speakerLabel}] ` : "";
+      return `${i + 1}
+${start} --> ${end}
+${label}${seg.text}`;
+    }).join(`
+
+`);
+  }
+  toVTT(segments) {
+    const header = `WEBVTT
+
+`;
+    const cues = segments.map((seg, i) => {
+      const start = this.formatTimeVTT(seg.startMs);
+      const end = this.formatTimeVTT(seg.endMs);
+      const label = seg.speakerLabel ? `<v ${seg.speakerLabel}>` : "";
+      return `${i + 1}
+${start} --> ${end}
+${label}${seg.text}`;
+    }).join(`
+
+`);
+    return header + cues;
+  }
+  formatTimeSRT(ms) {
+    const hours = Math.floor(ms / 3600000);
+    const minutes = Math.floor(ms % 3600000 / 60000);
+    const seconds = Math.floor(ms % 60000 / 1000);
+    const millis = ms % 1000;
+    return `${this.pad(hours, 2)}:${this.pad(minutes, 2)}:${this.pad(seconds, 2)},${this.pad(millis, 3)}`;
+  }
+  formatTimeVTT(ms) {
+    const hours = Math.floor(ms / 3600000);
+    const minutes = Math.floor(ms % 3600000 / 60000);
+    const seconds = Math.floor(ms % 60000 / 1000);
+    const millis = ms % 1000;
+    return `${this.pad(hours, 2)}:${this.pad(minutes, 2)}:${this.pad(seconds, 2)}.${this.pad(millis, 3)}`;
+  }
+  pad(value, length) {
+    return value.toString().padStart(length, "0");
+  }
+}
+
+// src/stt/transcriber.ts
+class Transcriber {
+  stt;
+  segmentSplitter = new SegmentSplitter;
+  diarizationMapper = new DiarizationMapper;
+  subtitleFormatter = new SubtitleFormatter;
+  constructor(options) {
+    this.stt = options.stt;
+  }
+  async transcribe(brief) {
+    const projectId = generateProjectId();
+    const chunks = this.segmentSplitter.split(brief.audio);
+    const allSegments = [];
+    let fullText = "";
+    let totalDurationMs = 0;
+    let offsetMs = 0;
+    for (const chunk of chunks) {
+      const result = await this.stt.transcribe({
+        audio: chunk,
+        language: brief.language,
+        diarize: brief.diarize,
+        speakerCount: brief.speakerCount,
+        wordTimestamps: true,
+        vocabularyHints: brief.vocabularyHints
+      });
+      const offsetSegments = result.segments.map((seg) => ({
+        text: seg.text,
+        startMs: seg.startMs + offsetMs,
+        endMs: seg.endMs + offsetMs,
+        speakerId: seg.speakerId,
+        speakerName: seg.speakerName,
+        confidence: seg.confidence
+      }));
+      allSegments.push(...offsetSegments);
+      fullText += (fullText ? " " : "") + result.text;
+      totalDurationMs += result.durationMs;
+      offsetMs += chunk.durationMs ?? 0;
+    }
+    let mappedSegments = allSegments;
+    let speakers;
+    if (brief.diarize) {
+      const mapping = this.diarizationMapper.map(allSegments);
+      mappedSegments = mapping.segments;
+      speakers = mapping.speakers;
+    }
+    const transcript = {
+      text: fullText,
+      segments: mappedSegments,
+      language: brief.language ?? "en",
+      durationMs: totalDurationMs
+    };
+    let subtitles;
+    const format = brief.subtitleFormat ?? "none";
+    if (format === "srt") {
+      subtitles = this.subtitleFormatter.toSRT(mappedSegments);
+    } else if (format === "vtt") {
+      subtitles = this.subtitleFormatter.toVTT(mappedSegments);
+    }
+    return {
+      id: projectId,
+      transcript,
+      subtitles,
+      speakers
+    };
+  }
+  async* transcribeStream(audio, options) {
+    if (!this.stt.transcribeStream) {
+      throw new Error("Streaming transcription not supported by the current STT provider");
+    }
+    const stream = this.stt.transcribeStream(audio, {
+      language: options?.language,
+      diarize: options?.diarize,
+      speakerCount: options?.speakerCount,
+      wordTimestamps: true,
+      vocabularyHints: options?.vocabularyHints
+    });
+    for await (const segment of stream) {
+      yield {
+        text: segment.text,
+        startMs: segment.startMs,
+        endMs: segment.endMs,
+        speakerId: segment.speakerId,
+        speakerLabel: segment.speakerName,
+        confidence: segment.confidence
+      };
+    }
+  }
+}
+function generateProjectId() {
+  const timestamp = Date.now().toString(36);
+  const random = Math.random().toString(36).slice(2, 8);
+  return `stt_${timestamp}_${random}`;
+}
+export {
+  Transcriber
+};

package/dist/browser/sync/duration-negotiator.js

@@ -0,0 +1,69 @@
+// src/sync/duration-negotiator.ts
+class DurationNegotiator {
+  static UPPER_THRESHOLD = 1.1;
+  static LOWER_THRESHOLD = 0.7;
+  static MAX_RATE = 1.3;
+  static MIN_RATE = 0.8;
+  negotiate(timingMap, sceneDurations) {
+    const adjustments = [];
+    const updatedSegments = timingMap.segments.map((seg) => {
+      const originalSceneDuration = sceneDurations.get(seg.sceneId);
+      if (originalSceneDuration === undefined) {
+        adjustments.push({
+          sceneId: seg.sceneId,
+          originalSceneDurationInFrames: seg.recommendedSceneDurationInFrames,
+          voiceDurationInFrames: seg.durationInFrames,
+          action: "no_change",
+          finalSceneDurationInFrames: seg.recommendedSceneDurationInFrames
+        });
+        return seg;
+      }
+      const ratio = seg.durationInFrames / originalSceneDuration;
+      if (ratio > DurationNegotiator.UPPER_THRESHOLD) {
+        const suggestedRate = Math.min(ratio, DurationNegotiator.MAX_RATE);
+        adjustments.push({
+          sceneId: seg.sceneId,
+          originalSceneDurationInFrames: originalSceneDuration,
+          voiceDurationInFrames: seg.durationInFrames,
+          action: ratio > DurationNegotiator.MAX_RATE ? "extend_scene" : "suggest_rate_change",
+          suggestedRate,
+          finalSceneDurationInFrames: seg.recommendedSceneDurationInFrames
+        });
+        return seg;
+      }
+      if (ratio < DurationNegotiator.LOWER_THRESHOLD) {
+        const suggestedRate = Math.max(ratio, DurationNegotiator.MIN_RATE);
+        adjustments.push({
+          sceneId: seg.sceneId,
+          originalSceneDurationInFrames: originalSceneDuration,
+          voiceDurationInFrames: seg.durationInFrames,
+          action: "pad_silence",
+          suggestedRate,
+          finalSceneDurationInFrames: originalSceneDuration
+        });
+        return {
+          ...seg,
+          recommendedSceneDurationInFrames: originalSceneDuration
+        };
+      }
+      adjustments.push({
+        sceneId: seg.sceneId,
+        originalSceneDurationInFrames: originalSceneDuration,
+        voiceDurationInFrames: seg.durationInFrames,
+        action: "no_change",
+        finalSceneDurationInFrames: seg.recommendedSceneDurationInFrames
+      });
+      return seg;
+    });
+    return {
+      timingMap: {
+        ...timingMap,
+        segments: updatedSegments
+      },
+      adjustments
+    };
+  }
+}
+export {
+  DurationNegotiator
+};