@storyteller-platform/align 0.1.19 → 0.1.20

package/dist/align/getSentenceRanges.cjs

@@ -18,11 +18,13 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var getSentenceRanges_exports = {};
 __export(getSentenceRanges_exports, {
+  collapseSentenceRangeGaps: () => collapseSentenceRangeGaps,
   expandEmptySentenceRanges: () => expandEmptySentenceRanges,
   findEndTimestamp: () => findEndTimestamp,
   getChapterDuration: () => getChapterDuration,
   getSentenceRanges: () => getSentenceRanges,
-  interpolateSentenceRanges: () => interpolateSentenceRanges
+  interpolateSentenceRanges: () => interpolateSentenceRanges,
+  mapTranscriptionTimeline: () => mapTranscriptionTimeline
 });
 module.exports = __toCommonJS(getSentenceRanges_exports);
 var import_itertools = require("itertools");
@@ -31,9 +33,9 @@ var import_ffmpeg = require("../common/ffmpeg.cjs");
 var import_errorAlign = require("../errorAlign/errorAlign.cjs");
 var import_utils = require("../errorAlign/utils.cjs");
 var import_slugify = require("./slugify.cjs");
-function findStartTimestamp(matchStartIndex, transcription) {
-  const entry = transcription.timeline.find(
-    (entry2) => (entry2.endOffsetUtf16 ?? 0) > matchStartIndex
+function findStartTimestamp(matchStartIndex, timeline) {
+  const entry = timeline.find(
+    (entry2) => entry2.mappedEndOffsetUtf16 > matchStartIndex
   );
   if (!entry) return null;
   return {
@@ -42,9 +44,9 @@ function findStartTimestamp(matchStartIndex, transcription) {
     audiofile: entry.audiofile
   };
 }
-function findEndTimestamp(matchEndIndex, transcription) {
-  const entry = transcription.timeline.findLast(
-    (entry2) => (entry2.startOffsetUtf16 ?? 0) < matchEndIndex
+function findEndTimestamp(matchEndIndex, timeline) {
+  const entry = timeline.findLast(
+    (entry2) => entry2.mappedStartOffsetUtf16 < matchEndIndex
   );
   if (!entry) return null;
   return {
@@ -135,17 +137,23 @@ function errorAlignWithNarrowing(refSentences, hyp, narrowStart, narrowEnd) {
   );
   return { alignments, slice: [slice[0] + narrowed[0], slice[0] + narrowed[1]] };
 }
-async function getSentenceRanges(transcription, sentences, chapterOffset, chapterEndOffset, locale) {
+function mapTranscriptionTimeline(transcription, mapping) {
+  return transcription.timeline.map((entry) => ({
+    ...entry,
+    mappedStartOffsetUtf16: mapping.map(entry.startOffsetUtf16 ?? 0, 1),
+    mappedEndOffsetUtf16: mapping.map(entry.endOffsetUtf16 ?? 0, -1)
+  }));
+}
+async function getSentenceRanges(transcriptionText, mappedTimeline, sentences, chapterId, chapterOffset, chapterEndOffset, granularity, locale) {
   const sentenceRanges = [];
-  const fullTranscript = transcription.transcript;
-  const chapterTranscript = fullTranscript.slice(
+  const wordRanges = [];
+  const slugifiedChapterTranscript = transcriptionText.slice(
     chapterOffset,
     chapterEndOffset
   );
-  const { result: slugifiedChapterTranscript, mapping: transcriptMapping } = await (0, import_slugify.slugify)(chapterTranscript, locale);
   const slugifiedChapterSentences = [];
   for (const s of sentences) {
-    const { result } = await (0, import_slugify.slugify)(s, locale);
+    const { result } = await (0, import_slugify.slugify)(s.text, locale);
     slugifiedChapterSentences.push(result);
   }
   let firstFoundSentence = 0;
@@ -214,21 +222,16 @@ async function getSentenceRanges(transcription, sentences, chapterOffset, chapte
       const sentenceLengthInSlugifiedTranscript = sentenceAlignments.filter((a) => a.opType !== "DELETE").map((a) => a.hyp).join("-").length;
       if (score > 0) {
         const start = findStartTimestamp(
-          chapterOffset + transcriptMapping.invert().map(
-            slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex,
-            1
-          ),
-          transcription
-        );
-        chapterTranscriptEndIndex = chapterOffset + transcriptMapping.invert().map(
-          slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex + sentenceLengthInSlugifiedTranscript,
-          -1
+          chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex,
+          mappedTimeline
         );
-        const end = findEndTimestamp(chapterTranscriptEndIndex, transcription);
+        chapterTranscriptEndIndex = chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex + sentenceLengthInSlugifiedTranscript;
+        const end = findEndTimestamp(chapterTranscriptEndIndex, mappedTimeline);
         if (start && end) {
           if (start.audiofile !== end.audiofile) {
             sentenceRanges.push({
               id: j + chapterSentenceIndex + slice[0],
+              chapterId,
               start: 0,
               audiofile: end.audiofile,
               end: end.end
@@ -236,12 +239,56 @@ async function getSentenceRanges(transcription, sentences, chapterOffset, chapte
           } else {
             sentenceRanges.push({
               id: j + chapterSentenceIndex + slice[0],
+              chapterId,
               start: start.start,
               audiofile: start.audiofile,
               end: end.end
             });
           }
         }
+        if (granularity === "word") {
+          const sentenceSegmentation = sentences[j + chapterSentenceIndex + slice[0]];
+          const words = [];
+          for (const entry of sentenceSegmentation.words.entries) {
+            if (!entry.text.match(/\S/)) continue;
+            const { result } = await (0, import_slugify.slugify)(entry.text, locale);
+            words.push(result);
+          }
+          let currentTranscriptWordWindowIndex = currentTranscriptWindowIndex;
+          let sentenceAlignmentIndex = 0;
+          const perSentenceWordRanges = [];
+          for (const [k, word] of (0, import_itertools.enumerate)(words)) {
+            if (!word) continue;
+            const { alignments: wordAlignments } = getAlignmentsForSentence(
+              word,
+              sentenceAlignments.slice(sentenceAlignmentIndex)
+            );
+            const wordLengthInSlugifiedTranscript = wordAlignments.filter((a) => a.opType !== "DELETE").map((a) => a.hyp).join("-").length;
+            const start2 = findStartTimestamp(
+              chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWordWindowIndex,
+              mappedTimeline
+            );
+            const end2 = findEndTimestamp(
+              chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWordWindowIndex + wordLengthInSlugifiedTranscript,
+              mappedTimeline
+            );
+            if (start2 && end2) {
+              perSentenceWordRanges.push({
+                id: k,
+                sentenceId: j + chapterSentenceIndex + slice[0],
+                start: end2.audiofile === start2.audiofile ? start2.start : 0,
+                audiofile: end2.audiofile,
+                end: end2.end
+              });
+            }
+            sentenceAlignmentIndex += wordAlignments.length;
+            currentTranscriptWordWindowIndex += wordLengthInSlugifiedTranscript;
+            if (slugifiedChapterTranscriptWindow[currentTranscriptWordWindowIndex] === "-") {
+              currentTranscriptWordWindowIndex++;
+            }
+          }
+          wordRanges.push(perSentenceWordRanges);
+        }
       }
       alignmentIndex += sentenceAlignments.length;
       currentTranscriptWindowIndex += sentenceLengthInSlugifiedTranscript;
@@ -257,6 +304,7 @@ async function getSentenceRanges(transcription, sentences, chapterOffset, chapte
   }
   return {
     sentenceRanges,
+    wordRanges,
     transcriptionOffset: chapterTranscriptEndIndex,
     firstFoundSentence,
     lastFoundSentence
@@ -268,66 +316,53 @@ async function getLargestGap(trailing, leading) {
   if (trailingGap > leadingGap) return [trailingGap, trailing.audiofile];
   return [leadingGap, leading.audiofile];
 }
-async function interpolateSentenceRanges(sentenceRanges, lastSentenceRange) {
+async function interpolateSentenceRanges(sentenceRanges, chapterSentenceCounts) {
   const interpolated = [];
-  const [first, ...rest] = sentenceRanges;
-  if (!first) return interpolated;
-  if (first.id !== 0) {
-    const count = first.id;
-    const crossesAudioBoundary = !lastSentenceRange || first.audiofile !== lastSentenceRange.audiofile;
-    let diff = crossesAudioBoundary ? first.start : first.start - lastSentenceRange.end;
-    if (!crossesAudioBoundary && diff <= 0) {
-      diff = 0.25;
-      lastSentenceRange.end = first.start - diff;
-    }
-    const interpolatedLength = diff / count;
-    const start = crossesAudioBoundary ? 0 : lastSentenceRange.end;
-    for (let i = 0; i < count; i++) {
-      interpolated.push({
-        id: i,
-        start: start + interpolatedLength * i,
-        end: start + interpolatedLength * (i + 1),
-        audiofile: first.audiofile
-      });
-    }
-    interpolated.push(first);
-  } else {
-    rest.unshift(first);
-  }
-  for (const sentenceRange of rest) {
-    if (interpolated.length === 0) {
-      interpolated.push(sentenceRange);
-      continue;
-    }
-    const lastSentenceRange2 = interpolated[interpolated.length - 1];
-    const count = sentenceRange.id - lastSentenceRange2.id - 1;
+  for (let i = 0; i < sentenceRanges.length; i++) {
+    const endRange = sentenceRanges[i];
+    const startRange = sentenceRanges[i - 1] ?? {
+      id: 0,
+      audiofile: endRange.audiofile,
+      chapterId: endRange.chapterId,
+      start: 0,
+      end: 0
+    };
+    const newChapter = startRange.chapterId !== endRange.chapterId;
+    const newAudiofile = startRange.audiofile !== endRange.audiofile;
+    const count = newChapter ? chapterSentenceCounts[startRange.chapterId] - startRange.id - 1 : endRange.id - startRange.id - 1;
     if (count === 0) {
-      interpolated.push(sentenceRange);
+      interpolated.push(endRange);
       continue;
     }
-    const crossesAudioBoundary = sentenceRange.audiofile !== lastSentenceRange2.audiofile;
-    let [diff, audiofile] = crossesAudioBoundary ? await getLargestGap(lastSentenceRange2, sentenceRange) : [sentenceRange.start - lastSentenceRange2.end, sentenceRange.audiofile];
+    let [diff, audiofile] = newAudiofile ? await getLargestGap(startRange, endRange) : [endRange.start - startRange.end, endRange.audiofile];
     if (diff <= 0) {
-      if (crossesAudioBoundary) {
-        const rangeLength = sentenceRange.end - sentenceRange.start;
+      if (newAudiofile) {
+        const rangeLength = endRange.end - endRange.start;
         diff = rangeLength < 0.5 ? rangeLength / 2 : 0.25;
-        sentenceRange.start = diff;
+        endRange.start = diff;
       } else {
         diff = 0.25;
-        lastSentenceRange2.end = sentenceRange.start - diff;
+        startRange.end = startRange.start - diff;
       }
     }
     const interpolatedLength = diff / count;
-    const start = crossesAudioBoundary ? 0 : lastSentenceRange2.end;
-    for (let i = 0; i < count; i++) {
+    const start = newAudiofile ? 0 : startRange.end;
+    for (let i2 = 0; i2 < count; i2++) {
+      let id = startRange.id + i2 + 1;
+      let chapterId = startRange.chapterId;
+      if (newChapter && i2 > chapterSentenceCounts[startRange.chapterId] - startRange.id) {
+        id = i2;
+        chapterId = endRange.chapterId;
+      }
       interpolated.push({
-        id: lastSentenceRange2.id + i + 1,
-        start: start + interpolatedLength * i,
-        end: start + interpolatedLength * (i + 1),
+        id,
+        chapterId,
+        start: start + interpolatedLength * i2,
+        end: start + interpolatedLength * (i2 + 1),
         audiofile
       });
     }
-    interpolated.push(sentenceRange);
+    interpolated.push(endRange);
   }
   return interpolated;
 }
@@ -345,6 +380,17 @@ function expandEmptySentenceRanges(sentenceRanges) {
   }
   return expandedRanges;
 }
+async function collapseSentenceRangeGaps(sentenceRanges) {
+  const collapsed = [];
+  for (const [i, sentenceRange] of (0, import_itertools.enumerate)(sentenceRanges)) {
+    const nextSentence = sentenceRanges[i + 1];
+    const prevSentence = sentenceRanges[i - 1];
+    const start = prevSentence?.audiofile !== sentenceRange.audiofile ? 0 : sentenceRange.start;
+    const end = nextSentence?.audiofile !== sentenceRange.audiofile ? await (0, import_ffmpeg.getTrackDuration)(sentenceRange.audiofile) : nextSentence.start;
+    collapsed.push({ ...sentenceRange, start, end });
+  }
+  return collapsed;
+}
 function getChapterDuration(sentenceRanges) {
   let i = 0;
   let duration = 0;
@@ -366,9 +412,11 @@ function getChapterDuration(sentenceRanges) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  collapseSentenceRangeGaps,
   expandEmptySentenceRanges,
   findEndTimestamp,
   getChapterDuration,
   getSentenceRanges,
-  interpolateSentenceRanges
+  interpolateSentenceRanges,
+  mapTranscriptionTimeline
 });

package/dist/align/getSentenceRanges.d.cts

@@ -1,4 +1,7 @@
+import * as _storyteller_platform_ghost_story from '@storyteller-platform/ghost-story';
 import { TimelineEntry } from '@storyteller-platform/ghost-story';
+import { SegmentationResult } from '@echogarden/text-segmentation';
+import { Mapping } from '@storyteller-platform/transliteration';
 
 type StorytellerTimelineEntry = TimelineEntry & {
     audiofile: string;
@@ -9,29 +12,56 @@ type StorytellerTranscription = {
 };
 type SentenceRange = {
     id: number;
+    chapterId: string;
     start: number;
     end: number;
     audiofile: string;
 };
-declare function findEndTimestamp(matchEndIndex: number, transcription: StorytellerTranscription): {
+type WordRange = {
+    id: number;
+    sentenceId: number;
+    start: number;
+    end: number;
+    audiofile: string;
+};
+declare function findEndTimestamp(matchEndIndex: number, timeline: MappedTimeline): {
     start: number;
     end: number;
     audiofile: string;
 } | null;
-declare function getSentenceRanges(transcription: StorytellerTranscription, sentences: string[], chapterOffset: number, chapterEndOffset: number, locale: Intl.Locale): Promise<{
+declare function mapTranscriptionTimeline(transcription: StorytellerTranscription, mapping: Mapping): {
+    mappedStartOffsetUtf16: number;
+    mappedEndOffsetUtf16: number;
+    type: _storyteller_platform_ghost_story.TimelineEntryType;
+    text: string;
+    startTime: number;
+    endTime: number;
+    startOffsetUtf16?: number;
+    endOffsetUtf16?: number;
+    startOffsetUtf32?: number;
+    endOffsetUtf32?: number;
+    confidence?: number;
+    id?: number;
+    timeline?: _storyteller_platform_ghost_story.Timeline;
+    audiofile: string;
+}[];
+type MappedTimeline = ReturnType<typeof mapTranscriptionTimeline>;
+declare function getSentenceRanges(transcriptionText: string, mappedTimeline: MappedTimeline, sentences: SegmentationResult["sentences"], chapterId: string, chapterOffset: number, chapterEndOffset: number, granularity: "sentence" | "word", locale: Intl.Locale): Promise<{
     sentenceRanges: SentenceRange[];
+    wordRanges: WordRange[][];
     transcriptionOffset: number;
     firstFoundSentence: number;
     lastFoundSentence: number;
 }>;
-declare function interpolateSentenceRanges(sentenceRanges: SentenceRange[], lastSentenceRange: SentenceRange | null): Promise<SentenceRange[]>;
+declare function interpolateSentenceRanges(sentenceRanges: SentenceRange[], chapterSentenceCounts: Record<string, number>): Promise<SentenceRange[]>;
 /**
  * Whisper sometimes provides words with no time information,
  * or start and end timestamps that are equal. EpubCheck complains
  * about these, so we nudge them out a bit to make sure that they're
  * not truly equal.
  */
-declare function expandEmptySentenceRanges(sentenceRanges: SentenceRange[]): SentenceRange[];
+declare function expandEmptySentenceRanges<Range extends SentenceRange | WordRange>(sentenceRanges: Range[]): Range[];
+declare function collapseSentenceRangeGaps(sentenceRanges: SentenceRange[]): Promise<SentenceRange[]>;
 declare function getChapterDuration(sentenceRanges: SentenceRange[]): number;
 
-export { type SentenceRange, type StorytellerTimelineEntry, type StorytellerTranscription, expandEmptySentenceRanges, findEndTimestamp, getChapterDuration, getSentenceRanges, interpolateSentenceRanges };
+export { type MappedTimeline, type SentenceRange, type StorytellerTimelineEntry, type StorytellerTranscription, type WordRange, collapseSentenceRangeGaps, expandEmptySentenceRanges, findEndTimestamp, getChapterDuration, getSentenceRanges, interpolateSentenceRanges, mapTranscriptionTimeline };

package/dist/align/getSentenceRanges.d.ts

@@ -1,4 +1,7 @@
+import * as _storyteller_platform_ghost_story from '@storyteller-platform/ghost-story';
 import { TimelineEntry } from '@storyteller-platform/ghost-story';
+import { SegmentationResult } from '@echogarden/text-segmentation';
+import { Mapping } from '@storyteller-platform/transliteration';
 
 type StorytellerTimelineEntry = TimelineEntry & {
     audiofile: string;
@@ -9,29 +12,56 @@ type StorytellerTranscription = {
 };
 type SentenceRange = {
     id: number;
+    chapterId: string;
     start: number;
     end: number;
     audiofile: string;
 };
-declare function findEndTimestamp(matchEndIndex: number, transcription: StorytellerTranscription): {
+type WordRange = {
+    id: number;
+    sentenceId: number;
+    start: number;
+    end: number;
+    audiofile: string;
+};
+declare function findEndTimestamp(matchEndIndex: number, timeline: MappedTimeline): {
     start: number;
     end: number;
     audiofile: string;
 } | null;
-declare function getSentenceRanges(transcription: StorytellerTranscription, sentences: string[], chapterOffset: number, chapterEndOffset: number, locale: Intl.Locale): Promise<{
+declare function mapTranscriptionTimeline(transcription: StorytellerTranscription, mapping: Mapping): {
+    mappedStartOffsetUtf16: number;
+    mappedEndOffsetUtf16: number;
+    type: _storyteller_platform_ghost_story.TimelineEntryType;
+    text: string;
+    startTime: number;
+    endTime: number;
+    startOffsetUtf16?: number;
+    endOffsetUtf16?: number;
+    startOffsetUtf32?: number;
+    endOffsetUtf32?: number;
+    confidence?: number;
+    id?: number;
+    timeline?: _storyteller_platform_ghost_story.Timeline;
+    audiofile: string;
+}[];
+type MappedTimeline = ReturnType<typeof mapTranscriptionTimeline>;
+declare function getSentenceRanges(transcriptionText: string, mappedTimeline: MappedTimeline, sentences: SegmentationResult["sentences"], chapterId: string, chapterOffset: number, chapterEndOffset: number, granularity: "sentence" | "word", locale: Intl.Locale): Promise<{
     sentenceRanges: SentenceRange[];
+    wordRanges: WordRange[][];
     transcriptionOffset: number;
     firstFoundSentence: number;
     lastFoundSentence: number;
 }>;
-declare function interpolateSentenceRanges(sentenceRanges: SentenceRange[], lastSentenceRange: SentenceRange | null): Promise<SentenceRange[]>;
+declare function interpolateSentenceRanges(sentenceRanges: SentenceRange[], chapterSentenceCounts: Record<string, number>): Promise<SentenceRange[]>;
 /**
  * Whisper sometimes provides words with no time information,
  * or start and end timestamps that are equal. EpubCheck complains
  * about these, so we nudge them out a bit to make sure that they're
  * not truly equal.
  */
-declare function expandEmptySentenceRanges(sentenceRanges: SentenceRange[]): SentenceRange[];
+declare function expandEmptySentenceRanges<Range extends SentenceRange | WordRange>(sentenceRanges: Range[]): Range[];
+declare function collapseSentenceRangeGaps(sentenceRanges: SentenceRange[]): Promise<SentenceRange[]>;
 declare function getChapterDuration(sentenceRanges: SentenceRange[]): number;
 
-export { type SentenceRange, type StorytellerTimelineEntry, type StorytellerTranscription, expandEmptySentenceRanges, findEndTimestamp, getChapterDuration, getSentenceRanges, interpolateSentenceRanges };
+export { type MappedTimeline, type SentenceRange, type StorytellerTimelineEntry, type StorytellerTranscription, type WordRange, collapseSentenceRangeGaps, expandEmptySentenceRanges, findEndTimestamp, getChapterDuration, getSentenceRanges, interpolateSentenceRanges, mapTranscriptionTimeline };