@storyteller-platform/align 0.1.9 → 0.1.11

package/dist/align/__tests__/align.test.cjs

@@ -131,13 +131,13 @@ async function assertAlignSnapshot(context, epub, transcriptionFilepaths) {
       "utf-8"
     );
     const chapterXml = import_epub.Epub.xhtmlParser.parse(chapterContents);
-    const segmentation = await (0, import_segmentation.getXhtmlSegmentation)(
+    const { result: segmentation } = await (0, import_segmentation.getXhtmlSegmentation)(
       import_epub.Epub.getXhtmlBody(chapterXml),
       {
         primaryLocale: new Intl.Locale("en-US")
       }
     );
-    const chapterSentences = segmentation.sentences.map((s) => s.text);
+    const chapterSentences = segmentation.map((s) => s.text).filter((s) => s.match(/\S/));
     for (const par of import_epub.Epub.getXmlChildren(seq)) {
       newSnapshot += `
 `;
@@ -151,14 +151,14 @@ async function assertAlignSnapshot(context, epub, transcriptionFilepaths) {
       if (sentenceId === void 0) continue;
       const textSentence = chapterSentences[parseInt(sentenceId)];
       if (!textSentence) continue;
-      newSnapshot += `Text:  ${textSentence}
+      newSnapshot += `Text:  ${textSentence.replace(/\n/, "")}
 `;
       const audioSrc = (_d = audio[":@"]) == null ? void 0 : _d["@_src"];
       if (!audioSrc) continue;
       const audioStart = (_e = audio[":@"]) == null ? void 0 : _e["@_clipBegin"];
       const audioEnd = (_f = audio[":@"]) == null ? void 0 : _f["@_clipEnd"];
       if (!audioStart || !audioEnd) continue;
-      const audioStartTime = parseFloat(audioStart.slice(0, -1));
+      const audioStartTime = parseFloat(audioStart.slice(0, -1)) - 2e-3;
       const audioEndTime = parseFloat(audioEnd.slice(0, -1));
       const audioFilename = (0, import_posix.basename)(audioSrc, (0, import_node_path.extname)(audioSrc));
       const transcriptionFilepath = transcriptionFilepaths.find(
@@ -271,7 +271,8 @@ void (0, import_node_test.describe)("align", () => {
         void 0,
         createTestLogger()
       );
-      await aligner.alignBook();
+      const timing = await aligner.alignBook();
+      if (!process.env["CI"]) timing.print();
       await assertAlignSnapshot(context, epub, transcriptionFilepaths);
     } catch (_) {
       var _error = _, _hasError = true;

package/dist/align/__tests__/align.test.js

@@ -67,13 +67,13 @@ async function assertAlignSnapshot(context, epub, transcriptionFilepaths) {
       "utf-8"
     );
     const chapterXml = Epub.xhtmlParser.parse(chapterContents);
-    const segmentation = await getXhtmlSegmentation(
+    const { result: segmentation } = await getXhtmlSegmentation(
       Epub.getXhtmlBody(chapterXml),
       {
         primaryLocale: new Intl.Locale("en-US")
       }
     );
-    const chapterSentences = segmentation.sentences.map((s) => s.text);
+    const chapterSentences = segmentation.map((s) => s.text).filter((s) => s.match(/\S/));
     for (const par of Epub.getXmlChildren(seq)) {
       newSnapshot += `
 `;
@@ -87,14 +87,14 @@ async function assertAlignSnapshot(context, epub, transcriptionFilepaths) {
       if (sentenceId === void 0) continue;
       const textSentence = chapterSentences[parseInt(sentenceId)];
       if (!textSentence) continue;
-      newSnapshot += `Text:  ${textSentence}
+      newSnapshot += `Text:  ${textSentence.replace(/\n/, "")}
 `;
       const audioSrc = (_d = audio[":@"]) == null ? void 0 : _d["@_src"];
       if (!audioSrc) continue;
       const audioStart = (_e = audio[":@"]) == null ? void 0 : _e["@_clipBegin"];
       const audioEnd = (_f = audio[":@"]) == null ? void 0 : _f["@_clipEnd"];
       if (!audioStart || !audioEnd) continue;
-      const audioStartTime = parseFloat(audioStart.slice(0, -1));
+      const audioStartTime = parseFloat(audioStart.slice(0, -1)) - 2e-3;
       const audioEndTime = parseFloat(audioEnd.slice(0, -1));
       const audioFilename = posixBasename(audioSrc, extname(audioSrc));
       const transcriptionFilepath = transcriptionFilepaths.find(
@@ -207,7 +207,8 @@ void describe("align", () => {
         void 0,
         createTestLogger()
       );
-      await aligner.alignBook();
+      const timing = await aligner.alignBook();
+      if (!process.env["CI"]) timing.print();
       await assertAlignSnapshot(context, epub, transcriptionFilepaths);
     } catch (_) {
       var _error = _, _hasError = true;

package/dist/align/align.cjs

@@ -81,7 +81,9 @@ module.exports = __toCommonJS(align_exports);
 var import_promises = require("node:fs/promises");
 var import_node_path = require("node:path");
 var import_posix = require("node:path/posix");
+var import_itertools = require("itertools");
 var import_memoize = __toESM(require("memoize"), 1);
+var import_runes2 = require("runes2");
 var import_audiobook = require("@storyteller-platform/audiobook");
 var import_epub = require("@storyteller-platform/epub");
 var import_ghost_story = require("@storyteller-platform/ghost-story");
@@ -90,7 +92,6 @@ var import_segmentation = require("../markup/segmentation.cjs");
 var import_fuzzy = require("./fuzzy.cjs");
 var import_getSentenceRanges = require("./getSentenceRanges.cjs");
 var import_slugify = require("./slugify.cjs");
-const OFFSET_SEARCH_WINDOW_SIZE = 5e3;
 async function align(input, output, transcriptionsDir, audiobookDir, options) {
   var _stack = [];
   try {
@@ -141,6 +142,7 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
 class Aligner {
   constructor(epub, audiofiles, transcriptions, granularity, languageOverride, logger) {
     this.epub = epub;
+    this.audiofiles = audiofiles;
     this.languageOverride = languageOverride;
     this.logger = logger;
     this.transcription = concatTranscriptions(transcriptions, audiofiles);
@@ -155,71 +157,92 @@ class Aligner {
   report = {
     chapters: []
   };
-  findBestOffset(epubSentences, transcriptionText, lastMatchOffset, mapping) {
-    let i = 0;
-    while (i < transcriptionText.length) {
-      let startSentence = 0;
-      const proposedStartIndex = (lastMatchOffset + i) % transcriptionText.length;
-      const proposedEndIndex = (proposedStartIndex + OFFSET_SEARCH_WINDOW_SIZE) % transcriptionText.length;
-      const wrapping = proposedEndIndex < proposedStartIndex;
-      let endIndex = wrapping ? transcriptionText.length : proposedEndIndex;
-      let startIndex = proposedStartIndex;
-      let startSeen = null;
-      let endSeen = null;
-      for (const aligned of this.alignedChapters) {
-        const alignedStart = mapping.map(aligned.startOffset, -1);
-        const alignedEnd = mapping.map(aligned.endOffset, -1);
-        if (startSeen !== null && endSeen === alignedStart) {
-          endSeen = alignedEnd;
-        } else {
-          startSeen = alignedStart;
-          endSeen = alignedEnd;
-        }
-        if (startIndex >= startSeen && startIndex < endSeen) {
-          startIndex = endSeen;
-        }
-        if (endIndex >= startSeen && endIndex <= endSeen) {
-          endIndex = startSeen;
+  findBestOffset(epubSentences, transcriptionText, lastMatchOffset, dir = 1) {
+    const reverse = dir < 0;
+    if (dir < 0) {
+      epubSentences = epubSentences.toReversed().map((s) => (0, import_runes2.runes)(s).toReversed().join(""));
+      transcriptionText = (0, import_runes2.runes)(transcriptionText).toReversed().join("");
+      lastMatchOffset = transcriptionText.length - lastMatchOffset;
+    }
+    const flatSliceIndices = [
+      0,
+      ...this.alignedChapters.toSorted(
+        (a, b) => reverse ? transcriptionText.length - a.endOffset - (transcriptionText.length - b.endOffset) : a.startOffset - b.startOffset
+      ).flatMap((aligned) => [
+        reverse ? transcriptionText.length - aligned.endOffset : aligned.startOffset,
+        reverse ? transcriptionText.length - aligned.startOffset : aligned.endOffset
+      ]),
+      transcriptionText.length
+    ];
+    const sliceIndices = [];
+    for (let i = 0; i < flatSliceIndices.length - 1; i += 2) {
+      sliceIndices.push([flatSliceIndices[i], flatSliceIndices[i + 1]]);
+    }
+    const allSlices = [];
+    let startSlice = 0;
+    for (const [i, [start, end]] of (0, import_itertools.enumerate)(sliceIndices)) {
+      if (lastMatchOffset >= start && lastMatchOffset < end) {
+        if (!reverse) {
+          startSlice = i + 1;
+          allSlices.push({
+            start,
+            text: transcriptionText.slice(start, lastMatchOffset)
+          });
         }
+        allSlices.push({
+          start: lastMatchOffset,
+          text: transcriptionText.slice(lastMatchOffset, end)
+        });
+      } else if (!reverse) {
+        allSlices.push({ start, text: transcriptionText.slice(start, end) });
+      }
+    }
+    const slices = allSlices.filter((slice) => slice.text.length);
+    if (reverse && !slices.length) {
+      const indices = sliceIndices.find(([start]) => start > lastMatchOffset);
+      if (indices) {
+        slices.push({
+          start: indices[0],
+          text: transcriptionText.slice(...indices)
+        });
       }
-      if (startIndex < endIndex) {
-        const transcriptionTextSlice = transcriptionText.slice(
-          startIndex,
-          endIndex
+    }
+    for (const slice of slices.slice(startSlice).concat(slices.slice(0, startSlice))) {
+      let startSentence = 0;
+      while (startSentence < epubSentences.length) {
+        const needle = epubSentences.slice(startSentence, startSentence + 6).join("-");
+        const firstMatch = (0, import_fuzzy.findNearestMatch)(
+          needle,
+          slice.text,
+          Math.max(Math.floor(0.1 * needle.length), 1)
         );
-        while (startSentence < epubSentences.length) {
-          const queryString = epubSentences.slice(startSentence, startSentence + 6).join("-");
-          const firstMatch = (0, import_fuzzy.findNearestMatch)(
-            queryString.toLowerCase(),
-            transcriptionTextSlice.toLowerCase(),
-            Math.max(Math.floor(0.1 * queryString.length), 1)
-          );
-          if (firstMatch) {
-            return {
-              startSentence,
-              transcriptionOffset: (firstMatch.index + startIndex) % transcriptionText.length
-            };
-          }
-          startSentence += 3;
+        if (firstMatch) {
+          const start = reverse ? transcriptionText.length - (slice.start + firstMatch.index) : slice.start + firstMatch.index;
+          return {
+            startSentence: reverse ? epubSentences.length - startSentence : startSentence,
+            transcriptionOffset: start
+          };
         }
+        startSentence += 3;
       }
-      if (wrapping) {
-        i += transcriptionText.length - proposedStartIndex;
-      } else {
-        i += Math.floor(OFFSET_SEARCH_WINDOW_SIZE / 2);
-      }
+    }
+    if (reverse) {
+      return {
+        startSentence: epubSentences.length,
+        transcriptionOffset: slices[0] ? transcriptionText.length - slices[0].start : null
+      };
     }
     return { startSentence: 0, transcriptionOffset: null };
   }
   async getChapterSentences(chapterId) {
     const chapterXml = await this.epub.readXhtmlItemContents(chapterId);
-    const segmentation = await (0, import_segmentation.getXhtmlSegmentation)(
+    const { result: segmentation } = await (0, import_segmentation.getXhtmlSegmentation)(
       import_epub.Epub.getXhtmlBody(chapterXml),
       {
         primaryLocale: this.languageOverride ?? await this.epub.getLanguage()
       }
     );
-    return segmentation.sentences.map((s) => s.text);
+    return segmentation.map((s) => s.text).filter((s) => s.match(/\S/));
   }
   async writeAlignedChapter(alignedChapter) {
     const { chapter, sentenceRanges, xml } = alignedChapter;
@@ -314,7 +337,7 @@ class Aligner {
       }, [])
     });
   }
-  async alignChapter(startSentence, chapterId, transcriptionOffset, locale, lastSentenceRange) {
+  async alignChapter(startSentence, endSentence, chapterId, transcriptionOffset, transcriptionEndOffset, locale, mapping) {
     const timing = (0, import_ghost_story.createTiming)();
     timing.start("read contents");
     const manifest = await this.epub.getManifest();
@@ -331,20 +354,14 @@ class Aligner {
     timing.start("align sentences");
     const { sentenceRanges, transcriptionOffset: endTranscriptionOffset } = await (0, import_getSentenceRanges.getSentenceRanges)(
       startSentence,
+      endSentence,
       this.transcription,
       chapterSentences,
       transcriptionOffset,
-      locale,
-      lastSentenceRange
+      transcriptionEndOffset,
+      locale
     );
     timing.end("align sentences");
-    timing.start("expand ranges");
-    const interpolated = await (0, import_getSentenceRanges.interpolateSentenceRanges)(
-      sentenceRanges,
-      lastSentenceRange
-    );
-    const expanded = (0, import_getSentenceRanges.expandEmptySentenceRanges)(interpolated);
-    timing.end("expand ranges");
     const storytellerStylesheetUrl = (0, import_posix.relative)(
       (0, import_posix.dirname)(chapter.href),
       "Styles/storyteller-readaloud.css"
@@ -357,25 +374,25 @@ class Aligner {
     this.alignedChapters.push({
       chapter,
       xml: chapterXml,
-      sentenceRanges: expanded,
-      startOffset: transcriptionOffset,
-      endOffset: endTranscriptionOffset
+      sentenceRanges,
+      startOffset: mapping.map(transcriptionOffset),
+      endOffset: mapping.map(endTranscriptionOffset, -1)
     });
     this.addChapterReport(
       chapter,
       chapterSentences,
-      expanded,
+      sentenceRanges,
       startSentence,
       transcriptionOffset
     );
     return {
-      lastSentenceRange: expanded[expanded.length - 1] ?? null,
+      lastSentenceRange: sentenceRanges.at(-1) ?? null,
       endTranscriptionOffset,
       timing
     };
   }
   async alignBook(onProgress) {
-    var _a, _b, _c, _d, _e, _f, _g;
+    var _a, _b, _c, _d, _e, _f, _g, _h;
     const locale = this.languageOverride ?? await this.epub.getLanguage() ?? new Intl.Locale("en-US");
     this.timing.setMetadata("language", locale.toString());
     this.timing.setMetadata("granularity", this.granularity);
@@ -386,7 +403,6 @@ class Aligner {
       locale
     );
     let lastTranscriptionOffset = 0;
-    let lastSentenceRange = null;
     for (let index = 0; index < spine.length; index++) {
       onProgress == null ? void 0 : onProgress(index / spine.length);
       const spineItem = spine[index];
@@ -418,36 +434,72 @@ class Aligner {
       const { startSentence, transcriptionOffset: slugifiedOffset } = this.findBestOffset(
         slugifiedChapterSentences,
         transcriptionText,
-        mapping.map(lastTranscriptionOffset, -1),
-        mapping
+        mapping.map(lastTranscriptionOffset, -1)
       );
-      const transcriptionOffset = slugifiedOffset && mapping.invert().map(slugifiedOffset, -1);
-      if (transcriptionOffset === null) {
+      if (slugifiedOffset === null) {
         (_f = this.logger) == null ? void 0 : _f.info(
           `Couldn't find matching transcription for chapter #${index}`
         );
         continue;
       }
-      (_g = this.logger) == null ? void 0 : _g.info(
-        `Chapter #${index} best matches transcription at offset ${transcriptionOffset}, starting at sentence ${startSentence}`
+      const transcriptionOffset = mapping.invert().map(slugifiedOffset, -1);
+      const {
+        startSentence: startEndSentence,
+        transcriptionOffset: slugifiedEndOffset
+      } = this.findBestOffset(
+        slugifiedChapterSentences,
+        transcriptionText,
+        Math.min(
+          transcriptionText.length,
+          slugifiedOffset + Math.round(slugifiedChapterSentences.join("-").length * 1.2)
+        ),
+        -1
+      );
+      const endSentence = startEndSentence;
+      const endOffset = slugifiedEndOffset === null ? this.transcription.transcript.length : mapping.invert().map(slugifiedEndOffset, 1);
+      if (endSentence - startSentence < slugifiedChapterSentences.length / 2) {
+        (_g = this.logger) == null ? void 0 : _g.info(`Found less than half of chapter #${index}, skipping`);
+      }
+      (_h = this.logger) == null ? void 0 : _h.info(
+        `Chapter #${index} best matches transcription from ${transcriptionOffset} to ${endOffset}, from sentence ${startSentence} to ${endSentence} (of ${slugifiedChapterSentences.length}) in the book`
       );
       const result = await this.alignChapter(
         startSentence,
+        endSentence,
         chapterId,
         transcriptionOffset,
+        endOffset,
         locale,
-        lastSentenceRange
+        mapping
       );
-      lastSentenceRange = result.lastSentenceRange;
       lastTranscriptionOffset = result.endTranscriptionOffset;
       this.timing.add(result.timing.summary());
     }
-    if (lastSentenceRange) {
-      lastSentenceRange.end = await (0, import_ffmpeg.getTrackDuration)(
-        lastSentenceRange.audiofile
+    const audioOrderedChapters = this.alignedChapters.toSorted((a, b) => {
+      const firstRangeA = a.sentenceRanges[0];
+      const firstRangeB = b.sentenceRanges[0];
+      if (!firstRangeA) return 1;
+      if (!firstRangeB) return -1;
+      const firstAudiofileIndexA = this.audiofiles.indexOf(
+        firstRangeA.audiofile
       );
-    }
-    for (const alignedChapter of this.alignedChapters) {
+      const firstAudiofileIndexB = this.audiofiles.indexOf(
+        firstRangeB.audiofile
+      );
+      if (firstAudiofileIndexA === firstAudiofileIndexB) {
+        return firstRangeA.start - firstRangeB.start;
+      }
+      return firstAudiofileIndexA - firstAudiofileIndexB;
+    });
+    let lastSentenceRange = null;
+    for (const alignedChapter of audioOrderedChapters) {
+      const interpolated = await (0, import_getSentenceRanges.interpolateSentenceRanges)(
+        alignedChapter.sentenceRanges,
+        lastSentenceRange
+      );
+      const expanded = (0, import_getSentenceRanges.expandEmptySentenceRanges)(interpolated);
+      alignedChapter.sentenceRanges = expanded;
+      lastSentenceRange = expanded.at(-1) ?? null;
       await this.writeAlignedChapter(alignedChapter);
     }
     await this.epub.addMetadata({

package/dist/align/align.d.cts

@@ -37,6 +37,7 @@ interface AlignOptions {
 declare function align(input: string, output: string, transcriptionsDir: string, audiobookDir: string, options: AlignOptions): Promise<_storyteller_platform_ghost_story.TimingAggregator>;
 declare class Aligner {
     epub: Epub;
+    private audiofiles;
     private languageOverride?;
     private logger?;
     private transcription;

package/dist/align/align.d.ts

@@ -37,6 +37,7 @@ interface AlignOptions {
 declare function align(input: string, output: string, transcriptionsDir: string, audiobookDir: string, options: AlignOptions): Promise<_storyteller_platform_ghost_story.TimingAggregator>;
 declare class Aligner {
     epub: Epub;
+    private audiofiles;
     private languageOverride?;
     private logger?;
     private transcription;