@storyteller-platform/align 0.1.19 → 0.1.20

package/dist/align/align.cjs

@@ -81,6 +81,7 @@ module.exports = __toCommonJS(align_exports);
 var import_promises = require("node:fs/promises");
 var import_node_path = require("node:path");
 var import_posix = require("node:path/posix");
+var import_itertools = require("itertools");
 var import_memoize = __toESM(require("memoize"), 1);
 var import_audiobook = require("@storyteller-platform/audiobook");
 var import_epub = require("@storyteller-platform/epub");
@@ -173,10 +174,11 @@ class Aligner {
         primaryLocale: this.languageOverride ?? await this.epub.getLanguage()
       }
     );
-    return segmentation.map((s) => s.text).filter((s) => s.match(/\S/));
+    return segmentation.filter((s) => s.text.match(/\S/));
   }
   async writeAlignedChapter(alignedChapter) {
-    const { chapter, sentenceRanges, xml } = alignedChapter;
+    const { chapter, sentenceRanges, wordRanges, xml } = alignedChapter;
+    const wordRangeMap = new Map(wordRanges.map((w) => [w[0].sentenceId, w]));
     const audiofiles = Array.from(
       new Set(sentenceRanges.map(({ audiofile }) => audiofile))
     );
@@ -209,7 +211,12 @@ class Aligner {
         href: `MediaOverlays/${chapterStem}.smil`,
         mediaType: "application/smil+xml"
       },
-      createMediaOverlay(chapter, sentenceRanges),
+      createMediaOverlay(
+        chapter,
+        this.granularity,
+        sentenceRanges,
+        wordRangeMap
+      ),
       "xml"
     );
     await this.epub.updateManifestItem(chapter.id, {
@@ -246,17 +253,17 @@ class Aligner {
       },
       firstMatchedSentenceId: startSentence,
       firstMatchedSentenceContext: {
-        prevSentence: chapterSentences[startSentence - 1] ?? null,
+        prevSentence: chapterSentences[startSentence - 1]?.text ?? null,
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        matchedSentence: chapterSentences[startSentence],
-        nextSentence: chapterSentences[startSentence + 1] ?? null
+        matchedSentence: chapterSentences[startSentence].text,
+        nextSentence: chapterSentences[startSentence + 1]?.text ?? null
       },
       lastMatchedSentenceId: endSentence,
       lastMatchedSentenceContext: {
-        prevSentence: chapterSentences[endSentence - 1] ?? null,
+        prevSentence: chapterSentences[endSentence - 1]?.text ?? null,
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        matchedSentence: chapterSentences[endSentence],
-        nextSentence: chapterSentences[endSentence + 1] ?? null
+        matchedSentence: chapterSentences[endSentence].text,
+        nextSentence: chapterSentences[endSentence + 1]?.text ?? null
       },
       chapterSentenceCount: chapterSentences.length,
       alignedSentenceCount: sentenceRanges.length,
@@ -277,7 +284,7 @@ class Aligner {
       }, [])
     });
   }
-  async alignChapter(chapterId, transcriptionOffset, transcriptionEndOffset, locale, mapping) {
+  async alignChapter(chapterId, transcriptionText, transcriptionOffset, transcriptionEndOffset, locale, mappedTimeline) {
     const timing = (0, import_ghost_story.createTiming)();
     timing.start("read contents");
     const manifest = await this.epub.getManifest();
@@ -294,14 +301,18 @@ class Aligner {
     timing.start("align sentences");
     const {
       sentenceRanges,
+      wordRanges,
       transcriptionOffset: endTranscriptionOffset,
       firstFoundSentence,
       lastFoundSentence
     } = await (0, import_getSentenceRanges.getSentenceRanges)(
-      this.transcription,
+      transcriptionText,
+      mappedTimeline,
       chapterSentences,
+      chapterId,
       transcriptionOffset,
       transcriptionEndOffset,
+      this.granularity,
       locale
     );
     timing.end("align sentences");
@@ -318,8 +329,9 @@ class Aligner {
       chapter,
       xml: chapterXml,
       sentenceRanges,
-      startOffset: mapping.map(transcriptionOffset),
-      endOffset: mapping.map(endTranscriptionOffset, -1)
+      wordRanges,
+      startOffset: transcriptionOffset,
+      endOffset: endTranscriptionOffset
     });
     this.addChapterReport(
       chapter,
@@ -357,6 +369,7 @@ class Aligner {
       this.transcription.transcript,
       locale
     );
+    const mappedTimeline = (0, import_getSentenceRanges.mapTranscriptionTimeline)(this.transcription, mapping);
     for (let index = 0; index < spine.length; index++) {
       onProgress?.(index / spine.length);
       const spineItem = spine[index];
@@ -371,7 +384,7 @@ class Aligner {
       const slugifiedChapterSentences = [];
       for (const chapterSentence of chapterSentences) {
         slugifiedChapterSentences.push(
-          (await (0, import_slugify.slugify)(chapterSentence, locale)).result
+          (await (0, import_slugify.slugify)(chapterSentence.text, locale)).result
         );
       }
       if (chapterSentences.length === 0) {
@@ -379,7 +392,7 @@ class Aligner {
         continue;
       }
       if (chapterSentences.length < 2 && // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      chapterSentences[0].split(" ").length < 4) {
+      chapterSentences[0].words.length < 4) {
         this.logger?.info(
           `Chapter #${index} is fewer than four words; skipping`
         );
@@ -399,14 +412,13 @@ class Aligner {
       if (start === end) {
         continue;
       }
-      const transcriptionOffset = mapping.invert().map(Math.max(start, 0), -1);
-      const endOffset = mapping.invert().map(Math.min(end, transcriptionText.length), 1);
       const result = await this.alignChapter(
         chapterId,
-        transcriptionOffset,
-        endOffset,
+        transcriptionText,
+        Math.max(start, 0),
+        Math.min(end, transcriptionText.length),
         locale,
-        mapping
+        mappedTimeline
       );
       this.timing.add(result.timing.summary());
     }
@@ -426,16 +438,36 @@ class Aligner {
       }
       return firstAudiofileIndexA - firstAudiofileIndexB;
     });
-    let lastSentenceRange = null;
+    const sentenceRanges = [];
+    const chapterSentenceCounts = {};
     for (const alignedChapter of audioOrderedChapters) {
-      const interpolated = await (0, import_getSentenceRanges.interpolateSentenceRanges)(
-        alignedChapter.sentenceRanges,
-        lastSentenceRange
+      sentenceRanges.push(...alignedChapter.sentenceRanges);
+      const sentences = await this.getChapterSentences(
+        alignedChapter.chapter.id
       );
-      const expanded = (0, import_getSentenceRanges.expandEmptySentenceRanges)(interpolated);
-      alignedChapter.sentenceRanges = expanded;
-      lastSentenceRange = expanded.at(-1) ?? null;
+      chapterSentenceCounts[alignedChapter.chapter.id] = sentences.length;
+    }
+    const interpolated = await (0, import_getSentenceRanges.interpolateSentenceRanges)(
+      sentenceRanges,
+      chapterSentenceCounts
+    );
+    const expanded = (0, import_getSentenceRanges.expandEmptySentenceRanges)(interpolated);
+    const collapsed = await (0, import_getSentenceRanges.collapseSentenceRangeGaps)(expanded);
+    let collapsedStart = 0;
+    for (const alignedChapter of audioOrderedChapters) {
+      const sentences = await this.getChapterSentences(
+        alignedChapter.chapter.id
+      );
+      const finalSentenceRanges = collapsed.slice(
+        collapsedStart,
+        collapsedStart + sentences.length - 1
+      );
+      alignedChapter.sentenceRanges = finalSentenceRanges;
+      for (const [i, wordRanges] of (0, import_itertools.enumerate)(alignedChapter.wordRanges)) {
+        alignedChapter.wordRanges[i] = (0, import_getSentenceRanges.expandEmptySentenceRanges)(wordRanges);
+      }
       await this.writeAlignedChapter(alignedChapter);
+      collapsedStart += sentences.length - 1;
     }
     await this.epub.addMetadata({
       type: "meta",
@@ -463,7 +495,7 @@ class Aligner {
     return this.timing;
   }
 }
-function createMediaOverlay(chapter, sentenceRanges) {
+function createMediaOverlay(chapter, granularity, sentenceRanges, wordRanges) {
   return [
     import_epub.Epub.createXmlElement(
       "smil",
@@ -481,24 +513,53 @@ function createMediaOverlay(chapter, sentenceRanges) {
               "epub:textref": `../${chapter.href}`,
               "epub:type": "chapter"
             },
-            sentenceRanges.map(
-              (sentenceRange) => import_epub.Epub.createXmlElement(
-                "par",
+            sentenceRanges.map((sentenceRange) => {
+              if (granularity === "sentence" || !wordRanges.has(sentenceRange.id)) {
+                return import_epub.Epub.createXmlElement(
+                  "par",
+                  {
+                    id: `${chapter.id}-s${sentenceRange.id}`
+                  },
+                  [
+                    import_epub.Epub.createXmlElement("text", {
+                      src: `../${chapter.href}#${chapter.id}-s${sentenceRange.id}`
+                    }),
+                    import_epub.Epub.createXmlElement("audio", {
+                      src: `../Audio/${(0, import_posix.basename)(sentenceRange.audiofile)}`,
+                      clipBegin: `${sentenceRange.start.toFixed(3)}s`,
+                      clipEnd: `${sentenceRange.end.toFixed(3)}s`
+                    })
+                  ]
+                );
+              }
+              const words = wordRanges.get(sentenceRange.id);
+              return import_epub.Epub.createXmlElement(
+                "seq",
                 {
-                  id: `${chapter.id}-s${sentenceRange.id}`
+                  id: `${chapter.id}-s${sentenceRange.id}`,
+                  "epub:type": "text-range-small",
+                  "epub:textref": `../${chapter.href}#${chapter.id}-s${sentenceRange.id}`
                 },
-                [
-                  import_epub.Epub.createXmlElement("text", {
-                    src: `../${chapter.href}#${chapter.id}-s${sentenceRange.id}`
-                  }),
-                  import_epub.Epub.createXmlElement("audio", {
-                    src: `../Audio/${(0, import_posix.basename)(sentenceRange.audiofile)}`,
-                    clipBegin: `${sentenceRange.start.toFixed(3)}s`,
-                    clipEnd: `${sentenceRange.end.toFixed(3)}s`
-                  })
-                ]
-              )
-            )
+                words.map(
+                  (word) => import_epub.Epub.createXmlElement(
+                    "par",
+                    {
+                      id: `${chapter.id}-s${sentenceRange.id}-w${word.id}`
+                    },
+                    [
+                      import_epub.Epub.createXmlElement("text", {
+                        src: `../${chapter.href}#${chapter.id}-s${sentenceRange.id}-w${word.id}`
+                      }),
+                      import_epub.Epub.createXmlElement("audio", {
+                        src: `../Audio/${(0, import_posix.basename)(word.audiofile)}`,
+                        clipBegin: `${word.start.toFixed(3)}s`,
+                        clipEnd: `${word.end.toFixed(3)}s`
+                      })
+                    ]
+                  )
+                )
+              );
+            })
           )
         ])
       ]

package/dist/align/align.d.cts

@@ -3,6 +3,8 @@ import { Logger } from 'pino';
 import { Epub } from '@storyteller-platform/epub';
 import { RecognitionResult } from '@storyteller-platform/ghost-story/recognition';
 import { StorytellerTranscription } from './getSentenceRanges.cjs';
+import '@echogarden/text-segmentation';
+import '@storyteller-platform/transliteration';
 
 interface AudioFileContext {
     start: number;

package/dist/align/align.d.ts

@@ -3,6 +3,8 @@ import { Logger } from 'pino';
 import { Epub } from '@storyteller-platform/epub';
 import { RecognitionResult } from '@storyteller-platform/ghost-story/recognition';
 import { StorytellerTranscription } from './getSentenceRanges.js';
+import '@echogarden/text-segmentation';
+import '@storyteller-platform/transliteration';
 
 interface AudioFileContext {
     start: number;

package/dist/align/align.js

@@ -5,6 +5,7 @@ import {
 import { copyFile, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import { dirname as autoDirname, join as autoJoin } from "node:path";
 import { basename, dirname, parse, relative } from "node:path/posix";
+import { enumerate } from "itertools";
 import memoize from "memoize";
 import { isAudioFile, lookupAudioMime } from "@storyteller-platform/audiobook";
 import {
@@ -17,10 +18,12 @@ import {
 import { getTrackDuration } from "../common/ffmpeg.js";
 import { getXhtmlSegmentation } from "../markup/segmentation.js";
 import {
+  collapseSentenceRangeGaps,
   expandEmptySentenceRanges,
   getChapterDuration,
   getSentenceRanges,
-  interpolateSentenceRanges
+  interpolateSentenceRanges,
+  mapTranscriptionTimeline
 } from "./getSentenceRanges.js";
 import { findBoundaries } from "./search.js";
 import { slugify } from "./slugify.js";
@@ -107,10 +110,11 @@ class Aligner {
         primaryLocale: this.languageOverride ?? await this.epub.getLanguage()
       }
     );
-    return segmentation.map((s) => s.text).filter((s) => s.match(/\S/));
+    return segmentation.filter((s) => s.text.match(/\S/));
   }
   async writeAlignedChapter(alignedChapter) {
-    const { chapter, sentenceRanges, xml } = alignedChapter;
+    const { chapter, sentenceRanges, wordRanges, xml } = alignedChapter;
+    const wordRangeMap = new Map(wordRanges.map((w) => [w[0].sentenceId, w]));
     const audiofiles = Array.from(
       new Set(sentenceRanges.map(({ audiofile }) => audiofile))
     );
@@ -143,7 +147,12 @@ class Aligner {
         href: `MediaOverlays/${chapterStem}.smil`,
         mediaType: "application/smil+xml"
       },
-      createMediaOverlay(chapter, sentenceRanges),
+      createMediaOverlay(
+        chapter,
+        this.granularity,
+        sentenceRanges,
+        wordRangeMap
+      ),
       "xml"
     );
     await this.epub.updateManifestItem(chapter.id, {
@@ -180,17 +189,17 @@ class Aligner {
       },
       firstMatchedSentenceId: startSentence,
       firstMatchedSentenceContext: {
-        prevSentence: chapterSentences[startSentence - 1] ?? null,
+        prevSentence: chapterSentences[startSentence - 1]?.text ?? null,
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        matchedSentence: chapterSentences[startSentence],
-        nextSentence: chapterSentences[startSentence + 1] ?? null
+        matchedSentence: chapterSentences[startSentence].text,
+        nextSentence: chapterSentences[startSentence + 1]?.text ?? null
       },
       lastMatchedSentenceId: endSentence,
       lastMatchedSentenceContext: {
-        prevSentence: chapterSentences[endSentence - 1] ?? null,
+        prevSentence: chapterSentences[endSentence - 1]?.text ?? null,
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        matchedSentence: chapterSentences[endSentence],
-        nextSentence: chapterSentences[endSentence + 1] ?? null
+        matchedSentence: chapterSentences[endSentence].text,
+        nextSentence: chapterSentences[endSentence + 1]?.text ?? null
       },
       chapterSentenceCount: chapterSentences.length,
       alignedSentenceCount: sentenceRanges.length,
@@ -211,7 +220,7 @@ class Aligner {
       }, [])
     });
   }
-  async alignChapter(chapterId, transcriptionOffset, transcriptionEndOffset, locale, mapping) {
+  async alignChapter(chapterId, transcriptionText, transcriptionOffset, transcriptionEndOffset, locale, mappedTimeline) {
     const timing = createTiming();
     timing.start("read contents");
     const manifest = await this.epub.getManifest();
@@ -228,14 +237,18 @@ class Aligner {
     timing.start("align sentences");
     const {
       sentenceRanges,
+      wordRanges,
       transcriptionOffset: endTranscriptionOffset,
       firstFoundSentence,
       lastFoundSentence
     } = await getSentenceRanges(
-      this.transcription,
+      transcriptionText,
+      mappedTimeline,
       chapterSentences,
+      chapterId,
       transcriptionOffset,
       transcriptionEndOffset,
+      this.granularity,
       locale
     );
     timing.end("align sentences");
@@ -252,8 +265,9 @@ class Aligner {
       chapter,
       xml: chapterXml,
       sentenceRanges,
-      startOffset: mapping.map(transcriptionOffset),
-      endOffset: mapping.map(endTranscriptionOffset, -1)
+      wordRanges,
+      startOffset: transcriptionOffset,
+      endOffset: endTranscriptionOffset
     });
     this.addChapterReport(
       chapter,
@@ -291,6 +305,7 @@ class Aligner {
       this.transcription.transcript,
       locale
     );
+    const mappedTimeline = mapTranscriptionTimeline(this.transcription, mapping);
     for (let index = 0; index < spine.length; index++) {
       onProgress?.(index / spine.length);
       const spineItem = spine[index];
@@ -305,7 +320,7 @@ class Aligner {
       const slugifiedChapterSentences = [];
       for (const chapterSentence of chapterSentences) {
         slugifiedChapterSentences.push(
-          (await slugify(chapterSentence, locale)).result
+          (await slugify(chapterSentence.text, locale)).result
         );
       }
       if (chapterSentences.length === 0) {
@@ -313,7 +328,7 @@ class Aligner {
         continue;
       }
       if (chapterSentences.length < 2 && // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      chapterSentences[0].split(" ").length < 4) {
+      chapterSentences[0].words.length < 4) {
         this.logger?.info(
           `Chapter #${index} is fewer than four words; skipping`
         );
@@ -333,14 +348,13 @@ class Aligner {
       if (start === end) {
         continue;
       }
-      const transcriptionOffset = mapping.invert().map(Math.max(start, 0), -1);
-      const endOffset = mapping.invert().map(Math.min(end, transcriptionText.length), 1);
       const result = await this.alignChapter(
         chapterId,
-        transcriptionOffset,
-        endOffset,
+        transcriptionText,
+        Math.max(start, 0),
+        Math.min(end, transcriptionText.length),
         locale,
-        mapping
+        mappedTimeline
       );
       this.timing.add(result.timing.summary());
     }
@@ -360,16 +374,36 @@ class Aligner {
       }
       return firstAudiofileIndexA - firstAudiofileIndexB;
     });
-    let lastSentenceRange = null;
+    const sentenceRanges = [];
+    const chapterSentenceCounts = {};
     for (const alignedChapter of audioOrderedChapters) {
-      const interpolated = await interpolateSentenceRanges(
-        alignedChapter.sentenceRanges,
-        lastSentenceRange
+      sentenceRanges.push(...alignedChapter.sentenceRanges);
+      const sentences = await this.getChapterSentences(
+        alignedChapter.chapter.id
       );
-      const expanded = expandEmptySentenceRanges(interpolated);
-      alignedChapter.sentenceRanges = expanded;
-      lastSentenceRange = expanded.at(-1) ?? null;
+      chapterSentenceCounts[alignedChapter.chapter.id] = sentences.length;
+    }
+    const interpolated = await interpolateSentenceRanges(
+      sentenceRanges,
+      chapterSentenceCounts
+    );
+    const expanded = expandEmptySentenceRanges(interpolated);
+    const collapsed = await collapseSentenceRangeGaps(expanded);
+    let collapsedStart = 0;
+    for (const alignedChapter of audioOrderedChapters) {
+      const sentences = await this.getChapterSentences(
+        alignedChapter.chapter.id
+      );
+      const finalSentenceRanges = collapsed.slice(
+        collapsedStart,
+        collapsedStart + sentences.length - 1
+      );
+      alignedChapter.sentenceRanges = finalSentenceRanges;
+      for (const [i, wordRanges] of enumerate(alignedChapter.wordRanges)) {
+        alignedChapter.wordRanges[i] = expandEmptySentenceRanges(wordRanges);
+      }
       await this.writeAlignedChapter(alignedChapter);
+      collapsedStart += sentences.length - 1;
     }
     await this.epub.addMetadata({
       type: "meta",
@@ -397,7 +431,7 @@ class Aligner {
     return this.timing;
   }
 }
-function createMediaOverlay(chapter, sentenceRanges) {
+function createMediaOverlay(chapter, granularity, sentenceRanges, wordRanges) {
   return [
     Epub.createXmlElement(
       "smil",
@@ -415,24 +449,53 @@ function createMediaOverlay(chapter, sentenceRanges) {
               "epub:textref": `../${chapter.href}`,
               "epub:type": "chapter"
             },
-            sentenceRanges.map(
-              (sentenceRange) => Epub.createXmlElement(
-                "par",
+            sentenceRanges.map((sentenceRange) => {
+              if (granularity === "sentence" || !wordRanges.has(sentenceRange.id)) {
+                return Epub.createXmlElement(
+                  "par",
+                  {
+                    id: `${chapter.id}-s${sentenceRange.id}`
+                  },
+                  [
+                    Epub.createXmlElement("text", {
+                      src: `../${chapter.href}#${chapter.id}-s${sentenceRange.id}`
+                    }),
+                    Epub.createXmlElement("audio", {
+                      src: `../Audio/${basename(sentenceRange.audiofile)}`,
+                      clipBegin: `${sentenceRange.start.toFixed(3)}s`,
+                      clipEnd: `${sentenceRange.end.toFixed(3)}s`
+                    })
+                  ]
+                );
+              }
+              const words = wordRanges.get(sentenceRange.id);
+              return Epub.createXmlElement(
+                "seq",
                 {
-                  id: `${chapter.id}-s${sentenceRange.id}`
+                  id: `${chapter.id}-s${sentenceRange.id}`,
+                  "epub:type": "text-range-small",
+                  "epub:textref": `../${chapter.href}#${chapter.id}-s${sentenceRange.id}`
                 },
-                [
-                  Epub.createXmlElement("text", {
-                    src: `../${chapter.href}#${chapter.id}-s${sentenceRange.id}`
-                  }),
-                  Epub.createXmlElement("audio", {
-                    src: `../Audio/${basename(sentenceRange.audiofile)}`,
-                    clipBegin: `${sentenceRange.start.toFixed(3)}s`,
-                    clipEnd: `${sentenceRange.end.toFixed(3)}s`
-                  })
-                ]
-              )
-            )
+                words.map(
+                  (word) => Epub.createXmlElement(
+                    "par",
+                    {
+                      id: `${chapter.id}-s${sentenceRange.id}-w${word.id}`
+                    },
+                    [
+                      Epub.createXmlElement("text", {
+                        src: `../${chapter.href}#${chapter.id}-s${sentenceRange.id}-w${word.id}`
+                      }),
+                      Epub.createXmlElement("audio", {
+                        src: `../Audio/${basename(word.audiofile)}`,
+                        clipBegin: `${word.start.toFixed(3)}s`,
+                        clipEnd: `${word.end.toFixed(3)}s`
+                      })
+                    ]
+                  )
+                )
+              );
+            })
           )
         ])
       ]