@storyteller-platform/align 0.0.1

package/dist/align/align.js

@@ -0,0 +1,458 @@
+import {
+  __callDispose,
+  __using
+} from "../chunk-BIEQXUOY.js";
+import { copyFile, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { dirname as autoDirname, join as autoJoin } from "node:path";
+import { basename, dirname, parse, relative } from "node:path/posix";
+import memoize from "memoize";
+import { isAudioFile, lookupAudioMime } from "@storyteller-platform/audiobook";
+import {
+  Epub
+} from "@storyteller-platform/epub";
+import {
+  createAggregator,
+  createTiming
+} from "@storyteller-platform/ghost-story";
+import { getTrackDuration } from "../common/ffmpeg.js";
+import { getXhtmlSegmentation } from "../markup/segmentation.js";
+import { findNearestMatch } from "./fuzzy.js";
+import {
+  expandEmptySentenceRanges,
+  getChapterDuration,
+  getSentenceRanges,
+  interpolateSentenceRanges
+} from "./getSentenceRanges.js";
+const OFFSET_SEARCH_WINDOW_SIZE = 5e3;
+async function align(input, output, transcriptionsDir, audiobookDir, options) {
+  var _stack = [];
+  try {
+    await copyFile(input, output);
+    const audiobookFiles = await readdir(audiobookDir).then(
+      (filenames) => filenames.filter((f) => isAudioFile(f)).map((f) => autoJoin(audiobookDir, f))
+    );
+    const epub = __using(_stack, await Epub.from(output));
+    const transcriptions = await readdir(transcriptionsDir).then(
+      (filenames) => filenames.filter((f) => f.endsWith(".json")).map((f) => autoJoin(transcriptionsDir, f))
+    ).then(
+      (filepaths) => Promise.all(
+        filepaths.map(async (p) => readFile(p, { encoding: "utf-8" }))
+      )
+    ).then(
+      (contents) => contents.map(
+        (c) => JSON.parse(c)
+      )
+    );
+    const aligner = new Aligner(
+      epub,
+      audiobookFiles,
+      transcriptions,
+      options.granularity,
+      options.primaryLocale,
+      options.logger
+    );
+    const timing = await aligner.alignBook(options.onProgress);
+    if (options.reportsPath) {
+      await mkdir(autoDirname(options.reportsPath), { recursive: true });
+      await writeFile(
+        options.reportsPath,
+        JSON.stringify(aligner.report, null, 2),
+        {
+          encoding: "utf-8"
+        }
+      );
+    }
+    return timing;
+  } catch (_) {
+    var _error = _, _hasError = true;
+  } finally {
+    __callDispose(_stack, _error, _hasError);
+  }
+}
+class Aligner {
+  constructor(epub, audiofiles, transcriptions, granularity, languageOverride, logger) {
+    this.epub = epub;
+    this.languageOverride = languageOverride;
+    this.logger = logger;
+    this.transcription = concatTranscriptions(transcriptions, audiofiles);
+    this.getChapterSentences = memoize(this.getChapterSentences.bind(this));
+    this.granularity = granularity ?? "sentence";
+  }
+  transcription;
+  totalDuration = 0;
+  alignedChapters = [];
+  timing = createAggregator();
+  granularity;
+  report = {
+    chapters: []
+  };
+  findBestOffset(epubSentences, transcriptionText, lastMatchOffset) {
+    let i = 0;
+    while (i < transcriptionText.length) {
+      let startSentence = 0;
+      const proposedStartIndex = (lastMatchOffset + i) % transcriptionText.length;
+      const proposedEndIndex = (proposedStartIndex + OFFSET_SEARCH_WINDOW_SIZE) % transcriptionText.length;
+      const wrapping = proposedEndIndex < proposedStartIndex;
+      let endIndex = wrapping ? transcriptionText.length : proposedEndIndex;
+      let startIndex = proposedStartIndex;
+      let startSeen = null;
+      let endSeen = null;
+      for (const aligned of this.alignedChapters) {
+        if (startSeen !== null && endSeen === aligned.startOffset) {
+          endSeen = aligned.endOffset;
+        } else {
+          startSeen = aligned.startOffset;
+          endSeen = aligned.endOffset;
+        }
+        if (startIndex >= startSeen && startIndex < endSeen) {
+          startIndex = endSeen;
+        }
+        if (endIndex >= startSeen && endIndex <= endSeen) {
+          endIndex = startSeen;
+        }
+      }
+      if (startIndex < endIndex) {
+        const transcriptionTextSlice = transcriptionText.slice(
+          startIndex,
+          endIndex
+        );
+        while (startSentence < epubSentences.length) {
+          const queryString = epubSentences.slice(startSentence, startSentence + 6).join(" ");
+          const firstMatch = findNearestMatch(
+            queryString.toLowerCase(),
+            transcriptionTextSlice.toLowerCase(),
+            Math.max(Math.floor(0.1 * queryString.length), 1)
+          );
+          if (firstMatch) {
+            return {
+              startSentence,
+              transcriptionOffset: (firstMatch.index + startIndex) % transcriptionText.length
+            };
+          }
+          startSentence += 3;
+        }
+      }
+      if (wrapping) {
+        i += transcriptionText.length - proposedStartIndex;
+      } else {
+        i += Math.floor(OFFSET_SEARCH_WINDOW_SIZE / 2);
+      }
+    }
+    return { startSentence: 0, transcriptionOffset: null };
+  }
+  async getChapterSentences(chapterId) {
+    const chapterXml = await this.epub.readXhtmlItemContents(chapterId);
+    const segmentation = await getXhtmlSegmentation(
+      Epub.getXhtmlBody(chapterXml),
+      {
+        primaryLocale: this.languageOverride ?? await this.epub.getLanguage()
+      }
+    );
+    return segmentation.sentences.map((s) => s.text);
+  }
+  async writeAlignedChapter(alignedChapter) {
+    const { chapter, sentenceRanges, xml } = alignedChapter;
+    const audiofiles = Array.from(
+      new Set(sentenceRanges.map(({ audiofile }) => audiofile))
+    );
+    await Promise.all(
+      audiofiles.map(async (audiofile) => {
+        const { name, base } = parse(audiofile);
+        const id = `audio_${name}`;
+        const manifest = await this.epub.getManifest();
+        if (id in manifest) return;
+        const epubAudioFilename = `Audio/${base}`;
+        const duration = await getTrackDuration(audiofile);
+        this.totalDuration += duration;
+        const audio = await readFile(audiofile);
+        const mediaType = lookupAudioMime(base) ?? void 0;
+        await this.epub.addManifestItem(
+          {
+            id,
+            href: epubAudioFilename,
+            mediaType
+          },
+          audio
+        );
+      })
+    );
+    const { name: chapterStem } = parse(chapter.href);
+    const mediaOverlayId = `${chapter.id}_overlay`;
+    await this.epub.addManifestItem(
+      {
+        id: mediaOverlayId,
+        href: `MediaOverlays/${chapterStem}.smil`,
+        mediaType: "application/smil+xml"
+      },
+      createMediaOverlay(chapter, sentenceRanges),
+      "xml"
+    );
+    await this.epub.updateManifestItem(chapter.id, {
+      ...chapter,
+      mediaOverlay: mediaOverlayId
+    });
+    await this.epub.writeXhtmlItemContents(chapter.id, xml);
+    const chapterDuration = getChapterDuration(sentenceRanges);
+    await this.epub.addMetadata({
+      type: "meta",
+      properties: {
+        property: "media:duration",
+        refines: `#${mediaOverlayId}`
+      },
+      value: Epub.formatSmilDuration(chapterDuration)
+    });
+  }
+  addChapterReport(chapter, chapterSentences, sentenceRanges, startSentence, transcriptionOffset) {
+    this.report.chapters.push({
+      href: chapter.href,
+      transcriptionOffset,
+      transcriptionContext: {
+        before: this.transcription.transcript.slice(
+          Math.max(0, transcriptionOffset - 30),
+          transcriptionOffset
+        ),
+        after: this.transcription.transcript.slice(
+          transcriptionOffset,
+          Math.min(
+            transcriptionOffset + 30,
+            this.transcription.transcript.length - 1
+          )
+        )
+      },
+      firstMatchedSentenceId: startSentence,
+      firstMatchedSentenceContext: {
+        prevSentence: chapterSentences[startSentence - 1] ?? null,
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        matchedSentence: chapterSentences[startSentence],
+        nextSentence: chapterSentences[startSentence + 1] ?? null
+      },
+      audioFiles: sentenceRanges.reduce((acc, range) => {
+        const existing = acc.find(
+          (context) => context.filepath === range.audiofile
+        );
+        if (existing) {
+          existing.end = range.end;
+          return acc;
+        }
+        acc.push({
+          filepath: range.audiofile,
+          start: range.start,
+          end: range.end
+        });
+        return acc;
+      }, [])
+    });
+  }
+  async alignChapter(startSentence, chapterId, transcriptionOffset, lastSentenceRange) {
+    const timing = createTiming();
+    timing.start("read contents");
+    const manifest = await this.epub.getManifest();
+    const chapter = manifest[chapterId];
+    if (!chapter)
+      throw new Error(
+        `Failed to align chapter: could not find chapter with id ${chapterId} in manifest`
+      );
+    const chapterXml = await this.epub.readXhtmlItemContents(chapterId);
+    timing.end("read contents");
+    timing.start("split to sentences");
+    const chapterSentences = await this.getChapterSentences(chapterId);
+    timing.end("split to sentences");
+    timing.start("align sentences");
+    const { sentenceRanges, transcriptionOffset: endTranscriptionOffset } = await getSentenceRanges(
+      startSentence,
+      this.transcription,
+      chapterSentences,
+      transcriptionOffset,
+      lastSentenceRange
+    );
+    timing.end("align sentences");
+    timing.start("expand ranges");
+    const interpolated = await interpolateSentenceRanges(
+      sentenceRanges,
+      lastSentenceRange
+    );
+    const expanded = expandEmptySentenceRanges(interpolated);
+    timing.end("expand ranges");
+    const storytellerStylesheetUrl = relative(
+      dirname(chapter.href),
+      "Styles/storyteller-readaloud.css"
+    );
+    Epub.addLinkToXhtmlHead(chapterXml, {
+      rel: "stylesheet",
+      href: storytellerStylesheetUrl,
+      type: "text/css"
+    });
+    this.alignedChapters.push({
+      chapter,
+      xml: chapterXml,
+      sentenceRanges: expanded,
+      startOffset: transcriptionOffset,
+      endOffset: endTranscriptionOffset
+    });
+    this.addChapterReport(
+      chapter,
+      chapterSentences,
+      expanded,
+      startSentence,
+      transcriptionOffset
+    );
+    return {
+      lastSentenceRange: expanded[expanded.length - 1] ?? null,
+      endTranscriptionOffset,
+      timing
+    };
+  }
+  async alignBook(onProgress) {
+    var _a, _b, _c, _d, _e, _f;
+    this.timing.setMetadata(
+      "language",
+      ((_a = this.languageOverride ?? await this.epub.getLanguage()) == null ? void 0 : _a.language) ?? "unknown"
+    );
+    this.timing.setMetadata("granularity", this.granularity);
+    const spine = await this.epub.getSpineItems();
+    const transcriptionText = this.transcription.transcript;
+    let lastTranscriptionOffset = 0;
+    let lastSentenceRange = null;
+    for (let index = 0; index < spine.length; index++) {
+      onProgress == null ? void 0 : onProgress(index / spine.length);
+      const spineItem = spine[index];
+      (_b = this.logger) == null ? void 0 : _b.info(
+        `Aligning epub item #${index} : ${basename(spineItem.href)}`
+      );
+      const chapterId = spineItem.id;
+      const chapterSentences = await this.getChapterSentences(chapterId);
+      if (chapterSentences.length === 0) {
+        (_c = this.logger) == null ? void 0 : _c.info(`Chapter #${index} has no text; skipping`);
+        continue;
+      }
+      if (chapterSentences.length < 2 && // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      chapterSentences[0].split(" ").length < 4) {
+        (_d = this.logger) == null ? void 0 : _d.info(
+          `Chapter #${index} is fewer than four words; skipping`
+        );
+        continue;
+      }
+      const { startSentence, transcriptionOffset } = this.findBestOffset(
+        chapterSentences,
+        transcriptionText,
+        lastTranscriptionOffset
+      );
+      if (transcriptionOffset === null) {
+        (_e = this.logger) == null ? void 0 : _e.info(
+          `Couldn't find matching transcription for chapter #${index}`
+        );
+        continue;
+      }
+      (_f = this.logger) == null ? void 0 : _f.info(
+        `Chapter #${index} best matches transcription at offset ${transcriptionOffset}, starting at sentence ${startSentence}`
+      );
+      const result = await this.alignChapter(
+        startSentence,
+        chapterId,
+        transcriptionOffset,
+        lastSentenceRange
+      );
+      lastSentenceRange = result.lastSentenceRange;
+      lastTranscriptionOffset = result.endTranscriptionOffset;
+      this.timing.add(result.timing.summary());
+    }
+    if (lastSentenceRange) {
+      lastSentenceRange.end = await getTrackDuration(
+        lastSentenceRange.audiofile
+      );
+    }
+    for (const alignedChapter of this.alignedChapters) {
+      await this.writeAlignedChapter(alignedChapter);
+    }
+    await this.epub.addMetadata({
+      type: "meta",
+      properties: { property: "media:duration" },
+      value: Epub.formatSmilDuration(this.totalDuration)
+    });
+    await this.epub.addMetadata({
+      type: "meta",
+      properties: { property: "media:active-class" },
+      value: "-epub-media-overlay-active"
+    });
+    await this.epub.addManifestItem(
+      {
+        id: "storyteller_readaloud_styles",
+        href: "Styles/storyteller-readaloud.css",
+        mediaType: "text/css"
+      },
+      `
+.-epub-media-overlay-active {
+  background-color: #ffb;
+}
+      `,
+      "utf-8"
+    );
+    return this.timing;
+  }
+}
+function createMediaOverlay(chapter, sentenceRanges) {
+  return [
+    Epub.createXmlElement(
+      "smil",
+      {
+        xmlns: "https://www.w3.org/ns/SMIL",
+        "xmlns:epub": "http://www.idpf.org/2007/ops",
+        version: "3.0"
+      },
+      [
+        Epub.createXmlElement("body", {}, [
+          Epub.createXmlElement(
+            "seq",
+            {
+              id: `${chapter.id}_overlay`,
+              "epub:textref": `../${chapter.href}`,
+              "epub:type": "chapter"
+            },
+            sentenceRanges.map(
+              (sentenceRange) => Epub.createXmlElement(
+                "par",
+                {
+                  id: `${chapter.id}-s${sentenceRange.id}`
+                },
+                [
+                  Epub.createXmlElement("text", {
+                    src: `../${chapter.href}#${chapter.id}-s${sentenceRange.id}`
+                  }),
+                  Epub.createXmlElement("audio", {
+                    src: `../Audio/${basename(sentenceRange.audiofile)}`,
+                    clipBegin: `${sentenceRange.start.toFixed(3)}s`,
+                    clipEnd: `${sentenceRange.end.toFixed(3)}s`
+                  })
+                ]
+              )
+            )
+          )
+        ])
+      ]
+    )
+  ];
+}
+function concatTranscriptions(transcriptions, audiofiles) {
+  return transcriptions.reduce(
+    (acc, transcription, index) => ({
+      ...acc,
+      transcript: acc.transcript + " " + transcription.transcript,
+      timeline: [
+        ...acc.timeline,
+        ...transcription.timeline.map((entry) => ({
+          ...entry,
+          startOffsetUtf16: (entry.startOffsetUtf16 ?? 0) + acc.transcript.length + 1,
+          endOffsetUtf16: (entry.endOffsetUtf16 ?? 0) + acc.transcript.length + 1,
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          audiofile: audiofiles[index]
+        }))
+      ]
+    }),
+    { transcript: "", timeline: [] }
+  );
+}
+export {
+  Aligner,
+  align,
+  concatTranscriptions
+};

package/dist/align/fuzzy.cjs

@@ -0,0 +1,164 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var fuzzy_exports = {};
+__export(fuzzy_exports, {
+  findNearestMatch: () => findNearestMatch
+});
+module.exports = __toCommonJS(fuzzy_exports);
+function findNearestMatch(needle, haystack, maxDist) {
+  let nearest = null;
+  for (const match of levenshteinNgram(needle, haystack, maxDist)) {
+    if (!nearest || match.dist < nearest.dist) {
+      nearest = match;
+    }
+  }
+  return nearest && {
+    match: haystack.slice(nearest.start, nearest.end),
+    index: nearest.start
+  };
+}
+function reverse(str, from = str.length, to = 0) {
+  let reversed = "";
+  for (let i = from - 1; i >= to; i--) {
+    reversed = reversed + str[i];
+  }
+  return reversed;
+}
+function* searchExact(subsequence, sequence, startIndex = 0, endIndex = sequence.length) {
+  let index = sequence.indexOf(subsequence, startIndex);
+  while (index !== -1 && index + subsequence.length < endIndex) {
+    yield index;
+    index = sequence.indexOf(subsequence, index + 1);
+  }
+}
+function expand(subsequence, sequence, maxDist) {
+  const subsequenceLength = subsequence.length;
+  if (subsequenceLength === 0) {
+    return { index: 0, score: 0 };
+  }
+  const scores = Array.from({ length: subsequenceLength + 1 }).map((_, i) => i);
+  let minScore = subsequenceLength;
+  let minScoreIndex = -1;
+  let maxGoodScore = maxDist;
+  let newNeedleIndexRangeStart = 0;
+  let newNeedleIndexRangeEnd = subsequenceLength - 1;
+  for (let sequenceIndex = 0; sequenceIndex < sequence.length; sequenceIndex++) {
+    const char = sequence[sequenceIndex];
+    const needleIndexRangeStart = newNeedleIndexRangeStart;
+    const needleIndexRangeEnd = Math.min(
+      subsequenceLength,
+      newNeedleIndexRangeEnd + 1
+    );
+    let a = sequenceIndex;
+    let c = a + 1;
+    if (c <= maxGoodScore) {
+      newNeedleIndexRangeStart = 0;
+      newNeedleIndexRangeEnd = 0;
+    } else {
+      newNeedleIndexRangeStart = null;
+      newNeedleIndexRangeEnd = -1;
+    }
+    for (let subsequenceIndex = needleIndexRangeStart; subsequenceIndex < needleIndexRangeEnd; subsequenceIndex++) {
+      const b = scores[subsequenceIndex];
+      c = scores[subsequenceIndex] = Math.min(
+        a + (char === subsequence[subsequenceIndex] ? 0 : 1),
+        b + 1,
+        c + 1
+      );
+      a = b;
+      if (c <= maxGoodScore) {
+        if (newNeedleIndexRangeStart === null) {
+          newNeedleIndexRangeStart = subsequenceIndex;
+        }
+        newNeedleIndexRangeEnd = Math.max(
+          newNeedleIndexRangeEnd,
+          subsequenceIndex + 1 + (maxGoodScore - c)
+        );
+      }
+    }
+    if (newNeedleIndexRangeStart === null) {
+      break;
+    }
+    if (needleIndexRangeEnd === subsequenceLength && c <= minScore) {
+      minScore = c;
+      minScoreIndex = sequenceIndex;
+      if (minScore < maxGoodScore) {
+        maxGoodScore = minScore;
+      }
+    }
+  }
+  return minScore <= maxDist ? { score: minScore, index: minScoreIndex + 1 } : null;
+}
+function* levenshteinNgram(subsequence, sequence, maxDist) {
+  const subsequenceLength = subsequence.length;
+  const sequenceLength = sequence.length;
+  const ngramLength = Math.round(subsequenceLength / (maxDist + 1));
+  if (ngramLength === 0) {
+    throw new Error("The subsequence length must be greater than maxDist");
+  }
+  for (let ngramStart = 0; ngramStart < subsequenceLength - ngramLength + 1; ngramStart += ngramLength) {
+    const ngramEnd = ngramStart + ngramLength;
+    const subsequenceBeforeReversed = reverse(subsequence, ngramStart);
+    const subsequenceAfter = subsequence.slice(ngramEnd);
+    const startIndex = Math.max(0, ngramStart - maxDist);
+    const endIndex = Math.min(
+      sequenceLength,
+      sequenceLength - subsequenceLength + ngramEnd + maxDist
+    );
+    for (const index of searchExact(
+      subsequence.slice(ngramStart, ngramEnd),
+      sequence,
+      startIndex,
+      endIndex
+    )) {
+      const rightMatch = expand(
+        subsequenceAfter,
+        sequence.slice(
+          index + ngramLength,
+          index - ngramStart + subsequenceLength + maxDist
+        ),
+        maxDist
+      );
+      if (rightMatch === null) continue;
+      const { score: distRight, index: rightExpandSize } = rightMatch;
+      const leftMatch = expand(
+        subsequenceBeforeReversed,
+        reverse(
+          sequence,
+          index,
+          Math.max(0, index - ngramStart - (maxDist - distRight))
+        ),
+        maxDist - distRight
+      );
+      if (leftMatch === null) continue;
+      const { score: distLeft, index: leftExpandSize } = leftMatch;
+      const start = index - leftExpandSize;
+      yield {
+        start,
+        end: index + ngramLength + rightExpandSize,
+        // dist: distLeft + distRight + (start / sequenceLength) * maxDist,
+        dist: distLeft + distRight
+      };
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  findNearestMatch
+});

package/dist/align/fuzzy.d.cts

@@ -0,0 +1,6 @@
+declare function findNearestMatch(needle: string, haystack: string, maxDist: number): {
+    match: string;
+    index: number;
+} | null;
+
+export { findNearestMatch };

package/dist/align/fuzzy.d.ts

@@ -0,0 +1,6 @@
+declare function findNearestMatch(needle: string, haystack: string, maxDist: number): {
+    match: string;
+    index: number;
+} | null;
+
+export { findNearestMatch };