@storyteller-platform/align 0.0.1

package/dist/align/fuzzy.js

@@ -0,0 +1,141 @@
+import "../chunk-BIEQXUOY.js";
+function findNearestMatch(needle, haystack, maxDist) {
+  let nearest = null;
+  for (const match of levenshteinNgram(needle, haystack, maxDist)) {
+    if (!nearest || match.dist < nearest.dist) {
+      nearest = match;
+    }
+  }
+  return nearest && {
+    match: haystack.slice(nearest.start, nearest.end),
+    index: nearest.start
+  };
+}
+function reverse(str, from = str.length, to = 0) {
+  let reversed = "";
+  for (let i = from - 1; i >= to; i--) {
+    reversed = reversed + str[i];
+  }
+  return reversed;
+}
+function* searchExact(subsequence, sequence, startIndex = 0, endIndex = sequence.length) {
+  let index = sequence.indexOf(subsequence, startIndex);
+  while (index !== -1 && index + subsequence.length < endIndex) {
+    yield index;
+    index = sequence.indexOf(subsequence, index + 1);
+  }
+}
+function expand(subsequence, sequence, maxDist) {
+  const subsequenceLength = subsequence.length;
+  if (subsequenceLength === 0) {
+    return { index: 0, score: 0 };
+  }
+  const scores = Array.from({ length: subsequenceLength + 1 }).map((_, i) => i);
+  let minScore = subsequenceLength;
+  let minScoreIndex = -1;
+  let maxGoodScore = maxDist;
+  let newNeedleIndexRangeStart = 0;
+  let newNeedleIndexRangeEnd = subsequenceLength - 1;
+  for (let sequenceIndex = 0; sequenceIndex < sequence.length; sequenceIndex++) {
+    const char = sequence[sequenceIndex];
+    const needleIndexRangeStart = newNeedleIndexRangeStart;
+    const needleIndexRangeEnd = Math.min(
+      subsequenceLength,
+      newNeedleIndexRangeEnd + 1
+    );
+    let a = sequenceIndex;
+    let c = a + 1;
+    if (c <= maxGoodScore) {
+      newNeedleIndexRangeStart = 0;
+      newNeedleIndexRangeEnd = 0;
+    } else {
+      newNeedleIndexRangeStart = null;
+      newNeedleIndexRangeEnd = -1;
+    }
+    for (let subsequenceIndex = needleIndexRangeStart; subsequenceIndex < needleIndexRangeEnd; subsequenceIndex++) {
+      const b = scores[subsequenceIndex];
+      c = scores[subsequenceIndex] = Math.min(
+        a + (char === subsequence[subsequenceIndex] ? 0 : 1),
+        b + 1,
+        c + 1
+      );
+      a = b;
+      if (c <= maxGoodScore) {
+        if (newNeedleIndexRangeStart === null) {
+          newNeedleIndexRangeStart = subsequenceIndex;
+        }
+        newNeedleIndexRangeEnd = Math.max(
+          newNeedleIndexRangeEnd,
+          subsequenceIndex + 1 + (maxGoodScore - c)
+        );
+      }
+    }
+    if (newNeedleIndexRangeStart === null) {
+      break;
+    }
+    if (needleIndexRangeEnd === subsequenceLength && c <= minScore) {
+      minScore = c;
+      minScoreIndex = sequenceIndex;
+      if (minScore < maxGoodScore) {
+        maxGoodScore = minScore;
+      }
+    }
+  }
+  return minScore <= maxDist ? { score: minScore, index: minScoreIndex + 1 } : null;
+}
+function* levenshteinNgram(subsequence, sequence, maxDist) {
+  const subsequenceLength = subsequence.length;
+  const sequenceLength = sequence.length;
+  const ngramLength = Math.round(subsequenceLength / (maxDist + 1));
+  if (ngramLength === 0) {
+    throw new Error("The subsequence length must be greater than maxDist");
+  }
+  for (let ngramStart = 0; ngramStart < subsequenceLength - ngramLength + 1; ngramStart += ngramLength) {
+    const ngramEnd = ngramStart + ngramLength;
+    const subsequenceBeforeReversed = reverse(subsequence, ngramStart);
+    const subsequenceAfter = subsequence.slice(ngramEnd);
+    const startIndex = Math.max(0, ngramStart - maxDist);
+    const endIndex = Math.min(
+      sequenceLength,
+      sequenceLength - subsequenceLength + ngramEnd + maxDist
+    );
+    for (const index of searchExact(
+      subsequence.slice(ngramStart, ngramEnd),
+      sequence,
+      startIndex,
+      endIndex
+    )) {
+      const rightMatch = expand(
+        subsequenceAfter,
+        sequence.slice(
+          index + ngramLength,
+          index - ngramStart + subsequenceLength + maxDist
+        ),
+        maxDist
+      );
+      if (rightMatch === null) continue;
+      const { score: distRight, index: rightExpandSize } = rightMatch;
+      const leftMatch = expand(
+        subsequenceBeforeReversed,
+        reverse(
+          sequence,
+          index,
+          Math.max(0, index - ngramStart - (maxDist - distRight))
+        ),
+        maxDist - distRight
+      );
+      if (leftMatch === null) continue;
+      const { score: distLeft, index: leftExpandSize } = leftMatch;
+      const start = index - leftExpandSize;
+      yield {
+        start,
+        end: index + ngramLength + rightExpandSize,
+        // dist: distLeft + distRight + (start / sequenceLength) * maxDist,
+        dist: distLeft + distRight
+      };
+    }
+  }
+}
+export {
+  findNearestMatch
+};

package/dist/align/getSentenceRanges.cjs

@@ -0,0 +1,304 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var getSentenceRanges_exports = {};
+__export(getSentenceRanges_exports, {
+  expandEmptySentenceRanges: () => expandEmptySentenceRanges,
+  findEndTimestamp: () => findEndTimestamp,
+  getChapterDuration: () => getChapterDuration,
+  getSentenceRanges: () => getSentenceRanges,
+  interpolateSentenceRanges: () => interpolateSentenceRanges
+});
+module.exports = __toCommonJS(getSentenceRanges_exports);
+var import_text_segmentation = require("@echogarden/text-segmentation");
+var import_ffmpeg = require("../common/ffmpeg.cjs");
+var import_fuzzy = require("./fuzzy.cjs");
+async function getSentencesWithOffsets(text) {
+  const sentences = await (0, import_text_segmentation.segmentText)(text).then(
+    (r) => r.sentences.map((s) => s.text)
+  );
+  const sentencesWithOffsets = [];
+  let lastSentenceEnd = 0;
+  for (const sentence of sentences) {
+    const sentenceStart = text.indexOf(sentence, lastSentenceEnd);
+    if (sentenceStart > lastSentenceEnd) {
+      sentencesWithOffsets.push(text.slice(lastSentenceEnd, sentenceStart));
+    }
+    sentencesWithOffsets.push(sentence);
+    lastSentenceEnd = sentenceStart + sentence.length;
+  }
+  if (text.length > lastSentenceEnd) {
+    sentencesWithOffsets.push(text.slice(lastSentenceEnd));
+  }
+  return sentencesWithOffsets;
+}
+function findStartTimestamp(matchStartIndex, transcription) {
+  const entry = transcription.timeline.find(
+    (entry2) => (entry2.endOffsetUtf16 ?? 0) > matchStartIndex
+  );
+  if (!entry) return null;
+  return {
+    start: entry.startTime,
+    end: entry.endTime,
+    audiofile: entry.audiofile
+  };
+}
+function findEndTimestamp(matchEndIndex, transcription) {
+  const entry = transcription.timeline.findLast(
+    (entry2) => (entry2.startOffsetUtf16 ?? 0) < matchEndIndex
+  );
+  return (entry == null ? void 0 : entry.endTime) ?? null;
+}
+function getWindowIndexFromOffset(window, offset) {
+  let index = 0;
+  while (index < window.length - 1 && offset >= window[index].length) {
+    offset -= window[index].length;
+    index += 1;
+  }
+  return { index, offset };
+}
+function collapseWhitespace(input) {
+  return input.replaceAll(/\s+/g, " ");
+}
+async function getSentenceRanges(startSentence, transcription, sentences, chapterOffset, lastSentenceRange) {
+  const sentenceRanges = [];
+  const fullTranscriptionText = transcription.transcript;
+  const transcriptionText = fullTranscriptionText.slice(chapterOffset);
+  const transcriptionSentences = await getSentencesWithOffsets(
+    transcriptionText
+  ).then((s) => s.map((sentence) => sentence.toLowerCase()));
+  let startSentenceEntry = startSentence;
+  const sentenceEntries = sentences.map((sentence, index) => [index, sentence]).filter(([index, sentence]) => {
+    if (sentence.replaceAll(/[.-_()[\],/?!@#$%^^&*`~;:='"<>+ˌˈ]/g, "").length <= 3) {
+      if (index < startSentence) startSentenceEntry--;
+      return false;
+    }
+    return true;
+  });
+  let transcriptionWindowIndex = 0;
+  let transcriptionWindowOffset = 0;
+  let lastGoodTranscriptionWindow = 0;
+  let notFound = 0;
+  let sentenceIndex = startSentenceEntry;
+  let lastMatchEnd = chapterOffset;
+  while (sentenceIndex < sentenceEntries.length) {
+    const [sentenceId, sentence] = sentenceEntries[sentenceIndex];
+    const transcriptionWindowList = transcriptionSentences.slice(
+      transcriptionWindowIndex,
+      transcriptionWindowIndex + 10
+    );
+    const transcriptionWindow = transcriptionWindowList.join("").slice(transcriptionWindowOffset);
+    const query = collapseWhitespace(sentence.trim()).toLowerCase();
+    const firstMatch = (0, import_fuzzy.findNearestMatch)(
+      query,
+      transcriptionWindow,
+      Math.max(Math.floor(0.25 * query.length), 1)
+    );
+    if (!firstMatch) {
+      sentenceIndex += 1;
+      notFound += 1;
+      if (notFound === 3 || sentenceIndex === sentenceEntries.length) {
+        transcriptionWindowIndex += 1;
+        if (transcriptionWindowIndex == lastGoodTranscriptionWindow + 30) {
+          transcriptionWindowIndex = lastGoodTranscriptionWindow;
+          notFound = 0;
+          continue;
+        }
+        sentenceIndex -= notFound;
+        notFound = 0;
+      }
+      continue;
+    }
+    const transcriptionOffset = transcriptionSentences.slice(0, transcriptionWindowIndex).join("").length;
+    const startResult = findStartTimestamp(
+      firstMatch.index + transcriptionOffset + transcriptionWindowOffset + chapterOffset,
+      transcription
+    );
+    if (!startResult) {
+      sentenceIndex += 1;
+      continue;
+    }
+    let start = startResult.start;
+    const audiofile = startResult.audiofile;
+    const end = findEndTimestamp(
+      firstMatch.index + firstMatch.match.length + transcriptionOffset + transcriptionWindowOffset + chapterOffset,
+      transcription
+    ) ?? startResult.end;
+    if (sentenceRanges.length > 0) {
+      const previousSentenceRange = sentenceRanges[sentenceRanges.length - 1];
+      const previousAudiofile = previousSentenceRange.audiofile;
+      if (audiofile === previousAudiofile) {
+        if (previousSentenceRange.id === sentenceId - 1) {
+          previousSentenceRange.end = start;
+        }
+      } else {
+        if (previousSentenceRange.id === sentenceId - 1) {
+          const lastTrackDuration = await (0, import_ffmpeg.getTrackDuration)(previousAudiofile);
+          previousSentenceRange.end = lastTrackDuration;
+          start = 0;
+        }
+      }
+    } else if (lastSentenceRange !== null) {
+      if (audiofile === lastSentenceRange.audiofile) {
+        if (sentenceId === 0) {
+          lastSentenceRange.end = start;
+        }
+      } else {
+        const lastTrackDuration = await (0, import_ffmpeg.getTrackDuration)(
+          lastSentenceRange.audiofile
+        );
+        lastSentenceRange.end = lastTrackDuration;
+        if (sentenceId === 0) {
+          start = 0;
+        }
+      }
+    } else if (sentenceId === 0) {
+      start = 0;
+    }
+    sentenceRanges.push({
+      id: sentenceId,
+      start,
+      end,
+      audiofile
+    });
+    notFound = 0;
+    lastMatchEnd = firstMatch.index + firstMatch.match.length + transcriptionOffset + transcriptionWindowOffset + chapterOffset;
+    const windowIndexResult = getWindowIndexFromOffset(
+      transcriptionWindowList,
+      firstMatch.index + firstMatch.match.length + transcriptionWindowOffset
+    );
+    transcriptionWindowIndex += windowIndexResult.index;
+    transcriptionWindowOffset = windowIndexResult.offset;
+    lastGoodTranscriptionWindow = transcriptionWindowIndex;
+    sentenceIndex += 1;
+  }
+  return {
+    sentenceRanges,
+    transcriptionOffset: lastMatchEnd
+  };
+}
+async function getLargestGap(trailing, leading) {
+  const leadingGap = leading.start;
+  const trailingGap = await (0, import_ffmpeg.getTrackDuration)(trailing.audiofile) - trailing.end;
+  if (trailingGap > leadingGap) return [trailingGap, trailing.audiofile];
+  return [leadingGap, leading.audiofile];
+}
+async function interpolateSentenceRanges(sentenceRanges, lastSentenceRange) {
+  const interpolated = [];
+  const [first, ...rest] = sentenceRanges;
+  if (!first) return interpolated;
+  if (first.id !== 0) {
+    const count = first.id;
+    const crossesAudioBoundary = !lastSentenceRange || first.audiofile !== lastSentenceRange.audiofile;
+    let diff = crossesAudioBoundary ? first.start : first.start - lastSentenceRange.end;
+    if (!crossesAudioBoundary && diff <= 0) {
+      diff = 0.25;
+      lastSentenceRange.end = first.start - diff;
+    }
+    const interpolatedLength = diff / count;
+    const start = crossesAudioBoundary ? 0 : lastSentenceRange.end;
+    for (let i = 0; i < count; i++) {
+      interpolated.push({
+        id: i,
+        start: start + interpolatedLength * i,
+        end: start + interpolatedLength * (i + 1),
+        audiofile: first.audiofile
+      });
+    }
+    interpolated.push(first);
+  } else {
+    rest.unshift(first);
+  }
+  for (const sentenceRange of rest) {
+    if (interpolated.length === 0) {
+      interpolated.push(sentenceRange);
+      continue;
+    }
+    const lastSentenceRange2 = interpolated[interpolated.length - 1];
+    const count = sentenceRange.id - lastSentenceRange2.id - 1;
+    if (count === 0) {
+      interpolated.push(sentenceRange);
+      continue;
+    }
+    const crossesAudioBoundary = sentenceRange.audiofile !== lastSentenceRange2.audiofile;
+    let [diff, audiofile] = crossesAudioBoundary ? await getLargestGap(lastSentenceRange2, sentenceRange) : [sentenceRange.start - lastSentenceRange2.end, sentenceRange.audiofile];
+    if (diff <= 0) {
+      if (crossesAudioBoundary) {
+        const rangeLength = sentenceRange.end - sentenceRange.start;
+        diff = rangeLength < 0.5 ? rangeLength / 2 : 0.25;
+        sentenceRange.start = diff;
+      } else {
+        diff = 0.25;
+        lastSentenceRange2.end = sentenceRange.start - diff;
+      }
+    }
+    const interpolatedLength = diff / count;
+    const start = crossesAudioBoundary ? 0 : lastSentenceRange2.end;
+    for (let i = 0; i < count; i++) {
+      interpolated.push({
+        id: lastSentenceRange2.id + i + 1,
+        start: start + interpolatedLength * i,
+        end: start + interpolatedLength * (i + 1),
+        audiofile
+      });
+    }
+    interpolated.push(sentenceRange);
+  }
+  return interpolated;
+}
+function expandEmptySentenceRanges(sentenceRanges) {
+  const expandedRanges = [];
+  for (const sentenceRange of sentenceRanges) {
+    const previousSentenceRange = expandedRanges[expandedRanges.length - 1];
+    if (!previousSentenceRange) {
+      expandedRanges.push(sentenceRange);
+      continue;
+    }
+    const nudged = previousSentenceRange.end > sentenceRange.start && previousSentenceRange.audiofile === sentenceRange.audiofile ? { ...sentenceRange, start: previousSentenceRange.end } : sentenceRange;
+    const expanded = nudged.end <= nudged.start ? { ...nudged, end: nudged.start + 1e-3 } : nudged;
+    expandedRanges.push(expanded);
+  }
+  return expandedRanges;
+}
+function getChapterDuration(sentenceRanges) {
+  let i = 0;
+  let duration = 0;
+  let audiofile = null;
+  let start = 0;
+  let end = 0;
+  while (i < sentenceRanges.length) {
+    const sentenceRange = sentenceRanges[i];
+    if (sentenceRange.audiofile !== audiofile) {
+      duration += end - start;
+      start = sentenceRange.start;
+      audiofile = sentenceRange.audiofile;
+    }
+    end = sentenceRange.end;
+    i++;
+  }
+  duration += end - start;
+  return duration;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  expandEmptySentenceRanges,
+  findEndTimestamp,
+  getChapterDuration,
+  getSentenceRanges,
+  interpolateSentenceRanges
+});

package/dist/align/getSentenceRanges.d.cts

@@ -0,0 +1,31 @@
+import { TimelineEntry } from '@storyteller-platform/ghost-story';
+
+type StorytellerTimelineEntry = TimelineEntry & {
+    audiofile: string;
+};
+type StorytellerTranscription = {
+    transcript: string;
+    timeline: StorytellerTimelineEntry[];
+};
+type SentenceRange = {
+    id: number;
+    start: number;
+    end: number;
+    audiofile: string;
+};
+declare function findEndTimestamp(matchEndIndex: number, transcription: StorytellerTranscription): number | null;
+declare function getSentenceRanges(startSentence: number, transcription: StorytellerTranscription, sentences: string[], chapterOffset: number, lastSentenceRange: SentenceRange | null): Promise<{
+    sentenceRanges: SentenceRange[];
+    transcriptionOffset: number;
+}>;
+declare function interpolateSentenceRanges(sentenceRanges: SentenceRange[], lastSentenceRange: SentenceRange | null): Promise<SentenceRange[]>;
+/**
+ * Whisper sometimes provides words with no time information,
+ * or start and end timestamps that are equal. EpubCheck complains
+ * about these, so we nudge them out a bit to make sure that they're
+ * not truly equal.
+ */
+declare function expandEmptySentenceRanges(sentenceRanges: SentenceRange[]): SentenceRange[];
+declare function getChapterDuration(sentenceRanges: SentenceRange[]): number;
+
+export { type SentenceRange, type StorytellerTimelineEntry, type StorytellerTranscription, expandEmptySentenceRanges, findEndTimestamp, getChapterDuration, getSentenceRanges, interpolateSentenceRanges };

package/dist/align/getSentenceRanges.d.ts

@@ -0,0 +1,31 @@
+import { TimelineEntry } from '@storyteller-platform/ghost-story';
+
+type StorytellerTimelineEntry = TimelineEntry & {
+    audiofile: string;
+};
+type StorytellerTranscription = {
+    transcript: string;
+    timeline: StorytellerTimelineEntry[];
+};
+type SentenceRange = {
+    id: number;
+    start: number;
+    end: number;
+    audiofile: string;
+};
+declare function findEndTimestamp(matchEndIndex: number, transcription: StorytellerTranscription): number | null;
+declare function getSentenceRanges(startSentence: number, transcription: StorytellerTranscription, sentences: string[], chapterOffset: number, lastSentenceRange: SentenceRange | null): Promise<{
+    sentenceRanges: SentenceRange[];
+    transcriptionOffset: number;
+}>;
+declare function interpolateSentenceRanges(sentenceRanges: SentenceRange[], lastSentenceRange: SentenceRange | null): Promise<SentenceRange[]>;
+/**
+ * Whisper sometimes provides words with no time information,
+ * or start and end timestamps that are equal. EpubCheck complains
+ * about these, so we nudge them out a bit to make sure that they're
+ * not truly equal.
+ */
+declare function expandEmptySentenceRanges(sentenceRanges: SentenceRange[]): SentenceRange[];
+declare function getChapterDuration(sentenceRanges: SentenceRange[]): number;
+
+export { type SentenceRange, type StorytellerTimelineEntry, type StorytellerTranscription, expandEmptySentenceRanges, findEndTimestamp, getChapterDuration, getSentenceRanges, interpolateSentenceRanges };