@storyteller-platform/align 0.1.19 → 0.1.20

package/dist/align/getSentenceRanges.js

@@ -5,9 +5,9 @@ import { getTrackDuration } from "../common/ffmpeg.js";
 import { errorAlign } from "../errorAlign/errorAlign.js";
 import { Alignment, reversed } from "../errorAlign/utils.js";
 import { slugify } from "./slugify.js";
-function findStartTimestamp(matchStartIndex, transcription) {
-  const entry = transcription.timeline.find(
-    (entry2) => (entry2.endOffsetUtf16 ?? 0) > matchStartIndex
+function findStartTimestamp(matchStartIndex, timeline) {
+  const entry = timeline.find(
+    (entry2) => entry2.mappedEndOffsetUtf16 > matchStartIndex
   );
   if (!entry) return null;
   return {
@@ -16,9 +16,9 @@ function findStartTimestamp(matchStartIndex, transcription) {
     audiofile: entry.audiofile
   };
 }
-function findEndTimestamp(matchEndIndex, transcription) {
-  const entry = transcription.timeline.findLast(
-    (entry2) => (entry2.startOffsetUtf16 ?? 0) < matchEndIndex
+function findEndTimestamp(matchEndIndex, timeline) {
+  const entry = timeline.findLast(
+    (entry2) => entry2.mappedStartOffsetUtf16 < matchEndIndex
   );
   if (!entry) return null;
   return {
@@ -109,17 +109,23 @@ function errorAlignWithNarrowing(refSentences, hyp, narrowStart, narrowEnd) {
   );
   return { alignments, slice: [slice[0] + narrowed[0], slice[0] + narrowed[1]] };
 }
-async function getSentenceRanges(transcription, sentences, chapterOffset, chapterEndOffset, locale) {
+function mapTranscriptionTimeline(transcription, mapping) {
+  return transcription.timeline.map((entry) => ({
+    ...entry,
+    mappedStartOffsetUtf16: mapping.map(entry.startOffsetUtf16 ?? 0, 1),
+    mappedEndOffsetUtf16: mapping.map(entry.endOffsetUtf16 ?? 0, -1)
+  }));
+}
+async function getSentenceRanges(transcriptionText, mappedTimeline, sentences, chapterId, chapterOffset, chapterEndOffset, granularity, locale) {
   const sentenceRanges = [];
-  const fullTranscript = transcription.transcript;
-  const chapterTranscript = fullTranscript.slice(
+  const wordRanges = [];
+  const slugifiedChapterTranscript = transcriptionText.slice(
     chapterOffset,
     chapterEndOffset
   );
-  const { result: slugifiedChapterTranscript, mapping: transcriptMapping } = await slugify(chapterTranscript, locale);
   const slugifiedChapterSentences = [];
   for (const s of sentences) {
-    const { result } = await slugify(s, locale);
+    const { result } = await slugify(s.text, locale);
     slugifiedChapterSentences.push(result);
   }
   let firstFoundSentence = 0;
@@ -188,21 +194,16 @@ async function getSentenceRanges(transcription, sentences, chapterOffset, chapte
       const sentenceLengthInSlugifiedTranscript = sentenceAlignments.filter((a) => a.opType !== "DELETE").map((a) => a.hyp).join("-").length;
       if (score > 0) {
         const start = findStartTimestamp(
-          chapterOffset + transcriptMapping.invert().map(
-            slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex,
-            1
-          ),
-          transcription
-        );
-        chapterTranscriptEndIndex = chapterOffset + transcriptMapping.invert().map(
-          slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex + sentenceLengthInSlugifiedTranscript,
-          -1
+          chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex,
+          mappedTimeline
         );
-        const end = findEndTimestamp(chapterTranscriptEndIndex, transcription);
+        chapterTranscriptEndIndex = chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWindowIndex + sentenceLengthInSlugifiedTranscript;
+        const end = findEndTimestamp(chapterTranscriptEndIndex, mappedTimeline);
         if (start && end) {
           if (start.audiofile !== end.audiofile) {
             sentenceRanges.push({
               id: j + chapterSentenceIndex + slice[0],
+              chapterId,
               start: 0,
               audiofile: end.audiofile,
               end: end.end
@@ -210,12 +211,56 @@ async function getSentenceRanges(transcription, sentences, chapterOffset, chapte
           } else {
             sentenceRanges.push({
               id: j + chapterSentenceIndex + slice[0],
+              chapterId,
               start: start.start,
               audiofile: start.audiofile,
               end: end.end
             });
           }
         }
+        if (granularity === "word") {
+          const sentenceSegmentation = sentences[j + chapterSentenceIndex + slice[0]];
+          const words = [];
+          for (const entry of sentenceSegmentation.words.entries) {
+            if (!entry.text.match(/\S/)) continue;
+            const { result } = await slugify(entry.text, locale);
+            words.push(result);
+          }
+          let currentTranscriptWordWindowIndex = currentTranscriptWindowIndex;
+          let sentenceAlignmentIndex = 0;
+          const perSentenceWordRanges = [];
+          for (const [k, word] of enumerate(words)) {
+            if (!word) continue;
+            const { alignments: wordAlignments } = getAlignmentsForSentence(
+              word,
+              sentenceAlignments.slice(sentenceAlignmentIndex)
+            );
+            const wordLengthInSlugifiedTranscript = wordAlignments.filter((a) => a.opType !== "DELETE").map((a) => a.hyp).join("-").length;
+            const start2 = findStartTimestamp(
+              chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWordWindowIndex,
+              mappedTimeline
+            );
+            const end2 = findEndTimestamp(
+              chapterOffset + slugifiedChapterTranscriptWindowStartIndex + currentTranscriptWordWindowIndex + wordLengthInSlugifiedTranscript,
+              mappedTimeline
+            );
+            if (start2 && end2) {
+              perSentenceWordRanges.push({
+                id: k,
+                sentenceId: j + chapterSentenceIndex + slice[0],
+                start: end2.audiofile === start2.audiofile ? start2.start : 0,
+                audiofile: end2.audiofile,
+                end: end2.end
+              });
+            }
+            sentenceAlignmentIndex += wordAlignments.length;
+            currentTranscriptWordWindowIndex += wordLengthInSlugifiedTranscript;
+            if (slugifiedChapterTranscriptWindow[currentTranscriptWordWindowIndex] === "-") {
+              currentTranscriptWordWindowIndex++;
+            }
+          }
+          wordRanges.push(perSentenceWordRanges);
+        }
       }
       alignmentIndex += sentenceAlignments.length;
       currentTranscriptWindowIndex += sentenceLengthInSlugifiedTranscript;
@@ -231,6 +276,7 @@ async function getSentenceRanges(transcription, sentences, chapterOffset, chapte
   }
   return {
     sentenceRanges,
+    wordRanges,
     transcriptionOffset: chapterTranscriptEndIndex,
     firstFoundSentence,
     lastFoundSentence
@@ -242,66 +288,53 @@ async function getLargestGap(trailing, leading) {
   if (trailingGap > leadingGap) return [trailingGap, trailing.audiofile];
   return [leadingGap, leading.audiofile];
 }
-async function interpolateSentenceRanges(sentenceRanges, lastSentenceRange) {
+async function interpolateSentenceRanges(sentenceRanges, chapterSentenceCounts) {
   const interpolated = [];
-  const [first, ...rest] = sentenceRanges;
-  if (!first) return interpolated;
-  if (first.id !== 0) {
-    const count = first.id;
-    const crossesAudioBoundary = !lastSentenceRange || first.audiofile !== lastSentenceRange.audiofile;
-    let diff = crossesAudioBoundary ? first.start : first.start - lastSentenceRange.end;
-    if (!crossesAudioBoundary && diff <= 0) {
-      diff = 0.25;
-      lastSentenceRange.end = first.start - diff;
-    }
-    const interpolatedLength = diff / count;
-    const start = crossesAudioBoundary ? 0 : lastSentenceRange.end;
-    for (let i = 0; i < count; i++) {
-      interpolated.push({
-        id: i,
-        start: start + interpolatedLength * i,
-        end: start + interpolatedLength * (i + 1),
-        audiofile: first.audiofile
-      });
-    }
-    interpolated.push(first);
-  } else {
-    rest.unshift(first);
-  }
-  for (const sentenceRange of rest) {
-    if (interpolated.length === 0) {
-      interpolated.push(sentenceRange);
-      continue;
-    }
-    const lastSentenceRange2 = interpolated[interpolated.length - 1];
-    const count = sentenceRange.id - lastSentenceRange2.id - 1;
+  for (let i = 0; i < sentenceRanges.length; i++) {
+    const endRange = sentenceRanges[i];
+    const startRange = sentenceRanges[i - 1] ?? {
+      id: 0,
+      audiofile: endRange.audiofile,
+      chapterId: endRange.chapterId,
+      start: 0,
+      end: 0
+    };
+    const newChapter = startRange.chapterId !== endRange.chapterId;
+    const newAudiofile = startRange.audiofile !== endRange.audiofile;
+    const count = newChapter ? chapterSentenceCounts[startRange.chapterId] - startRange.id - 1 : endRange.id - startRange.id - 1;
     if (count === 0) {
-      interpolated.push(sentenceRange);
+      interpolated.push(endRange);
       continue;
     }
-    const crossesAudioBoundary = sentenceRange.audiofile !== lastSentenceRange2.audiofile;
-    let [diff, audiofile] = crossesAudioBoundary ? await getLargestGap(lastSentenceRange2, sentenceRange) : [sentenceRange.start - lastSentenceRange2.end, sentenceRange.audiofile];
+    let [diff, audiofile] = newAudiofile ? await getLargestGap(startRange, endRange) : [endRange.start - startRange.end, endRange.audiofile];
     if (diff <= 0) {
-      if (crossesAudioBoundary) {
-        const rangeLength = sentenceRange.end - sentenceRange.start;
+      if (newAudiofile) {
+        const rangeLength = endRange.end - endRange.start;
         diff = rangeLength < 0.5 ? rangeLength / 2 : 0.25;
-        sentenceRange.start = diff;
+        endRange.start = diff;
       } else {
         diff = 0.25;
-        lastSentenceRange2.end = sentenceRange.start - diff;
+        startRange.end = startRange.start - diff;
       }
     }
     const interpolatedLength = diff / count;
-    const start = crossesAudioBoundary ? 0 : lastSentenceRange2.end;
-    for (let i = 0; i < count; i++) {
+    const start = newAudiofile ? 0 : startRange.end;
+    for (let i2 = 0; i2 < count; i2++) {
+      let id = startRange.id + i2 + 1;
+      let chapterId = startRange.chapterId;
+      if (newChapter && i2 > chapterSentenceCounts[startRange.chapterId] - startRange.id) {
+        id = i2;
+        chapterId = endRange.chapterId;
+      }
       interpolated.push({
-        id: lastSentenceRange2.id + i + 1,
-        start: start + interpolatedLength * i,
-        end: start + interpolatedLength * (i + 1),
+        id,
+        chapterId,
+        start: start + interpolatedLength * i2,
+        end: start + interpolatedLength * (i2 + 1),
         audiofile
       });
     }
-    interpolated.push(sentenceRange);
+    interpolated.push(endRange);
   }
   return interpolated;
 }
@@ -319,6 +352,17 @@ function expandEmptySentenceRanges(sentenceRanges) {
   }
   return expandedRanges;
 }
+async function collapseSentenceRangeGaps(sentenceRanges) {
+  const collapsed = [];
+  for (const [i, sentenceRange] of enumerate(sentenceRanges)) {
+    const nextSentence = sentenceRanges[i + 1];
+    const prevSentence = sentenceRanges[i - 1];
+    const start = prevSentence?.audiofile !== sentenceRange.audiofile ? 0 : sentenceRange.start;
+    const end = nextSentence?.audiofile !== sentenceRange.audiofile ? await getTrackDuration(sentenceRange.audiofile) : nextSentence.start;
+    collapsed.push({ ...sentenceRange, start, end });
+  }
+  return collapsed;
+}
 function getChapterDuration(sentenceRanges) {
   let i = 0;
   let duration = 0;
@@ -339,9 +383,11 @@ function getChapterDuration(sentenceRanges) {
   return duration;
 }
 export {
+  collapseSentenceRangeGaps,
   expandEmptySentenceRanges,
   findEndTimestamp,
   getChapterDuration,
   getSentenceRanges,
-  interpolateSentenceRanges
+  interpolateSentenceRanges,
+  mapTranscriptionTimeline
 };

package/dist/index.d.cts

@@ -11,3 +11,4 @@ import '@storyteller-platform/epub';
 import './markup/map.cjs';
 import '@storyteller-platform/ghost-story/recognition';
 import './align/getSentenceRanges.cjs';
+import '@storyteller-platform/transliteration';

package/dist/index.d.ts

@@ -11,3 +11,4 @@ import '@storyteller-platform/epub';
 import './markup/map.js';
 import '@storyteller-platform/ghost-story/recognition';
 import './align/getSentenceRanges.js';
+import '@storyteller-platform/transliteration';

package/dist/markup/markup.cjs

@@ -104,6 +104,7 @@ async function markup(input, output, options) {
       const { markedUp, timing: chapterTiming } = markupChapter(
         chapterId,
         chapterXml,
+        options.granularity ?? "sentence",
         segmentation,
         mapping
       );
@@ -118,7 +119,7 @@ async function markup(input, output, options) {
     __callDispose(_stack, _error, _hasError);
   }
 }
-function markupChapter(chapterId, chapterXml, segmentation, mapping) {
+function markupChapter(chapterId, chapterXml, granularity, segmentation, mapping) {
   const timing = (0, import_ghost_story.createTiming)();
   const html = import_epub.Epub.findXmlChildByName("html", chapterXml);
   if (!html) throw new Error("Invalid XHTML document: no html element");
@@ -132,6 +133,22 @@ function markupChapter(chapterId, chapterXml, segmentation, mapping) {
     let pos = 0;
     let i = 0;
     for (const sentence of segmentation) {
+      if (granularity === "word") {
+        let j = 0;
+        let wordPos = pos;
+        for (const word of sentence.words.entries) {
+          if (word.text.match(/\S/)) {
+            root = (0, import_transform.addMark)(
+              root,
+              mapping.invert().map(wordPos),
+              mapping.invert().map(wordPos + word.text.replace(/\n$/, "").length, -1),
+              new import_model.Mark("span", { id: `${chapterId}-s${i}-w${j}` })
+            );
+            j++;
+          }
+          wordPos += word.text.replace(/\n$/, "").length;
+        }
+      }
       if (sentence.text.match(/\S/)) {
         root = (0, import_transform.addMark)(
           root,

package/dist/markup/markup.d.cts

@@ -12,7 +12,7 @@ interface MarkupOptions {
     logger?: Logger;
 }
 declare function markup(input: string, output: string, options: MarkupOptions): Promise<TimingAggregator>;
-declare function markupChapter(chapterId: string, chapterXml: ParsedXml, segmentation: Sentence[], mapping: Mapping): {
+declare function markupChapter(chapterId: string, chapterXml: ParsedXml, granularity: "word" | "sentence", segmentation: Sentence[], mapping: Mapping): {
     markedUp: ParsedXml;
     timing: _storyteller_platform_ghost_story.Timing;
 };

package/dist/markup/markup.d.ts

@@ -12,7 +12,7 @@ interface MarkupOptions {
     logger?: Logger;
 }
 declare function markup(input: string, output: string, options: MarkupOptions): Promise<TimingAggregator>;
-declare function markupChapter(chapterId: string, chapterXml: ParsedXml, segmentation: Sentence[], mapping: Mapping): {
+declare function markupChapter(chapterId: string, chapterXml: ParsedXml, granularity: "word" | "sentence", segmentation: Sentence[], mapping: Mapping): {
     markedUp: ParsedXml;
     timing: _storyteller_platform_ghost_story.Timing;
 };

package/dist/markup/markup.js

@@ -42,6 +42,7 @@ async function markup(input, output, options) {
       const { markedUp, timing: chapterTiming } = markupChapter(
         chapterId,
         chapterXml,
+        options.granularity ?? "sentence",
         segmentation,
         mapping
       );
@@ -56,7 +57,7 @@ async function markup(input, output, options) {
     __callDispose(_stack, _error, _hasError);
   }
 }
-function markupChapter(chapterId, chapterXml, segmentation, mapping) {
+function markupChapter(chapterId, chapterXml, granularity, segmentation, mapping) {
   const timing = createTiming();
   const html = Epub.findXmlChildByName("html", chapterXml);
   if (!html) throw new Error("Invalid XHTML document: no html element");
@@ -70,6 +71,22 @@ function markupChapter(chapterId, chapterXml, segmentation, mapping) {
     let pos = 0;
     let i = 0;
     for (const sentence of segmentation) {
+      if (granularity === "word") {
+        let j = 0;
+        let wordPos = pos;
+        for (const word of sentence.words.entries) {
+          if (word.text.match(/\S/)) {
+            root = addMark(
+              root,
+              mapping.invert().map(wordPos),
+              mapping.invert().map(wordPos + word.text.replace(/\n$/, "").length, -1),
+              new Mark("span", { id: `${chapterId}-s${i}-w${j}` })
+            );
+            j++;
+          }
+          wordPos += word.text.replace(/\n$/, "").length;
+        }
+      }
       if (sentence.text.match(/\S/)) {
         root = addMark(
           root,

package/dist/markup/serializeDom.cjs

@@ -18,70 +18,97 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var serializeDom_exports = {};
 __export(serializeDom_exports, {
-  serializeDom: () => serializeDom,
-  serializeDomNode: () => serializeDomNode
+  serializeDom: () => serializeDom
 });
 module.exports = __toCommonJS(serializeDom_exports);
 var import_epub = require("@storyteller-platform/epub");
 var import_model = require("./model.cjs");
-function serializeDom(doc) {
-  return doc.children.map((child) => serializeDomNode(child));
-}
-function serializeDomNode(node) {
-  if (node instanceof import_model.TextNode) {
-    return import_epub.Epub.createXmlTextNode(node.text);
+class Serializer {
+  constructor(doc) {
+    this.doc = doc;
   }
-  return import_epub.Epub.createXmlElement(
-    node.tagName,
-    node.attrs,
-    serializeDomNodes(node.children)
-  );
-}
-function serializeDomNodes(nodes) {
-  const partitioned = nodes.reduce((acc, child) => {
-    const lastPartition = acc.at(-1);
-    if (!lastPartition) {
-      return [[child]];
-    }
-    const lastChild = lastPartition.at(-1);
-    if (!lastChild) {
-      return [...acc.slice(0, acc.length), [child]];
+  serializedIds = /* @__PURE__ */ new Set();
+  serialize() {
+    return this.doc.children.map((child) => this.serializeDomNode(child));
+  }
+  serializeDomNode(node) {
+    if (node instanceof import_model.TextNode) {
+      return import_epub.Epub.createXmlTextNode(node.text);
     }
-    const childFirstMark = child.marks[0];
-    const lastChildFirstMark = lastChild.marks[0];
-    if (childFirstMark === lastChildFirstMark || childFirstMark?.eq(lastChildFirstMark)) {
-      return [
-        ...acc.slice(0, acc.length - 1),
-        [...lastPartition.slice(0, lastPartition.length), child]
-      ];
+    let attrs = node.attrs;
+    const id = node.attrs["id"];
+    if (id) {
+      if (this.serializedIds.has(id)) {
+        const { id: _id, ...remaining } = node.attrs;
+        attrs = remaining;
+      } else {
+        this.serializedIds.add(id);
+      }
     }
-    return [...acc, [child]];
-  }, []);
-  const xmlChildren = [];
-  for (const partition of partitioned) {
-    xmlChildren.push(...serializePartition(partition));
+    return import_epub.Epub.createXmlElement(
+      node.tagName,
+      attrs,
+      this.serializeDomNodes(node.children)
+    );
   }
-  return xmlChildren;
-}
-function serializePartition(nodes) {
-  const firstChild = nodes[0];
-  if (!firstChild) return [];
-  const firstMark = firstChild.marks[0];
-  if (!firstMark) {
-    return nodes.map((child) => serializeDomNode(child));
+  serializeDomNodes(nodes) {
+    const partitioned = nodes.reduce((acc, child) => {
+      const lastPartition = acc.at(-1);
+      if (!lastPartition) {
+        return [[child]];
+      }
+      const lastChild = lastPartition.at(-1);
+      if (!lastChild) {
+        return [...acc.slice(0, acc.length), [child]];
+      }
+      const childFirstMark = child.marks[0];
+      const lastChildFirstMark = lastChild.marks[0];
+      if (childFirstMark === lastChildFirstMark || childFirstMark?.eq(lastChildFirstMark)) {
+        return [
+          ...acc.slice(0, acc.length - 1),
+          [...lastPartition.slice(0, lastPartition.length), child]
+        ];
+      }
+      return [...acc, [child]];
+    }, []);
+    const xmlChildren = [];
+    for (const partition of partitioned) {
+      xmlChildren.push(...this.serializePartition(partition));
+    }
+    return xmlChildren;
   }
-  return [
-    import_epub.Epub.createXmlElement(
-      firstMark.tagName,
-      firstMark.attrs,
-      serializeDomNodes(
-        nodes.map((node) => node.copy({ marks: node.marks.slice(1) }))
+  serializePartition(nodes) {
+    const firstChild = nodes[0];
+    if (!firstChild) return [];
+    const firstMark = firstChild.marks[0];
+    if (!firstMark) {
+      return nodes.map((child) => this.serializeDomNode(child));
+    }
+    let attrs = firstMark.attrs;
+    const id = firstMark.attrs["id"];
+    if (id) {
+      if (this.serializedIds.has(id)) {
+        const { id: _id, ...remaining } = firstMark.attrs;
+        attrs = remaining;
+      } else {
+        this.serializedIds.add(id);
+      }
+    }
+    return [
+      import_epub.Epub.createXmlElement(
+        firstMark.tagName,
+        attrs,
+        this.serializeDomNodes(
+          nodes.map((node) => node.copy({ marks: node.marks.slice(1) }))
+        )
       )
-    )
-  ];
+    ];
+  }
+}
+function serializeDom(doc) {
+  return new Serializer(doc).serialize();
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  serializeDom,
-  serializeDomNode
+  serializeDom
 });

package/dist/markup/serializeDom.d.cts

@@ -1,7 +1,6 @@
-import { ParsedXml, XmlNode } from '@storyteller-platform/epub';
-import { Root, Node, TextNode } from './model.cjs';
+import { ParsedXml } from '@storyteller-platform/epub';
+import { Root } from './model.cjs';
 
 declare function serializeDom(doc: Root): ParsedXml;
-declare function serializeDomNode(node: Node | TextNode): XmlNode;
 
-export { serializeDom, serializeDomNode };
+export { serializeDom };

package/dist/markup/serializeDom.d.ts

@@ -1,7 +1,6 @@
-import { ParsedXml, XmlNode } from '@storyteller-platform/epub';
-import { Root, Node, TextNode } from './model.js';
+import { ParsedXml } from '@storyteller-platform/epub';
+import { Root } from './model.js';
 
 declare function serializeDom(doc: Root): ParsedXml;
-declare function serializeDomNode(node: Node | TextNode): XmlNode;
 
-export { serializeDom, serializeDomNode };
+export { serializeDom };