@storyteller-platform/align 0.1.8 → 0.1.10

package/dist/markup/markup.cjs

@@ -63,7 +63,6 @@ var __callDispose = (stack, error, hasError) => {
 };
 var markup_exports = {};
 __export(markup_exports, {
-  appendTextNode: () => appendTextNode,
   markup: () => markup,
   markupChapter: () => markupChapter
 });
@@ -72,8 +71,11 @@ var import_promises = require("node:fs/promises");
 var import_posix = require("node:path/posix");
 var import_epub = require("@storyteller-platform/epub");
 var import_ghost_story = require("@storyteller-platform/ghost-story");
+var import_model = require("./model.cjs");
+var import_parseDom = require("./parseDom.cjs");
 var import_segmentation = require("./segmentation.cjs");
-var import_semantics = require("./semantics.cjs");
+var import_serializeDom = require("./serializeDom.cjs");
+var import_transform = require("./transform.cjs");
 async function markup(input, output, options) {
   var _a, _b, _c, _d;
   var _stack = [];
@@ -96,14 +98,15 @@ async function markup(input, output, options) {
         continue;
       }
       const chapterXml = await epub.readXhtmlItemContents(chapterId);
-      const segmentation = await (0, import_segmentation.getXhtmlSegmentation)(
+      const { result: segmentation, mapping } = await (0, import_segmentation.getXhtmlSegmentation)(
         import_epub.Epub.getXhtmlBody(chapterXml),
         { primaryLocale }
       );
       const { markedUp, timing: chapterTiming } = markupChapter(
         chapterId,
         chapterXml,
-        segmentation
+        segmentation,
+        mapping
       );
       timing.add(chapterTiming.summary());
       await epub.writeXhtmlItemContents(chapterId, markedUp);
@@ -116,7 +119,7 @@ async function markup(input, output, options) {
     __callDispose(_stack, _error, _hasError);
   }
 }
-function markupChapter(chapterId, chapterXml, segmentation) {
+function markupChapter(chapterId, chapterXml, segmentation, mapping) {
   const timing = (0, import_ghost_story.createTiming)();
   const html = import_epub.Epub.findXmlChildByName("html", chapterXml);
   if (!html) throw new Error("Invalid XHTML document: no html element");
@@ -125,205 +128,25 @@ function markupChapter(chapterId, chapterXml, segmentation) {
   clearBodyElement(chapterXml);
   const taggedHtml = import_epub.Epub.findXmlChildByName("html", chapterXml);
   const taggedBody = import_epub.Epub.findXmlChildByName("body", taggedHtml["html"]);
-  taggedBody["body"] = [];
   timing.time("mark up", () => {
-    markupBySegmentation(
-      chapterId,
-      {
-        currentSentenceIndex: 0,
-        currentNodeProgress: 0,
-        currentSentenceProgress: 0
-      },
-      segmentation,
-      body,
-      /* @__PURE__ */ new Set(),
-      [],
-      import_epub.Epub.getXmlChildren(taggedBody)
-    );
-  });
-  return { markedUp: chapterXml, timing };
-}
-function markupBySegmentation(chapterId, state, segmentation, currentNode, taggedSentences, marks, taggedXml) {
-  if (import_epub.Epub.isXmlTextNode(currentNode)) {
-    const remainingSentence = segmentation.sentences[state.currentSentenceIndex].text.slice(state.currentSentenceProgress);
-    const text = currentNode["#text"];
-    const remainingNodeText = text.slice(state.currentNodeProgress);
-    const index = remainingNodeText.indexOf(remainingSentence[0]);
-    if (index === -1) {
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText,
-        marks,
-        taggedSentences
-      );
-      return {
-        ...state,
-        currentNodeProgress: -1
-      };
-    }
-    if (remainingNodeText.slice(index).length < remainingSentence.length) {
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText.slice(0, index),
-        marks,
-        taggedSentences
-      );
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText.slice(index),
-        marks,
-        taggedSentences,
-        state.currentSentenceIndex
-      );
-      return {
-        ...state,
-        currentSentenceProgress: state.currentSentenceProgress + remainingNodeText.length - index,
-        currentNodeProgress: -1
-      };
-    }
-    appendTextNode(
-      chapterId,
-      taggedXml,
-      remainingNodeText.slice(0, index),
-      marks,
-      taggedSentences
-    );
-    appendTextNode(
-      chapterId,
-      taggedXml,
-      remainingSentence,
-      marks,
-      taggedSentences,
-      state.currentSentenceIndex
-    );
-    if (state.currentSentenceIndex + 1 === segmentation.sentences.length) {
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText.slice(index + remainingSentence.length),
-        marks,
-        taggedSentences
-      );
-    }
-    const mapping = mapWhitespace(remainingNodeText);
-    const mapped = mapThrough(
-      remainingSentence.length,
-      mapping.filter(([start]) => start >= index)
-    );
-    return {
-      currentSentenceIndex: state.currentSentenceIndex + 1,
-      currentSentenceProgress: 0,
-      currentNodeProgress: state.currentNodeProgress + mapped + index
-    };
-  }
-  let nextState = {
-    ...state
-  };
-  const children = import_epub.Epub.getXmlChildren(currentNode);
-  for (const child of children) {
-    if (nextState.currentSentenceIndex > segmentation.sentences.length + 1) {
-      taggedXml.push(child);
-      continue;
-    }
-    nextState.currentNodeProgress = 0;
-    let nextTaggedXml = taggedXml;
-    const nextMarks = [...marks];
-    if (!import_epub.Epub.isXmlTextNode(child)) {
-      const childTagName = import_epub.Epub.getXmlElementName(child);
-      const isTextContent = import_semantics.BLOCKS.includes(childTagName.toLowerCase());
-      if (import_epub.Epub.getXmlChildren(child).length === 0) {
-        appendLeafNode(
-          chapterId,
-          taggedXml,
-          child,
-          nextMarks,
-          taggedSentences,
-          isTextContent || nextState.currentSentenceProgress === 0 ? void 0 : nextState.currentSentenceIndex
+    let root = (0, import_parseDom.parseDom)(import_epub.Epub.getXmlChildren(body));
+    let pos = 0;
+    let i = 0;
+    for (const sentence of segmentation) {
+      if (sentence.text.match(/\S/)) {
+        root = (0, import_transform.addMark)(
+          root,
+          mapping.invert().map(pos),
+          mapping.invert().map(pos + sentence.text.replace(/\n$/, "").length, -1),
+          new import_model.Mark("span", { id: `${chapterId}-s${i}` })
         );
-        continue;
-      }
-      if (isTextContent) {
-        const block = {
-          [childTagName]: [],
-          ...child[":@"] && { ":@": child[":@"] }
-        };
-        nextTaggedXml.push(block);
-        nextTaggedXml = import_epub.Epub.getXmlChildren(block);
-      } else {
-        nextMarks.push({
-          elementName: childTagName,
-          attributes: child[":@"]
-        });
+        i++;
       }
+      pos += sentence.text.replace(/\n$/, "").length;
     }
-    while (nextState.currentSentenceIndex < segmentation.sentences.length && nextState.currentNodeProgress !== -1) {
-      nextState = markupBySegmentation(
-        chapterId,
-        nextState,
-        segmentation,
-        child,
-        taggedSentences,
-        nextMarks,
-        nextTaggedXml
-      );
-    }
-  }
-  nextState.currentNodeProgress = -1;
-  return nextState;
-}
-function mapWhitespace(text) {
-  const re = /(\s\s+)/g;
-  const mapping = [];
-  let match = null;
-  while ((match = re.exec(text)) !== null) {
-    mapping.push([match.index, match[0].length, 1]);
-  }
-  return mapping;
-}
-function mapThrough(position, mapping) {
-  let result = position;
-  let index = 0;
-  while (index < mapping.length && mapping[index][0] < result) {
-    const map = mapping[index];
-    result += map[1] - map[2];
-    index++;
-  }
-  return result;
-}
-function appendTextNode(chapterId, xml, text, marks, taggedSentences, sentenceId) {
-  if (text.length === 0) return;
-  const textNode = { "#text": text };
-  appendLeafNode(chapterId, xml, textNode, marks, taggedSentences, sentenceId);
-}
-function appendLeafNode(chapterId, xml, node, marks, taggedSentences, sentenceId) {
-  var _a, _b;
-  const tagId = `${chapterId}-s${sentenceId}`;
-  const markedNode = [...marks].reverse().reduce(
-    (acc, mark) => ({
-      [mark.elementName]: [acc],
-      ":@": mark.attributes
-    }),
-    node
-  );
-  const lastNode = xml[xml.length - 1];
-  if (lastNode && !import_epub.Epub.isXmlTextNode(lastNode) && ((_a = lastNode[":@"]) == null ? void 0 : _a["@_id"]) && lastNode[":@"]["@_id"] === tagId) {
-    const tagName = import_epub.Epub.getXmlElementName(lastNode);
-    (_b = lastNode[tagName]) == null ? void 0 : _b.push(markedNode);
-    return;
-  }
-  if (sentenceId === void 0 || taggedSentences.has(sentenceId)) {
-    xml.push(markedNode);
-    return;
-  }
-  const taggedNode = {
-    span: [markedNode],
-    ":@": { "@_id": tagId }
-  };
-  taggedSentences.add(sentenceId);
-  xml.push(taggedNode);
+    taggedBody["body"] = (0, import_serializeDom.serializeDom)(root);
+  });
+  return { markedUp: chapterXml, timing };
 }
 function clearBodyElement(xml) {
   const html = import_epub.Epub.findXmlChildByName("html", xml);
@@ -338,7 +161,6 @@ function clearBodyElement(xml) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  appendTextNode,
   markup,
   markupChapter
 });

package/dist/markup/markup.d.cts

@@ -1,8 +1,9 @@
 import * as _storyteller_platform_ghost_story from '@storyteller-platform/ghost-story';
 import { TimingAggregator } from '@storyteller-platform/ghost-story';
-import { SegmentationResult } from '@echogarden/text-segmentation';
+import { Sentence } from '@echogarden/text-segmentation';
 import { Logger } from 'pino';
-import { ParsedXml, ElementName } from '@storyteller-platform/epub';
+import { ParsedXml } from '@storyteller-platform/epub';
+import { Mapping } from './map.cjs';
 
 interface MarkupOptions {
     granularity?: "word" | "sentence";
@@ -11,14 +12,9 @@ interface MarkupOptions {
     logger?: Logger;
 }
 declare function markup(input: string, output: string, options: MarkupOptions): Promise<TimingAggregator>;
-declare function markupChapter(chapterId: string, chapterXml: ParsedXml, segmentation: SegmentationResult): {
+declare function markupChapter(chapterId: string, chapterXml: ParsedXml, segmentation: Sentence[], mapping: Mapping): {
     markedUp: ParsedXml;
     timing: _storyteller_platform_ghost_story.Timing;
 };
-type Mark = {
-    elementName: ElementName;
-    attributes: Record<string, string> | undefined;
-};
-declare function appendTextNode(chapterId: string, xml: ParsedXml, text: string, marks: Mark[], taggedSentences: Set<number>, sentenceId?: number): void;
 
-export { type MarkupOptions, appendTextNode, markup, markupChapter };
+export { type MarkupOptions, markup, markupChapter };

package/dist/markup/markup.d.ts

@@ -1,8 +1,9 @@
 import * as _storyteller_platform_ghost_story from '@storyteller-platform/ghost-story';
 import { TimingAggregator } from '@storyteller-platform/ghost-story';
-import { SegmentationResult } from '@echogarden/text-segmentation';
+import { Sentence } from '@echogarden/text-segmentation';
 import { Logger } from 'pino';
-import { ParsedXml, ElementName } from '@storyteller-platform/epub';
+import { ParsedXml } from '@storyteller-platform/epub';
+import { Mapping } from './map.js';
 
 interface MarkupOptions {
     granularity?: "word" | "sentence";
@@ -11,14 +12,9 @@ interface MarkupOptions {
     logger?: Logger;
 }
 declare function markup(input: string, output: string, options: MarkupOptions): Promise<TimingAggregator>;
-declare function markupChapter(chapterId: string, chapterXml: ParsedXml, segmentation: SegmentationResult): {
+declare function markupChapter(chapterId: string, chapterXml: ParsedXml, segmentation: Sentence[], mapping: Mapping): {
     markedUp: ParsedXml;
     timing: _storyteller_platform_ghost_story.Timing;
 };
-type Mark = {
-    elementName: ElementName;
-    attributes: Record<string, string> | undefined;
-};
-declare function appendTextNode(chapterId: string, xml: ParsedXml, text: string, marks: Mark[], taggedSentences: Set<number>, sentenceId?: number): void;
 
-export { type MarkupOptions, appendTextNode, markup, markupChapter };
+export { type MarkupOptions, markup, markupChapter };

package/dist/markup/markup.js

@@ -4,15 +4,16 @@ import {
 } from "../chunk-BIEQXUOY.js";
 import { copyFile } from "node:fs/promises";
 import { basename } from "node:path/posix";
-import {
-  Epub
-} from "@storyteller-platform/epub";
+import { Epub } from "@storyteller-platform/epub";
 import {
   createAggregator,
   createTiming
 } from "@storyteller-platform/ghost-story";
+import { Mark } from "./model.js";
+import { parseDom } from "./parseDom.js";
 import { getXhtmlSegmentation } from "./segmentation.js";
-import { BLOCKS } from "./semantics.js";
+import { serializeDom } from "./serializeDom.js";
+import { addMark } from "./transform.js";
 async function markup(input, output, options) {
   var _a, _b, _c, _d;
   var _stack = [];
@@ -35,14 +36,15 @@ async function markup(input, output, options) {
         continue;
       }
       const chapterXml = await epub.readXhtmlItemContents(chapterId);
-      const segmentation = await getXhtmlSegmentation(
+      const { result: segmentation, mapping } = await getXhtmlSegmentation(
         Epub.getXhtmlBody(chapterXml),
         { primaryLocale }
       );
       const { markedUp, timing: chapterTiming } = markupChapter(
         chapterId,
         chapterXml,
-        segmentation
+        segmentation,
+        mapping
       );
       timing.add(chapterTiming.summary());
       await epub.writeXhtmlItemContents(chapterId, markedUp);
@@ -55,7 +57,7 @@ async function markup(input, output, options) {
     __callDispose(_stack, _error, _hasError);
   }
 }
-function markupChapter(chapterId, chapterXml, segmentation) {
+function markupChapter(chapterId, chapterXml, segmentation, mapping) {
   const timing = createTiming();
   const html = Epub.findXmlChildByName("html", chapterXml);
   if (!html) throw new Error("Invalid XHTML document: no html element");
@@ -64,205 +66,25 @@ function markupChapter(chapterId, chapterXml, segmentation) {
   clearBodyElement(chapterXml);
   const taggedHtml = Epub.findXmlChildByName("html", chapterXml);
   const taggedBody = Epub.findXmlChildByName("body", taggedHtml["html"]);
-  taggedBody["body"] = [];
   timing.time("mark up", () => {
-    markupBySegmentation(
-      chapterId,
-      {
-        currentSentenceIndex: 0,
-        currentNodeProgress: 0,
-        currentSentenceProgress: 0
-      },
-      segmentation,
-      body,
-      /* @__PURE__ */ new Set(),
-      [],
-      Epub.getXmlChildren(taggedBody)
-    );
-  });
-  return { markedUp: chapterXml, timing };
-}
-function markupBySegmentation(chapterId, state, segmentation, currentNode, taggedSentences, marks, taggedXml) {
-  if (Epub.isXmlTextNode(currentNode)) {
-    const remainingSentence = segmentation.sentences[state.currentSentenceIndex].text.slice(state.currentSentenceProgress);
-    const text = currentNode["#text"];
-    const remainingNodeText = text.slice(state.currentNodeProgress);
-    const index = remainingNodeText.indexOf(remainingSentence[0]);
-    if (index === -1) {
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText,
-        marks,
-        taggedSentences
-      );
-      return {
-        ...state,
-        currentNodeProgress: -1
-      };
-    }
-    if (remainingNodeText.slice(index).length < remainingSentence.length) {
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText.slice(0, index),
-        marks,
-        taggedSentences
-      );
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText.slice(index),
-        marks,
-        taggedSentences,
-        state.currentSentenceIndex
-      );
-      return {
-        ...state,
-        currentSentenceProgress: state.currentSentenceProgress + remainingNodeText.length - index,
-        currentNodeProgress: -1
-      };
-    }
-    appendTextNode(
-      chapterId,
-      taggedXml,
-      remainingNodeText.slice(0, index),
-      marks,
-      taggedSentences
-    );
-    appendTextNode(
-      chapterId,
-      taggedXml,
-      remainingSentence,
-      marks,
-      taggedSentences,
-      state.currentSentenceIndex
-    );
-    if (state.currentSentenceIndex + 1 === segmentation.sentences.length) {
-      appendTextNode(
-        chapterId,
-        taggedXml,
-        remainingNodeText.slice(index + remainingSentence.length),
-        marks,
-        taggedSentences
-      );
-    }
-    const mapping = mapWhitespace(remainingNodeText);
-    const mapped = mapThrough(
-      remainingSentence.length,
-      mapping.filter(([start]) => start >= index)
-    );
-    return {
-      currentSentenceIndex: state.currentSentenceIndex + 1,
-      currentSentenceProgress: 0,
-      currentNodeProgress: state.currentNodeProgress + mapped + index
-    };
-  }
-  let nextState = {
-    ...state
-  };
-  const children = Epub.getXmlChildren(currentNode);
-  for (const child of children) {
-    if (nextState.currentSentenceIndex > segmentation.sentences.length + 1) {
-      taggedXml.push(child);
-      continue;
-    }
-    nextState.currentNodeProgress = 0;
-    let nextTaggedXml = taggedXml;
-    const nextMarks = [...marks];
-    if (!Epub.isXmlTextNode(child)) {
-      const childTagName = Epub.getXmlElementName(child);
-      const isTextContent = BLOCKS.includes(childTagName.toLowerCase());
-      if (Epub.getXmlChildren(child).length === 0) {
-        appendLeafNode(
-          chapterId,
-          taggedXml,
-          child,
-          nextMarks,
-          taggedSentences,
-          isTextContent || nextState.currentSentenceProgress === 0 ? void 0 : nextState.currentSentenceIndex
+    let root = parseDom(Epub.getXmlChildren(body));
+    let pos = 0;
+    let i = 0;
+    for (const sentence of segmentation) {
+      if (sentence.text.match(/\S/)) {
+        root = addMark(
+          root,
+          mapping.invert().map(pos),
+          mapping.invert().map(pos + sentence.text.replace(/\n$/, "").length, -1),
+          new Mark("span", { id: `${chapterId}-s${i}` })
         );
-        continue;
-      }
-      if (isTextContent) {
-        const block = {
-          [childTagName]: [],
-          ...child[":@"] && { ":@": child[":@"] }
-        };
-        nextTaggedXml.push(block);
-        nextTaggedXml = Epub.getXmlChildren(block);
-      } else {
-        nextMarks.push({
-          elementName: childTagName,
-          attributes: child[":@"]
-        });
+        i++;
       }
+      pos += sentence.text.replace(/\n$/, "").length;
     }
-    while (nextState.currentSentenceIndex < segmentation.sentences.length && nextState.currentNodeProgress !== -1) {
-      nextState = markupBySegmentation(
-        chapterId,
-        nextState,
-        segmentation,
-        child,
-        taggedSentences,
-        nextMarks,
-        nextTaggedXml
-      );
-    }
-  }
-  nextState.currentNodeProgress = -1;
-  return nextState;
-}
-function mapWhitespace(text) {
-  const re = /(\s\s+)/g;
-  const mapping = [];
-  let match = null;
-  while ((match = re.exec(text)) !== null) {
-    mapping.push([match.index, match[0].length, 1]);
-  }
-  return mapping;
-}
-function mapThrough(position, mapping) {
-  let result = position;
-  let index = 0;
-  while (index < mapping.length && mapping[index][0] < result) {
-    const map = mapping[index];
-    result += map[1] - map[2];
-    index++;
-  }
-  return result;
-}
-function appendTextNode(chapterId, xml, text, marks, taggedSentences, sentenceId) {
-  if (text.length === 0) return;
-  const textNode = { "#text": text };
-  appendLeafNode(chapterId, xml, textNode, marks, taggedSentences, sentenceId);
-}
-function appendLeafNode(chapterId, xml, node, marks, taggedSentences, sentenceId) {
-  var _a, _b;
-  const tagId = `${chapterId}-s${sentenceId}`;
-  const markedNode = [...marks].reverse().reduce(
-    (acc, mark) => ({
-      [mark.elementName]: [acc],
-      ":@": mark.attributes
-    }),
-    node
-  );
-  const lastNode = xml[xml.length - 1];
-  if (lastNode && !Epub.isXmlTextNode(lastNode) && ((_a = lastNode[":@"]) == null ? void 0 : _a["@_id"]) && lastNode[":@"]["@_id"] === tagId) {
-    const tagName = Epub.getXmlElementName(lastNode);
-    (_b = lastNode[tagName]) == null ? void 0 : _b.push(markedNode);
-    return;
-  }
-  if (sentenceId === void 0 || taggedSentences.has(sentenceId)) {
-    xml.push(markedNode);
-    return;
-  }
-  const taggedNode = {
-    span: [markedNode],
-    ":@": { "@_id": tagId }
-  };
-  taggedSentences.add(sentenceId);
-  xml.push(taggedNode);
+    taggedBody["body"] = serializeDom(root);
+  });
+  return { markedUp: chapterXml, timing };
 }
 function clearBodyElement(xml) {
   const html = Epub.findXmlChildByName("html", xml);
@@ -276,7 +98,6 @@ function clearBodyElement(xml) {
   });
 }
 export {
-  appendTextNode,
   markup,
   markupChapter
 };