@storyteller-platform/align 0.1.8 → 0.1.10

package/dist/markup/model.cjs

@@ -0,0 +1,172 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var model_exports = {};
+__export(model_exports, {
+  Mark: () => Mark,
+  Node: () => Node,
+  Root: () => Root,
+  TextNode: () => TextNode,
+  descendants: () => descendants
+});
+module.exports = __toCommonJS(model_exports);
+var import_itertools = require("itertools");
+var import_semantics = require("./semantics.cjs");
+class Root {
+  constructor(children) {
+    this.children = children;
+  }
+  isInline = false;
+  isBlock = true;
+  get border() {
+    return 0;
+  }
+  get textContent() {
+    return this.children.reduce((acc, child) => acc + child.textContent, "");
+  }
+  split(at) {
+    const children = [];
+    let pos = this.border;
+    for (const child of this.children) {
+      if (at > pos && at < pos + child.nodeSize) {
+        children.push(
+          ...child instanceof TextNode ? child.split(at - pos) : [child.split(at - pos)]
+        );
+      } else {
+        children.push(child);
+      }
+      pos += child.nodeSize;
+    }
+    return this.copy({ children });
+  }
+  copy(opts = {}) {
+    return new Root(opts.children ?? this.children);
+  }
+}
+class Node {
+  constructor(tagName, attrs = {}, children = [], marks = []) {
+    this.tagName = tagName;
+    this.attrs = attrs;
+    this.children = children;
+    this.marks = marks;
+  }
+  get isLeaf() {
+    return !this.children.length;
+  }
+  get isInline() {
+    return !this.isBlock;
+  }
+  get isBlock() {
+    return import_semantics.BLOCKS.includes(this.tagName);
+  }
+  get border() {
+    return this.isLeaf ? 0 : 1;
+  }
+  get nodeSize() {
+    return this.border + (this.children.reduce((acc, child) => acc + child.nodeSize, 0) || 1) + this.border;
+  }
+  get textContent() {
+    return this.children.reduce((acc, child) => acc + child.textContent, "");
+  }
+  split(at) {
+    if (at === this.border) return this;
+    if (at === this.nodeSize - this.border) return this;
+    const children = [];
+    let pos = this.border;
+    for (const child of this.children) {
+      if (at > pos && at < pos + child.nodeSize) {
+        if (child instanceof TextNode) {
+          children.push(...child.split(at - pos));
+        } else {
+          children.push(child.split(at - pos));
+        }
+      } else {
+        children.push(child);
+      }
+      pos += child.nodeSize;
+    }
+    return this.copy({ children });
+  }
+  copy(opts = {}) {
+    return new Node(
+      this.tagName,
+      opts.attrs ?? this.attrs,
+      opts.children ?? this.children,
+      opts.marks ?? this.marks
+    );
+  }
+}
+class Mark {
+  constructor(tagName, attrs = {}) {
+    this.tagName = tagName;
+    this.attrs = attrs;
+  }
+  eq(other) {
+    if (!other) return false;
+    if (Object.keys(this.attrs).length !== Object.keys(other.attrs).length)
+      return false;
+    for (const [key, value] of Object.entries(this.attrs)) {
+      if (other.attrs[key] !== value) return false;
+    }
+    return this.tagName === other.tagName;
+  }
+}
+class TextNode {
+  constructor(text, marks = []) {
+    this.text = text;
+    this.marks = marks;
+  }
+  isLeaf = true;
+  isInline = true;
+  isBlock = false;
+  border = 0;
+  get nodeSize() {
+    return this.text.length;
+  }
+  get textContent() {
+    return this.text;
+  }
+  split(at) {
+    if (at === 0) return [this];
+    if (at === this.text.length) return [this];
+    return [
+      new TextNode(this.text.slice(0, at), this.marks),
+      new TextNode(this.text.slice(at), this.marks)
+    ];
+  }
+  copy(opts = {}) {
+    return new TextNode(this.text, opts.marks ?? this.marks);
+  }
+}
+function descendants(root, cb, pos = 0) {
+  for (const [i, child] of (0, import_itertools.enumerate)(root.children)) {
+    const descend = cb(child, pos, root, i);
+    if (descend && !child.isLeaf) {
+      descendants(child, cb, pos + child.border);
+    }
+    pos += child.nodeSize;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  Mark,
+  Node,
+  Root,
+  TextNode,
+  descendants
+});

package/dist/markup/model.d.cts

@@ -0,0 +1,57 @@
+import { ElementName } from '@storyteller-platform/epub';
+
+declare class Root {
+    children: (Node | TextNode)[];
+    constructor(children: (Node | TextNode)[]);
+    isInline: boolean;
+    isBlock: boolean;
+    get border(): number;
+    get textContent(): string;
+    split(at: number): Root;
+    copy(opts?: {
+        children?: (Node | TextNode)[];
+    }): Root;
+}
+declare class Node {
+    tagName: ElementName;
+    attrs: Record<string, string>;
+    children: (Node | TextNode)[];
+    marks: Mark[];
+    constructor(tagName: ElementName, attrs?: Record<string, string>, children?: (Node | TextNode)[], marks?: Mark[]);
+    get isLeaf(): boolean;
+    get isInline(): boolean;
+    get isBlock(): boolean;
+    get border(): 0 | 1;
+    get nodeSize(): number;
+    get textContent(): string;
+    split(at: number): Node;
+    copy(opts?: {
+        attrs?: Record<string, string>;
+        children?: (Node | TextNode)[];
+        marks?: Mark[];
+    }): Node;
+}
+declare class Mark {
+    tagName: ElementName;
+    attrs: Record<string, string>;
+    constructor(tagName: ElementName, attrs?: Record<string, string>);
+    eq(other: Mark | undefined): boolean;
+}
+declare class TextNode {
+    text: string;
+    marks: Mark[];
+    constructor(text: string, marks?: Mark[]);
+    isLeaf: boolean;
+    isInline: boolean;
+    isBlock: boolean;
+    border: number;
+    get nodeSize(): number;
+    get textContent(): string;
+    split(at: number): TextNode[];
+    copy(opts?: {
+        marks?: Mark[];
+    }): TextNode;
+}
+declare function descendants(root: Root | Node, cb: (node: Node | TextNode, pos: number, parent: Node | Root, index: number) => boolean, pos?: number): void;
+
+export { Mark, Node, Root, TextNode, descendants };

package/dist/markup/model.d.ts

@@ -0,0 +1,57 @@
+import { ElementName } from '@storyteller-platform/epub';
+
+declare class Root {
+    children: (Node | TextNode)[];
+    constructor(children: (Node | TextNode)[]);
+    isInline: boolean;
+    isBlock: boolean;
+    get border(): number;
+    get textContent(): string;
+    split(at: number): Root;
+    copy(opts?: {
+        children?: (Node | TextNode)[];
+    }): Root;
+}
+declare class Node {
+    tagName: ElementName;
+    attrs: Record<string, string>;
+    children: (Node | TextNode)[];
+    marks: Mark[];
+    constructor(tagName: ElementName, attrs?: Record<string, string>, children?: (Node | TextNode)[], marks?: Mark[]);
+    get isLeaf(): boolean;
+    get isInline(): boolean;
+    get isBlock(): boolean;
+    get border(): 0 | 1;
+    get nodeSize(): number;
+    get textContent(): string;
+    split(at: number): Node;
+    copy(opts?: {
+        attrs?: Record<string, string>;
+        children?: (Node | TextNode)[];
+        marks?: Mark[];
+    }): Node;
+}
+declare class Mark {
+    tagName: ElementName;
+    attrs: Record<string, string>;
+    constructor(tagName: ElementName, attrs?: Record<string, string>);
+    eq(other: Mark | undefined): boolean;
+}
+declare class TextNode {
+    text: string;
+    marks: Mark[];
+    constructor(text: string, marks?: Mark[]);
+    isLeaf: boolean;
+    isInline: boolean;
+    isBlock: boolean;
+    border: number;
+    get nodeSize(): number;
+    get textContent(): string;
+    split(at: number): TextNode[];
+    copy(opts?: {
+        marks?: Mark[];
+    }): TextNode;
+}
+declare function descendants(root: Root | Node, cb: (node: Node | TextNode, pos: number, parent: Node | Root, index: number) => boolean, pos?: number): void;
+
+export { Mark, Node, Root, TextNode, descendants };

package/dist/markup/model.js

@@ -0,0 +1,145 @@
+import "../chunk-BIEQXUOY.js";
+import { enumerate } from "itertools";
+import { BLOCKS } from "./semantics.js";
+class Root {
+  constructor(children) {
+    this.children = children;
+  }
+  isInline = false;
+  isBlock = true;
+  get border() {
+    return 0;
+  }
+  get textContent() {
+    return this.children.reduce((acc, child) => acc + child.textContent, "");
+  }
+  split(at) {
+    const children = [];
+    let pos = this.border;
+    for (const child of this.children) {
+      if (at > pos && at < pos + child.nodeSize) {
+        children.push(
+          ...child instanceof TextNode ? child.split(at - pos) : [child.split(at - pos)]
+        );
+      } else {
+        children.push(child);
+      }
+      pos += child.nodeSize;
+    }
+    return this.copy({ children });
+  }
+  copy(opts = {}) {
+    return new Root(opts.children ?? this.children);
+  }
+}
+class Node {
+  constructor(tagName, attrs = {}, children = [], marks = []) {
+    this.tagName = tagName;
+    this.attrs = attrs;
+    this.children = children;
+    this.marks = marks;
+  }
+  get isLeaf() {
+    return !this.children.length;
+  }
+  get isInline() {
+    return !this.isBlock;
+  }
+  get isBlock() {
+    return BLOCKS.includes(this.tagName);
+  }
+  get border() {
+    return this.isLeaf ? 0 : 1;
+  }
+  get nodeSize() {
+    return this.border + (this.children.reduce((acc, child) => acc + child.nodeSize, 0) || 1) + this.border;
+  }
+  get textContent() {
+    return this.children.reduce((acc, child) => acc + child.textContent, "");
+  }
+  split(at) {
+    if (at === this.border) return this;
+    if (at === this.nodeSize - this.border) return this;
+    const children = [];
+    let pos = this.border;
+    for (const child of this.children) {
+      if (at > pos && at < pos + child.nodeSize) {
+        if (child instanceof TextNode) {
+          children.push(...child.split(at - pos));
+        } else {
+          children.push(child.split(at - pos));
+        }
+      } else {
+        children.push(child);
+      }
+      pos += child.nodeSize;
+    }
+    return this.copy({ children });
+  }
+  copy(opts = {}) {
+    return new Node(
+      this.tagName,
+      opts.attrs ?? this.attrs,
+      opts.children ?? this.children,
+      opts.marks ?? this.marks
+    );
+  }
+}
+class Mark {
+  constructor(tagName, attrs = {}) {
+    this.tagName = tagName;
+    this.attrs = attrs;
+  }
+  eq(other) {
+    if (!other) return false;
+    if (Object.keys(this.attrs).length !== Object.keys(other.attrs).length)
+      return false;
+    for (const [key, value] of Object.entries(this.attrs)) {
+      if (other.attrs[key] !== value) return false;
+    }
+    return this.tagName === other.tagName;
+  }
+}
+class TextNode {
+  constructor(text, marks = []) {
+    this.text = text;
+    this.marks = marks;
+  }
+  isLeaf = true;
+  isInline = true;
+  isBlock = false;
+  border = 0;
+  get nodeSize() {
+    return this.text.length;
+  }
+  get textContent() {
+    return this.text;
+  }
+  split(at) {
+    if (at === 0) return [this];
+    if (at === this.text.length) return [this];
+    return [
+      new TextNode(this.text.slice(0, at), this.marks),
+      new TextNode(this.text.slice(at), this.marks)
+    ];
+  }
+  copy(opts = {}) {
+    return new TextNode(this.text, opts.marks ?? this.marks);
+  }
+}
+function descendants(root, cb, pos = 0) {
+  for (const [i, child] of enumerate(root.children)) {
+    const descend = cb(child, pos, root, i);
+    if (descend && !child.isLeaf) {
+      descendants(child, cb, pos + child.border);
+    }
+    pos += child.nodeSize;
+  }
+}
+export {
+  Mark,
+  Node,
+  Root,
+  TextNode,
+  descendants
+};

package/dist/markup/parseDom.cjs

@@ -0,0 +1,59 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var parseDom_exports = {};
+__export(parseDom_exports, {
+  parseDom: () => parseDom,
+  parseDomNode: () => parseDomNode
+});
+module.exports = __toCommonJS(parseDom_exports);
+var import_epub = require("@storyteller-platform/epub");
+var import_model = require("./model.cjs");
+var import_semantics = require("./semantics.cjs");
+function parseDom(xml) {
+  const children = xml.flatMap((node) => parseDomNode(node));
+  return new import_model.Root(children);
+}
+function parseDomNode(xmlNode, marks) {
+  if (import_epub.Epub.isXmlTextNode(xmlNode)) {
+    return new import_model.TextNode(xmlNode["#text"], marks);
+  }
+  const tagName = import_epub.Epub.getXmlElementName(xmlNode);
+  if (import_semantics.BLOCKS.includes(tagName)) {
+    return new import_model.Node(
+      tagName,
+      import_epub.Epub.getXmlAttributes(xmlNode),
+      import_epub.Epub.getXmlChildren(xmlNode).flatMap((child) => parseDomNode(child)),
+      marks
+    );
+  }
+  if (!import_epub.Epub.getXmlChildren(xmlNode).length) {
+    return new import_model.Node(tagName, import_epub.Epub.getXmlAttributes(xmlNode), [], marks);
+  }
+  return import_epub.Epub.getXmlChildren(xmlNode).flatMap(
+    (child) => parseDomNode(child, [
+      ...marks ?? [],
+      new import_model.Mark(tagName, import_epub.Epub.getXmlAttributes(xmlNode))
+    ])
+  );
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parseDom,
+  parseDomNode
+});

package/dist/markup/parseDom.d.cts

@@ -0,0 +1,7 @@
+import { ParsedXml, XmlNode } from '@storyteller-platform/epub';
+import { Root, Mark, Node, TextNode } from './model.cjs';
+
+declare function parseDom(xml: ParsedXml): Root;
+declare function parseDomNode(xmlNode: XmlNode, marks?: Mark[]): Node | TextNode | (Node | TextNode)[];
+
+export { parseDom, parseDomNode };

package/dist/markup/parseDom.d.ts

@@ -0,0 +1,7 @@
+import { ParsedXml, XmlNode } from '@storyteller-platform/epub';
+import { Root, Mark, Node, TextNode } from './model.js';
+
+declare function parseDom(xml: ParsedXml): Root;
+declare function parseDomNode(xmlNode: XmlNode, marks?: Mark[]): Node | TextNode | (Node | TextNode)[];
+
+export { parseDom, parseDomNode };

package/dist/markup/parseDom.js

@@ -0,0 +1,35 @@
+import "../chunk-BIEQXUOY.js";
+import { Epub } from "@storyteller-platform/epub";
+import { Mark, Node, Root, TextNode } from "./model.js";
+import { BLOCKS } from "./semantics.js";
+function parseDom(xml) {
+  const children = xml.flatMap((node) => parseDomNode(node));
+  return new Root(children);
+}
+function parseDomNode(xmlNode, marks) {
+  if (Epub.isXmlTextNode(xmlNode)) {
+    return new TextNode(xmlNode["#text"], marks);
+  }
+  const tagName = Epub.getXmlElementName(xmlNode);
+  if (BLOCKS.includes(tagName)) {
+    return new Node(
+      tagName,
+      Epub.getXmlAttributes(xmlNode),
+      Epub.getXmlChildren(xmlNode).flatMap((child) => parseDomNode(child)),
+      marks
+    );
+  }
+  if (!Epub.getXmlChildren(xmlNode).length) {
+    return new Node(tagName, Epub.getXmlAttributes(xmlNode), [], marks);
+  }
+  return Epub.getXmlChildren(xmlNode).flatMap(
+    (child) => parseDomNode(child, [
+      ...marks ?? [],
+      new Mark(tagName, Epub.getXmlAttributes(xmlNode))
+    ])
+  );
+}
+export {
+  parseDom,
+  parseDomNode
+};

package/dist/markup/segmentation.cjs

@@ -22,64 +22,18 @@ __export(segmentation_exports, {
 });
 module.exports = __toCommonJS(segmentation_exports);
 var import_text_segmentation = require("@echogarden/text-segmentation");
-var import_epub = require("@storyteller-platform/epub");
-var import_semantics = require("./semantics.cjs");
+var import_parseDom = require("./parseDom.cjs");
+var import_transform = require("./transform.cjs");
 async function getXhtmlSegmentation(xml, options) {
-  const result = {
-    words: new import_text_segmentation.WordSequence(),
-    sentences: [],
-    segmentSentenceRanges: []
-  };
-  let stagedText = "";
-  for (const child of xml) {
-    if (import_epub.Epub.isXmlTextNode(child)) {
-      stagedText += child["#text"];
-      continue;
-    }
-    const childName = import_epub.Epub.getXmlElementName(child);
-    if (!import_semantics.BLOCKS.includes(childName)) {
-      stagedText += import_epub.Epub.getXhtmlTextContent(import_epub.Epub.getXmlChildren(child));
-      continue;
-    }
-    mergeSegmentations(
-      result,
-      await (0, import_text_segmentation.segmentText)(collapseWhitespace(stagedText), {
-        ...options.primaryLocale && {
-          language: options.primaryLocale.language
-        },
-        enableEastAsianPostprocessing: true
-      })
-    );
-    stagedText = "";
-    mergeSegmentations(
-      result,
-      await getXhtmlSegmentation(import_epub.Epub.getXmlChildren(child), options)
-    );
-  }
-  mergeSegmentations(
-    result,
-    await (0, import_text_segmentation.segmentText)(collapseWhitespace(stagedText), {
-      ...options.primaryLocale && {
-        language: options.primaryLocale.language
-      },
-      enableEastAsianPostprocessing: true
-    })
-  );
-  return result;
-}
-function collapseWhitespace(text) {
-  return text.replace(/^\s*/, "").replace(/\s*$/, "").replaceAll(/\s+/g, " ");
-}
-function mergeSegmentations(first, second) {
-  for (const wordEntry of second.words.entries) {
-    first.words.addWord(
-      wordEntry.text,
-      wordEntry.startOffset,
-      wordEntry.isPunctuation
-    );
-  }
-  first.sentences.push(...second.sentences);
-  first.segmentSentenceRanges.push(...second.segmentSentenceRanges);
+  const root = (0, import_parseDom.parseDom)(xml);
+  const { result: text, mapping } = (0, import_transform.liftText)(root);
+  const result = await (0, import_text_segmentation.segmentText)(text, {
+    ...options.primaryLocale && {
+      language: options.primaryLocale.language
+    },
+    enableEastAsianPostprocessing: true
+  });
+  return { result: result.sentences, mapping };
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/markup/segmentation.d.cts

@@ -1,8 +1,12 @@
 import { SegmentationResult } from '@echogarden/text-segmentation';
 import { ParsedXml } from '@storyteller-platform/epub';
+import { Mapping } from './map.cjs';
 
 declare function getXhtmlSegmentation(xml: ParsedXml, options: {
-    primaryLocale?: Intl.Locale | null;
-}): Promise<SegmentationResult>;
+    primaryLocale?: Intl.Locale | null | undefined;
+}): Promise<{
+    result: SegmentationResult["sentences"];
+    mapping: Mapping;
+}>;
 
 export { getXhtmlSegmentation };

package/dist/markup/segmentation.d.ts

@@ -1,8 +1,12 @@
 import { SegmentationResult } from '@echogarden/text-segmentation';
 import { ParsedXml } from '@storyteller-platform/epub';
+import { Mapping } from './map.js';
 
 declare function getXhtmlSegmentation(xml: ParsedXml, options: {
-    primaryLocale?: Intl.Locale | null;
-}): Promise<SegmentationResult>;
+    primaryLocale?: Intl.Locale | null | undefined;
+}): Promise<{
+    result: SegmentationResult["sentences"];
+    mapping: Mapping;
+}>;
 
 export { getXhtmlSegmentation };