@storyteller-platform/align 0.1.13 → 0.1.14

package/dist/align/search.cjs

@@ -0,0 +1,122 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var search_exports = {};
+__export(search_exports, {
+  buildNgramIndex: () => buildNgramIndex,
+  collectBoundaryVotes: () => collectBoundaryVotes,
+  findBoundaries: () => findBoundaries,
+  ngrams: () => ngrams
+});
+module.exports = __toCommonJS(search_exports);
+var import_itertools = require("itertools");
+function buildNgramIndex(text) {
+  const index = /* @__PURE__ */ new Map();
+  for (const [ngram, pos] of ngrams(text)) {
+    const positions = index.get(ngram);
+    if (positions) {
+      positions.push(pos);
+    } else {
+      index.set(ngram, [pos]);
+    }
+  }
+  return index;
+}
+function* ngrams(text) {
+  const words = text.split("-");
+  let pos = 0;
+  for (const i of (0, import_itertools.range)(words.length - 4)) {
+    const ngram = words.slice(i, i + 5).join("-");
+    yield [ngram, pos];
+    pos += words[i].length + 1;
+  }
+}
+function collectBoundaryVotes(query, document) {
+  const documentIndex = buildNgramIndex(document);
+  let skippedNgrams = 0;
+  let totalNgrams = 0;
+  const startVotes = [];
+  const endVotes = [];
+  for (const [ngram, start] of ngrams(query)) {
+    totalNgrams++;
+    const documentStarts = documentIndex.get(ngram);
+    if (!documentStarts) {
+      skippedNgrams++;
+      continue;
+    }
+    for (const documentStart of documentStarts) {
+      startVotes.push(documentStart - start);
+      endVotes.push(documentStart + (query.length - start));
+    }
+  }
+  if (skippedNgrams > totalNgrams / 2) {
+    return null;
+  }
+  return { startVotes, endVotes };
+}
+const BIN_SIZE = 1e3;
+function binBoundaryVotes(votes) {
+  const start = (0, import_itertools.min)(votes);
+  const bins = /* @__PURE__ */ new Map();
+  if (start === void 0) return bins;
+  for (const vote of votes) {
+    const binIndex = Math.floor((vote - start) / BIN_SIZE);
+    const bin = bins.get(binIndex);
+    if (bin) {
+      bin.push(vote);
+    } else {
+      bins.set(binIndex, [vote]);
+    }
+  }
+  return bins;
+}
+function chooseBestFromBins(bins, dir) {
+  const totalLength = Array.from(bins.values()).reduce(
+    (acc, bin) => acc + bin.length,
+    0
+  );
+  const best = (0, import_itertools.max)(bins.values(), (bin) => bin.length);
+  if (!best) return null;
+  if (best.length / totalLength < 0.2) {
+    return null;
+  }
+  return dir > 0 ? (0, import_itertools.max)(best) ?? null : (0, import_itertools.min)(best) ?? null;
+}
+function findBoundaries(query, document) {
+  const boundaryVotes = collectBoundaryVotes(query, document);
+  if (!boundaryVotes) return null;
+  const { startVotes, endVotes } = boundaryVotes;
+  const startBins = binBoundaryVotes(startVotes);
+  const bestStart = chooseBestFromBins(startBins, -1);
+  if (bestStart === null) {
+    return null;
+  }
+  const endBins = binBoundaryVotes(endVotes);
+  const bestEnd = chooseBestFromBins(endBins, 1);
+  if (bestEnd === null) {
+    return null;
+  }
+  return { start: bestStart, end: bestEnd };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  buildNgramIndex,
+  collectBoundaryVotes,
+  findBoundaries,
+  ngrams
+});

package/dist/align/search.d.cts

@@ -0,0 +1,12 @@
+declare function buildNgramIndex(text: string): Map<string, number[]>;
+declare function ngrams(text: string): Generator<readonly [string, number], void, unknown>;
+declare function collectBoundaryVotes(query: string, document: string): {
+    startVotes: number[];
+    endVotes: number[];
+} | null;
+declare function findBoundaries(query: string, document: string): {
+    start: number;
+    end: number;
+} | null;
+
+export { buildNgramIndex, collectBoundaryVotes, findBoundaries, ngrams };

package/dist/align/search.d.ts

@@ -0,0 +1,12 @@
+declare function buildNgramIndex(text: string): Map<string, number[]>;
+declare function ngrams(text: string): Generator<readonly [string, number], void, unknown>;
+declare function collectBoundaryVotes(query: string, document: string): {
+    startVotes: number[];
+    endVotes: number[];
+} | null;
+declare function findBoundaries(query: string, document: string): {
+    start: number;
+    end: number;
+} | null;
+
+export { buildNgramIndex, collectBoundaryVotes, findBoundaries, ngrams };

package/dist/align/search.js

@@ -0,0 +1,96 @@
+import "../chunk-BIEQXUOY.js";
+import { max, min, range } from "itertools";
+function buildNgramIndex(text) {
+  const index = /* @__PURE__ */ new Map();
+  for (const [ngram, pos] of ngrams(text)) {
+    const positions = index.get(ngram);
+    if (positions) {
+      positions.push(pos);
+    } else {
+      index.set(ngram, [pos]);
+    }
+  }
+  return index;
+}
+function* ngrams(text) {
+  const words = text.split("-");
+  let pos = 0;
+  for (const i of range(words.length - 4)) {
+    const ngram = words.slice(i, i + 5).join("-");
+    yield [ngram, pos];
+    pos += words[i].length + 1;
+  }
+}
+function collectBoundaryVotes(query, document) {
+  const documentIndex = buildNgramIndex(document);
+  let skippedNgrams = 0;
+  let totalNgrams = 0;
+  const startVotes = [];
+  const endVotes = [];
+  for (const [ngram, start] of ngrams(query)) {
+    totalNgrams++;
+    const documentStarts = documentIndex.get(ngram);
+    if (!documentStarts) {
+      skippedNgrams++;
+      continue;
+    }
+    for (const documentStart of documentStarts) {
+      startVotes.push(documentStart - start);
+      endVotes.push(documentStart + (query.length - start));
+    }
+  }
+  if (skippedNgrams > totalNgrams / 2) {
+    return null;
+  }
+  return { startVotes, endVotes };
+}
+const BIN_SIZE = 1e3;
+function binBoundaryVotes(votes) {
+  const start = min(votes);
+  const bins = /* @__PURE__ */ new Map();
+  if (start === void 0) return bins;
+  for (const vote of votes) {
+    const binIndex = Math.floor((vote - start) / BIN_SIZE);
+    const bin = bins.get(binIndex);
+    if (bin) {
+      bin.push(vote);
+    } else {
+      bins.set(binIndex, [vote]);
+    }
+  }
+  return bins;
+}
+function chooseBestFromBins(bins, dir) {
+  const totalLength = Array.from(bins.values()).reduce(
+    (acc, bin) => acc + bin.length,
+    0
+  );
+  const best = max(bins.values(), (bin) => bin.length);
+  if (!best) return null;
+  if (best.length / totalLength < 0.2) {
+    return null;
+  }
+  return dir > 0 ? max(best) ?? null : min(best) ?? null;
+}
+function findBoundaries(query, document) {
+  const boundaryVotes = collectBoundaryVotes(query, document);
+  if (!boundaryVotes) return null;
+  const { startVotes, endVotes } = boundaryVotes;
+  const startBins = binBoundaryVotes(startVotes);
+  const bestStart = chooseBestFromBins(startBins, -1);
+  if (bestStart === null) {
+    return null;
+  }
+  const endBins = binBoundaryVotes(endVotes);
+  const bestEnd = chooseBestFromBins(endBins, 1);
+  if (bestEnd === null) {
+    return null;
+  }
+  return { start: bestStart, end: bestEnd };
+}
+export {
+  buildNgramIndex,
+  collectBoundaryVotes,
+  findBoundaries,
+  ngrams
+};

package/dist/errorAlign/utils.d.cts

@@ -21,7 +21,7 @@ declare class Alignment {
  * @returns All possible combinations of operation types.
  */
 declare function opTypePowerset(): IterableIterator<NonNullable<"DELETE" | "MATCH" | "INSERT" | "SUBSTITUTE">[]>;
-declare function reversed<T>(iterable: IterableIterator<T>): T[];
+declare function reversed<T>(iterable: Iterable<T>): T[];
 declare const START_DELIMITER = "<";
 declare const END_DELIMITER = ">";
 declare const DELIMITERS: Set<string>;

package/dist/errorAlign/utils.d.ts

@@ -21,7 +21,7 @@ declare class Alignment {
  * @returns All possible combinations of operation types.
  */
 declare function opTypePowerset(): IterableIterator<NonNullable<"DELETE" | "MATCH" | "INSERT" | "SUBSTITUTE">[]>;
-declare function reversed<T>(iterable: IterableIterator<T>): T[];
+declare function reversed<T>(iterable: Iterable<T>): T[];
 declare const START_DELIMITER = "<";
 declare const END_DELIMITER = ">";
 declare const DELIMITERS: Set<string>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@storyteller-platform/align",
-  "version": "0.1.13",
+  "version": "0.1.14",
   "description": "A library and CLI for automatically aligning audiobooks and EPUBs to produce Media Overlays",
   "author": "Shane Friedman",
   "license": "MIT",
@@ -57,9 +57,9 @@
     "@esfx/async-semaphore": "^1.0.0",
     "@optique/core": "^0.10.7",
     "@optique/run": "^0.10.7",
-    "@storyteller-platform/audiobook": "^0.3.9",
+    "@storyteller-platform/audiobook": "^0.3.10",
     "@storyteller-platform/epub": "^0.4.8",
-    "@storyteller-platform/ghost-story": "^0.1.5",
+    "@storyteller-platform/ghost-story": "^0.1.6",
     "@storyteller-platform/transliteration": "^3.1.0",
     "chalk": "^5.4.1",
     "cli-progress": "^3.12.0",

package/dist/align/fuzzy.cjs

@@ -1,164 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var fuzzy_exports = {};
-__export(fuzzy_exports, {
-  findNearestMatch: () => findNearestMatch
-});
-module.exports = __toCommonJS(fuzzy_exports);
-function findNearestMatch(needle, haystack, maxDist) {
-  let nearest = null;
-  for (const match of levenshteinNgram(needle, haystack, maxDist)) {
-    if (!nearest || match.dist < nearest.dist) {
-      nearest = match;
-    }
-  }
-  return nearest && {
-    match: haystack.slice(nearest.start, nearest.end),
-    index: nearest.start
-  };
-}
-function reverse(str, from = str.length, to = 0) {
-  let reversed = "";
-  for (let i = from - 1; i >= to; i--) {
-    reversed = reversed + str[i];
-  }
-  return reversed;
-}
-function* searchExact(subsequence, sequence, startIndex = 0, endIndex = sequence.length) {
-  let index = sequence.indexOf(subsequence, startIndex);
-  while (index !== -1 && index + subsequence.length < endIndex) {
-    yield index;
-    index = sequence.indexOf(subsequence, index + 1);
-  }
-}
-function expand(subsequence, sequence, maxDist) {
-  const subsequenceLength = subsequence.length;
-  if (subsequenceLength === 0) {
-    return { index: 0, score: 0 };
-  }
-  const scores = Array.from({ length: subsequenceLength + 1 }).map((_, i) => i);
-  let minScore = subsequenceLength;
-  let minScoreIndex = -1;
-  let maxGoodScore = maxDist;
-  let newNeedleIndexRangeStart = 0;
-  let newNeedleIndexRangeEnd = subsequenceLength - 1;
-  for (let sequenceIndex = 0; sequenceIndex < sequence.length; sequenceIndex++) {
-    const char = sequence[sequenceIndex];
-    const needleIndexRangeStart = newNeedleIndexRangeStart;
-    const needleIndexRangeEnd = Math.min(
-      subsequenceLength,
-      newNeedleIndexRangeEnd + 1
-    );
-    let a = sequenceIndex;
-    let c = a + 1;
-    if (c <= maxGoodScore) {
-      newNeedleIndexRangeStart = 0;
-      newNeedleIndexRangeEnd = 0;
-    } else {
-      newNeedleIndexRangeStart = null;
-      newNeedleIndexRangeEnd = -1;
-    }
-    for (let subsequenceIndex = needleIndexRangeStart; subsequenceIndex < needleIndexRangeEnd; subsequenceIndex++) {
-      const b = scores[subsequenceIndex];
-      c = scores[subsequenceIndex] = Math.min(
-        a + (char === subsequence[subsequenceIndex] ? 0 : 1),
-        b + 1,
-        c + 1
-      );
-      a = b;
-      if (c <= maxGoodScore) {
-        if (newNeedleIndexRangeStart === null) {
-          newNeedleIndexRangeStart = subsequenceIndex;
-        }
-        newNeedleIndexRangeEnd = Math.max(
-          newNeedleIndexRangeEnd,
-          subsequenceIndex + 1 + (maxGoodScore - c)
-        );
-      }
-    }
-    if (newNeedleIndexRangeStart === null) {
-      break;
-    }
-    if (needleIndexRangeEnd === subsequenceLength && c <= minScore) {
-      minScore = c;
-      minScoreIndex = sequenceIndex;
-      if (minScore < maxGoodScore) {
-        maxGoodScore = minScore;
-      }
-    }
-  }
-  return minScore <= maxDist ? { score: minScore, index: minScoreIndex + 1 } : null;
-}
-function* levenshteinNgram(subsequence, sequence, maxDist) {
-  const subsequenceLength = subsequence.length;
-  const sequenceLength = sequence.length;
-  const ngramLength = Math.floor(subsequenceLength / (maxDist + 1));
-  if (ngramLength === 0) {
-    throw new Error("The subsequence length must be greater than maxDist");
-  }
-  for (let ngramStart = 0; ngramStart < subsequenceLength - ngramLength + 1; ngramStart += ngramLength) {
-    const ngramEnd = ngramStart + ngramLength;
-    const subsequenceBeforeReversed = reverse(subsequence, ngramStart);
-    const subsequenceAfter = subsequence.slice(ngramEnd);
-    const startIndex = Math.max(0, ngramStart - maxDist);
-    const endIndex = Math.min(
-      sequenceLength,
-      sequenceLength - subsequenceLength + ngramEnd + maxDist
-    );
-    for (const index of searchExact(
-      subsequence.slice(ngramStart, ngramEnd),
-      sequence,
-      startIndex,
-      endIndex
-    )) {
-      const rightMatch = expand(
-        subsequenceAfter,
-        sequence.slice(
-          index + ngramLength,
-          index - ngramStart + subsequenceLength + maxDist
-        ),
-        maxDist
-      );
-      if (rightMatch === null) continue;
-      const { score: distRight, index: rightExpandSize } = rightMatch;
-      const leftMatch = expand(
-        subsequenceBeforeReversed,
-        reverse(
-          sequence,
-          index,
-          Math.max(0, index - ngramStart - (maxDist - distRight))
-        ),
-        maxDist - distRight
-      );
-      if (leftMatch === null) continue;
-      const { score: distLeft, index: leftExpandSize } = leftMatch;
-      const start = index - leftExpandSize;
-      yield {
-        start,
-        end: index + ngramLength + rightExpandSize,
-        // dist: distLeft + distRight + (start / sequenceLength) * maxDist,
-        dist: distLeft + distRight
-      };
-    }
-  }
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  findNearestMatch
-});

package/dist/align/fuzzy.d.cts

@@ -1,6 +0,0 @@
-declare function findNearestMatch(needle: string, haystack: string, maxDist: number): {
-    match: string;
-    index: number;
-} | null;
-
-export { findNearestMatch };

package/dist/align/fuzzy.d.ts

@@ -1,6 +0,0 @@
-declare function findNearestMatch(needle: string, haystack: string, maxDist: number): {
-    match: string;
-    index: number;
-} | null;
-
-export { findNearestMatch };

package/dist/align/fuzzy.js

@@ -1,141 +0,0 @@
-import "../chunk-BIEQXUOY.js";
-function findNearestMatch(needle, haystack, maxDist) {
-  let nearest = null;
-  for (const match of levenshteinNgram(needle, haystack, maxDist)) {
-    if (!nearest || match.dist < nearest.dist) {
-      nearest = match;
-    }
-  }
-  return nearest && {
-    match: haystack.slice(nearest.start, nearest.end),
-    index: nearest.start
-  };
-}
-function reverse(str, from = str.length, to = 0) {
-  let reversed = "";
-  for (let i = from - 1; i >= to; i--) {
-    reversed = reversed + str[i];
-  }
-  return reversed;
-}
-function* searchExact(subsequence, sequence, startIndex = 0, endIndex = sequence.length) {
-  let index = sequence.indexOf(subsequence, startIndex);
-  while (index !== -1 && index + subsequence.length < endIndex) {
-    yield index;
-    index = sequence.indexOf(subsequence, index + 1);
-  }
-}
-function expand(subsequence, sequence, maxDist) {
-  const subsequenceLength = subsequence.length;
-  if (subsequenceLength === 0) {
-    return { index: 0, score: 0 };
-  }
-  const scores = Array.from({ length: subsequenceLength + 1 }).map((_, i) => i);
-  let minScore = subsequenceLength;
-  let minScoreIndex = -1;
-  let maxGoodScore = maxDist;
-  let newNeedleIndexRangeStart = 0;
-  let newNeedleIndexRangeEnd = subsequenceLength - 1;
-  for (let sequenceIndex = 0; sequenceIndex < sequence.length; sequenceIndex++) {
-    const char = sequence[sequenceIndex];
-    const needleIndexRangeStart = newNeedleIndexRangeStart;
-    const needleIndexRangeEnd = Math.min(
-      subsequenceLength,
-      newNeedleIndexRangeEnd + 1
-    );
-    let a = sequenceIndex;
-    let c = a + 1;
-    if (c <= maxGoodScore) {
-      newNeedleIndexRangeStart = 0;
-      newNeedleIndexRangeEnd = 0;
-    } else {
-      newNeedleIndexRangeStart = null;
-      newNeedleIndexRangeEnd = -1;
-    }
-    for (let subsequenceIndex = needleIndexRangeStart; subsequenceIndex < needleIndexRangeEnd; subsequenceIndex++) {
-      const b = scores[subsequenceIndex];
-      c = scores[subsequenceIndex] = Math.min(
-        a + (char === subsequence[subsequenceIndex] ? 0 : 1),
-        b + 1,
-        c + 1
-      );
-      a = b;
-      if (c <= maxGoodScore) {
-        if (newNeedleIndexRangeStart === null) {
-          newNeedleIndexRangeStart = subsequenceIndex;
-        }
-        newNeedleIndexRangeEnd = Math.max(
-          newNeedleIndexRangeEnd,
-          subsequenceIndex + 1 + (maxGoodScore - c)
-        );
-      }
-    }
-    if (newNeedleIndexRangeStart === null) {
-      break;
-    }
-    if (needleIndexRangeEnd === subsequenceLength && c <= minScore) {
-      minScore = c;
-      minScoreIndex = sequenceIndex;
-      if (minScore < maxGoodScore) {
-        maxGoodScore = minScore;
-      }
-    }
-  }
-  return minScore <= maxDist ? { score: minScore, index: minScoreIndex + 1 } : null;
-}
-function* levenshteinNgram(subsequence, sequence, maxDist) {
-  const subsequenceLength = subsequence.length;
-  const sequenceLength = sequence.length;
-  const ngramLength = Math.floor(subsequenceLength / (maxDist + 1));
-  if (ngramLength === 0) {
-    throw new Error("The subsequence length must be greater than maxDist");
-  }
-  for (let ngramStart = 0; ngramStart < subsequenceLength - ngramLength + 1; ngramStart += ngramLength) {
-    const ngramEnd = ngramStart + ngramLength;
-    const subsequenceBeforeReversed = reverse(subsequence, ngramStart);
-    const subsequenceAfter = subsequence.slice(ngramEnd);
-    const startIndex = Math.max(0, ngramStart - maxDist);
-    const endIndex = Math.min(
-      sequenceLength,
-      sequenceLength - subsequenceLength + ngramEnd + maxDist
-    );
-    for (const index of searchExact(
-      subsequence.slice(ngramStart, ngramEnd),
-      sequence,
-      startIndex,
-      endIndex
-    )) {
-      const rightMatch = expand(
-        subsequenceAfter,
-        sequence.slice(
-          index + ngramLength,
-          index - ngramStart + subsequenceLength + maxDist
-        ),
-        maxDist
-      );
-      if (rightMatch === null) continue;
-      const { score: distRight, index: rightExpandSize } = rightMatch;
-      const leftMatch = expand(
-        subsequenceBeforeReversed,
-        reverse(
-          sequence,
-          index,
-          Math.max(0, index - ngramStart - (maxDist - distRight))
-        ),
-        maxDist - distRight
-      );
-      if (leftMatch === null) continue;
-      const { score: distLeft, index: leftExpandSize } = leftMatch;
-      const start = index - leftExpandSize;
-      yield {
-        start,
-        end: index + ngramLength + rightExpandSize,
-        // dist: distLeft + distRight + (start / sequenceLength) * maxDist,
-        dist: distLeft + distRight
-      };
-    }
-  }
-}
-export {
-  findNearestMatch
-};