@storyteller-platform/align 0.1.56 → 0.1.58

package/dist/align/search.cjs

@@ -21,13 +21,15 @@ __export(search_exports, {
   buildNgramIndex: () => buildNgramIndex,
   collectBoundaryVotes: () => collectBoundaryVotes,
   findBoundaries: () => findBoundaries,
+  findCtcBoundaries: () => findCtcBoundaries,
   ngrams: () => ngrams
 });
 module.exports = __toCommonJS(search_exports);
 var import_itertools = require("itertools");
-function buildNgramIndex(text) {
+var import_mapping = require("@storyteller-platform/mapping");
+function buildNgramIndex(text, ngramSize, gramDelimiter) {
   const index = /* @__PURE__ */ new Map();
-  for (const [ngram, pos] of ngrams(text)) {
+  for (const [ngram, pos] of ngrams(text, ngramSize, gramDelimiter)) {
     const positions = index.get(ngram);
     if (positions) {
       positions.push(pos);
@@ -37,22 +39,21 @@ function buildNgramIndex(text) {
   }
   return index;
 }
-const NGRAM_SIZE = 5;
-function* ngrams(text) {
-  const words = text.split("-");
-  for (const i of (0, import_itertools.range)(words.length - NGRAM_SIZE - 1)) {
-    const ngram = words.slice(i, i + NGRAM_SIZE).join("-");
+function* ngrams(text, ngramSize, gramDelimiter) {
+  const grams = text.split(gramDelimiter);
+  for (const i of (0, import_itertools.range)(grams.length - ngramSize + 1)) {
+    const ngram = grams.slice(i, i + ngramSize).join(gramDelimiter);
     yield [ngram, i];
   }
 }
-function collectBoundaryVotes(query, document) {
-  const queryWords = query.split("-");
-  const documentIndex = buildNgramIndex(document);
+function collectBoundaryVotes(query, document, ngramSize, gramDelimiter) {
+  const queryGrams = query.split(gramDelimiter);
+  const documentIndex = buildNgramIndex(document, ngramSize, gramDelimiter);
   let skippedNgrams = 0;
   let totalNgrams = 0;
   const startVotes = [];
   const endVotes = [];
-  for (const [ngram, start] of ngrams(query)) {
+  for (const [ngram, start] of ngrams(query, ngramSize, gramDelimiter)) {
     totalNgrams++;
     const documentStarts = documentIndex.get(ngram);
     if (!documentStarts) {
@@ -61,7 +62,7 @@ function collectBoundaryVotes(query, document) {
     }
     for (const documentStart of documentStarts) {
       startVotes.push(documentStart - start);
-      endVotes.push(documentStart + (queryWords.length - start));
+      endVotes.push(documentStart + (queryGrams.length - start));
     }
   }
   if (skippedNgrams > totalNgrams / 2) {
@@ -69,13 +70,12 @@ function collectBoundaryVotes(query, document) {
   }
   return { startVotes, endVotes };
 }
-const BIN_SIZE = 1e3;
-function binBoundaryVotes(votes) {
+function binBoundaryVotes(votes, binSize) {
   const start = (0, import_itertools.min)(votes);
   const bins = /* @__PURE__ */ new Map();
   if (start === void 0) return bins;
   for (const vote of votes) {
-    const binIndex = Math.floor((vote - start) / BIN_SIZE);
+    const binIndex = Math.floor((vote - start) / binSize);
     const bin = bins.get(binIndex);
     if (bin) {
       bin.push(vote);
@@ -97,31 +97,83 @@ function chooseBestFromBins(bins, dir) {
   }
   return dir > 0 ? (0, import_itertools.max)(best) ?? null : (0, import_itertools.min)(best) ?? null;
 }
-function getOffsetFromWordIndex(wordIndex, document) {
-  const words = document.split("-");
+function getOffsetFromWordIndex(wordIndex, document, gramDelimiter) {
+  const words = document.split(gramDelimiter);
   let offset = 0;
   for (const i of (0, import_itertools.range)(Math.min(words.length, Math.max(0, wordIndex)))) {
     offset += words[i].length + 1;
   }
   return offset;
 }
-function findBoundaries(query, document) {
-  const boundaryVotes = collectBoundaryVotes(query, document);
+function findBoundariesInternal(query, document, ngramSize, binSize, gramDelimiter) {
+  const boundaryVotes = collectBoundaryVotes(
+    query,
+    document,
+    ngramSize,
+    gramDelimiter
+  );
   if (!boundaryVotes) return null;
   const { startVotes, endVotes } = boundaryVotes;
-  const startBins = binBoundaryVotes(startVotes);
+  const startBins = binBoundaryVotes(startVotes, binSize);
   const bestStart = chooseBestFromBins(startBins, -1);
   if (bestStart === null) {
     return null;
   }
-  const endBins = binBoundaryVotes(endVotes);
+  const endBins = binBoundaryVotes(endVotes, binSize);
   const bestEnd = chooseBestFromBins(endBins, 1);
   if (bestEnd === null) {
     return null;
   }
+  return { bestStart, bestEnd };
+}
+function findBoundaries(query, document) {
+  const ngramSize = 5;
+  const binSize = 1e3;
+  const gramDelimiter = "-";
+  const result = findBoundariesInternal(
+    query,
+    document,
+    ngramSize,
+    binSize,
+    gramDelimiter
+  );
+  if (!result) return null;
+  const { bestStart, bestEnd } = result;
+  return {
+    start: getOffsetFromWordIndex(bestStart, document, gramDelimiter),
+    end: getOffsetFromWordIndex(bestEnd, document, gramDelimiter)
+  };
+}
+function smooshWords(input) {
+  const mapping = new import_mapping.Mapping();
+  let smooshed = "";
+  for (const [i, c] of (0, import_itertools.enumerate)(input)) {
+    if (c !== "-") {
+      smooshed += c;
+      continue;
+    }
+    mapping.insertMap(i, 1, 0);
+  }
+  return { smooshed, mapping };
+}
+function findCtcBoundaries(query, document) {
+  const ngramSize = 10;
+  const binSize = 5e3;
+  const gramDelimiter = "";
+  const { smooshed: smooshedQuery } = smooshWords(query);
+  const { smooshed: smooshedDocument, mapping: documentMapping } = smooshWords(document);
+  const result = findBoundariesInternal(
+    smooshedQuery,
+    smooshedDocument,
+    ngramSize,
+    binSize,
+    gramDelimiter
+  );
+  if (!result) return null;
+  const { bestStart, bestEnd } = result;
   return {
-    start: getOffsetFromWordIndex(bestStart, document),
-    end: getOffsetFromWordIndex(bestEnd, document)
+    start: documentMapping.cursor().map(Math.max(bestStart, 0), "start"),
+    end: documentMapping.cursor().map(Math.min(bestEnd, document.length - 1), "end")
   };
 }
 // Annotate the CommonJS export names for ESM import in node:
@@ -129,5 +181,6 @@ function findBoundaries(query, document) {
   buildNgramIndex,
   collectBoundaryVotes,
   findBoundaries,
+  findCtcBoundaries,
   ngrams
 });

package/dist/align/search.d.cts

@@ -1,6 +1,6 @@
-declare function buildNgramIndex(text: string): Map<string, number[]>;
-declare function ngrams(text: string): Generator<readonly [string, number], void, unknown>;
-declare function collectBoundaryVotes(query: string, document: string): {
+declare function buildNgramIndex(text: string, ngramSize: number, gramDelimiter: string): Map<string, number[]>;
+declare function ngrams(text: string, ngramSize: number, gramDelimiter: string): Generator<readonly [string, number], void, unknown>;
+declare function collectBoundaryVotes(query: string, document: string, ngramSize: number, gramDelimiter: string): {
     startVotes: number[];
     endVotes: number[];
 } | null;
@@ -8,5 +8,9 @@ declare function findBoundaries(query: string, document: string): {
     start: number;
     end: number;
 } | null;
+declare function findCtcBoundaries(query: string, document: string): {
+    start: number;
+    end: number;
+} | null;
 
-export { buildNgramIndex, collectBoundaryVotes, findBoundaries, ngrams };
+export { buildNgramIndex, collectBoundaryVotes, findBoundaries, findCtcBoundaries, ngrams };

package/dist/align/search.d.ts

@@ -1,6 +1,6 @@
-declare function buildNgramIndex(text: string): Map<string, number[]>;
-declare function ngrams(text: string): Generator<readonly [string, number], void, unknown>;
-declare function collectBoundaryVotes(query: string, document: string): {
+declare function buildNgramIndex(text: string, ngramSize: number, gramDelimiter: string): Map<string, number[]>;
+declare function ngrams(text: string, ngramSize: number, gramDelimiter: string): Generator<readonly [string, number], void, unknown>;
+declare function collectBoundaryVotes(query: string, document: string, ngramSize: number, gramDelimiter: string): {
     startVotes: number[];
     endVotes: number[];
 } | null;
@@ -8,5 +8,9 @@ declare function findBoundaries(query: string, document: string): {
     start: number;
     end: number;
 } | null;
+declare function findCtcBoundaries(query: string, document: string): {
+    start: number;
+    end: number;
+} | null;
 
-export { buildNgramIndex, collectBoundaryVotes, findBoundaries, ngrams };
+export { buildNgramIndex, collectBoundaryVotes, findBoundaries, findCtcBoundaries, ngrams };

package/dist/align/search.js

@@ -1,8 +1,9 @@
 import "../chunk-BIEQXUOY.js";
-import { max, min, range } from "itertools";
-function buildNgramIndex(text) {
+import { enumerate, max, min, range } from "itertools";
+import { Mapping } from "@storyteller-platform/mapping";
+function buildNgramIndex(text, ngramSize, gramDelimiter) {
   const index = /* @__PURE__ */ new Map();
-  for (const [ngram, pos] of ngrams(text)) {
+  for (const [ngram, pos] of ngrams(text, ngramSize, gramDelimiter)) {
     const positions = index.get(ngram);
     if (positions) {
       positions.push(pos);
@@ -12,22 +13,21 @@ function buildNgramIndex(text) {
   }
   return index;
 }
-const NGRAM_SIZE = 5;
-function* ngrams(text) {
-  const words = text.split("-");
-  for (const i of range(words.length - NGRAM_SIZE - 1)) {
-    const ngram = words.slice(i, i + NGRAM_SIZE).join("-");
+function* ngrams(text, ngramSize, gramDelimiter) {
+  const grams = text.split(gramDelimiter);
+  for (const i of range(grams.length - ngramSize + 1)) {
+    const ngram = grams.slice(i, i + ngramSize).join(gramDelimiter);
     yield [ngram, i];
   }
 }
-function collectBoundaryVotes(query, document) {
-  const queryWords = query.split("-");
-  const documentIndex = buildNgramIndex(document);
+function collectBoundaryVotes(query, document, ngramSize, gramDelimiter) {
+  const queryGrams = query.split(gramDelimiter);
+  const documentIndex = buildNgramIndex(document, ngramSize, gramDelimiter);
   let skippedNgrams = 0;
   let totalNgrams = 0;
   const startVotes = [];
   const endVotes = [];
-  for (const [ngram, start] of ngrams(query)) {
+  for (const [ngram, start] of ngrams(query, ngramSize, gramDelimiter)) {
     totalNgrams++;
     const documentStarts = documentIndex.get(ngram);
     if (!documentStarts) {
@@ -36,7 +36,7 @@ function collectBoundaryVotes(query, document) {
     }
     for (const documentStart of documentStarts) {
       startVotes.push(documentStart - start);
-      endVotes.push(documentStart + (queryWords.length - start));
+      endVotes.push(documentStart + (queryGrams.length - start));
     }
   }
   if (skippedNgrams > totalNgrams / 2) {
@@ -44,13 +44,12 @@ function collectBoundaryVotes(query, document) {
   }
   return { startVotes, endVotes };
 }
-const BIN_SIZE = 1e3;
-function binBoundaryVotes(votes) {
+function binBoundaryVotes(votes, binSize) {
   const start = min(votes);
   const bins = /* @__PURE__ */ new Map();
   if (start === void 0) return bins;
   for (const vote of votes) {
-    const binIndex = Math.floor((vote - start) / BIN_SIZE);
+    const binIndex = Math.floor((vote - start) / binSize);
     const bin = bins.get(binIndex);
     if (bin) {
       bin.push(vote);
@@ -72,36 +71,89 @@ function chooseBestFromBins(bins, dir) {
   }
   return dir > 0 ? max(best) ?? null : min(best) ?? null;
 }
-function getOffsetFromWordIndex(wordIndex, document) {
-  const words = document.split("-");
+function getOffsetFromWordIndex(wordIndex, document, gramDelimiter) {
+  const words = document.split(gramDelimiter);
   let offset = 0;
   for (const i of range(Math.min(words.length, Math.max(0, wordIndex)))) {
     offset += words[i].length + 1;
   }
   return offset;
 }
-function findBoundaries(query, document) {
-  const boundaryVotes = collectBoundaryVotes(query, document);
+function findBoundariesInternal(query, document, ngramSize, binSize, gramDelimiter) {
+  const boundaryVotes = collectBoundaryVotes(
+    query,
+    document,
+    ngramSize,
+    gramDelimiter
+  );
   if (!boundaryVotes) return null;
   const { startVotes, endVotes } = boundaryVotes;
-  const startBins = binBoundaryVotes(startVotes);
+  const startBins = binBoundaryVotes(startVotes, binSize);
   const bestStart = chooseBestFromBins(startBins, -1);
   if (bestStart === null) {
     return null;
   }
-  const endBins = binBoundaryVotes(endVotes);
+  const endBins = binBoundaryVotes(endVotes, binSize);
   const bestEnd = chooseBestFromBins(endBins, 1);
   if (bestEnd === null) {
     return null;
   }
+  return { bestStart, bestEnd };
+}
+function findBoundaries(query, document) {
+  const ngramSize = 5;
+  const binSize = 1e3;
+  const gramDelimiter = "-";
+  const result = findBoundariesInternal(
+    query,
+    document,
+    ngramSize,
+    binSize,
+    gramDelimiter
+  );
+  if (!result) return null;
+  const { bestStart, bestEnd } = result;
+  return {
+    start: getOffsetFromWordIndex(bestStart, document, gramDelimiter),
+    end: getOffsetFromWordIndex(bestEnd, document, gramDelimiter)
+  };
+}
+function smooshWords(input) {
+  const mapping = new Mapping();
+  let smooshed = "";
+  for (const [i, c] of enumerate(input)) {
+    if (c !== "-") {
+      smooshed += c;
+      continue;
+    }
+    mapping.insertMap(i, 1, 0);
+  }
+  return { smooshed, mapping };
+}
+function findCtcBoundaries(query, document) {
+  const ngramSize = 10;
+  const binSize = 5e3;
+  const gramDelimiter = "";
+  const { smooshed: smooshedQuery } = smooshWords(query);
+  const { smooshed: smooshedDocument, mapping: documentMapping } = smooshWords(document);
+  const result = findBoundariesInternal(
+    smooshedQuery,
+    smooshedDocument,
+    ngramSize,
+    binSize,
+    gramDelimiter
+  );
+  if (!result) return null;
+  const { bestStart, bestEnd } = result;
   return {
-    start: getOffsetFromWordIndex(bestStart, document),
-    end: getOffsetFromWordIndex(bestEnd, document)
+    start: documentMapping.cursor().map(Math.max(bestStart, 0), "start"),
+    end: documentMapping.cursor().map(Math.min(bestEnd, document.length - 1), "end")
   };
 }
 export {
   buildNgramIndex,
   collectBoundaryVotes,
   findBoundaries,
+  findCtcBoundaries,
   ngrams
 };

package/dist/align/slugify.cjs

@@ -128,13 +128,14 @@ function createReplacers(locale) {
     [numberRegex, numberReplacer]
   ];
 }
-async function slugify(text, locale) {
+async function slugify(text, locale, separator = "-") {
   const replacers = replacerMap.get(locale) ?? createReplacers(locale);
   replacerMap.set(locale, replacers);
   const { result, mapping } = await (0, import_transliteration.slugify)(text, {
     allowedChars: "a-zA-Z0-9",
     locale,
-    replace: replacers
+    replace: replacers,
+    separator
   });
   return { result, mapping };
 }

package/dist/align/slugify.d.cts

@@ -1,6 +1,6 @@
 import * as _storyteller_platform_mapping from '@storyteller-platform/mapping';
 
-declare function slugify(text: string, locale: Intl.Locale): Promise<{
+declare function slugify(text: string, locale: Intl.Locale, separator?: string): Promise<{
     result: string;
     mapping: _storyteller_platform_mapping.Mapping;
 }>;

package/dist/align/slugify.d.ts

@@ -1,6 +1,6 @@
 import * as _storyteller_platform_mapping from '@storyteller-platform/mapping';
 
-declare function slugify(text: string, locale: Intl.Locale): Promise<{
+declare function slugify(text: string, locale: Intl.Locale, separator?: string): Promise<{
     result: string;
     mapping: _storyteller_platform_mapping.Mapping;
 }>;

package/dist/align/slugify.js

@@ -106,13 +106,14 @@ function createReplacers(locale) {
     [numberRegex, numberReplacer]
   ];
 }
-async function slugify(text, locale) {
+async function slugify(text, locale, separator = "-") {
   const replacers = replacerMap.get(locale) ?? createReplacers(locale);
   replacerMap.set(locale, replacers);
   const { result, mapping } = await transliterateSlugify(text, {
     allowedChars: "a-zA-Z0-9",
     locale,
-    replace: replacers
+    replace: replacers,
+    separator
   });
   return { result, mapping };
 }