@storyteller-platform/align 0.1.8 → 0.1.10

package/dist/errorAlign/errorAlign.cjs

@@ -0,0 +1,159 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var errorAlign_exports = {};
+__export(errorAlign_exports, {
+  errorAlign: () => errorAlign
+});
+module.exports = __toCommonJS(errorAlign_exports);
+var import_itertools = require("itertools");
+var import_backtraceGraph = require("./backtraceGraph.cjs");
+var import_graphMetadata = require("./graphMetadata.cjs");
+var import_native = require("./native.cjs");
+var import_pathToAlignment = require("./pathToAlignment.cjs");
+var import_utils = require("./utils.cjs");
+function errorAlign(ref, hyp, tokenizer = import_utils.basicTokenizer, normalizer = import_utils.basicNormalizer, beamSize = 100, wordLevelPass = true) {
+  const graphMetadata = prepareGraphMetadata(ref, hyp, tokenizer, normalizer);
+  if (graphMetadata.refNorm === graphMetadata.hypNorm) {
+    return alignIdenticalInputs(graphMetadata);
+  }
+  if (!wordLevelPass) {
+    return alignBeamSearch(graphMetadata, beamSize);
+  }
+  return alignWithWordLevelPass(graphMetadata, beamSize);
+}
+function prepareGraphMetadata(ref, hyp, tokenizer = import_utils.basicTokenizer, normalizer = import_utils.basicNormalizer) {
+  const unpackedTokenizer = (0, import_utils.unpackRegexMatch)(tokenizer);
+  const refTokenMatches = unpackedTokenizer(ref);
+  const hypTokenMatches = unpackedTokenizer(hyp);
+  const ensuredNormalizer = (0, import_utils.ensureLengthPreservation)(normalizer);
+  const refNorm = refTokenMatches.map(([r]) => ensuredNormalizer(r));
+  const hypNorm = hypTokenMatches.map(([h]) => ensuredNormalizer(h));
+  return {
+    refRaw: ref,
+    hypRaw: hyp,
+    refTokenMatches,
+    hypTokenMatches,
+    refNorm,
+    hypNorm
+  };
+}
+function alignIdenticalInputs(graphMetadata) {
+  const alignments = [];
+  for (const i of (0, import_itertools.range)(graphMetadata.refTokenMatches.length)) {
+    const alignment = getMatchAlignmentFromTokenIndices(graphMetadata, {
+      refIndex: i,
+      hypIndex: i
+    });
+    alignments.push(alignment);
+  }
+  return alignments;
+}
+function alignBeamSearch(graphMetadata, beamSize, refStart, refEnd, hypStart, hypEnd) {
+  const src = new import_graphMetadata.SubgraphMetadata(
+    graphMetadata.refRaw,
+    graphMetadata.hypRaw,
+    graphMetadata.refTokenMatches.slice(refStart, refEnd),
+    graphMetadata.hypTokenMatches.slice(hypStart, hypEnd),
+    graphMetadata.refNorm.slice(refStart, refEnd),
+    graphMetadata.hypNorm.slice(hypStart, hypEnd)
+  );
+  const path = (0, import_native.errorAlignBeamSearch)(src, beamSize);
+  return (0, import_pathToAlignment.getAlignments)(path);
+}
+function alignWithWordLevelPass(graphMetadata, beamSize) {
+  const { backtraceMatrix } = (0, import_native.computeLevenshteinDistanceMatrix)(
+    graphMetadata.refNorm,
+    graphMetadata.hypNorm,
+    true
+  );
+  const backtraceGraph = new import_backtraceGraph.BacktraceGraph(backtraceMatrix);
+  const matchIndices = backtraceGraph.getUnambiguousNodeMatches();
+  matchIndices.push([
+    graphMetadata.hypNorm.length,
+    graphMetadata.refNorm.length
+  ]);
+  let hypStart = 0;
+  let refStart = 0;
+  const alignments = [];
+  const endIndex = matchIndices.length - 1;
+  for (const [i, [hypEnd, refEnd]] of (0, import_itertools.enumerate)(matchIndices)) {
+    const refIsEmpty = refStart === refEnd;
+    const hypIsEmpty = hypStart === hypEnd;
+    if (!refIsEmpty && !hypIsEmpty) {
+      alignments.push(
+        ...alignBeamSearch(
+          graphMetadata,
+          beamSize,
+          refStart,
+          refEnd,
+          hypStart,
+          hypEnd
+        )
+      );
+    } else if (refIsEmpty && !hypIsEmpty) {
+      for (const tokenIndex of (0, import_itertools.range)(hypStart, hypEnd)) {
+        alignments.push(
+          getInsertAlignmentFromTokenIndex(graphMetadata, tokenIndex)
+        );
+      }
+    } else if (hypIsEmpty && !refIsEmpty) {
+      for (const tokenIndex of (0, import_itertools.range)(refStart, refEnd)) {
+        alignments.push(
+          getDeleteAlignmentFromTokenIndex(graphMetadata, tokenIndex)
+        );
+      }
+    }
+    if (i < endIndex) {
+      alignments.push(
+        getMatchAlignmentFromTokenIndices(graphMetadata, {
+          refIndex: refEnd,
+          hypIndex: hypEnd
+        })
+      );
+    }
+    refStart = refEnd + 1;
+    hypStart = hypEnd + 1;
+  }
+  return alignments;
+}
+function getMatchAlignmentFromTokenIndices(graphMetadata, { refIndex, hypIndex }) {
+  const refSlice = graphMetadata.refTokenMatches[refIndex][1];
+  const hypSlice = graphMetadata.hypTokenMatches[hypIndex][1];
+  return new import_utils.Alignment(
+    "MATCH",
+    refSlice,
+    hypSlice,
+    graphMetadata.refRaw.slice(...refSlice),
+    graphMetadata.hypRaw.slice(...hypSlice)
+  );
+}
+function getInsertAlignmentFromTokenIndex(graphMetadata, hypIndex) {
+  const slice = graphMetadata.hypTokenMatches[hypIndex][1];
+  const token = graphMetadata.hypRaw.slice(...slice);
+  return new import_utils.Alignment("INSERT", null, slice, null, token);
+}
+function getDeleteAlignmentFromTokenIndex(graphMetadata, refIndex) {
+  const slice = graphMetadata.refTokenMatches[refIndex][1];
+  const token = graphMetadata.refRaw.slice(...slice);
+  return new import_utils.Alignment("DELETE", slice, null, token);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  errorAlign
+});

package/dist/errorAlign/errorAlign.d.cts

@@ -0,0 +1,15 @@
+import { basicTokenizer, basicNormalizer, Alignment } from './utils.cjs';
+
+/**
+ * Run error alignment between reference and hypothesis texts.
+ *
+ * @param ref The reference sequence/transcript.
+ * @param hyp The hypothesis sequence/transcript.
+ * @param tokenizer A function to tokenize the sequence. Must be regex-based and return Match objects.
+ * @param normalizer A function to normalize the tokens. Defaults to basicNormalizer.
+ * @param beamSize The beam size for the beam search alignment.
+ * @param wordLevelPass Whether to perform a word-level aligment pass to identify unambiguous matches.
+ */
+declare function errorAlign(ref: string, hyp: string, tokenizer?: typeof basicTokenizer, normalizer?: typeof basicNormalizer, beamSize?: number, wordLevelPass?: boolean): Alignment[];
+
+export { errorAlign };

package/dist/errorAlign/errorAlign.d.ts

@@ -0,0 +1,15 @@
+import { basicTokenizer, basicNormalizer, Alignment } from './utils.js';
+
+/**
+ * Run error alignment between reference and hypothesis texts.
+ *
+ * @param ref The reference sequence/transcript.
+ * @param hyp The hypothesis sequence/transcript.
+ * @param tokenizer A function to tokenize the sequence. Must be regex-based and return Match objects.
+ * @param normalizer A function to normalize the tokens. Defaults to basicNormalizer.
+ * @param beamSize The beam size for the beam search alignment.
+ * @param wordLevelPass Whether to perform a word-level aligment pass to identify unambiguous matches.
+ */
+declare function errorAlign(ref: string, hyp: string, tokenizer?: typeof basicTokenizer, normalizer?: typeof basicNormalizer, beamSize?: number, wordLevelPass?: boolean): Alignment[];
+
+export { errorAlign };

package/dist/errorAlign/errorAlign.js

@@ -0,0 +1,145 @@
+import "../chunk-BIEQXUOY.js";
+import { enumerate, range } from "itertools";
+import { BacktraceGraph } from "./backtraceGraph.js";
+import { SubgraphMetadata } from "./graphMetadata.js";
+import {
+  computeLevenshteinDistanceMatrix,
+  errorAlignBeamSearch
+} from "./native.js";
+import { getAlignments } from "./pathToAlignment.js";
+import {
+  Alignment,
+  basicNormalizer,
+  basicTokenizer,
+  ensureLengthPreservation,
+  unpackRegexMatch
+} from "./utils.js";
+function errorAlign(ref, hyp, tokenizer = basicTokenizer, normalizer = basicNormalizer, beamSize = 100, wordLevelPass = true) {
+  const graphMetadata = prepareGraphMetadata(ref, hyp, tokenizer, normalizer);
+  if (graphMetadata.refNorm === graphMetadata.hypNorm) {
+    return alignIdenticalInputs(graphMetadata);
+  }
+  if (!wordLevelPass) {
+    return alignBeamSearch(graphMetadata, beamSize);
+  }
+  return alignWithWordLevelPass(graphMetadata, beamSize);
+}
+function prepareGraphMetadata(ref, hyp, tokenizer = basicTokenizer, normalizer = basicNormalizer) {
+  const unpackedTokenizer = unpackRegexMatch(tokenizer);
+  const refTokenMatches = unpackedTokenizer(ref);
+  const hypTokenMatches = unpackedTokenizer(hyp);
+  const ensuredNormalizer = ensureLengthPreservation(normalizer);
+  const refNorm = refTokenMatches.map(([r]) => ensuredNormalizer(r));
+  const hypNorm = hypTokenMatches.map(([h]) => ensuredNormalizer(h));
+  return {
+    refRaw: ref,
+    hypRaw: hyp,
+    refTokenMatches,
+    hypTokenMatches,
+    refNorm,
+    hypNorm
+  };
+}
+function alignIdenticalInputs(graphMetadata) {
+  const alignments = [];
+  for (const i of range(graphMetadata.refTokenMatches.length)) {
+    const alignment = getMatchAlignmentFromTokenIndices(graphMetadata, {
+      refIndex: i,
+      hypIndex: i
+    });
+    alignments.push(alignment);
+  }
+  return alignments;
+}
+function alignBeamSearch(graphMetadata, beamSize, refStart, refEnd, hypStart, hypEnd) {
+  const src = new SubgraphMetadata(
+    graphMetadata.refRaw,
+    graphMetadata.hypRaw,
+    graphMetadata.refTokenMatches.slice(refStart, refEnd),
+    graphMetadata.hypTokenMatches.slice(hypStart, hypEnd),
+    graphMetadata.refNorm.slice(refStart, refEnd),
+    graphMetadata.hypNorm.slice(hypStart, hypEnd)
+  );
+  const path = errorAlignBeamSearch(src, beamSize);
+  return getAlignments(path);
+}
+function alignWithWordLevelPass(graphMetadata, beamSize) {
+  const { backtraceMatrix } = computeLevenshteinDistanceMatrix(
+    graphMetadata.refNorm,
+    graphMetadata.hypNorm,
+    true
+  );
+  const backtraceGraph = new BacktraceGraph(backtraceMatrix);
+  const matchIndices = backtraceGraph.getUnambiguousNodeMatches();
+  matchIndices.push([
+    graphMetadata.hypNorm.length,
+    graphMetadata.refNorm.length
+  ]);
+  let hypStart = 0;
+  let refStart = 0;
+  const alignments = [];
+  const endIndex = matchIndices.length - 1;
+  for (const [i, [hypEnd, refEnd]] of enumerate(matchIndices)) {
+    const refIsEmpty = refStart === refEnd;
+    const hypIsEmpty = hypStart === hypEnd;
+    if (!refIsEmpty && !hypIsEmpty) {
+      alignments.push(
+        ...alignBeamSearch(
+          graphMetadata,
+          beamSize,
+          refStart,
+          refEnd,
+          hypStart,
+          hypEnd
+        )
+      );
+    } else if (refIsEmpty && !hypIsEmpty) {
+      for (const tokenIndex of range(hypStart, hypEnd)) {
+        alignments.push(
+          getInsertAlignmentFromTokenIndex(graphMetadata, tokenIndex)
+        );
+      }
+    } else if (hypIsEmpty && !refIsEmpty) {
+      for (const tokenIndex of range(refStart, refEnd)) {
+        alignments.push(
+          getDeleteAlignmentFromTokenIndex(graphMetadata, tokenIndex)
+        );
+      }
+    }
+    if (i < endIndex) {
+      alignments.push(
+        getMatchAlignmentFromTokenIndices(graphMetadata, {
+          refIndex: refEnd,
+          hypIndex: hypEnd
+        })
+      );
+    }
+    refStart = refEnd + 1;
+    hypStart = hypEnd + 1;
+  }
+  return alignments;
+}
+function getMatchAlignmentFromTokenIndices(graphMetadata, { refIndex, hypIndex }) {
+  const refSlice = graphMetadata.refTokenMatches[refIndex][1];
+  const hypSlice = graphMetadata.hypTokenMatches[hypIndex][1];
+  return new Alignment(
+    "MATCH",
+    refSlice,
+    hypSlice,
+    graphMetadata.refRaw.slice(...refSlice),
+    graphMetadata.hypRaw.slice(...hypSlice)
+  );
+}
+function getInsertAlignmentFromTokenIndex(graphMetadata, hypIndex) {
+  const slice = graphMetadata.hypTokenMatches[hypIndex][1];
+  const token = graphMetadata.hypRaw.slice(...slice);
+  return new Alignment("INSERT", null, slice, null, token);
+}
+function getDeleteAlignmentFromTokenIndex(graphMetadata, refIndex) {
+  const slice = graphMetadata.refTokenMatches[refIndex][1];
+  const token = graphMetadata.refRaw.slice(...slice);
+  return new Alignment("DELETE", slice, null, token);
+}
+export {
+  errorAlign
+};

package/dist/errorAlign/graphMetadata.cjs

@@ -0,0 +1,97 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var graphMetadata_exports = {};
+__export(graphMetadata_exports, {
+  SubgraphMetadata: () => SubgraphMetadata
+});
+module.exports = __toCommonJS(graphMetadata_exports);
+var import_itertools = require("itertools");
+var import_memoize = __toESM(require("memoize"), 1);
+var import_backtraceGraph = require("./backtraceGraph.cjs");
+var import_editDistance = require("./editDistance.cjs");
+var import_utils = require("./utils.cjs");
+class SubgraphMetadata {
+  constructor(refRaw, hypRaw, refTokenMatches, hypTokenMatches, refNorm, hypNorm) {
+    this.refRaw = refRaw;
+    this.hypRaw = hypRaw;
+    this.refTokenMatches = refTokenMatches;
+    this.hypTokenMatches = hypTokenMatches;
+    this.refNorm = refNorm;
+    this.hypNorm = hypNorm;
+    this.ref = embedTokens(refNorm);
+    this.hyp = embedTokens(hypNorm);
+    this.refMaxIndex = this.ref.length - 1;
+    this.hypMaxIndex = this.hyp.length - 1;
+    this.refCharTypes = getCharTypes(this.ref);
+    this.hypCharTypes = getCharTypes(this.hyp);
+    this.refIndexMap = createIndexMap(refTokenMatches);
+    this.hypIndexMap = createIndexMap(hypTokenMatches);
+    const { backtraceMatrix } = (0, import_editDistance.computeErrorAlignDistanceMatrix)(
+      this.ref,
+      this.hyp,
+      true
+    );
+    this.backtraceGraph = new import_backtraceGraph.BacktraceGraph(backtraceMatrix);
+    this.backtraceNodeSet = this.backtraceGraph.getNodeSet();
+    this.unambiguousMatches = this.backtraceGraph.getUnambiguousTokenSpanMatches(this.ref);
+  }
+  ref;
+  hyp;
+  refMaxIndex;
+  hypMaxIndex;
+  refCharTypes;
+  hypCharTypes;
+  refIndexMap;
+  hypIndexMap;
+  backtraceGraph;
+  backtraceNodeSet;
+  unambiguousMatches;
+}
+function embedTokens(textTokens) {
+  return textTokens.map((t) => `${import_utils.START_DELIMITER}${t}${import_utils.END_DELIMITER}`).join("");
+}
+const categorizeCharCached = (0, import_memoize.default)(function categorizeCharCached2(c) {
+  return (0, import_utils.categorizeChar)(c);
+});
+function getCharTypes(text) {
+  return text.split("").map((c) => categorizeCharCached(c));
+}
+function createIndexMap(textTokens) {
+  const indexMap = [];
+  for (const [_, span] of textTokens) {
+    indexMap.push(-1);
+    indexMap.push(...(0, import_itertools.range)(...span));
+    indexMap.push(-1);
+  }
+  return indexMap;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SubgraphMetadata
+});

package/dist/errorAlign/graphMetadata.d.cts

@@ -0,0 +1,44 @@
+import { BacktraceGraph, Index } from './backtraceGraph.cjs';
+import './utils.cjs';
+
+type TokenWithSpan = [string, [number, number]];
+interface GraphMetadata {
+    refRaw: string;
+    hypRaw: string;
+    refTokenMatches: TokenWithSpan[];
+    hypTokenMatches: TokenWithSpan[];
+    refNorm: string[];
+    hypNorm: string[];
+}
+/**
+ * Data class to hold information needed for beam search alignment.
+ *
+ * This data class encapsulates all necessary infomation about a subgraph
+ * derived from the reference and hypothesis texts, including their tokenized
+ * and normalized forms, as well as derived attributes used during
+ * the alignment process.
+ *
+ * It works as a reference for the `Path` class during beam search alignment.
+ */
+declare class SubgraphMetadata {
+    refRaw: string;
+    hypRaw: string;
+    refTokenMatches: [string, [number, number]][];
+    hypTokenMatches: [string, [number, number]][];
+    refNorm: string[];
+    hypNorm: string[];
+    ref: string;
+    hyp: string;
+    refMaxIndex: number;
+    hypMaxIndex: number;
+    refCharTypes: number[];
+    hypCharTypes: number[];
+    refIndexMap: number[];
+    hypIndexMap: number[];
+    backtraceGraph: BacktraceGraph;
+    backtraceNodeSet: Set<Index>;
+    unambiguousMatches: Set<Index>;
+    constructor(refRaw: string, hypRaw: string, refTokenMatches: [string, [number, number]][], hypTokenMatches: [string, [number, number]][], refNorm: string[], hypNorm: string[]);
+}
+
+export { type GraphMetadata, SubgraphMetadata, type TokenWithSpan };

package/dist/errorAlign/graphMetadata.d.ts

@@ -0,0 +1,44 @@
+import { BacktraceGraph, Index } from './backtraceGraph.js';
+import './utils.js';
+
+type TokenWithSpan = [string, [number, number]];
+interface GraphMetadata {
+    refRaw: string;
+    hypRaw: string;
+    refTokenMatches: TokenWithSpan[];
+    hypTokenMatches: TokenWithSpan[];
+    refNorm: string[];
+    hypNorm: string[];
+}
+/**
+ * Data class to hold information needed for beam search alignment.
+ *
+ * This data class encapsulates all necessary infomation about a subgraph
+ * derived from the reference and hypothesis texts, including their tokenized
+ * and normalized forms, as well as derived attributes used during
+ * the alignment process.
+ *
+ * It works as a reference for the `Path` class during beam search alignment.
+ */
+declare class SubgraphMetadata {
+    refRaw: string;
+    hypRaw: string;
+    refTokenMatches: [string, [number, number]][];
+    hypTokenMatches: [string, [number, number]][];
+    refNorm: string[];
+    hypNorm: string[];
+    ref: string;
+    hyp: string;
+    refMaxIndex: number;
+    hypMaxIndex: number;
+    refCharTypes: number[];
+    hypCharTypes: number[];
+    refIndexMap: number[];
+    hypIndexMap: number[];
+    backtraceGraph: BacktraceGraph;
+    backtraceNodeSet: Set<Index>;
+    unambiguousMatches: Set<Index>;
+    constructor(refRaw: string, hypRaw: string, refTokenMatches: [string, [number, number]][], hypTokenMatches: [string, [number, number]][], refNorm: string[], hypNorm: string[]);
+}
+
+export { type GraphMetadata, SubgraphMetadata, type TokenWithSpan };

package/dist/errorAlign/graphMetadata.js

@@ -0,0 +1,64 @@
+import "../chunk-BIEQXUOY.js";
+import { range } from "itertools";
+import memoize from "memoize";
+import { BacktraceGraph } from "./backtraceGraph.js";
+import { computeErrorAlignDistanceMatrix } from "./editDistance.js";
+import { END_DELIMITER, START_DELIMITER, categorizeChar } from "./utils.js";
+class SubgraphMetadata {
+  constructor(refRaw, hypRaw, refTokenMatches, hypTokenMatches, refNorm, hypNorm) {
+    this.refRaw = refRaw;
+    this.hypRaw = hypRaw;
+    this.refTokenMatches = refTokenMatches;
+    this.hypTokenMatches = hypTokenMatches;
+    this.refNorm = refNorm;
+    this.hypNorm = hypNorm;
+    this.ref = embedTokens(refNorm);
+    this.hyp = embedTokens(hypNorm);
+    this.refMaxIndex = this.ref.length - 1;
+    this.hypMaxIndex = this.hyp.length - 1;
+    this.refCharTypes = getCharTypes(this.ref);
+    this.hypCharTypes = getCharTypes(this.hyp);
+    this.refIndexMap = createIndexMap(refTokenMatches);
+    this.hypIndexMap = createIndexMap(hypTokenMatches);
+    const { backtraceMatrix } = computeErrorAlignDistanceMatrix(
+      this.ref,
+      this.hyp,
+      true
+    );
+    this.backtraceGraph = new BacktraceGraph(backtraceMatrix);
+    this.backtraceNodeSet = this.backtraceGraph.getNodeSet();
+    this.unambiguousMatches = this.backtraceGraph.getUnambiguousTokenSpanMatches(this.ref);
+  }
+  ref;
+  hyp;
+  refMaxIndex;
+  hypMaxIndex;
+  refCharTypes;
+  hypCharTypes;
+  refIndexMap;
+  hypIndexMap;
+  backtraceGraph;
+  backtraceNodeSet;
+  unambiguousMatches;
+}
+function embedTokens(textTokens) {
+  return textTokens.map((t) => `${START_DELIMITER}${t}${END_DELIMITER}`).join("");
+}
+const categorizeCharCached = memoize(function categorizeCharCached2(c) {
+  return categorizeChar(c);
+});
+function getCharTypes(text) {
+  return text.split("").map((c) => categorizeCharCached(c));
+}
+function createIndexMap(textTokens) {
+  const indexMap = [];
+  for (const [_, span] of textTokens) {
+    indexMap.push(-1);
+    indexMap.push(...range(...span));
+    indexMap.push(-1);
+  }
+  return indexMap;
+}
+export {
+  SubgraphMetadata
+};