@mailwoman/core 2.0.0

package/out/solvers/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+export * from "./ExclusiveCartesianSolver.js";
+export * from "./HouseNumberPositionPenalty.js";
+export * from "./LeadingAreaDeclassifier.js";
+export * from "./MultiStreetSolver.js";
+export * from "./OrphanedLevelTypeDeclassifier.js";
+export * from "./OrphanedUnitTypeDeclassifier.js";
+export * from "./PostcodePositionPenalty.js";
+export * from "./VenueCaptureSolver.js";
+//# sourceMappingURL=index.d.ts.map

package/out/solvers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../solvers/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,+BAA+B,CAAA;AAC7C,cAAc,iCAAiC,CAAA;AAC/C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,wBAAwB,CAAA;AACtC,cAAc,oCAAoC,CAAA;AAClD,cAAc,mCAAmC,CAAA;AACjD,cAAc,8BAA8B,CAAA;AAC5C,cAAc,yBAAyB,CAAA"}

package/out/solvers/index.js

@@ -0,0 +1,14 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+export * from "./ExclusiveCartesianSolver.js";
+export * from "./HouseNumberPositionPenalty.js";
+export * from "./LeadingAreaDeclassifier.js";
+export * from "./MultiStreetSolver.js";
+export * from "./OrphanedLevelTypeDeclassifier.js";
+export * from "./OrphanedUnitTypeDeclassifier.js";
+export * from "./PostcodePositionPenalty.js";
+export * from "./VenueCaptureSolver.js";
+//# sourceMappingURL=index.js.map

package/out/solvers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../solvers/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,+BAA+B,CAAA;AAC7C,cAAc,iCAAiC,CAAA;AAC/C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,wBAAwB,CAAA;AACtC,cAAc,oCAAoC,CAAA;AAClD,cAAc,mCAAmC,CAAA;AACjD,cAAc,8BAA8B,CAAA;AAC5C,cAAc,yBAAyB,CAAA"}

package/out/tokenization/Graph.d.ts

@@ -0,0 +1,35 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { Sequence } from "@mailwoman/core/resources";
+export type GraphNodeCallback<G> = (node: G) => boolean;
+/**
+ * A graph structure for storing relationships between nodes.
+ */
+export declare class Graph<GraphNode extends WeakKey> {
+    /**
+     * Parents of the graph, i.e. upward connections to the graph.
+     */
+    parents: Sequence<GraphNode>;
+    /**
+     * Children of the graph, i.e. downward connections to the graph.
+     */
+    children: Sequence<GraphNode>;
+    /**
+     * Previous siblings of the graph, i.e. leftward connections to the graph.
+     */
+    readonly previousSiblings: Sequence<GraphNode>;
+    /**
+     * Next siblings of the graph, i.e. rightward connections to the graph.
+     */
+    readonly nextSiblings: Sequence<GraphNode>;
+    /**
+     * Phrases identified in the graph, i.e. a sequence of nodes composing a classification.
+     */
+    readonly phrases: Sequence<GraphNode>;
+    get nextSibling(): GraphNode | null;
+    get previousSibling(): GraphNode | null;
+}
+//# sourceMappingURL=Graph.d.ts.map

package/out/tokenization/Graph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Graph.d.ts","sourceRoot":"","sources":["../../tokenization/Graph.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAEpD,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAA;AAEvD;;GAEG;AACH,qBAAa,KAAK,CAAC,SAAS,SAAS,OAAO;IAC3C;;OAEG;IACI,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAEpD;;OAEG;IACI,QAAQ,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAErD;;OAEG;IACH,SAAgB,gBAAgB,sBAA4B;IAC5D;;OAEG;IACH,SAAgB,YAAY,sBAA4B;IAExD;;OAEG;IACH,SAAgB,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,CAA4B;IAExE,IAAW,WAAW,IAAI,SAAS,GAAG,IAAI,CAEzC;IAED,IAAW,eAAe,IAAI,SAAS,GAAG,IAAI,CAE7C;CACD"}

package/out/tokenization/Graph.js

@@ -0,0 +1,38 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { Sequence } from "@mailwoman/core/resources";
+/**
+ * A graph structure for storing relationships between nodes.
+ */
+export class Graph {
+    /**
+     * Parents of the graph, i.e. upward connections to the graph.
+     */
+    parents = new Sequence();
+    /**
+     * Children of the graph, i.e. downward connections to the graph.
+     */
+    children = new Sequence();
+    /**
+     * Previous siblings of the graph, i.e. leftward connections to the graph.
+     */
+    previousSiblings = new Sequence();
+    /**
+     * Next siblings of the graph, i.e. rightward connections to the graph.
+     */
+    nextSiblings = new Sequence();
+    /**
+     * Phrases identified in the graph, i.e. a sequence of nodes composing a classification.
+     */
+    phrases = new Sequence();
+    get nextSibling() {
+        return this.nextSiblings.first;
+    }
+    get previousSibling() {
+        return this.previousSiblings.first;
+    }
+}
+//# sourceMappingURL=Graph.js.map

package/out/tokenization/Graph.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Graph.js","sourceRoot":"","sources":["../../tokenization/Graph.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAIpD;;GAEG;AACH,MAAM,OAAO,KAAK;IACjB;;OAEG;IACI,OAAO,GAAwB,IAAI,QAAQ,EAAE,CAAA;IAEpD;;OAEG;IACI,QAAQ,GAAwB,IAAI,QAAQ,EAAE,CAAA;IAErD;;OAEG;IACa,gBAAgB,GAAG,IAAI,QAAQ,EAAa,CAAA;IAC5D;;OAEG;IACa,YAAY,GAAG,IAAI,QAAQ,EAAa,CAAA;IAExD;;OAEG;IACa,OAAO,GAAwB,IAAI,QAAQ,EAAa,CAAA;IAExE,IAAW,WAAW;QACrB,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAA;IAC/B,CAAC;IAED,IAAW,eAAe;QACzB,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAA;IACnC,CAAC;CACD"}

package/out/tokenization/Span.d.ts

@@ -0,0 +1,115 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { type Classification, type ClassificationMatch, ClassificationsMatchMap } from "../classification/Classification.js";
+import type { Displayable } from "../resources/debugging.js";
+import type { Alpha3bLanguageCode } from "../resources/languages/index.js";
+import type { LibPostalLanguageCode } from "../resources/libpostal.js";
+import { Graph } from "./Graph.js";
+export interface SpanCreationOptions {
+    start?: number;
+    classifications?: Iterable<Classification>;
+    children?: Iterable<Span>;
+}
+declare const kSpanID: unique symbol;
+export interface SerializedSpan {
+    body: string;
+    start: number;
+    end: number;
+    normalized: string;
+    classifications: ClassificationMatch[];
+    children: SerializedSpan[];
+    phrases: SerializedSpan[];
+}
+/**
+ * A span of text, i.e. a token or a phrase.
+ */
+export declare class Span extends Graph<Span> {
+    #private;
+    /**
+     * The start index of the span.
+     */
+    start: number;
+    /**
+     * The end index of the span.
+     */
+    end: number;
+    /**
+     * The unique identifier for this span.
+     */
+    protected static IDCounter: number;
+    /**
+     * The unique identifier for this span.
+     */
+    readonly [kSpanID]: number;
+    get id(): number;
+    /**
+     * The normalized body of the span.
+     */
+    normalized: string;
+    /**
+     * Classifications for this span.
+     */
+    readonly classifications: ClassificationsMatchMap;
+    /**
+     * Boolean-like indications that hint at the nature of the span.
+     *
+     * Unlike classifications, these are not exposed in the final output.
+     */
+    get flags(): ReadonlySet<SpanFlag>;
+    is(classification: Classification): boolean;
+    static from(input?: string, options?: SpanCreationOptions): Span;
+    static from(input: Span, options?: Omit<SpanCreationOptions, "start">): Span;
+    static from(input: Span | string, options?: SpanCreationOptions): Span;
+    constructor(body?: string, start?: number);
+    get body(): string;
+    /**
+     * Set the body of the Span
+     */
+    set body(nextBody: string);
+    /**
+     * Predicate to determine if this Span intersects another Span
+     */
+    intersects(that: Pick<Span, "start" | "end">): boolean;
+    /**
+     * Predicate to determine if this Span covers another Span
+     */
+    covers(that: Pick<Span, "start" | "end">): boolean;
+    /**
+     * Returns the distance between two Spans
+     *
+     * @todo Use graph to find prev and next spans for a more accurate result
+     *
+     * @todo Or base 'distance' on word distance (slop) rather than characters
+     */
+    distance(that: Pick<Span, "start" | "end">): number;
+    /**
+     * Returns the coverage of the span, i.e. the number of characters covered by the span and its
+     * children.
+     */
+    get coverage(): number;
+    /**
+     * The combined languages of the span's children.
+     */
+    get languages(): Displayable<ReadonlySet<LibPostalLanguageCode | Alpha3bLanguageCode>>;
+    /**
+     * Serialize the span to JSON.
+     */
+    toJSON(): SerializedSpan;
+    toString(): string;
+    /**
+     * Connect siblings in the graph.
+     */
+    static connectSiblings(...spans: Span[]): Span[];
+}
+export type SpanFlag = "ends_with_period" | "numeric" | "alpha" | "alphanumeric" | "numeral" | "punctuation";
+/**
+ * Patterns to test and apply classifications to spans.
+ *
+ * Note that order here is important, as the first pattern that matches will be used.
+ */
+export declare const PatternMatchers: readonly [pattern: RegExp, flag: SpanFlag][];
+export {};
+//# sourceMappingURL=Span.d.ts.map

package/out/tokenization/Span.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Span.d.ts","sourceRoot":"","sources":["../../tokenization/Span.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EACN,KAAK,cAAc,EACnB,KAAK,mBAAmB,EACxB,uBAAuB,EACvB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAA;AACtE,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAIlC,MAAM,WAAW,mBAAmB;IACnC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,eAAe,CAAC,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAA;IAC1C,QAAQ,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAA;CACzB;AAED,QAAA,MAAM,OAAO,EAAE,OAAO,MAAyB,CAAA;AAE/C,MAAM,WAAW,cAAc;IAE9B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,mBAAmB,EAAE,CAAA;IACtC,QAAQ,EAAE,cAAc,EAAE,CAAA;IAC1B,OAAO,EAAE,cAAc,EAAE,CAAA;CACzB;AAED;;GAEG;AACH,qBAAa,IAAK,SAAQ,KAAK,CAAC,IAAI,CAAC;;IAEpC;;OAEG;IACI,KAAK,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACI,GAAG,EAAG,MAAM,CAAA;IAEnB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,SAAS,SAAI;IAE9B;;OAEG;IACH,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAG,MAAM,CAAA;IAE3B,IAAW,EAAE,IAAI,MAAM,CAEtB;IAED;;OAEG;IACI,UAAU,SAAK;IAEtB;;OAEG;IACH,SAAgB,eAAe,EAAE,uBAAuB,CAAgC;IAIxF;;;;OAIG;IACH,IAAW,KAAK,IAAI,WAAW,CAAC,QAAQ,CAAC,CAExC;IAEM,EAAE,CAAC,cAAc,EAAE,cAAc,GAAG,OAAO;IAIlD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,IAAI;IAChE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,CAAC,GAAG,IAAI;IAC5E,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,IAAI;gBAa1D,IAAI,SAAK,EAAE,KAAK,SAAI;IAgBhC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;OAEG;IACH,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAsBxB;IAED;;OAEG;IACI,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC,GAAG,OAAO;IAI7D;;OAEG;IACI,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC,GAAG,OAAO;IAIzD;;;;;;OAMG;IACI,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC,GAAG,MAAM;IAS1D;;;OAGG;IACH,IAAW,QAAQ,IAAI,MAAM,CAU5B;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,WAAW,CAAC,WAAW,CAAC,qBAAqB,GAAG,mBAAmB,CAAC,CAAC,CAwB5F;IAED;;OAEG;IACI,MAAM,IAAI,cAAc;IAiBf,QAAQ;IAWxB;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE;CAavC;AAED,MAAM,MAAM,QAAQ,GAAG,kBAAkB,GAAG,SAAS,GAAG,OAAO,GAAG,cAAc,GAAG,SAAS,GAAG,aAAa,CAAA;AAE5G;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,EAYvE,CAAA"}

package/out/tokenization/Span.js

@@ -0,0 +1,223 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+// Imported via deep relative path (not @mailwoman/core/classification) to avoid a runtime cycle:
+// classification/index.ts re-exports SectionClassifier / WordClassifier which themselves import
+// Span from @mailwoman/core/tokenization, creating a TDZ that surfaces as "Class extends value
+// undefined" when the source-mode test runner loads tokenization first.
+import { ClassificationsMatchMap, } from "../classification/Classification.js";
+import { Graph } from "./Graph.js";
+const MAX_SPAN_LENGTH = 140;
+const kSpanID = Symbol("SpanID");
+/**
+ * A span of text, i.e. a token or a phrase.
+ */
+export class Span extends Graph {
+    #body = "";
+    /**
+     * The start index of the span.
+     */
+    start;
+    /**
+     * The end index of the span.
+     */
+    end;
+    /**
+     * The unique identifier for this span.
+     */
+    static IDCounter = 0;
+    /**
+     * The unique identifier for this span.
+     */
+    [kSpanID];
+    get id() {
+        return this[kSpanID];
+    }
+    /**
+     * The normalized body of the span.
+     */
+    normalized = "";
+    /**
+     * Classifications for this span.
+     */
+    classifications = new ClassificationsMatchMap();
+    #flags = new Set();
+    /**
+     * Boolean-like indications that hint at the nature of the span.
+     *
+     * Unlike classifications, these are not exposed in the final output.
+     */
+    get flags() {
+        return this.#flags;
+    }
+    is(classification) {
+        return this.classifications.has(classification);
+    }
+    static from(input = "", options = {}) {
+        const span = input instanceof Span ? input : new Span(input, options.start);
+        for (const classification of options.classifications ?? []) {
+            span.classifications.add(classification);
+        }
+        span.children.add(...(options.children ?? []));
+        return span;
+    }
+    constructor(body = "", start = 0) {
+        super();
+        // this[kSpanID] = Span.IDCounter++
+        Object.defineProperty(this, kSpanID, {
+            value: Span.IDCounter++,
+            writable: false,
+            enumerable: false,
+            configurable: false,
+        });
+        // Note that `start` should be set first to ensure that `end` is calculated correctly.
+        this.start = start;
+        this.body = body;
+    }
+    get body() {
+        return this.#body;
+    }
+    /**
+     * Set the body of the Span
+     */
+    set body(nextBody) {
+        this.#flags.clear();
+        this.#body = nextBody.slice(0, MAX_SPAN_LENGTH);
+        this.normalized = this.#body.toLowerCase();
+        this.end = this.start + this.#body.length;
+        for (const [pattern, flag] of PatternMatchers) {
+            if (pattern.test(this.normalized)) {
+                this.#flags.add(flag);
+                break;
+            }
+        }
+        if (this.flags.has("numeric") || this.flags.has("alphanumeric")) {
+            this.#flags.add("numeral");
+        }
+        if (this.#body.slice(-1) === ".") {
+            this.#flags.add("ends_with_period");
+        }
+    }
+    /**
+     * Predicate to determine if this Span intersects another Span
+     */
+    intersects(that) {
+        return this.start < that.end && this.end > that.start;
+    }
+    /**
+     * Predicate to determine if this Span covers another Span
+     */
+    covers(that) {
+        return this.start <= that.start && this.end >= that.end;
+    }
+    /**
+     * Returns the distance between two Spans
+     *
+     * @todo Use graph to find prev and next spans for a more accurate result
+     *
+     * @todo Or base 'distance' on word distance (slop) rather than characters
+     */
+    distance(that) {
+        if (this.intersects(that))
+            return 0;
+        if (this.end < that.start) {
+            return that.start - this.end;
+        }
+        return this.start - that.end;
+    }
+    /**
+     * Returns the coverage of the span, i.e. the number of characters covered by the span and its
+     * children.
+     */
+    get coverage() {
+        if (this.children.size) {
+            return (Iterator
+                // ---
+                .from(this.children)
+                .reduce((sum, child) => sum + (child.end - child.start), 0));
+        }
+        return this.end - this.start;
+    }
+    /**
+     * The combined languages of the span's children.
+     */
+    get languages() {
+        // Spread children langs to the parent...
+        const languages = new Set();
+        const displayNames = new Set();
+        for (const child of this.children) {
+            for (const classification of child.classifications.values()) {
+                const childLanguages = classification.languages;
+                if (!childLanguages)
+                    continue;
+                if (childLanguages.displayName) {
+                    displayNames.add(childLanguages.displayName);
+                }
+                for (const language of childLanguages) {
+                    languages.add(language);
+                }
+            }
+        }
+        languages.displayName = displayNames.size ? Array.from(displayNames).join(", ") : undefined;
+        return languages;
+    }
+    /**
+     * Serialize the span to JSON.
+     */
+    toJSON() {
+        return {
+            // [kSpanID]: this.id,
+            body: this.body,
+            start: this.start,
+            end: this.end,
+            normalized: this.normalized,
+            classifications: this.classifications.toJSON(),
+            children: Iterator.from(this.children)
+                .map((c) => c.toJSON())
+                .toArray(),
+            phrases: Iterator.from(this.phrases)
+                .map((p) => p.toJSON())
+                .toArray(),
+        };
+    }
+    toString() {
+        const classifications = Array.from(this.classifications.keys()).join(", ");
+        const flags = Array.from(this.#flags).join(", ");
+        return `Span(${this.children.size}) "${this.body}" [${classifications}][${flags}]`;
+    }
+    [Symbol.for("nodejs.util.inspect.custom")]() {
+        return this.toString();
+    }
+    /**
+     * Connect siblings in the graph.
+     */
+    static connectSiblings(...spans) {
+        for (const [i, span] of spans.entries()) {
+            if (spans[i - 1]) {
+                span.previousSiblings.add(spans[i - 1]);
+            }
+            if (spans[i + 1]) {
+                span.nextSiblings.add(spans[i + 1]);
+            }
+        }
+        return spans;
+    }
+}
+/**
+ * Patterns to test and apply classifications to spans.
+ *
+ * Note that order here is important, as the first pattern that matches will be used.
+ */
+export const PatternMatchers = [
+    // Entirely numeric, i.e. contains only digits.
+    [/^\d+$/, "numeric"],
+    // Entirely special characters, i.e. contains only punctuation, symbols, or other non-alphanumeric characters.
+    [/^[@&/\\#,+()$~%.!^'";:*?[\]<>{}]+$/, "punctuation"],
+    // Entirely alpha, i.e. contains only letters.
+    [/^[A-Za-z\W]+$/, "alpha"],
+    // Entirely alphanumeric, i.e. contains only letters and digits.
+    [/^[A-Za-z0-9\W]+$/, "alphanumeric"],
+];
+//# sourceMappingURL=Span.js.map

package/out/tokenization/Span.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Span.js","sourceRoot":"","sources":["../../tokenization/Span.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,iGAAiG;AACjG,gGAAgG;AAChG,+FAA+F;AAC/F,wEAAwE;AACxE,OAAO,EAGN,uBAAuB,GACvB,MAAM,qCAAqC,CAAA;AAI5C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,MAAM,eAAe,GAAG,GAAG,CAAA;AAQ3B,MAAM,OAAO,GAAkB,MAAM,CAAC,QAAQ,CAAC,CAAA;AAa/C;;GAEG;AACH,MAAM,OAAO,IAAK,SAAQ,KAAW;IACpC,KAAK,GAAW,EAAE,CAAA;IAClB;;OAEG;IACI,KAAK,CAAQ;IAEpB;;OAEG;IACI,GAAG,CAAS;IAEnB;;OAEG;IACO,MAAM,CAAC,SAAS,GAAG,CAAC,CAAA;IAE9B;;OAEG;IACM,CAAC,OAAO,CAAC,CAAS;IAE3B,IAAW,EAAE;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED;;OAEG;IACI,UAAU,GAAG,EAAE,CAAA;IAEtB;;OAEG;IACa,eAAe,GAA4B,IAAI,uBAAuB,EAAE,CAAA;IAE/E,MAAM,GAAG,IAAI,GAAG,EAAY,CAAA;IAErC;;;;OAIG;IACH,IAAW,KAAK;QACf,OAAO,IAAI,CAAC,MAAM,CAAA;IACnB,CAAC;IAEM,EAAE,CAAC,cAA8B;QACvC,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAChD,CAAC;IAKD,MAAM,CAAC,IAAI,CAAC,QAAuB,EAAE,EAAE,UAA+B,EAAE;QACvE,MAAM,IAAI,GAAG,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;QAE3E,KAAK,MAAM,cAAc,IAAI,OAAO,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;QACzC,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAA;QAE9C,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,YAAY,IAAI,GAAG,EAAE,EAAE,KAAK,GAAG,CAAC;QAC/B,KAAK,EAAE,CAAA;QAEP,mCAAmC;QACnC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE;YACpC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE;YACvB,QAAQ,EAAE,KAAK;YACf,UAAU,EAAE,KAAK;YACjB,YAAY,EAAE,KAAK;SACnB,CAAC,CAAA;QAEF,sFAAsF;QACtF,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;IACjB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAA;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,IAAI,CAAC,QAAgB;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;QACnB,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAA;QAE/C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAA;QAC1C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;QAEzC,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,eAAe,EAAE,CAAC;YAC/C,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBAErB,MAAK;YACN,CAAC;QACF,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAA;QACpC,CAAC;IACF,CAAC;IAED;;OAEG;IACI,UAAU,CAAC,IAAiC;QAClD,OAAO,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAA;IACtD,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,IAAiC;QAC9C,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAA;IACxD,CAAC;IAED;;;;;;OAMG;IACI,QAAQ,CAAC,IAAiC;QAChD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAA;QAEnC,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAA;QAC7B,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAA;IAC7B,CAAC;IAED;;;OAGG;IACH,IAAW,QAAQ;QAClB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxB,OAAO,CACN,QAAQ;gBACP,MAAM;iBACL,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;iBACnB,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAC5D,CAAA;QACF,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAA;IAC7B,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QACnB,yCAAyC;QACzC,MAAM,SAAS,GAAkE,IAAI,GAAG,EAAE,CAAA;QAC1F,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAA;QAEtC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,KAAK,MAAM,cAAc,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC7D,MAAM,cAAc,GAAG,cAAc,CAAC,SAAS,CAAA;gBAE/C,IAAI,CAAC,cAAc;oBAAE,SAAQ;gBAE7B,IAAI,cAAc,CAAC,WAAW,EAAE,CAAC;oBAChC,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;gBAC7C,CAAC;gBAED,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;oBACvC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;gBACxB,CAAC;YACF,CAAC;QACF,CAAC;QAED,SAAS,CAAC,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QAE3F,OAAO,SAAS,CAAA;IACjB,CAAC;IAED;;OAEG;IACI,MAAM;QACZ,OAAO;YACN,sBAAsB;YACtB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE;YAC9C,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;iBACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACtB,OAAO,EAAE;YACX,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;iBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACtB,OAAO,EAAE;SACc,CAAA;IAC3B,CAAC;IAEe,QAAQ;QACvB,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1E,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEhD,OAAO,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,eAAe,KAAK,KAAK,GAAG,CAAA;IACnF,CAAC;IAEM,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAA;IACvB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,GAAG,KAAa;QACtC,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACzC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAClB,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAA;YACzC,CAAC;YAED,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAClB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAA;YACrC,CAAC;QACF,CAAC;QAED,OAAO,KAAK,CAAA;IACb,CAAC;;AAKF;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAiD;IAC5E,+CAA+C;IAC/C,CAAC,OAAO,EAAE,SAAS,CAAC;IAEpB,8GAA8G;IAC9G,CAAC,oCAAoC,EAAE,aAAa,CAAC;IAErD,8CAA8C;IAC9C,CAAC,eAAe,EAAE,OAAO,CAAC;IAE1B,gEAAgE;IAChE,CAAC,kBAAkB,EAAE,cAAc,CAAC;CACpC,CAAA"}

package/out/tokenization/context.d.ts

@@ -0,0 +1,39 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { type SerializedSolution, type Solution } from "@mailwoman/core/solver";
+import { type SerializedSpan, Span } from "./Span.js";
+export interface SerializedTokenContext {
+    span: SerializedSpan;
+    solutions: SerializedSolution[];
+    sections: SerializedSpan[];
+    readonly coverage: number;
+}
+/**
+ * Tokenizes a string into sections and phrases.
+ */
+export declare class TokenContext {
+    #private;
+    get solutions(): Solution[];
+    set solutions(nextSolutions: Solution[]);
+    get span(): Span;
+    /**
+     * Sections of the tokenization.
+     */
+    get sections(): Span[];
+    constructor(input?: string);
+    /**
+     * Evaluate and rank the solutions.
+     *
+     * This method is called after the solver has generated solutions, modifying the solutions
+     * in-place.
+     */
+    evaluateAndRank(solutionLimit: number): void;
+    /**
+     * Serialize the tokenizer.
+     */
+    toJSON(): SerializedTokenContext;
+}
+//# sourceMappingURL=context.d.ts.map

package/out/tokenization/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../tokenization/context.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAGN,KAAK,kBAAkB,EACvB,KAAK,QAAQ,EACb,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EAAE,KAAK,cAAc,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAGrD,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,cAAc,CAAA;IACpB,SAAS,EAAE,kBAAkB,EAAE,CAAA;IAC/B,QAAQ,EAAE,cAAc,EAAE,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,qBAAa,YAAY;;IAOxB,IAAW,SAAS,IAAI,QAAQ,EAAE,CAEjC;IAED,IAAW,SAAS,CAAC,aAAa,EAAE,QAAQ,EAAE,EAW7C;IAED,IAAW,IAAI,IAAI,IAAI,CAEtB;IAED;;OAEG;IACH,IAAW,QAAQ,IAAI,IAAI,EAAE,CAE5B;gBAEW,KAAK,GAAE,MAAW;IAuB9B;;;;;OAKG;IACI,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI;IAkBnD;;OAEG;IACI,MAAM,IAAI,sBAAsB;CAQvC"}

package/out/tokenization/context.js

@@ -0,0 +1,114 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { compareMatchesByStart, rankingSolutionsByAreaClassification, } from "@mailwoman/core/solver";
+import { permutate } from "./permutate.js";
+import { Span } from "./Span.js";
+import { fieldsFuncBoundary, fieldsFuncHyphenOrWhiteSpace, fieldsFuncWhiteSpace, splitByField } from "./split.js";
+/**
+ * Tokenizes a string into sections and phrases.
+ */
+export class TokenContext {
+    #span;
+    #sections = [];
+    #coverage = 0;
+    #solutions = [];
+    get solutions() {
+        return this.#solutions;
+    }
+    set solutions(nextSolutions) {
+        // if (this.#solutions.length && nextSolutions.length < this.#solutions.length) {
+        // 	console.log(`--------------------------------------`)
+        // 	console.log(`${this.#solutions.length} Solution(s)`)
+        // 	for (const [i, solution] of this.#solutions.entries()) {
+        // 		console.log(`Solution ${i + 1}`, solution.toJSON())
+        // 	}
+        // }
+        this.#solutions = nextSolutions;
+    }
+    get span() {
+        return this.#span;
+    }
+    /**
+     * Sections of the tokenization.
+     */
+    get sections() {
+        return this.#sections;
+    }
+    constructor(input = "") {
+        this.#span = Span.from(input);
+        // Split the input into sections.
+        const sections = splitByField(this.#span, fieldsFuncBoundary);
+        for (const section of sections) {
+            // Then, split each section into phrases.
+            section.children.add(...splitByField(section, fieldsFuncWhiteSpace));
+            section.children.add(...splitByField(section, fieldsFuncHyphenOrWhiteSpace));
+        }
+        this.#sections = sections;
+        this.#coverage = computeCoverage(sections);
+        // Permute the phrases of each section.
+        for (const section of this.sections) {
+            const permutations = permutate(section.children, { from: 0, to: 10 });
+            section.phrases.add(...permutations);
+        }
+    }
+    /**
+     * Evaluate and rank the solutions.
+     *
+     * This method is called after the solver has generated solutions, modifying the solutions
+     * in-place.
+     */
+    evaluateAndRank(solutionLimit) {
+        if (this.#solutions.length === 0)
+            return;
+        for (const solution of this.#solutions) {
+            // Re-compute scores.
+            solution.computeScore(this.#coverage);
+            // Re-sort matches.
+            solution.matches.sort(compareMatchesByStart);
+        }
+        // Re-sort the solutions.
+        this.#solutions.sort(rankingSolutionsByAreaClassification);
+        // Finally, we keep only the best solutions.
+        this.#solutions = this.#solutions.slice(0, solutionLimit);
+    }
+    /**
+     * Serialize the tokenizer.
+     */
+    toJSON() {
+        return {
+            span: this.span.toJSON(),
+            sections: this.sections.map((s) => s.toJSON()),
+            solutions: this.#solutions.map((s) => s.toJSON()),
+            coverage: this.#coverage,
+        };
+    }
+}
+/**
+ * Compute the coverage of the tokenization.
+ */
+function computeCoverageRec(sum, currentSpan) {
+    if (!currentSpan)
+        return sum;
+    sum += currentSpan.end - currentSpan.start;
+    if (currentSpan.end < currentSpan.start) {
+        throw new Error(`Tokenizer: invalid span ${currentSpan.start} ${currentSpan.end}`);
+    }
+    return computeCoverageRec(sum, currentSpan.nextSibling);
+}
+/**
+ * Compute the coverage of the tokenization.
+ */
+function computeCoverage(sections) {
+    let coverage = 0;
+    for (const [i, section] of sections.entries()) {
+        const firstChild = section.children.first;
+        if (!firstChild)
+            throw new Error(`Tokenizer: section ${i} has no children`);
+        coverage += computeCoverageRec(0, firstChild);
+    }
+    return coverage;
+}
+//# sourceMappingURL=context.js.map

package/out/tokenization/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../../tokenization/context.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACN,qBAAqB,EACrB,oCAAoC,GAGpC,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAuB,IAAI,EAAE,MAAM,WAAW,CAAA;AACrD,OAAO,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AASjH;;GAEG;AACH,MAAM,OAAO,YAAY;IACxB,KAAK,CAAO;IACZ,SAAS,GAAW,EAAE,CAAA;IACtB,SAAS,GAAW,CAAC,CAAA;IAErB,UAAU,GAAe,EAAE,CAAA;IAE3B,IAAW,SAAS;QACnB,OAAO,IAAI,CAAC,UAAU,CAAA;IACvB,CAAC;IAED,IAAW,SAAS,CAAC,aAAyB;QAC7C,iFAAiF;QACjF,yDAAyD;QACzD,wDAAwD;QAExD,4DAA4D;QAC5D,wDAAwD;QACxD,KAAK;QACL,IAAI;QAEJ,IAAI,CAAC,UAAU,GAAG,aAAa,CAAA;IAChC,CAAC;IAED,IAAW,IAAI;QACd,OAAO,IAAI,CAAC,KAAK,CAAA;IAClB,CAAC;IAED;;OAEG;IACH,IAAW,QAAQ;QAClB,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAED,YAAY,QAAgB,EAAE;QAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAE7B,iCAAiC;QACjC,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAA;QAE7D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,yCAAyC;YACzC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAA;YACpE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAC,CAAA;QAC7E,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,SAAS,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;QAE1C,uCAAuC;QACvC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAA;YAErE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,CAAA;QACrC,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACI,eAAe,CAAC,aAAqB;QAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAExC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,qBAAqB;YACrB,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAErC,mBAAmB;YACnB,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;QAC7C,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAA;QAE1D,4CAA4C;QAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAA;IAC1D,CAAC;IAED;;OAEG;IACI,MAAM;QACZ,OAAO;YACN,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAC9C,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACjD,QAAQ,EAAE,IAAI,CAAC,SAAS;SACxB,CAAA;IACF,CAAC;CACD;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,GAAW,EAAE,WAAwB;IAChE,IAAI,CAAC,WAAW;QAAE,OAAO,GAAG,CAAA;IAE5B,GAAG,IAAI,WAAW,CAAC,GAAG,GAAG,WAAW,CAAC,KAAK,CAAA;IAE1C,IAAI,WAAW,CAAC,GAAG,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,2BAA2B,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC,CAAA;IACnF,CAAC;IAED,OAAO,kBAAkB,CAAC,GAAG,EAAE,WAAW,CAAC,WAAW,CAAC,CAAA;AACxD,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,QAAgB;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAA;QAEzC,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,CAAA;QAE3E,QAAQ,IAAI,kBAAkB,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;IAC9C,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/tokenization/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+export * from "./context.js";
+export * from "./Graph.js";
+export * from "./normalizer.js";
+export * from "./permutate.js";
+export * from "./Span.js";
+//# sourceMappingURL=index.d.ts.map

package/out/tokenization/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../tokenization/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAA;AAC5B,cAAc,YAAY,CAAA;AAC1B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,WAAW,CAAA"}

package/out/tokenization/index.js

@@ -0,0 +1,11 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+export * from "./context.js";
+export * from "./Graph.js";
+export * from "./normalizer.js";
+export * from "./permutate.js";
+export * from "./Span.js";
+//# sourceMappingURL=index.js.map