@concordance-wiki/nlp 0.1.0

package/dist/locale/pack.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=pack.js.map

package/dist/locale/pack.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.js","sourceRoot":"","sources":["../../src/locale/pack.ts"],"names":[],"mappings":""}

package/dist/locale/registry.d.ts

@@ -0,0 +1,16 @@
+import type { Locale } from "@concordance-wiki/core";
+import type { LanguagePack } from "./pack.js";
+/**
+ * The pack of a locale: the exact tag first, then its language alone, so that `fr-CA`
+ * reads the `fr` pack until a plugin registers a more specific one.
+ */
+export declare function languagePack(locale: Locale): LanguagePack;
+export declare function registerLanguagePack(pack: LanguagePack): void;
+export declare function availableLocales(): Locale[];
+/** The locale of a source: its own, then the project's, then `en`; always canonical. */
+export declare function resolveLocale(source: {
+    locale?: Locale;
+}, project: {
+    locale?: Locale;
+}): Locale;
+//# sourceMappingURL=registry.d.ts.map

package/dist/locale/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/locale/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAGrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAc9C;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CASzD;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI,CAK7D;AAED,wBAAgB,gBAAgB,IAAI,MAAM,EAAE,CAE3C;AAED,wFAAwF;AACxF,wBAAgB,aAAa,CAAC,MAAM,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE/F"}

package/dist/locale/registry.js

@@ -0,0 +1,35 @@
+import { loadLanguagePack } from "./load-pack.js";
+import { canonicalLocale } from "./tag.js";
+// Resolves from both src/locale/ and dist/locale/, which sit at the same depth.
+const shipped = ["en", "fr"].map((locale) => loadLanguagePack(new URL(`../../locales/${locale}/`, import.meta.url)));
+const packs = new Map(shipped.map((pack) => [pack.locale, pack]));
+// Code-unit order, not locale order: the output must not depend on the collation data of the runtime.
+function byCodeUnit(a, b) {
+    return Number(a > b) - Number(a < b);
+}
+/**
+ * The pack of a locale: the exact tag first, then its language alone, so that `fr-CA`
+ * reads the `fr` pack until a plugin registers a more specific one.
+ */
+export function languagePack(locale) {
+    const tag = canonicalLocale(locale);
+    const pack = packs.get(tag) ?? packs.get(new Intl.Locale(tag).language);
+    if (pack === undefined) {
+        throw new Error(`no language pack for locale "${tag}": the engine ships ${availableLocales().join(", ")}; other locales come from plugins`);
+    }
+    return pack;
+}
+export function registerLanguagePack(pack) {
+    if (packs.has(pack.locale)) {
+        throw new Error(`a language pack for locale "${pack.locale}" is already registered`);
+    }
+    packs.set(pack.locale, pack);
+}
+export function availableLocales() {
+    return [...packs.keys()].sort(byCodeUnit);
+}
+/** The locale of a source: its own, then the project's, then `en`; always canonical. */
+export function resolveLocale(source, project) {
+    return canonicalLocale(source.locale ?? project.locale ?? "en");
+}
+//# sourceMappingURL=registry.js.map

package/dist/locale/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/locale/registry.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,gFAAgF;AAChF,MAAM,OAAO,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC1C,gBAAgB,CAAC,IAAI,GAAG,CAAC,iBAAiB,MAAM,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CACvE,CAAC;AACF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAuB,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AAExF,sGAAsG;AACtG,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,GAAG,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IACxE,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,gCAAgC,GAAG,uBAAuB,gBAAgB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAmC,CAC3H,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAkB;IACrD,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,MAAM,yBAAyB,CAAC,CAAC;IACvF,CAAC;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAC5C,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,aAAa,CAAC,MAA2B,EAAE,OAA4B;IACrF,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;AAClE,CAAC"}

package/dist/locale/stopwords.d.ts

@@ -0,0 +1,3 @@
+/** One word per line, `#` starts a comment; returns the unique lowercase words in code unit order. */
+export declare function loadStopwords(text: string): string[];
+//# sourceMappingURL=stopwords.d.ts.map

package/dist/locale/stopwords.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopwords.d.ts","sourceRoot":"","sources":["../../src/locale/stopwords.ts"],"names":[],"mappings":"AAKA,sGAAsG;AACtG,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAUpD"}

package/dist/locale/stopwords.js

@@ -0,0 +1,17 @@
+// Code-unit order, not locale order: the output must not depend on the collation data of the runtime.
+function byCodeUnit(a, b) {
+    return Number(a > b) - Number(a < b);
+}
+/** One word per line, `#` starts a comment; returns the unique lowercase words in code unit order. */
+export function loadStopwords(text) {
+    const words = new Set();
+    for (const line of text.split("\n")) {
+        const hash = line.indexOf("#");
+        const word = (hash === -1 ? line : line.slice(0, hash)).trim().toLowerCase();
+        if (word !== "") {
+            words.add(word);
+        }
+    }
+    return [...words].sort(byCodeUnit);
+}
+//# sourceMappingURL=stopwords.js.map

package/dist/locale/stopwords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopwords.js","sourceRoot":"","sources":["../../src/locale/stopwords.ts"],"names":[],"mappings":"AAAA,sGAAsG;AACtG,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,sGAAsG;AACtG,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7E,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YAChB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC"}

package/dist/locale/suffixes.d.ts

@@ -0,0 +1,7 @@
+/**
+ * The inflected-form suffixes of a pack's `suffixes.txt`: one suffix per line, an optional
+ * leading hyphen, `#` starts a comment; returns the unique suffixes in comparison form
+ * (through the pack's normaliser), in code unit order.
+ */
+export declare function loadSuffixes(text: string, normalize: (text: string) => string): string[];
+//# sourceMappingURL=suffixes.d.ts.map

package/dist/locale/suffixes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"suffixes.d.ts","sourceRoot":"","sources":["../../src/locale/suffixes.ts"],"names":[],"mappings":"AAKA;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,CAUxF"}

package/dist/locale/suffixes.js

@@ -0,0 +1,21 @@
+// Code-unit order, not locale order: the output must not depend on the collation data of the runtime.
+function byCodeUnit(a, b) {
+    return Number(a > b) - Number(a < b);
+}
+/**
+ * The inflected-form suffixes of a pack's `suffixes.txt`: one suffix per line, an optional
+ * leading hyphen, `#` starts a comment; returns the unique suffixes in comparison form
+ * (through the pack's normaliser), in code unit order.
+ */
+export function loadSuffixes(text, normalize) {
+    const suffixes = new Set();
+    for (const line of text.split("\n")) {
+        const hash = line.indexOf("#");
+        const suffix = normalize((hash === -1 ? line : line.slice(0, hash)).trim().replace(/^-/, ""));
+        if (suffix !== "") {
+            suffixes.add(suffix);
+        }
+    }
+    return [...suffixes].sort(byCodeUnit);
+}
+//# sourceMappingURL=suffixes.js.map

package/dist/locale/suffixes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suffixes.js","sourceRoot":"","sources":["../../src/locale/suffixes.ts"],"names":[],"mappings":"AAAA,sGAAsG;AACtG,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,SAAmC;IAC5E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9F,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACxC,CAAC"}

package/dist/locale/tag.d.ts

@@ -0,0 +1,4 @@
+import type { Locale } from "@concordance-wiki/core";
+/** The canonical form of a BCP 47 tag (`fr-ca` → `fr-CA`); throws a RangeError on a malformed tag. */
+export declare function canonicalLocale(tag: string): Locale;
+//# sourceMappingURL=tag.d.ts.map

package/dist/locale/tag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag.d.ts","sourceRoot":"","sources":["../../src/locale/tag.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,sGAAsG;AACtG,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEnD"}

package/dist/locale/tag.js

@@ -0,0 +1,5 @@
+/** The canonical form of a BCP 47 tag (`fr-ca` → `fr-CA`); throws a RangeError on a malformed tag. */
+export function canonicalLocale(tag) {
+    return new Intl.Locale(tag).toString();
+}
+//# sourceMappingURL=tag.js.map

package/dist/locale/tag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag.js","sourceRoot":"","sources":["../../src/locale/tag.ts"],"names":[],"mappings":"AAEA,sGAAsG;AACtG,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;AACzC,CAAC"}

package/dist/scan/automaton.d.ts

@@ -0,0 +1,46 @@
+/** A sequence of words to recognise, identified by `key`. */
+export interface Pattern<K = string> {
+    key: K;
+    words: readonly string[];
+}
+/** A pattern found in a token sequence: `start` inclusive, `end` exclusive, as token indexes. */
+export interface RawMatch<K = string> {
+    key: K;
+    start: number;
+    end: number;
+}
+interface Output<K> {
+    key: K;
+    length: number;
+}
+/** One state of the automaton: the words read so far, as a node of the pattern trie. */
+export interface AutomatonState<K = string> {
+    readonly next: ReadonlyMap<string, AutomatonState<K>>;
+    /** The longest proper suffix of this state that is also a state; the root fails to itself. */
+    readonly fail: AutomatonState<K>;
+    /** Every pattern ending here, own or through failure links, longest first. */
+    readonly output: readonly Output<K>[];
+}
+export interface Automaton<K = string> {
+    readonly root: AutomatonState<K>;
+}
+/**
+ * Aho-Corasick over words: the trie of the patterns with failure links, so that a text is read
+ * once whatever the number of patterns. A pattern without words is ignored, and a key given
+ * twice for the same words is kept once.
+ */
+export declare function buildAutomaton<K = string>(patterns: readonly Pattern<K>[]): Automaton<K>;
+/** Every pattern occurrence in the tokens, in one pass, in order of end then longest first. */
+export declare function scan<K>(automaton: Automaton<K>, tokens: readonly {
+    word: string;
+}[]): RawMatch<K>[];
+/**
+ * Resolves overlaps: a match contained in another one is dropped, so that among the matches
+ * sharing a start the longest wins and "keyword page" is not counted again as "page"; matches
+ * that only partly overlap, from different starts, are all kept since each may name another
+ * entity. Two keys on exactly the same span both stay: they are two spellings of the same
+ * words. The result is in text order; equal spans keep their input order.
+ */
+export declare function longestMatches<K>(matches: readonly RawMatch<K>[]): RawMatch<K>[];
+export {};
+//# sourceMappingURL=automaton.d.ts.map

package/dist/scan/automaton.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"automaton.d.ts","sourceRoot":"","sources":["../../src/scan/automaton.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,MAAM,WAAW,OAAO,CAAC,CAAC,GAAG,MAAM;IACjC,GAAG,EAAE,CAAC,CAAC;IACP,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1B;AAED,iGAAiG;AACjG,MAAM,WAAW,QAAQ,CAAC,CAAC,GAAG,MAAM;IAClC,GAAG,EAAE,CAAC,CAAC;IACP,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,UAAU,MAAM,CAAC,CAAC;IAChB,GAAG,EAAE,CAAC,CAAC;IACP,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wFAAwF;AACxF,MAAM,WAAW,cAAc,CAAC,CAAC,GAAG,MAAM;IACxC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,8FAA8F;IAC9F,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;IACjC,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,SAAS,CAAC,CAAC,GAAG,MAAM;IACnC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;CAClC;AA0BD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,SAAS,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAuBxF;AAED,+FAA+F;AAC/F,wBAAgB,IAAI,CAAC,CAAC,EACpB,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,EACvB,MAAM,EAAE,SAAS;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,GAClC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAYf;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAgBhF"}

package/dist/scan/automaton.js

@@ -0,0 +1,92 @@
+class State {
+    next = new Map();
+    fail;
+    output = [];
+    constructor(fail) {
+        this.fail = fail ?? this;
+    }
+}
+/** The state the words lead to from the root, created along the way when missing. */
+function stateOf(root, words) {
+    let state = root;
+    for (const word of words) {
+        let child = state.next.get(word);
+        if (child === undefined) {
+            child = new State(root);
+            state.next.set(word, child);
+        }
+        state = child;
+    }
+    return state;
+}
+/**
+ * Aho-Corasick over words: the trie of the patterns with failure links, so that a text is read
+ * once whatever the number of patterns. A pattern without words is ignored, and a key given
+ * twice for the same words is kept once.
+ */
+export function buildAutomaton(patterns) {
+    const root = new State();
+    for (const { key, words } of patterns) {
+        if (words.length === 0)
+            continue;
+        const state = stateOf(root, words);
+        if (!state.output.some((output) => output.key === key)) {
+            state.output.push({ key, length: words.length });
+        }
+    }
+    // Breadth-first, so that the failure link of a state is final before its children use it.
+    // The array iterator reads the elements pushed during the walk.
+    const queue = [...root.next.values()];
+    for (const state of queue) {
+        for (const [word, child] of state.next) {
+            let fail = state.fail;
+            while (fail !== root && !fail.next.has(word))
+                fail = fail.fail;
+            child.fail = fail.next.get(word) ?? root;
+            child.output = [...child.output, ...child.fail.output];
+            queue.push(child);
+        }
+    }
+    return { root };
+}
+/** Every pattern occurrence in the tokens, in one pass, in order of end then longest first. */
+export function scan(automaton, tokens) {
+    const { root } = automaton;
+    const matches = [];
+    let state = root;
+    for (const [index, token] of tokens.entries()) {
+        while (state !== root && !state.next.has(token.word))
+            state = state.fail;
+        state = state.next.get(token.word) ?? root;
+        for (const { key, length } of state.output) {
+            matches.push({ key, start: index + 1 - length, end: index + 1 });
+        }
+    }
+    return matches;
+}
+/**
+ * Resolves overlaps: a match contained in another one is dropped, so that among the matches
+ * sharing a start the longest wins and "keyword page" is not counted again as "page"; matches
+ * that only partly overlap, from different starts, are all kept since each may name another
+ * entity. Two keys on exactly the same span both stay: they are two spellings of the same
+ * words. The result is in text order; equal spans keep their input order.
+ */
+export function longestMatches(matches) {
+    const byStart = [...matches].sort((a, b) => a.start - b.start || b.end - a.end);
+    const kept = [];
+    // Farthest end of the matches starting before the current group, and of the group itself.
+    let outerEnd = -1;
+    let groupStart = -1;
+    let groupEnd = -1;
+    for (const match of byStart) {
+        if (match.start !== groupStart) {
+            outerEnd = Math.max(outerEnd, groupEnd);
+            groupStart = match.start;
+            groupEnd = match.end;
+        }
+        if (match.end === groupEnd && match.end > outerEnd)
+            kept.push(match);
+    }
+    return kept;
+}
+//# sourceMappingURL=automaton.js.map

package/dist/scan/automaton.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"automaton.js","sourceRoot":"","sources":["../../src/scan/automaton.ts"],"names":[],"mappings":"AA+BA,MAAM,KAAK;IACA,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC5C,IAAI,CAAW;IACf,MAAM,GAAgB,EAAE,CAAC;IAEzB,YAAY,IAAe;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC;IAC3B,CAAC;CACF;AAED,qFAAqF;AACrF,SAAS,OAAO,CAAI,IAAc,EAAE,KAAwB;IAC1D,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG,IAAI,KAAK,CAAI,IAAI,CAAC,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,KAAK,GAAG,KAAK,CAAC;IAChB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAa,QAA+B;IACxE,MAAM,IAAI,GAAG,IAAI,KAAK,EAAK,CAAC;IAC5B,KAAK,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACjC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YACvD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,0FAA0F;IAC1F,gEAAgE;IAChE,MAAM,KAAK,GAAe,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACtB,OAAO,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YAC/D,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;YACzC,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,CAAC;AAClB,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,IAAI,CAClB,SAAuB,EACvB,MAAmC;IAEnC,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC;IAC3B,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC;QACzE,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;QAC3C,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAI,OAA+B;IAC/D,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAChF,MAAM,IAAI,GAAkB,EAAE,CAAC;IAC/B,0FAA0F;IAC1F,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC/B,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACxC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;YACzB,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC;QACvB,CAAC;QACD,IAAI,KAAK,CAAC,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,GAAG,GAAG,QAAQ;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/scan/context.d.ts

@@ -0,0 +1,20 @@
+/** An inline code span the scanned text leaves out: what was written, at the offset it stood at. */
+export interface ElidedCode {
+    /** Offset in the scanned text where the code stood; the code precedes the character there. */
+    at: number;
+    text: string;
+}
+/** What the context of an occurrence is cut from: the scanned text and the code it leaves out. */
+export interface QuotedText {
+    text: string;
+    code?: readonly ElidedCode[];
+}
+/** How many characters of the text an occurrence quotes. */
+export declare const CONTEXT_WIDTH = 80;
+/**
+ * The context of an occurrence: a window of `width` characters of the text as written, the
+ * inline code the scan skipped put back in place, centred on the match at `start`–`end` of the
+ * scanned text. A code span at the start of the match precedes it, one at its end follows it.
+ */
+export declare function occurrenceContext(quoted: QuotedText, start: number, end: number, width?: number): string;
+//# sourceMappingURL=context.d.ts.map

package/dist/scan/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/scan/context.ts"],"names":[],"mappings":"AAEA,oGAAoG;AACpG,MAAM,WAAW,UAAU;IACzB,8FAA8F;IAC9F,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;CACd;AAED,kGAAkG;AAClG,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;CAC9B;AAED,4DAA4D;AAC5D,eAAO,MAAM,aAAa,KAAK,CAAC;AAEhC;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,KAAK,SAAgB,GACpB,MAAM,CAcR"}

package/dist/scan/context.js

@@ -0,0 +1,26 @@
+import { contextAround } from "../text/context.js";
+/** How many characters of the text an occurrence quotes. */
+export const CONTEXT_WIDTH = 80;
+/**
+ * The context of an occurrence: a window of `width` characters of the text as written, the
+ * inline code the scan skipped put back in place, centred on the match at `start`–`end` of the
+ * scanned text. A code span at the start of the match precedes it, one at its end follows it.
+ */
+export function occurrenceContext(quoted, start, end, width = CONTEXT_WIDTH) {
+    const code = quoted.code ?? [];
+    let written = "";
+    let cursor = 0;
+    let from = start;
+    let to = end;
+    for (const span of code) {
+        written += quoted.text.slice(cursor, span.at) + span.text;
+        cursor = span.at;
+        if (span.at <= start)
+            from += span.text.length;
+        if (span.at < end)
+            to += span.text.length;
+    }
+    written += quoted.text.slice(cursor);
+    return contextAround(written, from, to, width);
+}
+//# sourceMappingURL=context.js.map

package/dist/scan/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/scan/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAenD,4DAA4D;AAC5D,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEhC;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAkB,EAClB,KAAa,EACb,GAAW,EACX,KAAK,GAAG,aAAa;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,EAAE,GAAG,GAAG,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAC1D,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,EAAE,IAAI,KAAK;YAAE,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC/C,IAAI,IAAI,CAAC,EAAE,GAAG,GAAG;YAAE,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5C,CAAC;IACD,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;AACjD,CAAC"}

package/dist/scan/occurrences.d.ts

@@ -0,0 +1,64 @@
+import type { Dictionary, DictionaryTarget } from "../dictionary/types.js";
+import type { LanguagePack } from "../locale/pack.js";
+import { type ElidedCode } from "./context.js";
+/** The `confidence.glossary_occurrence` block of the profile, every key resolved. */
+export interface OccurrenceScale {
+    base: number;
+    per_occurrence: number;
+    cap: number;
+    homonym_factor: number;
+    type_prefix_bonus: number;
+}
+export interface ScannedParagraph {
+    line: number;
+    /** The text the scan reads, inline code left out. */
+    text: string;
+    /** Heading of the enclosing section, when any. */
+    section?: string;
+    /** The inline code the text leaves out, in text order, quoted back in the context of an occurrence. */
+    code?: readonly ElidedCode[];
+}
+/** A document reduced to the text units the scan reads, excluded zones already removed. */
+export interface ScannedDocument {
+    path: string;
+    paragraphs: readonly ScannedParagraph[];
+}
+export interface Occurrence {
+    /** The dictionary key that matched. */
+    key: string;
+    target: Pick<DictionaryTarget, "id" | "kind">;
+    source: string;
+    path: string;
+    line: number;
+    /** Code unit offset of the match in the paragraph text. */
+    position: number;
+    /** The match as written, from its first to its last token. */
+    text: string;
+    section?: string;
+    /** 80 characters of the text as written, inline code included, centred on the match, an ellipsis marking each cut. */
+    context: string;
+    /** The type slug announced by the word right before the match, when it is a type prefix. */
+    expectedType?: string;
+    confidence: number;
+}
+export interface ScanDocumentInput {
+    document: ScannedDocument;
+    source: string;
+    dictionary: Dictionary;
+    pack: LanguagePack;
+    /** Type slug to the words announcing it, for the locale of the dictionary. */
+    typePrefixes: Readonly<Record<string, readonly string[]>>;
+    scale: OccurrenceScale;
+}
+/** Canonical order of occurrences: path, line, position, target, then key. */
+export declare function compareOccurrences(a: Occurrence, b: Occurrence): number;
+/**
+ * Every mention of a dictionary entry in the paragraphs of a document, one per target of the
+ * entry, in canonical order. Confidence is the base of the scale, plus
+ * the type prefix bonus when a prefix announces the mention, halved (or whatever the scale
+ * says) for a homonym; the increment per further occurrence belongs to the combination step.
+ */
+export declare function scanDocument(input: ScanDocumentInput): Occurrence[];
+/** Confidence of an entry mentioned `count` times in a document: the increments up to the cap. */
+export declare function occurrenceConfidence(count: number, scale: OccurrenceScale): number;
+//# sourceMappingURL=occurrences.d.ts.map

package/dist/scan/occurrences.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"occurrences.d.ts","sourceRoot":"","sources":["../../src/scan/occurrences.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAmB,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC5F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAQtD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,cAAc,CAAC;AAGlE,qFAAqF;AACrF,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uGAAuG;IACvG,IAAI,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;CAC9B;AAED,2FAA2F;AAC3F,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,CAAC;CACzC;AAED,MAAM,WAAW,UAAU;IACzB,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,IAAI,GAAG,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sHAAsH;IACtH,OAAO,EAAE,MAAM,CAAC;IAChB,4FAA4F;IAC5F,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,UAAU,CAAC;IACvB,IAAI,EAAE,YAAY,CAAC;IACnB,8EAA8E;IAC9E,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1D,KAAK,EAAE,eAAe,CAAC;CACxB;AA4DD,8EAA8E;AAC9E,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,GAAG,MAAM,CAQvE;AAqCD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,UAAU,EAAE,CAanE;AAED,kGAAkG;AAClG,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,GAAG,MAAM,CAElF"}

package/dist/scan/occurrences.js

@@ -0,0 +1,109 @@
+import { buildAutomaton, longestMatches, scan, } from "./automaton.js";
+import { occurrenceContext } from "./context.js";
+import { tokenize } from "./tokens.js";
+function byCodeUnit(a, b) {
+    return Number(a > b) - Number(a < b);
+}
+// The same dictionary is scanned against every document: its automaton is built once.
+const automatons = new WeakMap();
+function automatonOf(dictionary, pack) {
+    let byPack = automatons.get(dictionary);
+    if (byPack === undefined) {
+        byPack = new WeakMap();
+        automatons.set(dictionary, byPack);
+    }
+    let automaton = byPack.get(pack);
+    if (automaton === undefined) {
+        // Keys are tokenised like the texts, so that "mot-clé" matches its two tokens.
+        const patterns = [...dictionary.entries.values()].map((entry) => ({
+            key: entry,
+            words: tokenize(entry.key, pack).map((token) => token.word),
+        }));
+        automaton = buildAutomaton(patterns);
+        byPack.set(pack, automaton);
+    }
+    return automaton;
+}
+/**
+ * Comparison form of every prefix word to the type it announces; a word listed under several
+ * types announces none. A prefix is a single word: the token right before the match.
+ */
+function prefixTypes(typePrefixes, pack) {
+    const types = new Map();
+    for (const [type, words] of Object.entries(typePrefixes)) {
+        for (const prefix of words) {
+            const word = tokenize(prefix, pack)
+                .map((token) => token.word)
+                .join(" ");
+            types.set(word, types.has(word) ? undefined : type);
+        }
+    }
+    return types;
+}
+// The automaton only reports spans inside the tokens it read, so the slice is never empty.
+function spanOf(tokens, match) {
+    const span = tokens.slice(match.start, match.end);
+    return {
+        start: Math.min(...span.map((token) => token.start)),
+        end: Math.max(...span.map((token) => token.end)),
+    };
+}
+/** Canonical order of occurrences: path, line, position, target, then key. */
+export function compareOccurrences(a, b) {
+    return (byCodeUnit(a.path, b.path) ||
+        a.line - b.line ||
+        a.position - b.position ||
+        byCodeUnit(a.target.id, b.target.id) ||
+        byCodeUnit(a.key, b.key));
+}
+/** The occurrences of one match: one per target of the entry, at the same confidence. */
+function occurrencesOf(scan, match) {
+    const { input, paragraph, tokens, prefixes } = scan;
+    const { scale } = input;
+    const span = spanOf(tokens, match);
+    const before = tokens[match.start - 1];
+    const expectedType = before === undefined ? undefined : prefixes.get(before.word);
+    const announced = expectedType === undefined ? 0 : scale.type_prefix_bonus;
+    const factor = match.key.homonym ? scale.homonym_factor : 1;
+    const base = {
+        key: match.key.key,
+        source: input.source,
+        path: input.document.path,
+        line: paragraph.line,
+        position: span.start,
+        text: paragraph.text.slice(span.start, span.end),
+        ...(paragraph.section === undefined ? {} : { section: paragraph.section }),
+        context: occurrenceContext(paragraph, span.start, span.end),
+        ...(expectedType === undefined ? {} : { expectedType }),
+        confidence: (scale.base + announced) * factor,
+    };
+    return match.key.targets.map((target) => ({
+        ...base,
+        target: { id: target.id, kind: target.kind },
+    }));
+}
+/**
+ * Every mention of a dictionary entry in the paragraphs of a document, one per target of the
+ * entry, in canonical order. Confidence is the base of the scale, plus
+ * the type prefix bonus when a prefix announces the mention, halved (or whatever the scale
+ * says) for a homonym; the increment per further occurrence belongs to the combination step.
+ */
+export function scanDocument(input) {
+    const { document, dictionary, pack } = input;
+    const automaton = automatonOf(dictionary, pack);
+    const prefixes = prefixTypes(input.typePrefixes, pack);
+    const occurrences = [];
+    for (const paragraph of document.paragraphs) {
+        const tokens = tokenize(paragraph.text, pack);
+        const scanned = { input, paragraph, tokens, prefixes };
+        for (const match of longestMatches(scan(automaton, tokens))) {
+            occurrences.push(...occurrencesOf(scanned, match));
+        }
+    }
+    return occurrences.sort(compareOccurrences);
+}
+/** Confidence of an entry mentioned `count` times in a document: the increments up to the cap. */
+export function occurrenceConfidence(count, scale) {
+    return Math.min(scale.cap, scale.base + scale.per_occurrence * (count - 1));
+}
+//# sourceMappingURL=occurrences.js.map

package/dist/scan/occurrences.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"occurrences.js","sourceRoot":"","sources":["../../src/scan/occurrences.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,cAAc,EACd,cAAc,EACd,IAAI,GAGL,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAmB,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAc,MAAM,aAAa,CAAC;AAwDnD,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,GAAG,IAAI,OAAO,EAAiE,CAAC;AAEhG,SAAS,WAAW,CAAC,UAAsB,EAAE,IAAkB;IAC7D,IAAI,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QACvB,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,+EAA+E;QAC/E,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAChE,GAAG,EAAE,KAAK;YACV,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SAC5D,CAAC,CAAC,CAAC;QACJ,SAAS,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAClB,YAAyD,EACzD,IAAkB;IAElB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA8B,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QACzD,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;iBAChC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;iBAC1B,IAAI,CAAC,GAAG,CAAC,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2FAA2F;AAC3F,SAAS,MAAM,CACb,MAAwB,EACxB,KAAwB;IAExB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAClD,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,kBAAkB,CAAC,CAAa,EAAE,CAAa;IAC7D,OAAO,CACL,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;QAC1B,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;QACf,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;QACvB,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QACpC,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CACzB,CAAC;AACJ,CAAC;AAUD,yFAAyF;AACzF,SAAS,aAAa,CAAC,IAAmB,EAAE,KAAgC;IAC1E,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IACpD,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IACxB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACvC,MAAM,YAAY,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClF,MAAM,SAAS,GAAG,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC;IAC3E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAA+B;QACvC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG;QAClB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;QACzB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;QAChD,GAAG,CAAC,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC;QAC1E,OAAO,EAAE,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;QAC3D,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC;QACvD,UAAU,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,SAAS,CAAC,GAAG,MAAM;KAC9C,CAAC;IACF,OAAO,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxC,GAAG,IAAI;QACP,MAAM,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;KAC7C,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAAwB;IACnD,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAC7C,MAAM,SAAS,GAAG,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACvD,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAkB,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACtE,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;YAC5D,WAAW,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;AAC9C,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,oBAAoB,CAAC,KAAa,EAAE,KAAsB;IACxE,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,cAAc,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;AAC9E,CAAC"}

package/dist/scan/tokens.d.ts

@@ -0,0 +1,12 @@
+import type { LanguagePack } from "../locale/pack.js";
+/** A word of a text as the scan compares it, with its character span in the text as given. */
+export interface Token {
+    /** Comparison form of the word: normalised by the pack, then singularised. */
+    word: string;
+    start: number;
+    /** Exclusive, in code units of the original text. */
+    end: number;
+}
+/** The words of a text in text order, each in comparison form with its span in the text. */
+export declare function tokenize(text: string, pack: LanguagePack): Token[];
+//# sourceMappingURL=tokens.d.ts.map

package/dist/scan/tokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/scan/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAQ,MAAM,mBAAmB,CAAC;AAG5D,8FAA8F;AAC9F,MAAM,WAAW,KAAK;IACpB,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,GAAG,EAAE,MAAM,CAAC;CACb;AAyBD,4FAA4F;AAC5F,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,KAAK,EAAE,CAWlE"}

package/dist/scan/tokens.js

@@ -0,0 +1,34 @@
+import { singularize } from "../text/singular.js";
+const notLetterOrDigit = /[^\p{L}\p{M}\p{N}]/gu;
+// The segmenter cuts a hyphenated compound into its parts ("mot-clé" gives the tokens
+// "mot" and "clé", so that both "mot" and "mot clé" match there) but keeps
+// an apostrophe inside its word ("l'écran"): the parts an apostrophe binds are cut here the same
+// way, with the apostrophe characters the pack unifies.
+function pieces(segment, pack) {
+    const result = [];
+    let start = 0;
+    for (const match of segment.text.matchAll(notLetterOrDigit)) {
+        if (pack.normalize(match[0]) !== "'")
+            continue;
+        result.push({ text: segment.text.slice(start, match.index), start: segment.index + start });
+        start = match.index + 1;
+    }
+    result.push({ text: segment.text.slice(start), start: segment.index + start });
+    return result;
+}
+/** The words of a text in text order, each in comparison form with its span in the text. */
+export function tokenize(text, pack) {
+    const tokens = [];
+    for (const segment of pack.segment(text)) {
+        if (!segment.isWordLike)
+            continue;
+        for (const piece of pieces(segment, pack)) {
+            const word = singularize(pack.normalize(piece.text), pack.plural);
+            if (word === "")
+                continue;
+            tokens.push({ word, start: piece.start, end: piece.start + piece.text.length });
+        }
+    }
+    return tokens;
+}
+//# sourceMappingURL=tokens.js.map

package/dist/scan/tokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/scan/tokens.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAgBlD,MAAM,gBAAgB,GAAG,sBAAsB,CAAC;AAEhD,sFAAsF;AACtF,2EAA2E;AAC3E,iGAAiG;AACjG,wDAAwD;AACxD,SAAS,MAAM,CAAC,OAAa,EAAE,IAAkB;IAC/C,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAC5D,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG;YAAE,SAAS;QAC/C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC;QAC5F,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC;IAC/E,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,IAAkB;IACvD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,OAAO,CAAC,UAAU;YAAE,SAAS;QAClC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAClE,IAAI,IAAI,KAAK,EAAE;gBAAE,SAAS;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}