@concordance-wiki/nlp 0.1.0

package/dist/keywords/ngrams.js

@@ -0,0 +1,64 @@
+import { occurrenceContext } from "../scan/context.js";
+import { tokenize } from "../scan/tokens.js";
+const contextWidth = 160;
+const digitsOnly = /^\p{N}+$/u;
+/** The comparison form of a term as the n-grams carry it: its words, tokenised like the texts. */
+export function keywordForm(text, pack) {
+    return tokenize(text, pack)
+        .map((token) => token.word)
+        .join(" ");
+}
+/** The comparison forms of a list of terms. */
+export function keywordForms(forms, pack) {
+    return new Set([...forms].map((form) => keywordForm(form, pack)));
+}
+/** Whether the words so far form a candidate: enough of them, the last not a stopword, not digits alone, long enough. */
+function isCandidate(words, last, rules) {
+    if (words.length < rules.minWords || rules.stopwords.has(last))
+        return false;
+    if (words.every((word) => digitsOnly.test(word)))
+        return false;
+    return words.join(" ").length >= rules.minLength;
+}
+/** The n-grams of one unit starting at each token that is not a stopword, in text order. */
+function ngramsOf(unit, tokens, rules) {
+    const occurrences = [];
+    for (const [start, first] of tokens.entries()) {
+        // A candidate never starts with a stopword, whatever its length.
+        if (rules.stopwords.has(first.word))
+            continue;
+        const words = [];
+        for (const last of tokens.slice(start, start + rules.maxWords)) {
+            words.push(last.word);
+            if (!isCandidate(words, last.word, rules))
+                continue;
+            occurrences.push({
+                key: words.join(" "),
+                surface: unit.text.slice(first.start, last.end),
+                ...(unit.source === undefined ? {} : { source: unit.source }),
+                path: unit.path,
+                line: unit.line,
+                position: first.start,
+                context: occurrenceContext(unit, first.start, last.end, contextWidth),
+            });
+        }
+    }
+    return occurrences;
+}
+/**
+ * Every n-gram of `minWords` to `maxWords` words in the units, in unit then text order,
+ * except those starting or ending with a stopword, those made only of digits and those
+ * whose key is shorter than `minLength`. Words are the tokens of the text: normalised by
+ * the pack and singularised, so that "Build summaries" and "build summary"
+ * share a key while each keeps its surface form.
+ */
+export function extractNgrams(units, pack, options) {
+    const rules = {
+        minWords: options.minWords ?? 1,
+        maxWords: options.maxWords,
+        minLength: options.minLength,
+        stopwords: keywordForms(options.stopwords ?? pack.stopwords, pack),
+    };
+    return units.flatMap((unit) => ngramsOf(unit, tokenize(unit.text, pack), rules));
+}
+//# sourceMappingURL=ngrams.js.map

package/dist/keywords/ngrams.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ngrams.js","sourceRoot":"","sources":["../../src/keywords/ngrams.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAmB,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAc,MAAM,mBAAmB,CAAC;AAqCzD,MAAM,YAAY,GAAG,GAAG,CAAC;AACzB,MAAM,UAAU,GAAG,WAAW,CAAC;AAE/B,kGAAkG;AAClG,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,IAAkB;IAC1D,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;SACxB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SAC1B,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,YAAY,CAAC,KAAuB,EAAE,IAAkB;IACtE,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC;AAUD,yHAAyH;AACzH,SAAS,WAAW,CAAC,KAAwB,EAAE,IAAY,EAAE,KAAiB;IAC5E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC;AACnD,CAAC;AAED,4FAA4F;AAC5F,SAAS,QAAQ,CACf,IAAiB,EACjB,MAAwB,EACxB,KAAiB;IAEjB,MAAM,WAAW,GAAsB,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,iEAAiE;QACjE,IAAI,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC;gBAAE,SAAS;YACpD,WAAW,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;gBACpB,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;gBAC/C,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC7D,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,KAAK,CAAC,KAAK;gBACrB,OAAO,EAAE,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC;aACtE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,KAA6B,EAC7B,IAAkB,EAClB,OAA6B;IAE7B,MAAM,KAAK,GAAe;QACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,CAAC;QAC/B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,YAAY,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC;KACnE,CAAC;IACF,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACnF,CAAC"}

package/dist/keywords/options.d.ts

@@ -0,0 +1,22 @@
+import type { Config } from "@concordance-wiki/core";
+/** The part of `concordance.lock.yaml` the discovery reads. */
+export interface KeywordLock {
+    rejected_terms?: readonly string[];
+}
+/** Every knob of the discovery, resolved from the configuration and the lock. */
+export interface KeywordOptions {
+    minWords: number;
+    maxWords: number;
+    /** Below this length of the normalised expression, an n-gram is noise; not configurable. */
+    minLength: number;
+    minOccurrences: number;
+    minDocuments: number;
+    /** `inference.candidate_score`: from this score a candidate yields `W-TERM-UNDEFINED`. */
+    minScore: number;
+    /** The lock's `rejected_terms`, as written. */
+    rejected: ReadonlySet<string>;
+}
+export declare const keywordDefaults: Omit<KeywordOptions, "rejected">;
+/** `inference.ngrams`, `inference.candidate_score` and the lock's `rejected_terms`, with their defaults. */
+export declare function keywordOptions(config: Config, lock?: KeywordLock): KeywordOptions;
+//# sourceMappingURL=options.d.ts.map

package/dist/keywords/options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/keywords/options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,+DAA+D;AAC/D,MAAM,WAAW,WAAW;IAC1B,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAED,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,4FAA4F;IAC5F,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,0FAA0F;IAC1F,QAAQ,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC/B;AAED,eAAO,MAAM,eAAe,EAAE,IAAI,CAAC,cAAc,EAAE,UAAU,CAO5D,CAAC;AAEF,4GAA4G;AAC5G,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,cAAc,CAWjF"}

package/dist/keywords/options.js

@@ -0,0 +1,22 @@
+export const keywordDefaults = {
+    minWords: 1,
+    maxWords: 4,
+    minLength: 3,
+    minOccurrences: 3,
+    minDocuments: 2,
+    minScore: 4,
+};
+/** `inference.ngrams`, `inference.candidate_score` and the lock's `rejected_terms`, with their defaults. */
+export function keywordOptions(config, lock) {
+    const ngrams = config.inference?.ngrams ?? {};
+    return {
+        minWords: ngrams.min ?? keywordDefaults.minWords,
+        maxWords: ngrams.max ?? keywordDefaults.maxWords,
+        minLength: keywordDefaults.minLength,
+        minOccurrences: ngrams.min_occurrences ?? keywordDefaults.minOccurrences,
+        minDocuments: ngrams.min_documents ?? keywordDefaults.minDocuments,
+        minScore: config.inference?.candidate_score ?? keywordDefaults.minScore,
+        rejected: new Set(lock?.rejected_terms ?? []),
+    };
+}
+//# sourceMappingURL=options.js.map

package/dist/keywords/options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/keywords/options.ts"],"names":[],"mappings":"AAqBA,MAAM,CAAC,MAAM,eAAe,GAAqC;IAC/D,QAAQ,EAAE,CAAC;IACX,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,cAAc,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF,4GAA4G;AAC5G,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,IAAkB;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE,CAAC;IAC9C,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,GAAG,IAAI,eAAe,CAAC,QAAQ;QAChD,QAAQ,EAAE,MAAM,CAAC,GAAG,IAAI,eAAe,CAAC,QAAQ;QAChD,SAAS,EAAE,eAAe,CAAC,SAAS;QACpC,cAAc,EAAE,MAAM,CAAC,eAAe,IAAI,eAAe,CAAC,cAAc;QACxE,YAAY,EAAE,MAAM,CAAC,aAAa,IAAI,eAAe,CAAC,YAAY;QAClE,QAAQ,EAAE,MAAM,CAAC,SAAS,EAAE,eAAe,IAAI,eAAe,CAAC,QAAQ;QACvE,QAAQ,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,cAAc,IAAI,EAAE,CAAC;KAC9C,CAAC;AACJ,CAAC"}

package/dist/keywords/publish.d.ts

@@ -0,0 +1,50 @@
+import { type Config, type Entity } from "@concordance-wiki/core";
+import type { KeywordCandidate, KeywordMention } from "./score.js";
+/** A candidate above the publication threshold: the page the site generates for it. */
+export interface KeywordPage {
+    /** `keywords/<slug of the key>`, the stable address of the page. */
+    id: string;
+    key: string;
+    display: string;
+    occurrences: number;
+    documents: number;
+    score: number;
+    confidence: number;
+    mentions: KeywordMention[];
+}
+export interface KeywordPublicationOptions {
+    minOccurrences: number;
+    /** Distinct files the expression must appear in. */
+    minFiles: number;
+    /** Confidence, in [0, 1], from which an expression at the threshold gets a page. */
+    minConfidence: number;
+}
+export interface PublishedKeywords {
+    /** Sorted by identifier. */
+    published: KeywordPage[];
+    /** Under the threshold, best score first; searchable, but without a page. */
+    discarded: KeywordCandidate[];
+    /** At the threshold but under the confidence, best score first: suspected noise, searchable, without a page. */
+    withheld: KeywordCandidate[];
+}
+export interface KeywordEntitiesOptions {
+    locale: Entity["locale"];
+}
+export declare const keywordPublicationDefaults: KeywordPublicationOptions;
+/** The type of a keyword page: a term nobody has defined yet. */
+export declare const KEYWORD_TYPE = "term";
+/** `inference.keyword_pages`, with its defaults: three occurrences in two distinct files, a confidence of one half. */
+export declare function keywordPublicationOptions(config: Config): KeywordPublicationOptions;
+/**
+ * Splits the candidates into the pages to generate, the expressions the threshold
+ * discards and those the confidence withholds. Two keys with the same slug would share an
+ * address: the best-scored keeps the plain identifier and the next ones take a numeric
+ * suffix, so that no page is lost.
+ */
+export declare function publishKeywords(candidates: readonly KeywordCandidate[], options: KeywordPublicationOptions): PublishedKeywords;
+/**
+ * One entity per page, marked `keyword: true`, of the type of a term, located on the first
+ * mention of its expression, with its counts, score and confidence as attributes; in page order.
+ */
+export declare function keywordEntities(pages: readonly KeywordPage[], options: KeywordEntitiesOptions): Entity[];
+//# sourceMappingURL=publish.d.ts.map

package/dist/keywords/publish.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"publish.d.ts","sourceRoot":"","sources":["../../src/keywords/publish.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,KAAK,MAAM,EAAE,KAAK,MAAM,EAAqB,MAAM,wBAAwB,CAAC;AAE9F,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEnE,uFAAuF;AACvF,MAAM,WAAW,WAAW;IAC1B,oEAAoE;IACpE,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,yBAAyB;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,6EAA6E;IAC7E,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,gHAAgH;IAChH,QAAQ,EAAE,gBAAgB,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;CAC1B;AAED,eAAO,MAAM,0BAA0B,EAAE,yBAIxC,CAAC;AAEF,iEAAiE;AACjE,eAAO,MAAM,YAAY,SAAS,CAAC;AAanC,uHAAuH;AACvH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,yBAAyB,CAOnF;AASD;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,UAAU,EAAE,SAAS,gBAAgB,EAAE,EACvC,OAAO,EAAE,yBAAyB,GACjC,iBAAiB,CA6BnB;AAUD;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,SAAS,WAAW,EAAE,EAC7B,OAAO,EAAE,sBAAsB,GAC9B,MAAM,EAAE,CAmBV"}

package/dist/keywords/publish.js

@@ -0,0 +1,96 @@
+import { slugify } from "@concordance-wiki/core";
+export const keywordPublicationDefaults = {
+    minOccurrences: 3,
+    minFiles: 2,
+    minConfidence: 0.5,
+};
+/** The type of a keyword page: a term nobody has defined yet. */
+export const KEYWORD_TYPE = "term";
+/** The status the profile gives an entity that declares none. */
+const KEYWORD_STATUS = "valid";
+function byCodeUnit(a, b) {
+    return Number(a > b) - Number(a < b);
+}
+function compareByScore(a, b) {
+    return b.score - a.score || byCodeUnit(a.key, b.key);
+}
+/** `inference.keyword_pages`, with its defaults: three occurrences in two distinct files, a confidence of one half. */
+export function keywordPublicationOptions(config) {
+    const pages = config.inference?.keyword_pages ?? {};
+    return {
+        minOccurrences: pages.min_occurrences ?? keywordPublicationDefaults.minOccurrences,
+        minFiles: pages.min_files ?? keywordPublicationDefaults.minFiles,
+        minConfidence: pages.min_confidence ?? keywordPublicationDefaults.minConfidence,
+    };
+}
+function reachesThreshold(candidate, options) {
+    return candidate.occurrences >= options.minOccurrences && candidate.documents >= options.minFiles;
+}
+/**
+ * Splits the candidates into the pages to generate, the expressions the threshold
+ * discards and those the confidence withholds. Two keys with the same slug would share an
+ * address: the best-scored keeps the plain identifier and the next ones take a numeric
+ * suffix, so that no page is lost.
+ */
+export function publishKeywords(candidates, options) {
+    const published = [];
+    const discarded = [];
+    const withheld = [];
+    const taken = new Map();
+    for (const candidate of [...candidates].sort(compareByScore)) {
+        if (!reachesThreshold(candidate, options)) {
+            discarded.push(candidate);
+            continue;
+        }
+        if (candidate.confidence < options.minConfidence) {
+            withheld.push(candidate);
+            continue;
+        }
+        const base = `keywords/${slugify(candidate.key)}`;
+        const rank = (taken.get(base) ?? 0) + 1;
+        taken.set(base, rank);
+        published.push({
+            id: rank === 1 ? base : `${base}-${String(rank)}`,
+            key: candidate.key,
+            display: candidate.display,
+            occurrences: candidate.occurrences,
+            documents: candidate.documents,
+            score: candidate.score,
+            confidence: candidate.confidence,
+            mentions: candidate.mentions,
+        });
+    }
+    return { published: published.toSorted((a, b) => byCodeUnit(a.id, b.id)), discarded, withheld };
+}
+function locationOf(page) {
+    const [first] = page.mentions;
+    if (first === undefined) {
+        throw new Error(`keyword page ${page.id} has no mention to locate it`);
+    }
+    return { name: first.source ?? "", path: first.path, line: first.line };
+}
+/**
+ * One entity per page, marked `keyword: true`, of the type of a term, located on the first
+ * mention of its expression, with its counts, score and confidence as attributes; in page order.
+ */
+export function keywordEntities(pages, options) {
+    return pages.map((page) => ({
+        id: page.id,
+        type: KEYWORD_TYPE,
+        title: page.display,
+        aliases: [],
+        locale: options.locale,
+        status: KEYWORD_STATUS,
+        type_origin: "default",
+        graph: "full",
+        attributes: {
+            documents: page.documents,
+            occurrences: page.occurrences,
+            score: page.score,
+            confidence: page.confidence,
+        },
+        source: locationOf(page),
+        keyword: true,
+    }));
+}
+//# sourceMappingURL=publish.js.map

package/dist/keywords/publish.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"publish.js","sourceRoot":"","sources":["../../src/keywords/publish.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAA+C,MAAM,wBAAwB,CAAC;AAsC9F,MAAM,CAAC,MAAM,0BAA0B,GAA8B;IACnE,cAAc,EAAE,CAAC;IACjB,QAAQ,EAAE,CAAC;IACX,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF,iEAAiE;AACjE,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,CAAC;AAEnC,iEAAiE;AACjE,MAAM,cAAc,GAAG,OAAO,CAAC;AAE/B,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,cAAc,CAAC,CAAmB,EAAE,CAAmB;IAC9D,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;AACvD,CAAC;AAED,uHAAuH;AACvH,MAAM,UAAU,yBAAyB,CAAC,MAAc;IACtD,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,EAAE,aAAa,IAAI,EAAE,CAAC;IACpD,OAAO;QACL,cAAc,EAAE,KAAK,CAAC,eAAe,IAAI,0BAA0B,CAAC,cAAc;QAClF,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,0BAA0B,CAAC,QAAQ;QAChE,aAAa,EAAE,KAAK,CAAC,cAAc,IAAI,0BAA0B,CAAC,aAAa;KAChF,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,SAA2B,EAC3B,OAAkC;IAElC,OAAO,SAAS,CAAC,WAAW,IAAI,OAAO,CAAC,cAAc,IAAI,SAAS,CAAC,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC;AACpG,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAC7B,UAAuC,EACvC,OAAkC;IAElC,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,SAAS,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QAC7D,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YAC1C,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,IAAI,SAAS,CAAC,UAAU,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;YACjD,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,YAAY,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACxC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACtB,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;YACjD,GAAG,EAAE,SAAS,CAAC,GAAG;YAClB,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,QAAQ,EAAE,SAAS,CAAC,QAAQ;SAC7B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAClG,CAAC;AAED,SAAS,UAAU,CAAC,IAAiB;IACnC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,gBAAgB,IAAI,CAAC,EAAE,8BAA8B,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;AAC1E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,KAA6B,EAC7B,OAA+B;IAE/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,IAAI,CAAC,OAAO;QACnB,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,cAAc;QACtB,WAAW,EAAE,SAAS;QACtB,KAAK,EAAE,MAAM;QACb,UAAU,EAAE;YACV,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B;QACD,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC;QACxB,OAAO,EAAE,IAAI;KACd,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/keywords/score.d.ts

@@ -0,0 +1,60 @@
+import type { LanguagePack } from "../locale/pack.js";
+import { type ConfidencePenalty, type ConfidenceSignals } from "./confidence.js";
+import { type NgramOccurrence } from "./ngrams.js";
+export interface KeywordMention {
+    source?: string;
+    path: string;
+    line: number;
+    position: number;
+    /** The span as written in the text, which the context holds. */
+    surface: string;
+    context: string;
+}
+export interface KeywordCandidate {
+    key: string;
+    /** The most frequent surface form, the first mention deciding a tie. */
+    display: string;
+    words: number;
+    occurrences: number;
+    /** Distinct files the expression appears in. */
+    documents: number;
+    /** C-value × IDF, rounded to four decimals. */
+    score: number;
+    /** The product of the confidence signals, in [0, 1], rounded to four decimals. */
+    confidence: number;
+    /** What the confidence was read from, the shares rounded to four decimals. */
+    signals: ConfidenceSignals;
+    /** The signals that lowered the confidence, in the order of the formula. */
+    penalties: ConfidencePenalty[];
+    /** Sorted by source, path, line, then position. */
+    mentions: KeywordMention[];
+}
+export interface ScoreCandidatesOptions {
+    pack: LanguagePack;
+    /** Comparison forms of the dictionary entries: an expression already defined is no candidate. */
+    dictionaryKeys: ReadonlySet<string>;
+    /** The lock's `rejected_terms`, as written. */
+    rejected: ReadonlySet<string>;
+    minOccurrences: number;
+    minDocuments: number;
+    /**
+     * The n-grams of the prominent texts, headings, written links and frontmatter, extracted
+     * like the units: an expression put forward there gains confidence. None by default.
+     */
+    prominent?: readonly NgramOccurrence[];
+    /** Share of the files beyond which an expression belongs to the language; 0.5 by default. */
+    maxSpread?: number;
+}
+/** Canonical order of mentions: source, path, line, position. */
+export declare function compareMentions(a: Pick<NgramOccurrence, "source" | "path" | "line" | "position">, b: Pick<NgramOccurrence, "source" | "path" | "line" | "position">): number;
+/**
+ * The expressions worth a page: the n-grams grouped by key, scored by C-value × IDF, kept
+ * from `minOccurrences` occurrences in `minDocuments` distinct files, best score first,
+ * each with its confidence of being a term of the subject rather than a word of the
+ * language, read in the same pass from the shape of its distribution. An expression
+ * defined in the dictionary or rejected in the lock is never a candidate, but still
+ * penalises the n-grams it contains, so that the words of a defined term do not surface
+ * on their own.
+ */
+export declare function scoreCandidates(occurrences: readonly NgramOccurrence[], options: ScoreCandidatesOptions): KeywordCandidate[];
+//# sourceMappingURL=score.d.ts.map

package/dist/keywords/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/keywords/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAIL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAgB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,UAAU,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,OAAO,EAAE,iBAAiB,CAAC;IAC3B,4EAA4E;IAC5E,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,mDAAmD;IACnD,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,YAAY,CAAC;IACnB,iGAAiG;IACjG,cAAc,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,+CAA+C;IAC/C,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACvC,6FAA6F;IAC7F,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA0BD,iEAAiE;AACjE,wBAAgB,eAAe,CAC7B,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC,EACjE,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC,GAChE,MAAM,CAOR;AAkJD;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,OAAO,EAAE,sBAAsB,GAC9B,gBAAgB,EAAE,CA6BpB"}

package/dist/keywords/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/keywords/score.ts"],"names":[],"mappings":"AACA,OAAO,EACL,kBAAkB,EAClB,YAAY,EACZ,mBAAmB,GAGpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,YAAY,EAAwB,MAAM,aAAa,CAAC;AAiEjE,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,UAAU,CAAC,UAA6C;IAC/D,OAAO,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;AACzD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,eAAe,CAC7B,CAAiE,EACjE,CAAiE;IAEjE,OAAO,CACL,UAAU,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QAC1C,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;QAC1B,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;QACf,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAmB,EAAE,CAAmB;IACjE,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;AACvD,CAAC;AAED,kFAAkF;AAClF,SAAS,UAAU,CAAC,KAAwB;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QAClD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,KAAY,EAAE,UAAsB;IACtD,OAAO,CACL,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,UAAU,CAAC,cAAc;QACrD,KAAK,CAAC,SAAS,CAAC,IAAI,IAAI,UAAU,CAAC,YAAY,CAChD,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,IAAyB,EAAE,OAA4B;IACzE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oFAAoF;AACpF,uFAAuF;AACvF,qFAAqF;AACrF,SAAS,UAAU,CACjB,WAAuC,EACvC,UAAsB,EACtB,OAA4B;IAE5B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;IACxC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG;gBACN,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;gBAChC,WAAW,EAAE,EAAE;gBACf,SAAS,EAAE,IAAI,GAAG,EAAE;gBACpB,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,KAAK;aACjB,CAAC;YACF,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC;YAAE,SAAS;QAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5C,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC;QAC5B,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,SAAS,KAAK,OAAO,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,KAAY;IAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACxF,OAAO,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,8DAA8D;AAC9D,SAAS,eAAe,CAAC,SAAqC;IAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,SAAS,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,KAAwB,EAAE,QAA6B;IAC1E,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACzB,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CACrF,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;AAC7C,CAAC;AAED,yEAAyE;AACzE,SAAS,SAAS,CAChB,KAAY,EACZ,MAAc,EACd,SAAiB,EACjB,IAAkB;IAElB,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,GAAG,MAAM,CAAC;QAC5C,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC;QAC7D,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACpE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,WAAuC;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,IAAI,GAAG,OAAO,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,SAAS,CAAC,UAA2B;IAC5C,OAAO;QACL,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;QACzE,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,OAAO,EAAE,UAAU,CAAC,OAAO;QAC3B,OAAO,EAAE,UAAU,CAAC,OAAO;KAC5B,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAuC,EACvC,OAA+B;IAE/B,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,SAAS,EAAE,CAAC;IAC3F,MAAM,UAAU,GAAuB,EAAE,CAAC;IAE1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACpC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC;YAAE,SAAS;QACrE,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QACvF,MAAM,KAAK,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAClE,UAAU,CAAC,IAAI,CAAC;YACd,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;YACzB,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,IAAI;YAC/B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM;YACxD,UAAU,EAAE,YAAY,CAAC,KAAK,EAAE,iBAAiB,CAAC;YAClD,OAAO;YACP,SAAS,EAAE,mBAAmB,CAAC,KAAK,EAAE,iBAAiB,CAAC;YACxD,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AAC5C,CAAC"}

package/dist/keywords/similar.d.ts

@@ -0,0 +1,19 @@
+/** How many similar expressions a page offers as a lead. */
+export declare const SIMILAR_EXPRESSIONS_LIMIT = 5;
+/** What the comparison needs of an expression: its key, the words in comparison form joined by a space. */
+export interface KeyedExpression {
+    key: string;
+}
+/**
+ * Whether two expressions have a similar form: one is a contiguous part of the other, or they
+ * share at least half of the distinct words of the longer one. Keys are compared word by word
+ * in their comparison form, so that a plural and its singular already read alike.
+ */
+export declare function similarForm(a: string, b: string): boolean;
+/**
+ * The expressions of a similar form to `key`, the closest first: by decreasing share of common
+ * words, then by key; at most `limit` of them, the expression itself never among them and each
+ * key once. The result depends on the set of candidates alone.
+ */
+export declare function similarExpressions<T extends KeyedExpression>(key: string, candidates: readonly T[], limit?: number): T[];
+//# sourceMappingURL=similar.d.ts.map

package/dist/keywords/similar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"similar.d.ts","sourceRoot":"","sources":["../../src/keywords/similar.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C,2GAA2G;AAC3G,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;CACb;AAkBD;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAQzD;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,SAAS,eAAe,EAC1D,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,SAAS,CAAC,EAAE,EACxB,KAAK,GAAE,MAAkC,GACxC,CAAC,EAAE,CAgBL"}

package/dist/keywords/similar.js

@@ -0,0 +1,52 @@
+/** How many similar expressions a page offers as a lead. */
+export const SIMILAR_EXPRESSIONS_LIMIT = 5;
+function wordsOf(key) {
+    return key.split(" ");
+}
+/** Whether `part` is a contiguous run of the words of `whole`: a prefix, a suffix or an inner expression. */
+function contains(whole, part) {
+    return whole.some((_, start) => part.every((word, index) => whole[start + index] === word));
+}
+/** The share of the distinct words of both expressions they have in common. */
+function overlap(a, b) {
+    const words = new Set(a);
+    const shared = new Set(b.filter((word) => words.has(word))).size;
+    return shared / new Set([...a, ...b]).size;
+}
+/**
+ * Whether two expressions have a similar form: one is a contiguous part of the other, or they
+ * share at least half of the distinct words of the longer one. Keys are compared word by word
+ * in their comparison form, so that a plural and its singular already read alike.
+ */
+export function similarForm(a, b) {
+    const wordsA = wordsOf(a);
+    const wordsB = wordsOf(b);
+    if (contains(wordsA, wordsB) || contains(wordsB, wordsA))
+        return true;
+    const setA = new Set(wordsA);
+    const setB = new Set(wordsB);
+    const shared = [...setB].filter((word) => setA.has(word)).length;
+    return 2 * shared >= Math.max(setA.size, setB.size);
+}
+/**
+ * The expressions of a similar form to `key`, the closest first: by decreasing share of common
+ * words, then by key; at most `limit` of them, the expression itself never among them and each
+ * key once. The result depends on the set of candidates alone.
+ */
+export function similarExpressions(key, candidates, limit = SIMILAR_EXPRESSIONS_LIMIT) {
+    const words = wordsOf(key);
+    const seen = new Set([key]);
+    const similar = [];
+    for (const candidate of candidates) {
+        if (seen.has(candidate.key) || !similarForm(key, candidate.key))
+            continue;
+        seen.add(candidate.key);
+        similar.push({ candidate, overlap: overlap(words, wordsOf(candidate.key)) });
+    }
+    return (similar
+        // Keys are distinct here, so the tie-break never meets two equal ones.
+        .toSorted((a, b) => b.overlap - a.overlap || (a.candidate.key < b.candidate.key ? -1 : 1))
+        .slice(0, limit)
+        .map((item) => item.candidate));
+}
+//# sourceMappingURL=similar.js.map

package/dist/keywords/similar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"similar.js","sourceRoot":"","sources":["../../src/keywords/similar.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAO3C,SAAS,OAAO,CAAC,GAAW;IAC1B,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,6GAA6G;AAC7G,SAAS,QAAQ,CAAC,KAAwB,EAAE,IAAuB;IACjE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AAC9F,CAAC;AAED,+EAA+E;AAC/E,SAAS,OAAO,CAAC,CAAoB,EAAE,CAAoB;IACzD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACjE,OAAO,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,CAAS,EAAE,CAAS;IAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACtE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACjE,OAAO,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,GAAW,EACX,UAAwB,EACxB,QAAgB,yBAAyB;IAEzC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,OAAO,GAAwC,EAAE,CAAC;IACxD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,CACL,OAAO;QACL,uEAAuE;SACtE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACzF,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CACjC,CAAC;AACJ,CAAC"}

package/dist/locale/load-pack.d.ts

@@ -0,0 +1,11 @@
+import type { LanguagePack } from "./pack.js";
+export declare class LanguagePackError extends Error {
+    constructor(directory: string, detail: string);
+}
+/**
+ * Reads a language pack from a folder holding `pack.yaml` and `stopwords.txt`, and
+ * `suffixes.txt` when the pack lists inflected-form suffixes. The core packs and the ones
+ * shipped by plugins are read the same way.
+ */
+export declare function loadLanguagePack(directory: URL): LanguagePack;
+//# sourceMappingURL=load-pack.d.ts.map

package/dist/locale/load-pack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-pack.d.ts","sourceRoot":"","sources":["../../src/locale/load-pack.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAoB,MAAM,WAAW,CAAC;AAahE,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CAI9C;AAoCD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,GAAG,GAAG,YAAY,CA4B7D"}

package/dist/locale/load-pack.js

@@ -0,0 +1,82 @@
+import { readFileSync } from "node:fs";
+import { describeSchemaError, formatIssue, readSchema } from "@concordance-wiki/core";
+import { Ajv2020 } from "ajv/dist/2020.js";
+import { parse } from "yaml";
+import { normalizer } from "./normalize.js";
+import { canonicalLocale } from "./tag.js";
+import { loadStopwords } from "./stopwords.js";
+import { loadSuffixes } from "./suffixes.js";
+export class LanguagePackError extends Error {
+    constructor(directory, detail) {
+        super(`language pack at ${directory}: ${detail}`);
+        this.name = "LanguagePackError";
+    }
+}
+function readDocument(directory) {
+    const ajv = new Ajv2020({ allErrors: true, strict: true });
+    const validate = ajv.compile(readSchema("language-pack"));
+    const document = parse(readFileSync(new URL("pack.yaml", directory), "utf8"));
+    if (!validate(document)) {
+        // The validator fills `errors` whenever it returns false.
+        const detail = validate.errors
+            .map((error) => formatIssue(describeSchemaError(error, document), "pack.yaml"))
+            .join("; ");
+        throw new LanguagePackError(directory.pathname, detail);
+    }
+    // Validated against the schema the type mirrors.
+    return document;
+}
+function canonical(tag, directory) {
+    try {
+        return canonicalLocale(tag);
+    }
+    catch {
+        throw new LanguagePackError(directory.pathname, `"${tag}" is not a valid language tag`);
+    }
+}
+/** The text of an optional file of the pack: an absent file reads as empty, any other failure is raised. */
+function optionalText(url) {
+    try {
+        return readFileSync(url, "utf8");
+    }
+    catch (error) {
+        // A node file system error carries its code; any other value has none and is rethrown.
+        if (error.code === "ENOENT")
+            return "";
+        throw error;
+    }
+}
+/**
+ * Reads a language pack from a folder holding `pack.yaml` and `stopwords.txt`, and
+ * `suffixes.txt` when the pack lists inflected-form suffixes. The core packs and the ones
+ * shipped by plugins are read the same way.
+ */
+export function loadLanguagePack(directory) {
+    const base = directory.href.endsWith("/") ? directory : new URL(`${directory.href}/`);
+    const document = readDocument(base);
+    const locale = canonical(document.locale, base);
+    const collator = new Intl.Collator(locale, document.collation);
+    const segmenter = new Intl.Segmenter(locale, { granularity: "word" });
+    const normalize = normalizer(document.apostrophes ?? []);
+    const plural = document.plural.map((rule) => ({
+        ending: rule.ending,
+        singular: rule.singular,
+        minLength: rule.min_length ?? rule.ending.length + 1,
+    }));
+    return {
+        locale,
+        language: document.language,
+        normalize,
+        segment: (text) => [...segmenter.segment(text)].map((part) => ({
+            text: part.segment,
+            index: part.index,
+            isWordLike: part.isWordLike === true,
+        })),
+        stopwords: new Set(loadStopwords(readFileSync(new URL("stopwords.txt", base), "utf8"))),
+        suffixes: new Set(loadSuffixes(optionalText(new URL("suffixes.txt", base)), normalize)),
+        plural,
+        collator,
+        compare: (a, b) => collator.compare(a, b),
+    };
+}
+//# sourceMappingURL=load-pack.js.map

package/dist/locale/load-pack.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-pack.js","sourceRoot":"","sources":["../../src/locale/load-pack.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,UAAU,EAAe,MAAM,wBAAwB,CAAC;AACnG,OAAO,EAAE,OAAO,EAAoB,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAU7C,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,SAAiB,EAAE,MAAc;QAC3C,KAAK,CAAC,oBAAoB,SAAS,KAAK,MAAM,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED,SAAS,YAAY,CAAC,SAAc;IAClC,MAAM,GAAG,GAAG,IAAI,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAe,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IACvF,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxB,0DAA0D;QAC1D,MAAM,MAAM,GAAI,QAAQ,CAAC,MAAwB;aAC9C,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,WAAW,CAAC,CAAC;aAC9E,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,iBAAiB,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;IACD,iDAAiD;IACjD,OAAO,QAAwB,CAAC;AAClC,CAAC;AAED,SAAS,SAAS,CAAC,GAAW,EAAE,SAAc;IAC5C,IAAI,CAAC;QACH,OAAO,eAAe,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,iBAAiB,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,GAAG,+BAA+B,CAAC,CAAC;IAC1F,CAAC;AACH,CAAC;AAED,4GAA4G;AAC5G,SAAS,YAAY,CAAC,GAAQ;IAC5B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,uFAAuF;QACvF,IAAK,KAA4B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC/D,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAc;IAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,MAAM,GAAiB,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1D,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;KACrD,CAAC,CAAC,CAAC;IACJ,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,SAAS;QACT,OAAO,EAAE,CAAC,IAAI,EAAU,EAAE,CACxB,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,EAAE,IAAI,CAAC,OAAO;YAClB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU,KAAK,IAAI;SACrC,CAAC,CAAC;QACL,SAAS,EAAE,IAAI,GAAG,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACvF,QAAQ,EAAE,IAAI,GAAG,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACvF,MAAM;QACN,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;KAC1C,CAAC;AACJ,CAAC"}

package/dist/locale/normalize.d.ts

@@ -0,0 +1,3 @@
+/** Builds the normaliser of a locale from the apostrophe characters its pack unifies. */
+export declare function normalizer(apostrophes: readonly string[]): (text: string) => string;
+//# sourceMappingURL=normalize.d.ts.map

package/dist/locale/normalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../src/locale/normalize.ts"],"names":[],"mappings":"AAGA,yFAAyF;AACzF,wBAAgB,UAAU,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAUnF"}

package/dist/locale/normalize.js

@@ -0,0 +1,14 @@
+const combiningMarks = /\p{M}+/gu;
+const whitespace = /\s+/g;
+/** Builds the normaliser of a locale from the apostrophe characters its pack unifies. */
+export function normalizer(apostrophes) {
+    const variants = new Set(apostrophes);
+    return (text) => text
+        .normalize("NFD")
+        .replace(combiningMarks, "")
+        .toLowerCase()
+        .replace(/./gu, (character) => (variants.has(character) ? "'" : character))
+        .replace(whitespace, " ")
+        .trim();
+}
+//# sourceMappingURL=normalize.js.map

package/dist/locale/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../src/locale/normalize.ts"],"names":[],"mappings":"AAAA,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,UAAU,GAAG,MAAM,CAAC;AAE1B,yFAAyF;AACzF,MAAM,UAAU,UAAU,CAAC,WAA8B;IACvD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;IACtC,OAAO,CAAC,IAAI,EAAE,EAAE,CACd,IAAI;SACD,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,WAAW,EAAE;SACb,OAAO,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;SAC1E,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,IAAI,EAAE,CAAC;AACd,CAAC"}

package/dist/locale/pack.d.ts

@@ -0,0 +1,34 @@
+import type { Locale } from "@concordance-wiki/core";
+/** A suffix rule bringing a plural form back to its singular. */
+export interface PluralRule {
+    ending: string;
+    singular: string;
+    minLength: number;
+}
+/** A word of a text as cut by the locale's segmenter. */
+export interface Word {
+    text: string;
+    index: number;
+    /** False for punctuation and spaces. */
+    isWordLike: boolean;
+}
+/** What the engine knows about a language. Built from a `pack.yaml` and a stopword list, never from code. */
+export interface LanguagePack {
+    /** Canonical BCP 47 tag. */
+    locale: Locale;
+    /** Name of the language, in that language. */
+    language: string;
+    /** Lower-cases, strips combining marks, unifies the apostrophes and collapses whitespace. */
+    normalize: (text: string) => string;
+    /** Cuts a text into words with the locale's rules (Unicode segmentation). */
+    segment: (text: string) => Word[];
+    stopwords: ReadonlySet<string>;
+    /** Endings of the inflected forms of the language, in comparison form; empty when the pack lists none. */
+    suffixes: ReadonlySet<string>;
+    plural: readonly PluralRule[];
+    /** Locale-aware, accent-insensitive and numeric, for every alphabetical index. */
+    collator: Intl.Collator;
+    /** Usable directly as a sort comparator. */
+    compare: (a: string, b: string) => number;
+}
+//# sourceMappingURL=pack.d.ts.map

package/dist/locale/pack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.d.ts","sourceRoot":"","sources":["../../src/locale/pack.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,yDAAyD;AACzD,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,6FAA6F;IAC7F,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACpC,6EAA6E;IAC7E,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;IAClC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC/B,0GAA0G;IAC1G,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC;IAC9B,kFAAkF;IAClF,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;IACxB,4CAA4C;IAC5C,OAAO,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC;CAC3C"}