@concordance-wiki/nlp 0.1.0

package/README.md

@@ -0,0 +1,7 @@
+# @concordance-wiki/nlp
+
+Normalisation, Aho-Corasick, n-grams, C-value, MinHash and the language packs.
+
+Today: language packs for `en` and `fr` built from data files (`locales/<locale>/pack.yaml`, `stopwords.txt` and the optional `suffixes.txt` of inflected-form endings; the default stopwords are the words that name nothing on their own in any corpus, from the articles to every form of the auxiliaries and light verbs, the adverbs of prose, quantifiers, numerals, time words and the nouns that only count or place, never a word that could be a business term or end a technical compound, which a project lists under `inference.stopwords`), BCP 47 locale resolution with fallback to the language, Unicode word segmentation and collation, a registry for packs shipped by plugins (`loadLanguagePack`, `languagePack`, `registerLanguagePack`, `resolveLocale`, `canonicalLocale`, `loadStopwords`), and text normalisation: the comparison form of a text (lower-cased, accents stripped, apostrophes unified, each word singularised by the pack's suffix rules; `comparisonForm`, `comparisonWords`, `singularize`) and word boundaries from Unicode segmentation (`wordBoundaries`, `isOnWordBoundaries`); and the recognition dictionary: `buildDictionary` keys every title and alias of the entities of a locale by comparison form, with priority to glossary sources (`glossarySources`), leaves out stopwords (`dictionaryStopwords` merges the pack's defaults with the files of `inference.stopwords`) and terms shorter than three characters unless `inference.short_terms` allows them, and flags the forms shared by several entities as homonyms with an `I-TERM-HOMONYM` finding; and the occurrence scan: `tokenize` cuts a text into words in comparison form with their spans, `buildAutomaton`, `scan` and `longestMatches` run a word-level Aho-Corasick over them (one pass per text; an expression contained in a longer recognised one is dropped, while expressions that only partly overlap are all kept), and `scanDocument` turns the matches of a dictionary in the paragraphs of a document into occurrences with file, line, position, the matched text as written, section, an 80-character context, the type announced by a recognised prefix (`type_prefixes`) and a confidence from the `glossary_occurrence` scale (`occurrenceConfidence` gives the per-occurrence increments up to the cap; `compareOccurrences` is their canonical order). The scan reads the units that `scannableText` of the ingestion package produces, so code blocks, inline code, URLs, frontmatter and link targets never yield an occurrence; the visible text of a link does. And the keyword discovery: `extractNgrams` reads the same units and yields every n-gram of one to four words (`keywordOptions` resolves `inference.ngrams`, `inference.candidate_score` and the lock's `rejected_terms`), leaving out those starting or ending with a stopword, made only of digits or shorter than three characters, each with its surface form, position and a 160-character context; `scoreCandidates` groups them by comparison form, drops the dictionary entries and the rejected terms (which still penalise the n-grams they contain), scores each candidate by C-value × IDF and keeps those with three occurrences in two distinct files, best score first with its display form and its mentions (`compareMentions`), each with its confidence in [0, 1] of being a term of the subject rather than a word of the language, the product of five signals rounded to four decimals (`confidenceOf`, `confidenceFactors`, `confidencePenalties`, the measures in `signals` and the lowering signals in `penalties`): spread `df / N`, 1 up to `maxSpread` (0.5) then linear down to 0 for a word in every file, read only in a corpus of `SPREAD_MIN_FILES` (10); burst `occurrences / df`, 1 from `BURST_PER_FILE` (1.5) else `0.5 + (burst − 1)`, read only from `BURST_MIN_FILES` (5); position `POSITION_BASE` (0.8) plus 0.2 times the share of appearances in the prominent texts given as `prominent` n-grams (headings, written links, frontmatter), capped at one; neighbourhood 1 when the expression is, or is a component of, a frequent n-gram that also holds a defined term, `NEIGHBOURHOOD_BASE` (0.9) otherwise; morphology `MORPHOLOGY_PENALTY` (0.8) when a word ends with an inflected-form suffix of the pack (`suffixes.txt`, `loadSuffixes`; a pack without the file penalises no form), 1 otherwise; `undefinedTermFindings` turns the candidates at or above the score threshold into `W-TERM-UNDEFINED` findings; `similarExpressions` picks, among keyed expressions, those of a similar form to a key (`similarForm`: one a contiguous part of the other, or half the distinct words of the longer one shared), the closest first, five at most (`SIMILAR_EXPRESSIONS_LIMIT`), for the leads of a keyword page; `definedExpressions` lists the n-grams the dictionary already defines that reach the publication threshold, the expressions whose keyword page a note took over. And the publication threshold: `publishKeywords` splits the candidates into the pages to generate (`keywords/<slug of the key>`, sorted by identifier; two keys with the same slug take numeric suffixes in score order), the expressions discarded by the counts of `inference.keyword_pages` (`keywordPublicationOptions`: three occurrences in two distinct files by default) and those withheld by its `min_confidence` (one half by default), both staying in the output for the search index without a page; `keywordEntities` turns each page into a `term` entity marked `keyword: true`, located on the first mention of its expression, with its occurrences, files, score and confidence as attributes. The counts of the three lists go into the build summary (`keywords.published`, `keywords.discarded`, `keywords.withheld` in core's `summarize`), printed by the build as `keyword pages`, `expressions under the threshold` and `expressions set aside by confidence` once the build runs the discovery.
+
+Part of [Concordance](../../README.md).

package/dist/dictionary/build.d.ts

@@ -0,0 +1,19 @@
+import type { Dictionary, DictionarySource } from "./types.js";
+export declare const HOMONYM_CHECK = "I-TERM-HOMONYM";
+export interface BuildDictionaryInput {
+    entities: readonly DictionarySource[];
+    locale: string;
+    /** Names of the sources whose entities take priority. */
+    glossarySources: ReadonlySet<string>;
+    /** The pack's defaults merged with the configured files; see `dictionaryStopwords`. */
+    stopwords: ReadonlySet<string>;
+    /** Terms shorter than three characters that stay in the dictionary. */
+    shortTerms: ReadonlySet<string>;
+}
+/**
+ * The words to recognise in the texts of a locale: every title and alias of its entities,
+ * keyed by comparison form, minus stopwords and short terms. Pure: the same entities give
+ * the same dictionary whatever their order.
+ */
+export declare function buildDictionary(input: BuildDictionaryInput): Dictionary;
+//# sourceMappingURL=build.d.ts.map

package/dist/dictionary/build.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/dictionary/build.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAmB,gBAAgB,EAAoB,MAAM,YAAY,CAAC;AAElG,eAAO,MAAM,aAAa,mBAAmB,CAAC;AAK9C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,eAAe,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACrC,uFAAuF;IACvF,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC/B,uEAAuE;IACvE,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACjC;AA8BD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,UAAU,CAkCvE"}

package/dist/dictionary/build.js

@@ -0,0 +1,72 @@
+import { languagePack } from "../locale/registry.js";
+import { comparisonForm } from "../text/comparison-form.js";
+export const HOMONYM_CHECK = "I-TERM-HOMONYM";
+/** Below this length a term is noise unless the configuration allows it. */
+const minimumLength = 3;
+// Code-unit order, not locale order: the output must not depend on the collation data of the runtime.
+function byCodeUnit(a, b) {
+    return Number(a > b) - Number(a < b);
+}
+function comparisonSet(forms, pack) {
+    return new Set([...forms].map((form) => comparisonForm(form, pack)));
+}
+function homonymFinding(entry) {
+    const ids = entry.targets.map((target) => target.id).sort(byCodeUnit);
+    return {
+        check: HOMONYM_CHECK,
+        severity: "info",
+        message: `"${entry.key}" is the title or an alias of ${String(ids.length)} entities: ${ids.join(", ")}`,
+        remediation: "Occurrences link to each entity at half confidence. Give the entities distinct titles or aliases, or add a `## Not to be confused with` section to each note so that readers tell them apart.",
+    };
+}
+// Identifiers are unique within an entry and keys within the dictionary, so no comparison ties.
+function compareTargets(a, b) {
+    return a.priority - b.priority || (a.id < b.id ? -1 : 1);
+}
+function compareKeys([a], [b]) {
+    return a < b ? -1 : 1;
+}
+/**
+ * The words to recognise in the texts of a locale: every title and alias of its entities,
+ * keyed by comparison form, minus stopwords and short terms. Pure: the same entities give
+ * the same dictionary whatever their order.
+ */
+export function buildDictionary(input) {
+    const pack = languagePack(input.locale);
+    const stopwords = comparisonSet(input.stopwords, pack);
+    const shortTerms = comparisonSet(input.shortTerms, pack);
+    const candidates = new Map();
+    const consider = (entity, kind, form) => {
+        const key = comparisonForm(form, pack);
+        if (key === "")
+            return;
+        if (key.length < minimumLength && !shortTerms.has(key))
+            return;
+        if (stopwords.has(key) || key.split(" ").every((word) => stopwords.has(word)))
+            return;
+        const targets = candidates.get(key) ?? new Map();
+        if (!targets.has(entity.id)) {
+            const priority = input.glossarySources.has(entity.source) ? 0 : 1;
+            targets.set(entity.id, { id: entity.id, kind, form, priority });
+        }
+        candidates.set(key, targets);
+    };
+    for (const entity of input.entities) {
+        if (entity.locale !== input.locale)
+            continue;
+        consider(entity, "title", entity.title);
+        for (const alias of entity.aliases)
+            consider(entity, "alias", alias);
+    }
+    const entries = new Map();
+    const findings = [];
+    for (const [key, byEntity] of [...candidates].sort(compareKeys)) {
+        const targets = [...byEntity.values()].sort(compareTargets);
+        const entry = { key, targets, homonym: targets.length > 1 };
+        entries.set(key, entry);
+        if (entry.homonym)
+            findings.push(homonymFinding(entry));
+    }
+    return { locale: input.locale, entries, findings };
+}
+//# sourceMappingURL=build.js.map

package/dist/dictionary/build.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/dictionary/build.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAG5D,MAAM,CAAC,MAAM,aAAa,GAAG,gBAAgB,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,aAAa,GAAG,CAAC,CAAC;AAaxB,sGAAsG;AACtG,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,aAAa,CAAC,KAA0B,EAAE,IAAkB;IACnE,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,cAAc,CAAC,KAAsB;IAC5C,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACtE,OAAO;QACL,KAAK,EAAE,aAAa;QACpB,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,IAAI,KAAK,CAAC,GAAG,iCAAiC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvG,WAAW,EACT,+LAA+L;KAClM,CAAC;AACJ,CAAC;AAED,gGAAgG;AAChG,SAAS,cAAc,CAAC,CAAmB,EAAE,CAAmB;IAC9D,OAAO,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AACD,SAAS,WAAW,CAAC,CAAC,CAAC,CAAoB,EAAE,CAAC,CAAC,CAAoB;IACjE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAA2B;IACzD,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyC,CAAC;IAEpE,MAAM,QAAQ,GAAG,CAAC,MAAwB,EAAE,IAA8B,EAAE,IAAY,EAAE,EAAE;QAC1F,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO;QACvB,IAAI,GAAG,CAAC,MAAM,GAAG,aAAa,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAC/D,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAAE,OAAO;QACtF,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAA4B,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,SAAS;QAC7C,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO;YAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAoB,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,IAAI,KAAK,CAAC,OAAO;YAAE,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AACrD,CAAC"}

package/dist/dictionary/stopwords.d.ts

@@ -0,0 +1,16 @@
+import type { Config, FileSystem } from "@concordance-wiki/core";
+export interface DictionaryStopwordsInput {
+    locale: string;
+    config: Config;
+    /** The folder of `concordance.yaml`, against which `inference.stopwords` paths resolve. */
+    configDirectory: string;
+    fs: FileSystem;
+}
+/**
+ * The stopwords excluded from the dictionary of a locale: the pack's defaults plus every
+ * file listed under `inference.stopwords`. A missing file is a configuration mistake.
+ */
+export declare function dictionaryStopwords(input: DictionaryStopwordsInput): ReadonlySet<string>;
+/** The sources whose entities take priority: `inference.glossary_sources`, else those with `glossary: true`. */
+export declare function glossarySources(config: Config): ReadonlySet<string>;
+//# sourceMappingURL=stopwords.d.ts.map

package/dist/dictionary/stopwords.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopwords.d.ts","sourceRoot":"","sources":["../../src/dictionary/stopwords.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAKjE,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,2FAA2F;IAC3F,eAAe,EAAE,MAAM,CAAC;IACxB,EAAE,EAAE,UAAU,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,WAAW,CAAC,MAAM,CAAC,CAUxF;AAED,gHAAgH;AAChH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAInE"}

package/dist/dictionary/stopwords.js

@@ -0,0 +1,27 @@
+import { posix } from "node:path";
+import { languagePack } from "../locale/registry.js";
+import { loadStopwords } from "../locale/stopwords.js";
+/**
+ * The stopwords excluded from the dictionary of a locale: the pack's defaults plus every
+ * file listed under `inference.stopwords`. A missing file is a configuration mistake.
+ */
+export function dictionaryStopwords(input) {
+    const words = new Set(languagePack(input.locale).stopwords);
+    for (const file of input.config.inference?.stopwords ?? []) {
+        const path = posix.resolve(input.configDirectory, file);
+        if (!input.fs.exists(path)) {
+            throw new Error(`stopword file not found: ${path} (inference.stopwords lists "${file}")`);
+        }
+        for (const word of loadStopwords(input.fs.readText(path)))
+            words.add(word);
+    }
+    return words;
+}
+/** The sources whose entities take priority: `inference.glossary_sources`, else those with `glossary: true`. */
+export function glossarySources(config) {
+    const declared = config.inference?.glossary_sources;
+    if (declared !== undefined)
+        return new Set(declared);
+    return new Set(config.sources.filter((source) => source.glossary === true).map((s) => s.name));
+}
+//# sourceMappingURL=stopwords.js.map

package/dist/dictionary/stopwords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopwords.js","sourceRoot":"","sources":["../../src/dictionary/stopwords.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAIlC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAUvD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,CAAC;IAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,IAAI,EAAE,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,gCAAgC,IAAI,IAAI,CAAC,CAAC;QAC5F,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gHAAgH;AAChH,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,EAAE,gBAAgB,CAAC;IACpD,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACjG,CAAC"}

package/dist/dictionary/types.d.ts

@@ -0,0 +1,34 @@
+import type { Finding } from "@concordance-wiki/core";
+/** What the dictionary needs to know about an entity: a structural view, so that any model fits. */
+export interface DictionarySource {
+    id: string;
+    source: string;
+    type: string;
+    title: string;
+    aliases: readonly string[];
+    locale: string;
+}
+/** One entity reachable from a dictionary entry, through its title or one of its aliases. */
+export interface DictionaryTarget {
+    id: string;
+    kind: "title" | "alias";
+    /** The spelling as written on the entity, kept for display. */
+    form: string;
+    /** 0 for an entity of a glossary source, 1 otherwise. */
+    priority: number;
+}
+export interface DictionaryEntry {
+    /** The comparison form the occurrence scan matches on. */
+    key: string;
+    /** Sorted by priority, then by identifier. */
+    targets: DictionaryTarget[];
+    /** True when the targets belong to more than one entity. */
+    homonym: boolean;
+}
+export interface Dictionary {
+    locale: string;
+    /** Keyed by comparison form, in code unit order of the keys. */
+    entries: ReadonlyMap<string, DictionaryEntry>;
+    findings: Finding[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/dictionary/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/dictionary/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAEtD,oGAAoG;AACpG,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,6FAA6F;AAC7F,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC;IACxB,+DAA+D;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ,8CAA8C;IAC9C,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9C,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB"}

package/dist/dictionary/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/dictionary/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/dictionary/types.ts"],"names":[],"mappings":""}

package/dist/index.d.ts

@@ -0,0 +1,26 @@
+export { LanguagePackError, loadLanguagePack } from "./locale/load-pack.js";
+export type { LanguagePack, PluralRule, Word } from "./locale/pack.js";
+export { availableLocales, languagePack, registerLanguagePack, resolveLocale, } from "./locale/registry.js";
+export { canonicalLocale } from "./locale/tag.js";
+export { loadStopwords } from "./locale/stopwords.js";
+export { loadSuffixes } from "./locale/suffixes.js";
+export { isOnWordBoundaries, wordBoundaries, type WordBoundary } from "./text/boundaries.js";
+export { comparisonForm, comparisonWords } from "./text/comparison-form.js";
+export { searchTokens } from "./text/search-tokens.js";
+export { singularize } from "./text/singular.js";
+export { buildDictionary, HOMONYM_CHECK, type BuildDictionaryInput } from "./dictionary/build.js";
+export { dictionaryStopwords, glossarySources, type DictionaryStopwordsInput, } from "./dictionary/stopwords.js";
+export type { Dictionary, DictionaryEntry, DictionarySource, DictionaryTarget, } from "./dictionary/types.js";
+export { buildAutomaton, longestMatches, scan, type Automaton, type AutomatonState, type Pattern, type RawMatch, } from "./scan/automaton.js";
+export { tokenize, type Token } from "./scan/tokens.js";
+export { CONTEXT_WIDTH, occurrenceContext, type ElidedCode, type QuotedText, } from "./scan/context.js";
+export { compareOccurrences, occurrenceConfidence, scanDocument, type Occurrence, type OccurrenceScale, type ScanDocumentInput, type ScannedDocument, type ScannedParagraph, } from "./scan/occurrences.js";
+export { extractNgrams, keywordForm, keywordForms, type ExtractNgramsOptions, type KeywordUnit, type NgramOccurrence, } from "./keywords/ngrams.js";
+export { compareMentions, scoreCandidates, type KeywordCandidate, type KeywordMention, type ScoreCandidatesOptions, } from "./keywords/score.js";
+export { BURST_MIN_FILES, BURST_PER_FILE, confidenceDefaults, confidenceFactors, confidenceOf, confidencePenalties, MORPHOLOGY_PENALTY, NEIGHBOURHOOD_BASE, POSITION_BASE, SPREAD_MIN_FILES, type ConfidenceFactors, type ConfidenceInput, type ConfidenceOptions, type ConfidencePenalty, type ConfidenceSignals, } from "./keywords/confidence.js";
+export { UNDEFINED_TERM_CHECK, undefinedTermFindings, type UndefinedTermOptions, } from "./keywords/findings.js";
+export { keywordDefaults, keywordOptions, type KeywordLock, type KeywordOptions, } from "./keywords/options.js";
+export { SIMILAR_EXPRESSIONS_LIMIT, similarExpressions, similarForm, type KeyedExpression, } from "./keywords/similar.js";
+export { definedExpressions, type DefinedExpression, type DefinedExpressionsOptions, } from "./keywords/defined.js";
+export { KEYWORD_TYPE, keywordEntities, keywordPublicationDefaults, keywordPublicationOptions, publishKeywords, type KeywordEntitiesOptions, type KeywordPage, type KeywordPublicationOptions, type PublishedKeywords, } from "./keywords/publish.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC5E,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EACL,gBAAgB,EAChB,YAAY,EACZ,oBAAoB,EACpB,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC7F,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,KAAK,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAClG,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,KAAK,wBAAwB,GAC9B,MAAM,2BAA2B,CAAC;AACnC,YAAY,EACV,UAAU,EACV,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,cAAc,EACd,cAAc,EACd,IAAI,EACJ,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,OAAO,EACZ,KAAK,QAAQ,GACd,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,YAAY,EACZ,KAAK,UAAU,EACf,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,gBAAgB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,aAAa,EACb,WAAW,EACX,YAAY,EACZ,KAAK,oBAAoB,EACzB,KAAK,WAAW,EAChB,KAAK,eAAe,GACrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,eAAe,EACf,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,sBAAsB,GAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,EACZ,mBAAmB,EACnB,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,EACb,gBAAgB,EAChB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,GACvB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,KAAK,oBAAoB,GAC1B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,eAAe,EACf,cAAc,EACd,KAAK,WAAW,EAChB,KAAK,cAAc,GACpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,yBAAyB,EACzB,kBAAkB,EAClB,WAAW,EACX,KAAK,eAAe,GACrB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC/B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,YAAY,EACZ,eAAe,EACf,0BAA0B,EAC1B,yBAAyB,EACzB,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,WAAW,EAChB,KAAK,yBAAyB,EAC9B,KAAK,iBAAiB,GACvB,MAAM,uBAAuB,CAAC"}

package/dist/index.js

@@ -0,0 +1,24 @@
+export { LanguagePackError, loadLanguagePack } from "./locale/load-pack.js";
+export { availableLocales, languagePack, registerLanguagePack, resolveLocale, } from "./locale/registry.js";
+export { canonicalLocale } from "./locale/tag.js";
+export { loadStopwords } from "./locale/stopwords.js";
+export { loadSuffixes } from "./locale/suffixes.js";
+export { isOnWordBoundaries, wordBoundaries } from "./text/boundaries.js";
+export { comparisonForm, comparisonWords } from "./text/comparison-form.js";
+export { searchTokens } from "./text/search-tokens.js";
+export { singularize } from "./text/singular.js";
+export { buildDictionary, HOMONYM_CHECK } from "./dictionary/build.js";
+export { dictionaryStopwords, glossarySources, } from "./dictionary/stopwords.js";
+export { buildAutomaton, longestMatches, scan, } from "./scan/automaton.js";
+export { tokenize } from "./scan/tokens.js";
+export { CONTEXT_WIDTH, occurrenceContext, } from "./scan/context.js";
+export { compareOccurrences, occurrenceConfidence, scanDocument, } from "./scan/occurrences.js";
+export { extractNgrams, keywordForm, keywordForms, } from "./keywords/ngrams.js";
+export { compareMentions, scoreCandidates, } from "./keywords/score.js";
+export { BURST_MIN_FILES, BURST_PER_FILE, confidenceDefaults, confidenceFactors, confidenceOf, confidencePenalties, MORPHOLOGY_PENALTY, NEIGHBOURHOOD_BASE, POSITION_BASE, SPREAD_MIN_FILES, } from "./keywords/confidence.js";
+export { UNDEFINED_TERM_CHECK, undefinedTermFindings, } from "./keywords/findings.js";
+export { keywordDefaults, keywordOptions, } from "./keywords/options.js";
+export { SIMILAR_EXPRESSIONS_LIMIT, similarExpressions, similarForm, } from "./keywords/similar.js";
+export { definedExpressions, } from "./keywords/defined.js";
+export { KEYWORD_TYPE, keywordEntities, keywordPublicationDefaults, keywordPublicationOptions, publishKeywords, } from "./keywords/publish.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE5E,OAAO,EACL,gBAAgB,EAChB,YAAY,EACZ,oBAAoB,EACpB,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAqB,MAAM,sBAAsB,CAAC;AAC7F,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,aAAa,EAA6B,MAAM,uBAAuB,CAAC;AAClG,OAAO,EACL,mBAAmB,EACnB,eAAe,GAEhB,MAAM,2BAA2B,CAAC;AAOnC,OAAO,EACL,cAAc,EACd,cAAc,EACd,IAAI,GAKL,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAc,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,aAAa,EACb,iBAAiB,GAGlB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,YAAY,GAMb,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,aAAa,EACb,WAAW,EACX,YAAY,GAIb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,eAAe,GAIhB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,EACZ,mBAAmB,EACnB,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,EACb,gBAAgB,GAMjB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,oBAAoB,EACpB,qBAAqB,GAEtB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,eAAe,EACf,cAAc,GAGf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,yBAAyB,EACzB,kBAAkB,EAClB,WAAW,GAEZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,kBAAkB,GAGnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,YAAY,EACZ,eAAe,EACf,0BAA0B,EAC1B,yBAAyB,EACzB,eAAe,GAKhB,MAAM,uBAAuB,CAAC"}

package/dist/keywords/confidence.d.ts

@@ -0,0 +1,56 @@
+import type { TermPenalty, TermSignals } from "@concordance-wiki/core";
+/**
+ * What the confidence of a candidate is read from: five measures of the corpus alone, no
+ * learning; `spread` is `df / N`, `burst` is `occurrences / df`, `prominence` the appearances
+ * in a heading, a written link or the frontmatter per occurrence, capped at one.
+ */
+export type ConfidenceSignals = TermSignals;
+/** The signals with the counts that decide whether the spread and the burst read: the files of the corpus and of the expression. */
+export interface ConfidenceInput extends ConfidenceSignals {
+    /** Files of the corpus. */
+    corpus: number;
+    /** Distinct files holding the expression. */
+    files: number;
+}
+export interface ConfidenceOptions {
+    /** Share of the files beyond which the expression belongs to the language rather than the subject. */
+    maxSpread: number;
+}
+/** The five factors of the product, each in [0, 1], in the order of the formula. */
+export interface ConfidenceFactors {
+    spread: number;
+    burst: number;
+    position: number;
+    neighbourhood: number;
+    morphology: number;
+}
+/** A signal that lowered the confidence: what the to-do page words as the reason. */
+export type ConfidencePenalty = TermPenalty;
+export declare const confidenceDefaults: ConfidenceOptions;
+/** Files a corpus must hold before the spread of an expression is read: in a handful, every word is everywhere. */
+export declare const SPREAD_MIN_FILES = 10;
+/** Files an expression must reach before its occurrences per file are read. */
+export declare const BURST_MIN_FILES = 5;
+/** Occurrences per file from which an expression recurs where it is treated. */
+export declare const BURST_PER_FILE = 1.5;
+/** The position factor of an expression never put forward; the bonus fills the rest. */
+export declare const POSITION_BASE = 0.8;
+/** The neighbourhood factor of an expression no defined term accompanies. */
+export declare const NEIGHBOURHOOD_BASE = 0.9;
+/** The morphology factor of an inflected form: a soft penalty, never an elimination. */
+export declare const MORPHOLOGY_PENALTY = 0.8;
+/**
+ * The factors of the formula: spread falls linearly from 1 at `maxSpread` to 0 when the
+ * expression is in every file, and reads only in a corpus of `SPREAD_MIN_FILES` files; burst
+ * rises linearly from 0.5 at one occurrence per file to 1 at `BURST_PER_FILE`, and reads
+ * only from `BURST_MIN_FILES` files; position is
+ * `POSITION_BASE` plus the prominence share of the rest; neighbourhood is 1 with the bonus,
+ * `NEIGHBOURHOOD_BASE` without; morphology is `MORPHOLOGY_PENALTY` for an inflected form, 1
+ * otherwise.
+ */
+export declare function confidenceFactors(input: ConfidenceInput, options?: ConfidenceOptions): ConfidenceFactors;
+/** The product of the five factors, rounded to four decimals. */
+export declare function confidenceOf(input: ConfidenceInput, options?: ConfidenceOptions): number;
+/** The signals that lowered the confidence, in the order of the formula; the bonuses never count. */
+export declare function confidencePenalties(input: ConfidenceInput, options?: ConfidenceOptions): ConfidencePenalty[];
+//# sourceMappingURL=confidence.d.ts.map

package/dist/keywords/confidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"confidence.d.ts","sourceRoot":"","sources":["../../src/keywords/confidence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC;AAE5C,oIAAoI;AACpI,MAAM,WAAW,eAAgB,SAAQ,iBAAiB;IACxD,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,sGAAsG;IACtG,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,oFAAoF;AACpF,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qFAAqF;AACrF,MAAM,MAAM,iBAAiB,GAAG,WAAW,CAAC;AAE5C,eAAO,MAAM,kBAAkB,EAAE,iBAAsC,CAAC;AAExE,mHAAmH;AACnH,eAAO,MAAM,gBAAgB,KAAK,CAAC;AACnC,+EAA+E;AAC/E,eAAO,MAAM,eAAe,IAAI,CAAC;AACjC,gFAAgF;AAChF,eAAO,MAAM,cAAc,MAAM,CAAC;AAClC,wFAAwF;AACxF,eAAO,MAAM,aAAa,MAAM,CAAC;AACjC,6EAA6E;AAC7E,eAAO,MAAM,kBAAkB,MAAM,CAAC;AACtC,wFAAwF;AACxF,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAMtC;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,eAAe,EACtB,OAAO,GAAE,iBAAsC,GAC9C,iBAAiB,CAiBnB;AAED,iEAAiE;AACjE,wBAAgB,YAAY,CAC1B,KAAK,EAAE,eAAe,EACtB,OAAO,GAAE,iBAAsC,GAC9C,MAAM,CAKR;AAED,qGAAqG;AACrG,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,eAAe,EACtB,OAAO,GAAE,iBAAsC,GAC9C,iBAAiB,EAAE,CAOrB"}

package/dist/keywords/confidence.js

@@ -0,0 +1,59 @@
+export const confidenceDefaults = { maxSpread: 0.5 };
+/** Files a corpus must hold before the spread of an expression is read: in a handful, every word is everywhere. */
+export const SPREAD_MIN_FILES = 10;
+/** Files an expression must reach before its occurrences per file are read. */
+export const BURST_MIN_FILES = 5;
+/** Occurrences per file from which an expression recurs where it is treated. */
+export const BURST_PER_FILE = 1.5;
+/** The position factor of an expression never put forward; the bonus fills the rest. */
+export const POSITION_BASE = 0.8;
+/** The neighbourhood factor of an expression no defined term accompanies. */
+export const NEIGHBOURHOOD_BASE = 0.9;
+/** The morphology factor of an inflected form: a soft penalty, never an elimination. */
+export const MORPHOLOGY_PENALTY = 0.8;
+function round(value) {
+    return Math.round(value * 10_000) / 10_000;
+}
+/**
+ * The factors of the formula: spread falls linearly from 1 at `maxSpread` to 0 when the
+ * expression is in every file, and reads only in a corpus of `SPREAD_MIN_FILES` files; burst
+ * rises linearly from 0.5 at one occurrence per file to 1 at `BURST_PER_FILE`, and reads
+ * only from `BURST_MIN_FILES` files; position is
+ * `POSITION_BASE` plus the prominence share of the rest; neighbourhood is 1 with the bonus,
+ * `NEIGHBOURHOOD_BASE` without; morphology is `MORPHOLOGY_PENALTY` for an inflected form, 1
+ * otherwise.
+ */
+export function confidenceFactors(input, options = confidenceDefaults) {
+    const excess = input.corpus < SPREAD_MIN_FILES
+        ? 0
+        : Math.max(0, input.spread - options.maxSpread) / (1 - options.maxSpread);
+    // Below `BURST_MIN_FILES` files, once per file says nothing; an expression never occurs less than once per file.
+    const burst = input.files < BURST_MIN_FILES
+        ? 1
+        : Math.min(1, 0.5 + (0.5 * (input.burst - 1)) / (BURST_PER_FILE - 1));
+    return {
+        spread: 1 - excess,
+        burst,
+        position: POSITION_BASE + (1 - POSITION_BASE) * input.prominence,
+        neighbourhood: input.neighbour ? 1 : NEIGHBOURHOOD_BASE,
+        morphology: input.inflected ? MORPHOLOGY_PENALTY : 1,
+    };
+}
+/** The product of the five factors, rounded to four decimals. */
+export function confidenceOf(input, options = confidenceDefaults) {
+    const factors = confidenceFactors(input, options);
+    return round(factors.spread * factors.burst * factors.position * factors.neighbourhood * factors.morphology);
+}
+/** The signals that lowered the confidence, in the order of the formula; the bonuses never count. */
+export function confidencePenalties(input, options = confidenceDefaults) {
+    const factors = confidenceFactors(input, options);
+    const penalties = [];
+    if (factors.spread < 1)
+        penalties.push("spread");
+    if (factors.burst < 1)
+        penalties.push("burst");
+    if (factors.morphology < 1)
+        penalties.push("morphology");
+    return penalties;
+}
+//# sourceMappingURL=confidence.js.map

package/dist/keywords/confidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"confidence.js","sourceRoot":"","sources":["../../src/keywords/confidence.ts"],"names":[],"mappings":"AAkCA,MAAM,CAAC,MAAM,kBAAkB,GAAsB,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;AAExE,mHAAmH;AACnH,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,CAAC;AACnC,+EAA+E;AAC/E,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AACjC,gFAAgF;AAChF,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC;AAClC,wFAAwF;AACxF,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AACjC,6EAA6E;AAC7E,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACtC,wFAAwF;AACxF,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;AAC7C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAsB,EACtB,UAA6B,kBAAkB;IAE/C,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,GAAG,gBAAgB;QAC7B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,iHAAiH;IACjH,MAAM,KAAK,GACT,KAAK,CAAC,KAAK,GAAG,eAAe;QAC3B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO;QACL,MAAM,EAAE,CAAC,GAAG,MAAM;QAClB,KAAK;QACL,QAAQ,EAAE,aAAa,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,GAAG,KAAK,CAAC,UAAU;QAChE,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;QACvD,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,YAAY,CAC1B,KAAsB,EACtB,UAA6B,kBAAkB;IAE/C,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAClD,OAAO,KAAK,CACV,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,UAAU,CAC/F,CAAC;AACJ,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,mBAAmB,CACjC,KAAsB,EACtB,UAA6B,kBAAkB;IAE/C,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC;QAAE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC;QAAE,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/keywords/defined.d.ts

@@ -0,0 +1,26 @@
+import type { LanguagePack } from "../locale/pack.js";
+import { type NgramOccurrence } from "./ngrams.js";
+/** A recurring expression a note defines: what would be a keyword page, had the note not been written. */
+export interface DefinedExpression {
+    /** The comparison form of the dictionary entry, as the n-grams carry it. */
+    key: string;
+    occurrences: number;
+    /** Distinct files the expression appears in. */
+    documents: number;
+}
+export interface DefinedExpressionsOptions {
+    pack: LanguagePack;
+    /** Comparison forms of the dictionary entries. */
+    dictionaryKeys: ReadonlySet<string>;
+    minOccurrences: number;
+    /** Distinct files the expression must appear in. */
+    minDocuments: number;
+}
+/**
+ * The n-grams the dictionary already defines that reach the publication threshold, in key
+ * order. Such an expression gets no keyword page, its note being the page; the build keeps
+ * the address the page would have had, so that a word defined after its page was published
+ * keeps its URL.
+ */
+export declare function definedExpressions(occurrences: readonly NgramOccurrence[], options: DefinedExpressionsOptions): DefinedExpression[];
+//# sourceMappingURL=defined.d.ts.map

package/dist/keywords/defined.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"defined.d.ts","sourceRoot":"","sources":["../../src/keywords/defined.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAgB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,0GAA0G;AAC1G,MAAM,WAAW,iBAAiB;IAChC,4EAA4E;IAC5E,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,YAAY,CAAC;IACnB,kDAAkD;IAClD,cAAc,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;CACtB;AAOD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,OAAO,EAAE,yBAAyB,GACjC,iBAAiB,EAAE,CA4BrB"}

package/dist/keywords/defined.js

@@ -0,0 +1,33 @@
+import { keywordForms } from "./ngrams.js";
+/**
+ * The n-grams the dictionary already defines that reach the publication threshold, in key
+ * order. Such an expression gets no keyword page, its note being the page; the build keeps
+ * the address the page would have had, so that a word defined after its page was published
+ * keeps its URL.
+ */
+export function definedExpressions(occurrences, options) {
+    const defined = keywordForms(options.dictionaryKeys, options.pack);
+    const counts = new Map();
+    for (const occurrence of occurrences) {
+        if (!defined.has(occurrence.key))
+            continue;
+        let count = counts.get(occurrence.key);
+        if (count === undefined) {
+            count = { occurrences: 0, documents: new Set() };
+            counts.set(occurrence.key, count);
+        }
+        count.occurrences += 1;
+        count.documents.add(`${occurrence.source ?? ""}\n${occurrence.path}`);
+    }
+    return ([...counts]
+        .filter(([, count]) => count.occurrences >= options.minOccurrences &&
+        count.documents.size >= options.minDocuments)
+        .map(([key, count]) => ({
+        key,
+        occurrences: count.occurrences,
+        documents: count.documents.size,
+    }))
+        // Keys are distinct: the comparison never meets two equal ones.
+        .sort((a, b) => (a.key < b.key ? -1 : 1)));
+}
+//# sourceMappingURL=defined.js.map

package/dist/keywords/defined.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"defined.js","sourceRoot":"","sources":["../../src/keywords/defined.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAwB,MAAM,aAAa,CAAC;AAyBjE;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,WAAuC,EACvC,OAAkC;IAElC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC3C,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC;QACvB,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,KAAK,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,CACL,CAAC,GAAG,MAAM,CAAC;SACR,MAAM,CACL,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CACZ,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,cAAc;QAC3C,KAAK,CAAC,SAAS,CAAC,IAAI,IAAI,OAAO,CAAC,YAAY,CAC/C;SACA,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,GAAG;QACH,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,IAAI;KAChC,CAAC,CAAC;QACH,gEAAgE;SAC/D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC5C,CAAC;AACJ,CAAC"}

package/dist/keywords/findings.d.ts

@@ -0,0 +1,13 @@
+import { type Finding } from "@concordance-wiki/core";
+import type { KeywordCandidate } from "./score.js";
+export declare const UNDEFINED_TERM_CHECK = "W-TERM-UNDEFINED";
+export interface UndefinedTermOptions {
+    /** Score from which a candidate deserves a finding. */
+    minScore: number;
+}
+/**
+ * One `W-TERM-UNDEFINED` per candidate scoring at least `minScore`, placed on its first
+ * mention, in canonical order.
+ */
+export declare function undefinedTermFindings(candidates: readonly KeywordCandidate[], options: UndefinedTermOptions): Finding[];
+//# sourceMappingURL=findings.d.ts.map

package/dist/keywords/findings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"findings.d.ts","sourceRoot":"","sources":["../../src/keywords/findings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,gBAAgB,EAAkB,MAAM,YAAY,CAAC;AAEnE,eAAO,MAAM,oBAAoB,qBAAqB,CAAC;AAEvD,MAAM,WAAW,oBAAoB;IACnC,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAkBD;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,SAAS,gBAAgB,EAAE,EACvC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,EAAE,CAcX"}

package/dist/keywords/findings.js

@@ -0,0 +1,35 @@
+import { compareFindings } from "@concordance-wiki/core";
+export const UNDEFINED_TERM_CHECK = "W-TERM-UNDEFINED";
+function plural(count, noun) {
+    return `${String(count)} ${noun}${count === 1 ? "" : "s"}`;
+}
+function locationOf(mentions) {
+    const [first] = mentions;
+    if (first === undefined)
+        return {};
+    return {
+        ...(first.source === undefined ? {} : { source: first.source }),
+        path: first.path,
+        line: first.line,
+    };
+}
+/**
+ * One `W-TERM-UNDEFINED` per candidate scoring at least `minScore`, placed on its first
+ * mention, in canonical order.
+ */
+export function undefinedTermFindings(candidates, options) {
+    const findings = [];
+    for (const candidate of candidates) {
+        if (candidate.score < options.minScore)
+            continue;
+        findings.push({
+            check: UNDEFINED_TERM_CHECK,
+            severity: "warning",
+            message: `"${candidate.display}" is used ${plural(candidate.occurrences, "time")} in ${plural(candidate.documents, "file")} (score ${String(candidate.score)}) without a note defining it`,
+            remediation: "Create a term note in the glossary, or add the expression to `rejected_terms` in the lock file if it is not a business term.",
+            ...locationOf(candidate.mentions),
+        });
+    }
+    return findings.sort(compareFindings);
+}
+//# sourceMappingURL=findings.js.map

package/dist/keywords/findings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"findings.js","sourceRoot":"","sources":["../../src/keywords/findings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAgB,MAAM,wBAAwB,CAAC;AAIvE,MAAM,CAAC,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AAOvD,SAAS,MAAM,CAAC,KAAa,EAAE,IAAY;IACzC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;AAC7D,CAAC;AAED,SAAS,UAAU,CACjB,QAAmC;IAEnC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO;QACL,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QAC/D,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;KACjB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,UAAuC,EACvC,OAA6B;IAE7B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,KAAK,GAAG,OAAO,CAAC,QAAQ;YAAE,SAAS;QACjD,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,oBAAoB;YAC3B,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,IAAI,SAAS,CAAC,OAAO,aAAa,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC,OAAO,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,WAAW,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,8BAA8B;YAC1L,WAAW,EACT,8HAA8H;YAChI,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC;SAClC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;AACxC,CAAC"}

package/dist/keywords/ngrams.d.ts

@@ -0,0 +1,47 @@
+import type { LanguagePack } from "../locale/pack.js";
+import { type QuotedText } from "../scan/context.js";
+/** A text unit the discovery reads: a scannable unit of a document, with the file it comes from. */
+export interface KeywordUnit extends QuotedText {
+    /** Name of the source holding the file; two sources may hold the same path. */
+    source?: string;
+    path: string;
+    line: number;
+    /** The text the discovery reads, inline code left out. */
+    text: string;
+}
+export interface NgramOccurrence {
+    /** The words in comparison form, joined by a single space. */
+    key: string;
+    /** The span as written in the text. */
+    surface: string;
+    source?: string;
+    path: string;
+    line: number;
+    /** Code unit offset of the span in the unit text. */
+    position: number;
+    /** 160 characters of the unit text as written, inline code included, centred on the span, an ellipsis marking each cut. */
+    context: string;
+}
+export interface ExtractNgramsOptions {
+    /** Shortest n-gram, in words; 1 by default. */
+    minWords?: number;
+    /** Longest n-gram, in words. */
+    maxWords: number;
+    /** Below this length of the key, an n-gram is noise. */
+    minLength: number;
+    /** The pack's stopwords by default; given as written, compared in comparison form. */
+    stopwords?: ReadonlySet<string>;
+}
+/** The comparison form of a term as the n-grams carry it: its words, tokenised like the texts. */
+export declare function keywordForm(text: string, pack: LanguagePack): string;
+/** The comparison forms of a list of terms. */
+export declare function keywordForms(forms: Iterable<string>, pack: LanguagePack): Set<string>;
+/**
+ * Every n-gram of `minWords` to `maxWords` words in the units, in unit then text order,
+ * except those starting or ending with a stopword, those made only of digits and those
+ * whose key is shorter than `minLength`. Words are the tokens of the text: normalised by
+ * the pack and singularised, so that "Build summaries" and "build summary"
+ * share a key while each keeps its surface form.
+ */
+export declare function extractNgrams(units: readonly KeywordUnit[], pack: LanguagePack, options: ExtractNgramsOptions): NgramOccurrence[];
+//# sourceMappingURL=ngrams.d.ts.map

package/dist/keywords/ngrams.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ngrams.d.ts","sourceRoot":"","sources":["../../src/keywords/ngrams.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAGxE,oGAAoG;AACpG,MAAM,WAAW,WAAY,SAAQ,UAAU;IAC7C,+EAA+E;IAC/E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,2HAA2H;IAC3H,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACjC;AAKD,kGAAkG;AAClG,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,MAAM,CAIpE;AAED,+CAA+C;AAC/C,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,CAErF;AA6CD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,SAAS,WAAW,EAAE,EAC7B,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,oBAAoB,GAC5B,eAAe,EAAE,CAQnB"}