@concordance-wiki/nlp 0.1.0

package/dist/text/boundaries.d.ts

@@ -0,0 +1,12 @@
+import type { LanguagePack } from "../locale/pack.js";
+/** A word-like segment of a text, `end` exclusive, in code units of the text as given. */
+export interface WordBoundary {
+    start: number;
+    end: number;
+    word: string;
+}
+/** Every word of a text with its position, in text order, as the pack's segmenter cuts it. */
+export declare function wordBoundaries(text: string, pack: LanguagePack): WordBoundary[];
+/** True when a word starts at `start` and a word ends at `end` (exclusive), possibly different ones. */
+export declare function isOnWordBoundaries(text: string, start: number, end: number, pack: LanguagePack): boolean;
+//# sourceMappingURL=boundaries.d.ts.map

package/dist/text/boundaries.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"boundaries.d.ts","sourceRoot":"","sources":["../../src/text/boundaries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD,0FAA0F;AAC1F,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,CAK/E;AAED,wGAAwG;AACxG,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,YAAY,GACjB,OAAO,CAGT"}

package/dist/text/boundaries.js

@@ -0,0 +1,13 @@
+/** Every word of a text with its position, in text order, as the pack's segmenter cuts it. */
+export function wordBoundaries(text, pack) {
+    return pack
+        .segment(text)
+        .filter((word) => word.isWordLike)
+        .map((word) => ({ start: word.index, end: word.index + word.text.length, word: word.text }));
+}
+/** True when a word starts at `start` and a word ends at `end` (exclusive), possibly different ones. */
+export function isOnWordBoundaries(text, start, end, pack) {
+    const words = wordBoundaries(text, pack);
+    return words.some((word) => word.start === start) && words.some((word) => word.end === end);
+}
+//# sourceMappingURL=boundaries.js.map

package/dist/text/boundaries.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"boundaries.js","sourceRoot":"","sources":["../../src/text/boundaries.ts"],"names":[],"mappings":"AASA,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,IAAkB;IAC7D,OAAO,IAAI;SACR,OAAO,CAAC,IAAI,CAAC;SACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;SACjC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,KAAa,EACb,GAAW,EACX,IAAkB;IAElB,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;AAC9F,CAAC"}

package/dist/text/comparison-form.d.ts

@@ -0,0 +1,9 @@
+import type { LanguagePack } from "../locale/pack.js";
+/** The words of a text as they are compared: normalised by the pack, then singularised. */
+export declare function comparisonWords(text: string, pack: LanguagePack): string[];
+/**
+ * The form two spellings are compared on. The text given is never modified: the caller keeps
+ * the original form for display and only compares on the value returned here.
+ */
+export declare function comparisonForm(text: string, pack: LanguagePack): string;
+//# sourceMappingURL=comparison-form.d.ts.map

package/dist/text/comparison-form.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-form.d.ts","sourceRoot":"","sources":["../../src/text/comparison-form.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAWtD,2FAA2F;AAC3F,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,MAAM,EAAE,CAM1E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,MAAM,CAEvE"}

package/dist/text/comparison-form.js

@@ -0,0 +1,23 @@
+import { singularize } from "./singular.js";
+// Hyphens and apostrophes stay inside a word; each part they join is singularised on its own,
+// so that "couts-benefices" gives "cout-benefice" and "author's" keeps its "s".
+const parts = /[^-']+/gu;
+function singularizeWord(word, pack) {
+    return word.replace(parts, (part) => singularize(part, pack.plural));
+}
+/** The words of a text as they are compared: normalised by the pack, then singularised. */
+export function comparisonWords(text, pack) {
+    const normalised = pack.normalize(text);
+    if (normalised === "") {
+        return [];
+    }
+    return normalised.split(" ").map((word) => singularizeWord(word, pack));
+}
+/**
+ * The form two spellings are compared on. The text given is never modified: the caller keeps
+ * the original form for display and only compares on the value returned here.
+ */
+export function comparisonForm(text, pack) {
+    return comparisonWords(text, pack).join(" ");
+}
+//# sourceMappingURL=comparison-form.js.map

package/dist/text/comparison-form.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-form.js","sourceRoot":"","sources":["../../src/text/comparison-form.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,8FAA8F;AAC9F,gFAAgF;AAChF,MAAM,KAAK,GAAG,UAAU,CAAC;AAEzB,SAAS,eAAe,CAAC,IAAY,EAAE,IAAkB;IACvD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,IAAkB;IAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,IAAI,UAAU,KAAK,EAAE,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,IAAkB;IAC7D,OAAO,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,CAAC"}

package/dist/text/context.d.ts

@@ -0,0 +1,6 @@
+/**
+ * A window of `width` characters of the text centred on the span `start`–`end`, kept inside
+ * the text, an ellipsis marking each cut.
+ */
+export declare function contextAround(text: string, start: number, end: number, width: number): string;
+//# sourceMappingURL=context.d.ts.map

package/dist/text/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/text/context.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7F"}

package/dist/text/context.js

@@ -0,0 +1,12 @@
+const ellipsis = "…";
+/**
+ * A window of `width` characters of the text centred on the span `start`–`end`, kept inside
+ * the text, an ellipsis marking each cut.
+ */
+export function contextAround(text, start, end, width) {
+    const centre = Math.floor((start + end) / 2);
+    const from = Math.max(0, Math.min(centre - width / 2, text.length - width));
+    const to = Math.min(text.length, from + width);
+    return `${from > 0 ? ellipsis : ""}${text.slice(from, to)}${to < text.length ? ellipsis : ""}`;
+}
+//# sourceMappingURL=context.js.map

package/dist/text/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/text/context.ts"],"names":[],"mappings":"AAAA,MAAM,QAAQ,GAAG,GAAG,CAAC;AAErB;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW,EAAE,KAAa;IACnF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;IAC5E,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC;IAC/C,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACjG,CAAC"}

package/dist/text/search-tokens.d.ts

@@ -0,0 +1,10 @@
+import type { LanguagePack } from "../locale/pack.js";
+/**
+ * The tokens of a text as a prefix search index stores them: every word normalised by the pack
+ * and its comparison form, then, for a word with a hyphen, an apostrophe or another separator
+ * inside, each of its parts and their comparison forms; nothing shorter than `minLength`. A
+ * query matches a token by prefix, so both the written and the singular form are there for
+ * "checks" and "check" to find the same note.
+ */
+export declare function searchTokens(text: string, pack: LanguagePack, minLength?: number): Set<string>;
+//# sourceMappingURL=search-tokens.d.ts.map

package/dist/text/search-tokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-tokens.d.ts","sourceRoot":"","sources":["../../src/text/search-tokens.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAQtD;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,SAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAazF"}

package/dist/text/search-tokens.js

@@ -0,0 +1,29 @@
+import { comparisonForm } from "./comparison-form.js";
+// Punctuation at both ends of a word goes; the inside stays, so that a hyphenated word is one token.
+// The trailing run starts right after a letter or digit: retrying inside the run would make the runtime quadratic.
+const edges = /^[^\p{L}\p{N}]+|(?<![^\p{L}\p{N}])[^\p{L}\p{N}]+$/gu;
+const separators = /[^\p{L}\p{N}]+/u;
+/**
+ * The tokens of a text as a prefix search index stores them: every word normalised by the pack
+ * and its comparison form, then, for a word with a hyphen, an apostrophe or another separator
+ * inside, each of its parts and their comparison forms; nothing shorter than `minLength`. A
+ * query matches a token by prefix, so both the written and the singular form are there for
+ * "checks" and "check" to find the same note.
+ */
+export function searchTokens(text, pack, minLength = 2) {
+    const tokens = new Set();
+    const add = (word) => {
+        for (const form of [word, comparisonForm(word, pack)]) {
+            if (Array.from(form).length >= minLength)
+                tokens.add(form);
+        }
+    };
+    for (const raw of pack.normalize(text).split(" ")) {
+        const word = raw.replace(edges, "");
+        // An unsplit word is its own single part; the set keeps it once.
+        for (const part of [word, ...word.split(separators)])
+            add(part);
+    }
+    return tokens;
+}
+//# sourceMappingURL=search-tokens.js.map

package/dist/text/search-tokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-tokens.js","sourceRoot":"","sources":["../../src/text/search-tokens.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,qGAAqG;AACrG,mHAAmH;AACnH,MAAM,KAAK,GAAG,qDAAqD,CAAC;AACpE,MAAM,UAAU,GAAG,iBAAiB,CAAC;AAErC;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,IAAkB,EAAE,SAAS,GAAG,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,GAAG,GAAG,CAAC,IAAY,EAAQ,EAAE;QACjC,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;YACtD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,SAAS;gBAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACpC,iEAAiE;QACjE,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/text/singular.d.ts

@@ -0,0 +1,7 @@
+import type { PluralRule } from "../locale/pack.js";
+/**
+ * Brings a normalised word (lowercase, no accents) back to its singular: the first rule
+ * whose ending the word carries, and whose minimum length the word reaches, applies.
+ */
+export declare function singularize(word: string, rules: readonly PluralRule[]): string;
+//# sourceMappingURL=singular.d.ts.map

package/dist/text/singular.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"singular.d.ts","sourceRoot":"","sources":["../../src/text/singular.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CAQ9E"}

package/dist/text/singular.js

@@ -0,0 +1,12 @@
+/**
+ * Brings a normalised word (lowercase, no accents) back to its singular: the first rule
+ * whose ending the word carries, and whose minimum length the word reaches, applies.
+ */
+export function singularize(word, rules) {
+    const rule = rules.find(({ ending, minLength }) => word.length >= minLength && word.endsWith(ending));
+    if (rule === undefined) {
+        return word;
+    }
+    return word.slice(0, word.length - rule.ending.length) + rule.singular;
+}
+//# sourceMappingURL=singular.js.map

package/dist/text/singular.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"singular.js","sourceRoot":"","sources":["../../src/text/singular.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAA4B;IACpE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CACrB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC7E,CAAC;IACF,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;AACzE,CAAC"}

package/locales/en/pack.yaml

@@ -0,0 +1,18 @@
+# Language pack: everything the engine needs to know about a language, as data.
+locale: en
+language: English
+# Apostrophe characters unified to the plain one before matching.
+apostrophes: ["’", "ʼ"]
+# Intl.Collator options for every alphabetical order of this locale.
+collation: { sensitivity: base, numeric: true }
+# Suffix rules that bring a plural form back to its singular; the first matching
+# rule wins, `min_length` guards short words. Applied by the occurrence scan.
+plural:
+  - { ending: ies, singular: y, min_length: 5 }
+  - { ending: sses, singular: ss, min_length: 6 }
+  - { ending: shes, singular: sh, min_length: 6 }
+  - { ending: ches, singular: ch, min_length: 6 }
+  - { ending: xes, singular: x, min_length: 5 }
+  # A word in -ss is a singular ("address"): this rule keeps it from the -s rule below.
+  - { ending: ss, singular: ss }
+  - { ending: s, singular: "", min_length: 4 }