@agentshouse/mdruntime 0.1.0-ce5c5bb29be5a → 0.1.0-ce6bc9239a3ee

package/dist/words.js

@@ -1,156 +0,0 @@
-const SPACELESS = /(?:[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}\p{Script=Hangul}]\p{M}*)+/gu;
-const SPACELESS_TERM = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}\p{Script=Hangul}]/u;
-const TOKEN = /(?![\x00-\x7f])\p{S}\p{M}*|[\p{L}\p{N}\p{M}\p{Co}]+/gu;
-const SYMBOL = /(?![\x00-\x7f])\p{S}\p{M}*/gu;
-const MARKED = /([\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}])\p{M}+/gu;
-const INFLECTED = /^[\p{Script=Cyrillic}\p{Script=Greek}]+$/u;
-const FOLD = /\p{M}+/gu;
-const ASCII = /^[\x00-\x7f]$/;
-const PREFIXED_LETTERS = 2;
-const INFLECTED_LETTERS = 4;
-const PHRASE_TERMS = { least: 2, most: 6 };
-const GRAPHEMES = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
-function bigrams(run) {
-    const characters = Array.from(run);
-    if (characters.length < 2)
-        return ` ${run} `;
-    const pairs = [];
-    for (let index = 0; index + 1 < characters.length; index += 1) {
-        pairs.push(characters[index] + characters[index + 1]);
-    }
-    return ` ${pairs.join(' ')} `;
-}
-export function indexed(text) {
-    return text
-        .normalize('NFKD')
-        .replace(MARKED, '$1')
-        .normalize('NFKC')
-        .replace(SPACELESS, bigrams)
-        .replace(SYMBOL, (symbol) => ` ${symbol.replace(FOLD, '')} `);
-}
-function tokens(text) {
-    return [...text.matchAll(TOKEN)].map(([token]) => token.toLowerCase());
-}
-function quoted(token) {
-    return `"${token.replaceAll('"', '""')}"`;
-}
-function truncated(term) {
-    const letters = Array.from(term);
-    if (!INFLECTED.test(term) || letters.length < INFLECTED_LETTERS)
-        return null;
-    return letters.slice(0, letters.length - (letters.length >= 6 ? 2 : 1)).join('');
-}
-export function matchClauses(text) {
-    const terms = tokens(indexed(text));
-    if (terms.length === 0)
-        return null;
-    const unique = [...new Set(terms)];
-    const clauses = unique.map((term) => ({ text: quoted(term), kind: 'term', term }));
-    for (const term of unique) {
-        const stem = truncated(term);
-        if (stem !== null)
-            clauses.push({ text: `${quoted(stem)}*`, kind: 'stem', term });
-    }
-    const last = terms[terms.length - 1];
-    if (!/\s$/u.test(text) && Array.from(last).length >= PREFIXED_LETTERS) {
-        clauses.push({ text: `${quoted(last)}*`, kind: 'prefix', term: last });
-    }
-    if (terms.length >= PHRASE_TERMS.least && terms.length <= PHRASE_TERMS.most) {
-        clauses.push({ text: quoted(terms.join(' ')), kind: 'phrase', term: terms.join(' ') });
-    }
-    return clauses;
-}
-export function matchExpression(clauses) {
-    return clauses.map(({ text }) => text).join(' OR ');
-}
-function folded(text) {
-    return text.normalize('NFKD').replace(FOLD, '').toLowerCase();
-}
-export function queryTerms(text) {
-    return [...new Set(tokens(text.normalize('NFKC')).map(folded))];
-}
-function agrees(token, term) {
-    if (SPACELESS_TERM.test(term))
-        return token.includes(term);
-    if (token.startsWith(term))
-        return true;
-    return token.length >= 3 && term.startsWith(token.slice(0, Math.max(3, token.length - 2)));
-}
-function inside(token, offset, term) {
-    const pieces = [...GRAPHEMES.segment(token)];
-    const owner = [];
-    let joined = '';
-    for (const [at, { segment }] of pieces.entries()) {
-        const part = folded(segment);
-        joined += part;
-        for (let index = 0; index < part.length; index += 1)
-            owner.push(at);
-    }
-    const found = joined.indexOf(term);
-    if (found < 0)
-        return null;
-    const first = pieces[owner[found]];
-    const last = pieces[owner[found + term.length - 1]];
-    return { start: offset + first.index, end: offset + last.index + last.segment.length };
-}
-function locate(part, terms) {
-    for (const match of part.matchAll(TOKEN)) {
-        const token = folded(match[0]);
-        for (const term of terms) {
-            if (!agrees(token, term))
-                continue;
-            if (!SPACELESS_TERM.test(term)) {
-                return { start: match.index, end: match.index + match[0].length };
-            }
-            const found = inside(match[0], match.index, term);
-            if (found !== null)
-                return found;
-        }
-    }
-    return null;
-}
-function settled(part, index) {
-    if (index <= 0 || index >= part.length)
-        return true;
-    const before = part[index - 1];
-    const after = part[index];
-    if (before === '\n')
-        return true;
-    return ASCII.test(before) && ASCII.test(after) && !(before === '\r' && after === '\n');
-}
-function edgesAround(part, low, high) {
-    let from = Math.max(0, low);
-    while (!settled(part, from))
-        from -= 1;
-    let to = Math.min(part.length, high);
-    while (!settled(part, to))
-        to += 1;
-    const edges = [...GRAPHEMES.segment(part.slice(from, to))].map(({ index }) => from + index);
-    edges.push(to);
-    return edges;
-}
-export function snippet(part, terms, reach) {
-    const located = locate(part, terms);
-    if (located === null)
-        return null;
-    const edges = edgesAround(part, located.start - 2 * reach, located.end + 2 * reach);
-    const last = (limit) => edges.filter((edge) => edge <= limit).at(-1) ?? edges[0];
-    const first = (limit) => edges.find((edge) => edge >= limit) ?? edges[edges.length - 1];
-    const from = last(located.start);
-    const to = Math.max(first(Math.min(located.end, from + reach)), first(from + 1));
-    const start = first(Math.max(0, from - reach));
-    const end = last(Math.min(part.length, to + reach));
-    return { text: part.slice(start, end), at: [from - start, to - start] };
-}
-export function matched(part, terms) {
-    const found = new Set();
-    for (const match of part.matchAll(TOKEN)) {
-        const token = folded(match[0]);
-        for (const term of terms)
-            if (agrees(token, term))
-                found.add(term);
-        if (found.size === terms.length)
-            break;
-    }
-    return [...found];
-}