@agentshouse/mdruntime 0.1.0-cefbc8bb8a1c0 → 0.1.0-cf08ada271a7f

package/dist/fulltext.js

@@ -1,370 +0,0 @@
-import { parse } from '@agentshouse/mdmodel';
-import { selected } from './compile.js';
-import { markdownPath } from './rows.js';
-import { indexed, matchClauses, matched, matchExpression, queryTerms, snippet, } from './words.js';
-const TOKENIZER = "porter unicode61 remove_diacritics 2 categories 'L* N* Co M* S*' separators '$+<=>^`|~'";
-const CONTENTLESS = `content='', contentless_delete=1, detail=full, tokenize="${TOKENIZER}"`;
-const DOCUMENTS = 'replica_search';
-const MESSAGES = 'conversation_search';
-export const SEARCH_TABLES = [
-    `CREATE VIRTUAL TABLE ${DOCUMENTS} USING fts5(path, fields, sections, preamble, ${CONTENTLESS})`,
-    `CREATE VIRTUAL TABLE ${MESSAGES} USING fts5(text, ${CONTENTLESS})`,
-];
-const CHUNKS = 4096;
-const CHUNK_CHARACTERS = 32_768;
-const CANDIDATES = 5_000;
-const LEAST_WINDOW = 32;
-const SNIPPET_REACH = 80;
-function scalars(value) {
-    if (value === null || value === undefined)
-        return [];
-    if (Array.isArray(value))
-        return value.flatMap(scalars);
-    if (typeof value === 'object')
-        return Object.values(value).flatMap(scalars);
-    return [String(value)];
-}
-function partsOf(path, content) {
-    if (!markdownPath(path))
-        return null;
-    const read = parse(content, path);
-    if (!read.ok)
-        return null;
-    return {
-        fields: scalars(read.document.fields).join('\n'),
-        sections: Object.entries(read.document.sections)
-            .map(([heading, text]) => `${heading}\n${text}`)
-            .join('\n'),
-        preamble: read.document.preamble,
-        path,
-    };
-}
-function chunked(text) {
-    const chunks = [];
-    let start = 0;
-    do {
-        let end = Math.min(text.length, start + CHUNK_CHARACTERS);
-        if (end < text.length) {
-            const space = text.lastIndexOf(' ', end);
-            if (space > start + CHUNK_CHARACTERS / 2)
-                end = space;
-            else if (/[\ud800-\udbff]/u.test(text[end - 1]))
-                end -= 1;
-        }
-        chunks.push(text.slice(start, end));
-        start = end;
-    } while (start < text.length && chunks.length < CHUNKS);
-    return chunks;
-}
-function rowOf(id, chunk) {
-    return id * CHUNKS + chunk;
-}
-export function indexDocument(connection, id, path, content) {
-    const parts = partsOf(path, content);
-    if (parts === null)
-        return;
-    chunked(parts.sections).forEach((sections, chunk) => {
-        const head = chunk === 0;
-        connection.run(`INSERT INTO ${DOCUMENTS} (rowid, path, fields, sections, preamble) VALUES (?, ?, ?, ?, ?)`, [
-            rowOf(id, chunk),
-            head ? indexed(parts.path) : null,
-            head ? indexed(parts.fields) : null,
-            indexed(sections),
-            head ? indexed(parts.preamble) : null,
-        ]);
-    });
-}
-export function indexMessage(connection, id, text) {
-    chunked(text).forEach((chunk, at) => {
-        connection.run(`INSERT INTO ${MESSAGES} (rowid, text) VALUES (?, ?)`, [
-            rowOf(id, at),
-            indexed(chunk),
-        ]);
-    });
-}
-function unindexed(connection, table, ids) {
-    for (const id of ids) {
-        connection.run(`DELETE FROM ${table} WHERE rowid >= ? AND rowid < ?`, [
-            rowOf(id, 0),
-            rowOf(id + 1, 0),
-        ]);
-    }
-}
-export function unindexDocuments(connection, ids) {
-    unindexed(connection, DOCUMENTS, ids);
-}
-export function unindexMessages(connection, ids) {
-    unindexed(connection, MESSAGES, ids);
-}
-export function unindexAll(connection) {
-    for (const table of [DOCUMENTS, MESSAGES]) {
-        connection.run(`INSERT INTO ${table} (${table}) VALUES ('delete-all')`);
-    }
-}
-function pair(left, right) {
-    if (left[0] !== right[0])
-        return left[0] < right[0] ? -1 : 1;
-    if (left[1] !== right[1])
-        return left[1] < right[1] ? -1 : 1;
-    return 0;
-}
-function byRank(left, right) {
-    return right.score - left.score || pair(left.tie, right.tie);
-}
-function beyond(hit, cursor) {
-    if (cursor.length === 0)
-        return true;
-    return (hit.score < cursor[0] || (hit.score === cursor[0] && pair(hit.tie, [cursor[1], cursor[2]]) > 0));
-}
-function rankedWithin(connection, listed, sources, cursor, needed) {
-    const { table } = listed;
-    let window = Math.max(needed * 2, LEAST_WINDOW);
-    for (;;) {
-        const rows = [];
-        const boundaries = [];
-        for (const source of sources) {
-            const found = connection.rows(`SELECT rowid, bm25(${table}) AS score FROM ${table} WHERE ${table} MATCH ? ${source.sql} ORDER BY score, rowid LIMIT ?`, [source.expression, ...source.bind, window]);
-            for (const row of found)
-                rows.push([Number(row.rowid), Number(row.score)]);
-            if (found.length === window)
-                boundaries.push(-Number(found.at(-1).score));
-        }
-        rows.sort((left, right) => left[1] - right[1] || left[0] - right[0]);
-        const best = new Map();
-        for (const [row, score] of rows) {
-            const id = Math.floor(row / CHUNKS);
-            if (!best.has(id))
-                best.set(id, { chunk: row % CHUNKS, score: -score });
-        }
-        const ties = listed.ties([...best.keys()]);
-        const kept = [...best]
-            .flatMap(([id, held]) => {
-            const tie = ties.get(id);
-            return tie === undefined ? [] : [{ id, ...held, tie }];
-        })
-            .sort(byRank)
-            .filter((hit) => beyond(hit, cursor))
-            .slice(0, needed);
-        const last = kept.at(-1);
-        if (boundaries.length === 0)
-            return kept;
-        if (kept.length === needed &&
-            last !== undefined &&
-            boundaries.every((boundary) => boundary < last.score)) {
-            return kept;
-        }
-        window *= 4;
-    }
-}
-function reaches(connection, listed, expression, reach) {
-    const { table, scope } = listed;
-    const [counted] = connection.rows(`SELECT count(*) AS held FROM (SELECT rowid FROM ${table} WHERE ${table} MATCH ? AND ${scope.sql} LIMIT ?)`, [expression, ...scope.bind, reach + 1]);
-    return Number(counted?.held ?? 0) > reach;
-}
-function candidates(connection, listed, clauses, cursor, needed) {
-    const { table, scope } = listed;
-    const expression = matchExpression(clauses);
-    const within = { sql: `AND ${scope.sql}`, bind: scope.bind };
-    let reach = CANDIDATES;
-    for (;;) {
-        const [edge] = connection.rows(`SELECT rowid FROM ${table} WHERE ${table} MATCH ? ${within.sql} ORDER BY rowid LIMIT 1 OFFSET ?`, [expression, ...within.bind, reach]);
-        const floor = edge === undefined ? null : Number(edge.rowid);
-        const sources = [
-            floor === null
-                ? { expression, ...within }
-                : { expression, sql: `${within.sql} AND rowid < ?`, bind: [...within.bind, floor] },
-        ];
-        if (floor !== null) {
-            const common = new Set();
-            const selective = [];
-            const frequent = [];
-            for (const clause of clauses) {
-                if (clause.kind === 'phrase')
-                    continue;
-                if (clause.kind !== 'term' && common.has(clause.term)) {
-                    frequent.push(clause);
-                    continue;
-                }
-                if (reaches(connection, listed, clause.text, reach)) {
-                    if (clause.kind === 'term')
-                        common.add(clause.term);
-                    frequent.push(clause);
-                }
-                else
-                    selective.push(clause);
-            }
-            const phrase = clauses.find((clause) => clause.kind === 'phrase');
-            if (phrase !== undefined) {
-                const rare = !clauses
-                    .filter((clause) => clause.kind === 'term')
-                    .every((clause) => common.has(clause.term)) ||
-                    !reaches(connection, listed, phrase.text, reach);
-                (rare ? selective : frequent).push(phrase);
-            }
-            if (selective.length > 0) {
-                const chosen = `(${matchExpression(selective)})`;
-                const tail = { sql: `${within.sql} AND rowid >= ?`, bind: [...within.bind, floor] };
-                if (frequent.length === 0)
-                    sources.push({ ...tail, expression: chosen });
-                else {
-                    const rest = `(${matchExpression(frequent)})`;
-                    sources.push({ ...tail, expression: `${chosen} AND ${rest}` }, { ...tail, expression: `${chosen} NOT ${rest}` });
-                }
-            }
-        }
-        const kept = rankedWithin(connection, listed, sources, cursor, needed);
-        if (floor === null || kept.length >= needed)
-            return kept;
-        reach *= 4;
-    }
-}
-function windowOf(connection, listed, clauses, cursor, limit) {
-    if (cursor === null || listed === null || clauses === null)
-        return { kept: [], cursor: null };
-    const found = candidates(connection, listed, clauses, cursor, limit + 1);
-    if (found.length <= limit)
-        return { kept: found, cursor: null };
-    const kept = found.slice(0, limit);
-    const last = kept.at(-1);
-    return { kept, cursor: last === undefined ? cursor : [last.score, ...last.tie] };
-}
-function totalOf(connection, listed, clauses) {
-    if (listed === null || clauses === null)
-        return 0;
-    const { table, scope } = listed;
-    const [counted] = connection.rows(`SELECT count(DISTINCT rowid / ${CHUNKS}) AS held FROM ${table} WHERE ${table} MATCH ? AND ${scope.sql}`, [matchExpression(clauses), ...scope.bind]);
-    return Number(counted?.held ?? 0);
-}
-function ids(found) {
-    return JSON.stringify(found);
-}
-function documentsListed(connection, documents) {
-    if (documents.rooms.length === 0)
-        return null;
-    const filter = selected(connection, documents.plan, documents.rooms);
-    const outside = {
-        sql: "NOT EXISTS (SELECT 1 FROM json_each(?) r WHERE d.key = r.value ->> 0 AND (d.path = r.value ->> 1 OR (d.path >= (r.value ->> 1) || '/' AND d.path < (r.value ->> 1) || '0')))",
-        bind: [
-            JSON.stringify(documents.beneath.flatMap(({ key, roots }) => roots.map((root) => [key, root]))),
-        ],
-    };
-    const conditions = [filter, outside];
-    const locators = new Map(documents.rooms.map(({ key, locator }) => [key, locator]));
-    return {
-        table: DOCUMENTS,
-        scope: {
-            sql: `EXISTS (SELECT 1 FROM replica_entries d WHERE d.id = ${DOCUMENTS}.rowid / ${CHUNKS} AND d.type IS NOT NULL AND ${conditions.map(({ sql }) => `(${sql})`).join(' AND ')})`,
-            bind: conditions.flatMap(({ bind }) => bind),
-        },
-        ties(found) {
-            const tied = new Map();
-            for (const row of connection.rows('SELECT id, key, path FROM replica_entries WHERE id IN (SELECT value FROM json_each(?))', [ids(found)])) {
-                const locator = locators.get(Number(row.key));
-                if (locator !== undefined)
-                    tied.set(Number(row.id), [locator, String(row.path)]);
-            }
-            return tied;
-        },
-    };
-}
-function messagesListed(connection, conversations) {
-    if (conversations.length === 0)
-        return null;
-    return {
-        table: MESSAGES,
-        scope: {
-            sql: `EXISTS (SELECT 1 FROM conversation_messages m WHERE m.id = ${MESSAGES}.rowid / ${CHUNKS} AND m.conversation IN (SELECT value FROM json_each(?)))`,
-            bind: [JSON.stringify(conversations)],
-        },
-        ties(found) {
-            return new Map(connection
-                .rows('SELECT id, conversation, identity FROM conversation_messages WHERE id IN (SELECT value FROM json_each(?))', [ids(found)])
-                .map((row) => [Number(row.id), [String(row.conversation), String(row.identity)]]));
-        },
-    };
-}
-function cut(candidates, terms) {
-    for (const [part, text] of candidates) {
-        const found = snippet(text, terms, SNIPPET_REACH);
-        if (found !== null)
-            return { snippet: { part, text: found.text, at: [...found.at] } };
-    }
-    return {};
-}
-function documentHit(connection, hit, terms) {
-    const [row] = connection.rows('SELECT type, content FROM replica_entries WHERE id = ?', [hit.id]);
-    if (row === undefined)
-        return null;
-    const [room, path] = hit.tie;
-    const parts = typeof row.content === 'string' ? partsOf(path, row.content) : null;
-    const whole = parts === null
-        ? []
-        : [
-            ['fields', parts.fields],
-            ['sections', parts.sections],
-            ['preamble', parts.preamble],
-            ['path', parts.path],
-        ];
-    const matches = {};
-    for (const [part, text] of whole) {
-        for (const term of matched(text, terms))
-            (matches[term] ??= []).push(part);
-    }
-    const sections = parts === null ? [] : chunked(parts.sections);
-    const chunk = parts === null
-        ? []
-        : hit.chunk === 0
-            ? [
-                ['fields', parts.fields],
-                ['sections', sections[0] ?? ''],
-                ['preamble', parts.preamble],
-                ['path', parts.path],
-            ]
-            : [['sections', sections[hit.chunk] ?? '']];
-    return {
-        room,
-        path,
-        type: String(row.type),
-        score: hit.score,
-        matches,
-        ...cut(chunk, terms),
-    };
-}
-function messageHit(connection, hit, terms) {
-    const [row] = connection.rows('SELECT text FROM conversation_messages WHERE id = ?', [hit.id]);
-    const text = typeof row?.text === 'string' ? row.text : '';
-    const matches = {};
-    for (const term of matched(text, terms))
-        matches[term] = ['text'];
-    const [conversation, identity] = hit.tie;
-    return {
-        conversation,
-        identity,
-        score: hit.score,
-        matches,
-        ...cut([['text', chunked(text)[hit.chunk] ?? '']], terms),
-    };
-}
-export function searched(connection, ask) {
-    const clauses = matchClauses(ask.text);
-    const terms = queryTerms(ask.text);
-    const documents = ask.documents === null ? null : documentsListed(connection, ask.documents);
-    const messages = messagesListed(connection, ask.conversations);
-    const [documentsAfter, messagesAfter] = ask.after ?? [[], []];
-    const documentWindow = windowOf(connection, documents, clauses, documentsAfter, ask.limit);
-    const messageWindow = windowOf(connection, messages, clauses, messagesAfter, ask.limit);
-    const more = documentWindow.cursor !== null || messageWindow.cursor !== null;
-    return {
-        documents: documentWindow.kept.flatMap((hit) => documentHit(connection, hit, terms) ?? []),
-        messages: messageWindow.kept.map((hit) => messageHit(connection, hit, terms)),
-        ...(more ? { next: JSON.stringify([documentWindow.cursor, messageWindow.cursor]) } : {}),
-        ...(ask.total
-            ? {
-                total: {
-                    documents: totalOf(connection, documents, clauses),
-                    messages: totalOf(connection, messages, clauses),
-                },
-            }
-            : {}),
-    };
-}

package/dist/words.d.ts

@@ -1,19 +0,0 @@
-export type Clause = {
-    readonly text: string;
-    readonly kind: 'term' | 'stem' | 'prefix' | 'phrase';
-    readonly term: string;
-};
-export type Located = {
-    readonly start: number;
-    readonly end: number;
-};
-export declare function indexed(text: string): string;
-export declare function matchClauses(text: string): Clause[] | null;
-export declare function matchExpression(clauses: readonly Clause[]): string;
-export declare function queryTerms(text: string): string[];
-export declare function snippet(part: string, terms: readonly string[], reach: number): {
-    readonly text: string;
-    readonly at: readonly [number, number];
-} | null;
-export declare function matched(part: string, terms: readonly string[]): string[];
-//# sourceMappingURL=words.d.ts.map

package/dist/words.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"words.d.ts","sourceRoot":"","sources":["../src/words.ts"],"names":[],"mappings":"AA0BA,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,OAAO,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAYvE,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO5C;AAgBD,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,CAiB1D;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAElE;AAMD,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAEjD;AAyDD,wBAAgB,OAAO,CACrB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,KAAK,EAAE,MAAM,GACZ;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAAG,IAAI,CAW1E;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAQxE"}

package/dist/words.js

@@ -1,156 +0,0 @@
-const SPACELESS = /(?:[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}\p{Script=Hangul}]\p{M}*)+/gu;
-const SPACELESS_TERM = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}\p{Script=Hangul}]/u;
-const TOKEN = /(?![\x00-\x7f])\p{S}\p{M}*|[\p{L}\p{N}\p{M}\p{Co}]+/gu;
-const SYMBOL = /(?![\x00-\x7f])\p{S}\p{M}*/gu;
-const MARKED = /([\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}])\p{M}+/gu;
-const INFLECTED = /^[\p{Script=Cyrillic}\p{Script=Greek}]+$/u;
-const FOLD = /\p{M}+/gu;
-const ASCII = /^[\x00-\x7f]$/;
-const PREFIXED_LETTERS = 2;
-const INFLECTED_LETTERS = 4;
-const PHRASE_TERMS = { least: 2, most: 6 };
-const GRAPHEMES = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
-function bigrams(run) {
-    const characters = Array.from(run);
-    if (characters.length < 2)
-        return ` ${run} `;
-    const pairs = [];
-    for (let index = 0; index + 1 < characters.length; index += 1) {
-        pairs.push(characters[index] + characters[index + 1]);
-    }
-    return ` ${pairs.join(' ')} `;
-}
-export function indexed(text) {
-    return text
-        .normalize('NFKD')
-        .replace(MARKED, '$1')
-        .normalize('NFKC')
-        .replace(SPACELESS, bigrams)
-        .replace(SYMBOL, (symbol) => ` ${symbol.replace(FOLD, '')} `);
-}
-function tokens(text) {
-    return [...text.matchAll(TOKEN)].map(([token]) => token.toLowerCase());
-}
-function quoted(token) {
-    return `"${token.replaceAll('"', '""')}"`;
-}
-function truncated(term) {
-    const letters = Array.from(term);
-    if (!INFLECTED.test(term) || letters.length < INFLECTED_LETTERS)
-        return null;
-    return letters.slice(0, letters.length - (letters.length >= 6 ? 2 : 1)).join('');
-}
-export function matchClauses(text) {
-    const terms = tokens(indexed(text));
-    if (terms.length === 0)
-        return null;
-    const unique = [...new Set(terms)];
-    const clauses = unique.map((term) => ({ text: quoted(term), kind: 'term', term }));
-    for (const term of unique) {
-        const stem = truncated(term);
-        if (stem !== null)
-            clauses.push({ text: `${quoted(stem)}*`, kind: 'stem', term });
-    }
-    const last = terms[terms.length - 1];
-    if (!/\s$/u.test(text) && Array.from(last).length >= PREFIXED_LETTERS) {
-        clauses.push({ text: `${quoted(last)}*`, kind: 'prefix', term: last });
-    }
-    if (terms.length >= PHRASE_TERMS.least && terms.length <= PHRASE_TERMS.most) {
-        clauses.push({ text: quoted(terms.join(' ')), kind: 'phrase', term: terms.join(' ') });
-    }
-    return clauses;
-}
-export function matchExpression(clauses) {
-    return clauses.map(({ text }) => text).join(' OR ');
-}
-function folded(text) {
-    return text.normalize('NFKD').replace(FOLD, '').toLowerCase();
-}
-export function queryTerms(text) {
-    return [...new Set(tokens(text.normalize('NFKC')).map(folded))];
-}
-function agrees(token, term) {
-    if (SPACELESS_TERM.test(term))
-        return token.includes(term);
-    if (token.startsWith(term))
-        return true;
-    return token.length >= 3 && term.startsWith(token.slice(0, Math.max(3, token.length - 2)));
-}
-function inside(token, offset, term) {
-    const pieces = [...GRAPHEMES.segment(token)];
-    const owner = [];
-    let joined = '';
-    for (const [at, { segment }] of pieces.entries()) {
-        const part = folded(segment);
-        joined += part;
-        for (let index = 0; index < part.length; index += 1)
-            owner.push(at);
-    }
-    const found = joined.indexOf(term);
-    if (found < 0)
-        return null;
-    const first = pieces[owner[found]];
-    const last = pieces[owner[found + term.length - 1]];
-    return { start: offset + first.index, end: offset + last.index + last.segment.length };
-}
-function locate(part, terms) {
-    for (const match of part.matchAll(TOKEN)) {
-        const token = folded(match[0]);
-        for (const term of terms) {
-            if (!agrees(token, term))
-                continue;
-            if (!SPACELESS_TERM.test(term)) {
-                return { start: match.index, end: match.index + match[0].length };
-            }
-            const found = inside(match[0], match.index, term);
-            if (found !== null)
-                return found;
-        }
-    }
-    return null;
-}
-function settled(part, index) {
-    if (index <= 0 || index >= part.length)
-        return true;
-    const before = part[index - 1];
-    const after = part[index];
-    if (before === '\n')
-        return true;
-    return ASCII.test(before) && ASCII.test(after) && !(before === '\r' && after === '\n');
-}
-function edgesAround(part, low, high) {
-    let from = Math.max(0, low);
-    while (!settled(part, from))
-        from -= 1;
-    let to = Math.min(part.length, high);
-    while (!settled(part, to))
-        to += 1;
-    const edges = [...GRAPHEMES.segment(part.slice(from, to))].map(({ index }) => from + index);
-    edges.push(to);
-    return edges;
-}
-export function snippet(part, terms, reach) {
-    const located = locate(part, terms);
-    if (located === null)
-        return null;
-    const edges = edgesAround(part, located.start - 2 * reach, located.end + 2 * reach);
-    const last = (limit) => edges.filter((edge) => edge <= limit).at(-1) ?? edges[0];
-    const first = (limit) => edges.find((edge) => edge >= limit) ?? edges[edges.length - 1];
-    const from = last(located.start);
-    const to = Math.max(first(Math.min(located.end, from + reach)), first(from + 1));
-    const start = first(Math.max(0, from - reach));
-    const end = last(Math.min(part.length, to + reach));
-    return { text: part.slice(start, end), at: [from - start, to - start] };
-}
-export function matched(part, terms) {
-    const found = new Set();
-    for (const match of part.matchAll(TOKEN)) {
-        const token = folded(match[0]);
-        for (const term of terms)
-            if (agrees(token, term))
-                found.add(term);
-        if (found.size === terms.length)
-            break;
-    }
-    return [...found];
-}