@byline/search-analysis 4.9.0

package/dist/highlight.js

@@ -0,0 +1,161 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { encodeSqlToken } from './sql-token-codec.js';
+const DEFAULT_MAX_FRAGMENTS = 2;
+const DEFAULT_MAX_WORDS = 24;
+/**
+ * Build a backend-neutral matched snippet from portable logical-token offsets.
+ *
+ * The returned string contains `<mark>…</mark>` delimiters. Consumers must
+ * parse those delimiters and render the surrounding source text as text; they
+ * must not inject the complete result as trusted HTML.
+ */
+export function highlightPortableText({ text, plan, analyzer, maxFragments = DEFAULT_MAX_FRAGMENTS, maxWords = DEFAULT_MAX_WORDS, }) {
+    assertPositiveInteger(maxFragments, 'maxFragments');
+    assertPositiveInteger(maxWords, 'maxWords');
+    if (text.length === 0 || plan.concepts.length === 0)
+        return undefined;
+    if (plan.analyzerFingerprint !== analyzer.fingerprint) {
+        throw new TypeError('Portable highlight plan and analyzer fingerprints do not match');
+    }
+    const queryTerms = new Set(queryTokens(plan).map((token) => encodeSqlToken(token)));
+    const analyzed = analyzer.analyzeText({ text, locale: plan.locale });
+    const matches = mergeRanges(analyzed.tokens
+        .filter((token) => queryTerms.has(encodeSqlToken(token)))
+        .map(({ start, end }) => ({ start, end })));
+    if (matches.length === 0)
+        return undefined;
+    const sourceTerms = logicalSourceRanges([...analyzed.exactTokens, ...analyzed.identifierTokens]);
+    const fragments = selectFragments(text, matches, sourceTerms, maxFragments, maxWords);
+    if (fragments.length === 0)
+        return undefined;
+    const rendered = fragments.map((fragment) => renderFragment(text, fragment, matches));
+    let snippet = rendered.join(' … ');
+    if (text.slice(0, fragments[0]?.start ?? 0).trim().length > 0)
+        snippet = `… ${snippet}`;
+    if (text.slice(fragments.at(-1)?.end ?? text.length).trim().length > 0)
+        snippet += ' …';
+    return snippet;
+}
+function logicalSourceRanges(tokens) {
+    const byPosition = new Map();
+    for (const token of tokens.toSorted(compareTokenRanges)) {
+        const current = byPosition.get(token.position);
+        if (current == null) {
+            byPosition.set(token.position, { start: token.start, end: token.end });
+            continue;
+        }
+        current.start = Math.min(current.start, token.start);
+        current.end = Math.max(current.end, token.end);
+    }
+    return [...byPosition.values()].toSorted(compareRanges);
+}
+function queryTokens(plan) {
+    return [...plan.concepts.flatMap(conceptTokens), ...plan.gramSequences.flat()];
+}
+function conceptTokens(concept) {
+    return [
+        ...concept.exactTokens,
+        ...concept.stemTokens,
+        ...concept.lemmaTokens,
+        ...concept.normalizedTokens,
+        ...concept.identifierTokens,
+        ...concept.gramTokens,
+    ];
+}
+function selectFragments(text, matches, sourceTerms, maxFragments, maxWords) {
+    const candidates = uniqueRanges(matches.map((match) => fragmentAroundMatch(text, match, sourceTerms, maxWords)))
+        .map((range) => ({
+        ...range,
+        matches: matches.filter((match) => overlaps(range, match)).length,
+    }))
+        .toSorted((left, right) => right.matches - left.matches || left.start - right.start);
+    const selected = [];
+    for (const candidate of candidates) {
+        if (selected.some((fragment) => overlaps(fragment, candidate)))
+            continue;
+        selected.push(candidate);
+        if (selected.length === maxFragments)
+            break;
+    }
+    return selected.toSorted((left, right) => left.start - right.start);
+}
+function fragmentAroundMatch(text, match, sourceTerms, maxWords) {
+    const firstMatch = sourceTerms.findIndex((term) => overlaps(term, match));
+    if (firstMatch === -1)
+        return match;
+    let lastMatch = firstMatch;
+    while (lastMatch + 1 < sourceTerms.length && overlaps(sourceTerms[lastMatch + 1], match)) {
+        lastMatch += 1;
+    }
+    const matchedWords = lastMatch - firstMatch + 1;
+    const contextWords = Math.max(0, maxWords - matchedWords);
+    let first = Math.max(0, firstMatch - Math.floor(contextWords / 2));
+    let last = Math.min(sourceTerms.length - 1, lastMatch + Math.ceil(contextWords / 2));
+    const currentWords = last - first + 1;
+    if (currentWords < maxWords) {
+        const missing = maxWords - currentWords;
+        const extendLeft = Math.min(first, missing);
+        first -= extendLeft;
+        last = Math.min(sourceTerms.length - 1, last + missing - extendLeft);
+    }
+    return {
+        start: first === 0 ? 0 : (sourceTerms[first]?.start ?? match.start),
+        end: last === sourceTerms.length - 1 ? text.length : (sourceTerms[last]?.end ?? match.end),
+    };
+}
+function renderFragment(text, fragment, matches) {
+    const included = matches
+        .filter((match) => overlaps(fragment, match))
+        .map((match) => ({
+        start: Math.max(fragment.start, match.start),
+        end: Math.min(fragment.end, match.end),
+    }));
+    let cursor = fragment.start;
+    let rendered = '';
+    for (const match of included) {
+        rendered += text.slice(cursor, match.start);
+        rendered += `<mark>${text.slice(match.start, match.end)}</mark>`;
+        cursor = match.end;
+    }
+    rendered += text.slice(cursor, fragment.end);
+    return rendered.trim();
+}
+function mergeRanges(ranges) {
+    const merged = [];
+    for (const range of uniqueRanges(ranges)) {
+        const previous = merged.at(-1);
+        if (previous == null || range.start > previous.end) {
+            merged.push({ ...range });
+            continue;
+        }
+        previous.end = Math.max(previous.end, range.end);
+    }
+    return merged;
+}
+function uniqueRanges(ranges) {
+    const unique = new Map();
+    for (const range of ranges.toSorted(compareRanges)) {
+        unique.set(`${range.start}:${range.end}`, range);
+    }
+    return [...unique.values()];
+}
+function overlaps(left, right) {
+    return left.start < right.end && right.start < left.end;
+}
+function compareTokenRanges(left, right) {
+    return left.start - right.start || left.end - right.end;
+}
+function compareRanges(left, right) {
+    return left.start - right.start || left.end - right.end;
+}
+function assertPositiveInteger(value, name) {
+    if (!Number.isSafeInteger(value) || value < 1) {
+        throw new RangeError(`Portable highlight ${name} must be a positive safe integer`);
+    }
+}

package/dist/highlight.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/highlight.test.node.js

@@ -0,0 +1,102 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { describe, expect, it } from 'vitest';
+import { createPortableSearchAnalyzer } from './analyzer.js';
+import { highlightPortableText } from './highlight.js';
+describe('portable highlighting', () => {
+    it('preserves original text while marking normalized terms and protected identifiers', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const plan = analyzer.analyzeQuery({
+            query: 'alpha ffi Node.js',
+            locale: 'en',
+        });
+        expect(highlightPortableText({
+            text: 'Prelude words before Alpha compatibility ffi and Node.js finish.',
+            plan,
+            analyzer,
+        })).toBe('Prelude words before <mark>Alpha</mark> compatibility <mark>ffi</mark> and <mark>Node.js</mark> finish.');
+    });
+    it('marks source terms that matched through a language expansion', () => {
+        const expander = {
+            fingerprint: 'english-highlight-test1',
+            supports: (locale) => locale.startsWith('en'),
+            expand: (token) => token.value === 'running' || token.value === 'runs' ? [{ kind: 'stem', value: 'run' }] : [],
+        };
+        const analyzer = createPortableSearchAnalyzer({ expanders: [expander] });
+        const plan = analyzer.analyzeQuery({ query: 'runs', locale: 'en' });
+        expect(highlightPortableText({
+            text: 'Running through the forest.',
+            plan,
+            analyzer,
+        })).toBe('<mark>Running</mark> through the forest.');
+    });
+    it('marks only concepts present in a partial any-term match', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const plan = analyzer.analyzeQuery({
+            query: 'forest missing',
+            matching: { operator: 'any' },
+        });
+        expect(highlightPortableText({
+            text: 'Forest restoration field notes.',
+            plan,
+            analyzer,
+        })).toBe('<mark>Forest</mark> restoration field notes.');
+    });
+    it('merges overlapping Han token and gram ranges', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const plan = analyzer.analyzeQuery({ query: '数据库' });
+        expect(highlightPortableText({ text: '研究数据库系统', plan, analyzer })).toContain('<mark>数据库</mark>');
+    });
+    it('selects bounded fragments around distant matches', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const words = Array.from({ length: 60 }, (_, index) => `word${index}`);
+        words[2] = 'alpha';
+        words[55] = 'beta';
+        const plan = analyzer.analyzeQuery({
+            query: 'alpha beta',
+            matching: { operator: 'any' },
+        });
+        const highlighted = highlightPortableText({
+            text: words.join(' '),
+            plan,
+            analyzer,
+            maxFragments: 2,
+            maxWords: 8,
+        });
+        expect(highlighted).toContain('<mark>alpha</mark>');
+        expect(highlighted).toContain('<mark>beta</mark>');
+        expect(highlighted).toContain(' … ');
+        expect(highlighted?.split(/\s+/)).toHaveLength(17);
+    });
+    it('counts overlapping identifier constituents as one snippet term', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const plan = analyzer.analyzeQuery({ query: 'covid' });
+        expect(highlightPortableText({
+            text: 'COVID-19 one two three',
+            plan,
+            analyzer,
+            maxFragments: 1,
+            maxWords: 2,
+        })).toContain('one');
+    });
+    it('returns undefined when the stored text contains no query term', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const plan = analyzer.analyzeQuery({ query: 'forest' });
+        expect(highlightPortableText({ text: 'ocean', plan, analyzer })).toBeUndefined();
+    });
+    it('rejects incompatible analyzers and invalid limits', () => {
+        const analyzer = createPortableSearchAnalyzer({ defaultLocale: 'en' });
+        const plan = analyzer.analyzeQuery({ query: 'forest' });
+        expect(() => highlightPortableText({
+            text: 'forest',
+            plan,
+            analyzer: createPortableSearchAnalyzer({ defaultLocale: 'th' }),
+        })).toThrow(/fingerprints/);
+        expect(() => highlightPortableText({ text: 'forest', plan, analyzer, maxFragments: 0 })).toThrow(RangeError);
+    });
+});

package/dist/identifiers.d.ts

@@ -0,0 +1,23 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import type { SearchIdentifierKind } from './types.js';
+export interface IdentifierSpan {
+    kind: SearchIdentifierKind;
+    value: string;
+    start: number;
+    end: number;
+    /** Whether word segmentation should retain exact constituent terms. */
+    preserveConstituents: boolean;
+}
+/**
+ * Extract non-overlapping domain identifiers in rule-priority order. The input
+ * is already search-normalized, so every emitted value is canonical.
+ */
+export declare function extractIdentifierSpans(text: string): IdentifierSpan[];
+/** Replace identifier spans with equal-length spaces so word offsets survive. */
+export declare function maskIdentifierSpans(text: string, spans: readonly IdentifierSpan[]): string;

package/dist/identifiers.js

@@ -0,0 +1,102 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+const RULES = [
+    {
+        kind: 'url',
+        pattern: /https?:\/\/[^\s<>"']+/giu,
+        value: trimUrlPunctuation,
+    },
+    {
+        kind: 'email',
+        pattern: /[\p{L}\p{N}._%+-]{1,64}@[\p{L}\p{N}.-]{1,255}\.[\p{L}]{2,63}/giu,
+    },
+    {
+        kind: 'technical',
+        pattern: /(?:^|(?<=[^\p{L}\p{N}_]))(?:c\+\+|c#|node\.js)(?=$|[^\p{L}\p{N}_])/giu,
+    },
+    {
+        kind: 'sku',
+        pattern: /\b[\p{L}]{2,}[-_]\d+\b/giu,
+        preserveConstituents: true,
+    },
+    {
+        kind: 'version',
+        pattern: /\bv?\d+(?:\.\d+){1,3}(?:-[\p{L}\p{N}.-]+)?\b/giu,
+        preserveConstituents: true,
+    },
+    {
+        kind: 'mention',
+        pattern: /@[\p{L}\p{M}\p{N}_]+/gu,
+    },
+    {
+        kind: 'hashtag',
+        pattern: /#[\p{L}\p{M}\p{N}_]+/gu,
+    },
+];
+const TRAILING_URL_PUNCTUATION = /[.,;:!?]+$/u;
+const TRAILING_URL_BRACKETS = /[)\]}]+$/u;
+function trimUrlPunctuation(value) {
+    let trimmed = value.replace(TRAILING_URL_PUNCTUATION, '');
+    while (TRAILING_URL_BRACKETS.test(trimmed)) {
+        const final = trimmed.at(-1);
+        const opening = final === ')' ? '(' : final === ']' ? '[' : '{';
+        const closing = final;
+        const openingCount = [...trimmed].filter((char) => char === opening).length;
+        const closingCount = [...trimmed].filter((char) => char === closing).length;
+        if (closingCount <= openingCount)
+            break;
+        trimmed = trimmed.slice(0, -1);
+    }
+    return trimmed;
+}
+/**
+ * Extract non-overlapping domain identifiers in rule-priority order. The input
+ * is already search-normalized, so every emitted value is canonical.
+ */
+export function extractIdentifierSpans(text) {
+    const spans = [];
+    for (const rule of RULES) {
+        rule.pattern.lastIndex = 0;
+        for (const match of text.matchAll(rule.pattern)) {
+            const matched = match[0];
+            const matchStart = match.index;
+            if (matched == null || matchStart == null)
+                continue;
+            const value = rule.value?.(matched) ?? matched;
+            if (value.length === 0)
+                continue;
+            const start = matchStart;
+            const end = start + value.length;
+            if (spans.some((span) => start < span.end && end > span.start))
+                continue;
+            spans.push({
+                kind: rule.kind,
+                value,
+                start,
+                end,
+                preserveConstituents: rule.preserveConstituents ?? false,
+            });
+        }
+    }
+    return spans.sort((a, b) => a.start - b.start || b.end - a.end);
+}
+/** Replace identifier spans with equal-length spaces so word offsets survive. */
+export function maskIdentifierSpans(text, spans) {
+    if (spans.length === 0)
+        return text;
+    const characters = text.split('');
+    for (const span of spans) {
+        if (span.preserveConstituents)
+            continue;
+        for (let index = span.start; index < span.end; index++) {
+            if (characters[index] !== '\n')
+                characters[index] = ' ';
+        }
+    }
+    return characters.join('');
+}

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export { createPortableSearchAnalyzer, resolveMatching } from './analyzer.js';
+export { type HighlightPortableTextInput, highlightPortableText, type PortableHighlightOptions, } from './highlight.js';
+export { canonicalSegmenterLocale, detectSearchLocale, resolveSearchLocale, } from './locale.js';
+export { normalizeForSearch, type SearchNormalization } from './normalize.js';
+export { encodeSqlToken, encodeSqlTokens, type SqlTokenCodecOptions, } from './sql-token-codec.js';
+export type { AnalyzedText, AnalyzeQueryInput, AnalyzeTextInput, LogicalToken, LogicalTokenKind, PortableQueryPlan, PortableSearchAnalyzer, PortableSearchAnalyzerOptions, QueryConcept, QueryPhrase, ResolvedSearchMatching, SearchIdentifierKind, SearchTokenExpander, SearchTokenExpansion, } from './types.js';

package/dist/index.js

@@ -0,0 +1,12 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export { createPortableSearchAnalyzer, resolveMatching } from './analyzer.js';
+export { highlightPortableText, } from './highlight.js';
+export { canonicalSegmenterLocale, detectSearchLocale, resolveSearchLocale, } from './locale.js';
+export { normalizeForSearch } from './normalize.js';
+export { encodeSqlToken, encodeSqlTokens, } from './sql-token-codec.js';

package/dist/locale.d.ts

@@ -0,0 +1,16 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+/** Return one canonical locale when `Intl.Segmenter` supports the candidate. */
+export declare function canonicalSegmenterLocale(candidate: string | undefined): string | undefined;
+/** Script-based locale fallback for content without a usable declaration. */
+export declare function detectSearchLocale(text: string, hanLocale?: 'zh' | 'ja'): string | undefined;
+/**
+ * Declared locale wins, followed by script detection and the configured
+ * platform fallback. Every returned locale is canonical and Segmenter-safe.
+ */
+export declare function resolveSearchLocale(text: string, declaredLocale: string | undefined, defaultLocale?: string, hanLocale?: 'zh' | 'ja'): string;

package/dist/locale.js

@@ -0,0 +1,52 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+const SCRIPT_LOCALES = [
+    [/\p{Script=Thai}/u, 'th'],
+    [/\p{Script=Hiragana}|\p{Script=Katakana}/u, 'ja'],
+    [/\p{Script=Hangul}/u, 'ko'],
+    [/\p{Script=Lao}/u, 'lo'],
+    [/\p{Script=Khmer}/u, 'km'],
+    [/\p{Script=Myanmar}/u, 'my'],
+];
+/** Return one canonical locale when `Intl.Segmenter` supports the candidate. */
+export function canonicalSegmenterLocale(candidate) {
+    if (!candidate)
+        return undefined;
+    try {
+        const canonical = Intl.getCanonicalLocales(candidate)[0];
+        if (canonical == null)
+            return undefined;
+        return Intl.Segmenter.supportedLocalesOf([canonical]).length > 0 ? canonical : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Script-based locale fallback for content without a usable declaration. */
+export function detectSearchLocale(text, hanLocale = 'zh') {
+    for (const [pattern, locale] of SCRIPT_LOCALES) {
+        if (pattern.test(text))
+            return locale;
+    }
+    if (/\p{Script=Han}/u.test(text))
+        return hanLocale;
+    return undefined;
+}
+/**
+ * Declared locale wins, followed by script detection and the configured
+ * platform fallback. Every returned locale is canonical and Segmenter-safe.
+ */
+export function resolveSearchLocale(text, declaredLocale, defaultLocale = 'en', hanLocale = 'zh') {
+    const declared = canonicalSegmenterLocale(declaredLocale);
+    if (declared != null)
+        return declared;
+    const detected = canonicalSegmenterLocale(detectSearchLocale(text, hanLocale));
+    if (detected != null)
+        return detected;
+    return canonicalSegmenterLocale(defaultLocale) ?? 'en';
+}

package/dist/normalize.d.ts

@@ -0,0 +1,23 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+interface OriginalRange {
+    start: number;
+    end: number;
+}
+export interface SearchNormalization {
+    value: string;
+    originalRange(normalizedStart: number, normalizedEnd: number): OriginalRange;
+}
+/**
+ * Search-only NFKC + locale-insensitive lowercase normalization with a compact
+ * normalized-to-original offset map. Grapheme-grain processing keeps combining
+ * sequences together while mapping compatibility expansions (for example `ffi`
+ * → `ffi`) back to their original span.
+ */
+export declare function normalizeForSearch(original: string): SearchNormalization;
+export {};

package/dist/normalize.js

@@ -0,0 +1,50 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
+/**
+ * Search-only NFKC + locale-insensitive lowercase normalization with a compact
+ * normalized-to-original offset map. Grapheme-grain processing keeps combining
+ * sequences together while mapping compatibility expansions (for example `ffi`
+ * → `ffi`) back to their original span.
+ */
+export function normalizeForSearch(original) {
+    const normalizedParts = [];
+    const originalStarts = [];
+    const originalEnds = [];
+    const segments = [...graphemeSegmenter.segment(original)];
+    for (let index = 0; index < segments.length; index++) {
+        const segment = segments[index];
+        if (segment == null)
+            continue;
+        const originalStart = segment.index;
+        const originalEnd = segments[index + 1]?.index ?? original.length;
+        const normalized = segment.segment.normalize('NFKC').toLowerCase();
+        normalizedParts.push(normalized);
+        for (let offset = 0; offset < normalized.length; offset++) {
+            originalStarts.push(originalStart);
+            originalEnds.push(originalEnd);
+        }
+    }
+    const value = normalizedParts.join('');
+    return {
+        value,
+        originalRange(normalizedStart, normalizedEnd) {
+            if (normalizedStart < 0 || normalizedEnd < normalizedStart || normalizedEnd > value.length) {
+                throw new RangeError(`Normalized range ${normalizedStart}..${normalizedEnd} is outside 0..${value.length}`);
+            }
+            if (normalizedStart === normalizedEnd) {
+                const boundary = originalStarts[normalizedStart] ?? originalEnds[normalizedStart - 1] ?? original.length;
+                return { start: boundary, end: boundary };
+            }
+            return {
+                start: originalStarts[normalizedStart] ?? original.length,
+                end: originalEnds[normalizedEnd - 1] ?? original.length,
+            };
+        },
+    };
+}

package/dist/sql-token-codec.d.ts

@@ -0,0 +1,22 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import type { LogicalToken } from './types.js';
+export interface SqlTokenCodecOptions {
+    /**
+     * Maximum physical term length before a SHA-256 representation is used.
+     * The default stays well below common SQL full-text parser limits.
+     */
+    maxLength?: number;
+}
+/**
+ * Encode a logical token as lowercase ASCII alphanumeric text. Kind prefixes
+ * prevent exact/stem/gram collisions and ensure even one-character source
+ * terms exceed MySQL's default minimum token length.
+ */
+export declare function encodeSqlToken(token: Pick<LogicalToken, 'kind' | 'value'>, options?: SqlTokenCodecOptions): string;
+export declare function encodeSqlTokens(tokens: readonly Pick<LogicalToken, 'kind' | 'value'>[], options?: SqlTokenCodecOptions): string[];

package/dist/sql-token-codec.js

@@ -0,0 +1,55 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { createHash } from 'node:crypto';
+const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567';
+const KIND_PREFIX = {
+    exact: 'ex',
+    stem: 'st',
+    lemma: 'le',
+    normalized: 'no',
+    identifier: 'id',
+    gram: 'gr',
+};
+/**
+ * Encode a logical token as lowercase ASCII alphanumeric text. Kind prefixes
+ * prevent exact/stem/gram collisions and ensure even one-character source
+ * terms exceed MySQL's default minimum token length.
+ */
+export function encodeSqlToken(token, options = {}) {
+    if (token.value.length === 0)
+        throw new TypeError('Cannot encode an empty search token');
+    const prefix = KIND_PREFIX[token.kind];
+    const encoded = `${prefix}${base32(new TextEncoder().encode(token.value))}`;
+    const maxLength = options.maxLength ?? 80;
+    if (!Number.isSafeInteger(maxLength) || maxLength < 16) {
+        throw new RangeError('SQL token maxLength must be a safe integer of at least 16');
+    }
+    if (encoded.length <= maxLength)
+        return encoded;
+    const digest = createHash('sha256').update(token.value, 'utf8').digest();
+    return `${prefix}h${base32(digest)}`.slice(0, maxLength);
+}
+export function encodeSqlTokens(tokens, options = {}) {
+    return tokens.map((token) => encodeSqlToken(token, options));
+}
+function base32(bytes) {
+    let value = 0;
+    let bits = 0;
+    let output = '';
+    for (const byte of bytes) {
+        value = (value << 8) | byte;
+        bits += 8;
+        while (bits >= 5) {
+            output += BASE32_ALPHABET[(value >>> (bits - 5)) & 31];
+            bits -= 5;
+        }
+    }
+    if (bits > 0)
+        output += BASE32_ALPHABET[(value << (5 - bits)) & 31];
+    return output;
+}

package/dist/sql-token-codec.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};