@byline/search-analysis 4.9.0

package/dist/analyzer.js

@@ -0,0 +1,310 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { MAX_SEARCH_QUERY_LENGTH } from '@byline/core';
+import { extractIdentifierSpans, maskIdentifierSpans } from './identifiers.js';
+import { canonicalSegmenterLocale, resolveSearchLocale } from './locale.js';
+import { normalizeForSearch } from './normalize.js';
+const PIPELINE_VERSION = 'portable1';
+const NORMALIZATION_VERSION = 'nfkc-lower1';
+const LOCALE_VERSION = 'locale1';
+const IDENTIFIER_VERSION = 'identifiers2';
+const GRAM_VERSION = 'han-bigram1';
+const wordSegmenters = new Map();
+function wordSegmenter(locale) {
+    let segmenter = wordSegmenters.get(locale);
+    if (segmenter == null) {
+        segmenter = new Intl.Segmenter(locale, { granularity: 'word' });
+        wordSegmenters.set(locale, segmenter);
+    }
+    return segmenter;
+}
+export function createPortableSearchAnalyzer(options = {}) {
+    return new PortableSearchAnalyzerImpl(options);
+}
+class PortableSearchAnalyzerImpl {
+    fingerprint;
+    defaultLocale;
+    hanLocale;
+    hanBigrams;
+    expanders;
+    constructor(options) {
+        this.defaultLocale = canonicalSegmenterLocale(options.defaultLocale) ?? 'en';
+        this.hanLocale = options.hanLocale ?? 'zh';
+        this.hanBigrams = options.hanBigrams ?? true;
+        this.expanders = options.expanders ?? [];
+        this.fingerprint = [
+            PIPELINE_VERSION,
+            NORMALIZATION_VERSION,
+            `icu${process.versions.icu ?? 'unknown'}`,
+            LOCALE_VERSION,
+            `default-${this.defaultLocale}`,
+            `han-${this.hanLocale}`,
+            IDENTIFIER_VERSION,
+            this.hanBigrams ? GRAM_VERSION : 'han-bigram0',
+            ...this.expanders.map((expander, index) => `expander${index}-${encodeURIComponent(expander.fingerprint)}`),
+        ].join('+');
+    }
+    analyzeText(input) {
+        const normalization = normalizeForSearch(input.text);
+        const locale = resolveSearchLocale(normalization.value, input.locale, this.defaultLocale, this.hanLocale);
+        const identifiers = extractIdentifierSpans(normalization.value);
+        const masked = maskIdentifierSpans(normalization.value, identifiers);
+        const sourceTokens = [];
+        for (const segment of wordSegmenter(locale).segment(masked)) {
+            if (!segment.isWordLike || segment.segment.length === 0)
+                continue;
+            sourceTokens.push(logicalToken('exact', segment.segment, segment.index, segment.index + segment.segment.length, sourceTokens.length, locale, normalization));
+        }
+        for (const identifier of identifiers) {
+            const token = logicalToken('identifier', identifier.value, identifier.start, identifier.end, 0, locale, normalization);
+            token.identifierKind = identifier.kind;
+            sourceTokens.push(token);
+        }
+        sourceTokens.sort((a, b) => a.normalizedStart - b.normalizedStart || a.normalizedEnd - b.normalizedEnd);
+        assignLogicalPositions(sourceTokens);
+        const derivedTokens = [];
+        for (const token of sourceTokens) {
+            if (token.kind !== 'exact')
+                continue;
+            for (const expander of this.expanders) {
+                if (!expander.supports(locale))
+                    continue;
+                for (const expansion of expander.expand(token)) {
+                    if (expansion.value.length === 0 || expansion.value === token.value)
+                        continue;
+                    derivedTokens.push({ ...token, kind: expansion.kind, value: expansion.value });
+                }
+            }
+        }
+        const gramTokens = this.hanBigrams
+            ? buildHanBigrams(masked, locale, normalization, sourceTokens)
+            : [];
+        const exactTokens = sourceTokens.filter((token) => token.kind === 'exact');
+        const identifierTokens = sourceTokens.filter((token) => token.kind === 'identifier');
+        return {
+            original: input.text,
+            normalized: normalization.value,
+            locale,
+            tokens: [...sourceTokens, ...derivedTokens, ...gramTokens],
+            exactTokens,
+            derivedTokens,
+            identifierTokens,
+            gramTokens,
+            analyzerFingerprint: this.fingerprint,
+        };
+    }
+    analyzeQuery(input) {
+        if (input.query.length > MAX_SEARCH_QUERY_LENGTH) {
+            throw new RangeError(`Search query exceeds the maximum length of ${MAX_SEARCH_QUERY_LENGTH} UTF-16 code units`);
+        }
+        const matching = resolveMatching(input.matching);
+        const full = this.analyzeText({ text: input.query, locale: input.locale });
+        const parts = splitQueryParts(input.query);
+        const concepts = [];
+        const phrases = [];
+        const gramSequences = [];
+        for (const part of parts) {
+            const analyzed = this.analyzeText({ text: part.text, locale: full.locale });
+            const normalizedBase = normalizeForSearch(input.query.slice(0, part.start)).value.length;
+            const partConceptIndexes = [];
+            const sourceGroups = groupTokensByPosition([
+                ...analyzed.exactTokens,
+                ...analyzed.identifierTokens,
+            ]);
+            for (const sourceGroup of sourceGroups) {
+                const conceptIndex = concepts.length;
+                const queryToken = (token) => rebaseQueryToken(token, part.start, normalizedBase, conceptIndex);
+                const identifiers = sourceGroup.filter((token) => token.kind === 'identifier');
+                const exact = identifiers.length === 0 ? sourceGroup : [];
+                const source = identifiers[0] ?? exact[0];
+                if (source == null)
+                    continue;
+                partConceptIndexes.push(conceptIndex);
+                concepts.push({
+                    index: conceptIndex,
+                    position: conceptIndex,
+                    exactTokens: exact.map(queryToken),
+                    stemTokens: identifiers.length === 0
+                        ? analyzed.derivedTokens
+                            .filter((token) => token.kind === 'stem' && token.position === source.position)
+                            .map(queryToken)
+                        : [],
+                    lemmaTokens: identifiers.length === 0
+                        ? analyzed.derivedTokens
+                            .filter((token) => token.kind === 'lemma' && token.position === source.position)
+                            .map(queryToken)
+                        : [],
+                    normalizedTokens: identifiers.length === 0
+                        ? analyzed.derivedTokens
+                            .filter((token) => token.kind === 'normalized' && token.position === source.position)
+                            .map(queryToken)
+                        : [],
+                    identifierTokens: identifiers.map(queryToken),
+                    gramTokens: analyzed.gramTokens
+                        .filter((token) => token.normalizedStart >= source.normalizedStart &&
+                        token.normalizedEnd <= source.normalizedEnd)
+                        .map(queryToken),
+                });
+            }
+            if (analyzed.gramTokens.length > 0) {
+                gramSequences.push(analyzed.gramTokens.map((token) => rebaseQueryToken(token, part.start, normalizedBase, token.position)));
+            }
+            if (matching.phrase !== 'off' && part.quoted && partConceptIndexes.length > 0) {
+                phrases.push({ conceptIndexes: partConceptIndexes, explicit: true });
+            }
+        }
+        if (matching.phrase === 'required' && concepts.length > 0) {
+            phrases.length = 0;
+            phrases.push({
+                conceptIndexes: concepts.map((concept) => concept.index),
+                explicit: false,
+            });
+        }
+        return {
+            original: input.query,
+            normalized: full.normalized,
+            locale: full.locale,
+            concepts,
+            phrases,
+            gramSequences,
+            matching,
+            analyzerFingerprint: this.fingerprint,
+        };
+    }
+}
+function assignLogicalPositions(tokens) {
+    let position = -1;
+    let groupEnd = -1;
+    for (const token of tokens) {
+        if (position === -1 || token.normalizedStart >= groupEnd) {
+            position += 1;
+            groupEnd = token.normalizedEnd;
+        }
+        else {
+            groupEnd = Math.max(groupEnd, token.normalizedEnd);
+        }
+        token.position = position;
+    }
+}
+function groupTokensByPosition(tokens) {
+    const groups = new Map();
+    for (const token of tokens.toSorted((left, right) => left.position - right.position ||
+        left.normalizedStart - right.normalizedStart ||
+        left.normalizedEnd - right.normalizedEnd)) {
+        const group = groups.get(token.position);
+        if (group == null)
+            groups.set(token.position, [token]);
+        else
+            group.push(token);
+    }
+    return [...groups.values()];
+}
+function logicalToken(kind, value, normalizedStart, normalizedEnd, position, locale, normalization) {
+    const original = normalization.originalRange(normalizedStart, normalizedEnd);
+    return {
+        kind,
+        value,
+        normalizedStart,
+        normalizedEnd,
+        start: original.start,
+        end: original.end,
+        position,
+        locale,
+    };
+}
+function buildHanBigrams(text, locale, normalization, sourceTokens) {
+    const grams = [];
+    let sourceIndex = 0;
+    let previousPosition = 0;
+    for (const match of text.matchAll(/\p{Script=Han}+/gu)) {
+        const run = match[0];
+        const runStart = match.index;
+        if (run == null || runStart == null)
+            continue;
+        const characters = [];
+        let offset = 0;
+        for (const value of run) {
+            const start = runStart + offset;
+            offset += value.length;
+            characters.push({ value, start, end: runStart + offset });
+        }
+        for (let index = 0; index + 1 < characters.length; index++) {
+            const first = characters[index];
+            const second = characters[index + 1];
+            if (first == null || second == null)
+                continue;
+            while (sourceIndex < sourceTokens.length) {
+                const source = sourceTokens[sourceIndex];
+                if (source == null || source.normalizedEnd > first.start)
+                    break;
+                previousPosition = source.position;
+                sourceIndex += 1;
+            }
+            const candidate = sourceTokens[sourceIndex];
+            const position = candidate != null &&
+                candidate.normalizedStart <= first.start &&
+                candidate.normalizedEnd >= first.end
+                ? candidate.position
+                : previousPosition;
+            grams.push(logicalToken('gram', first.value + second.value, first.start, second.end, position, locale, normalization));
+        }
+    }
+    return grams;
+}
+export function resolveMatching(matching) {
+    const operator = matching?.operator ?? 'all';
+    const phrase = matching?.phrase ?? 'auto';
+    const minimumShouldMatch = matching?.minimumShouldMatch;
+    if (minimumShouldMatch != null &&
+        (!Number.isSafeInteger(minimumShouldMatch) || minimumShouldMatch < 1)) {
+        throw new RangeError('minimumShouldMatch must be a positive safe integer');
+    }
+    if (operator === 'all' && minimumShouldMatch != null) {
+        throw new TypeError("minimumShouldMatch is only valid with operator: 'any'");
+    }
+    return {
+        operator,
+        phrase,
+        ...(minimumShouldMatch != null ? { minimumShouldMatch } : {}),
+    };
+}
+/** Split balanced ASCII-quoted spans while treating unmatched quotes as text. */
+function splitQueryParts(query) {
+    const parts = [];
+    let cursor = 0;
+    while (cursor < query.length) {
+        const open = query.indexOf('"', cursor);
+        if (open < 0) {
+            pushPart(parts, query.slice(cursor), cursor, false);
+            break;
+        }
+        const close = query.indexOf('"', open + 1);
+        if (close < 0) {
+            pushPart(parts, query.slice(cursor).replaceAll('"', ' '), cursor, false);
+            break;
+        }
+        pushPart(parts, query.slice(cursor, open), cursor, false);
+        pushPart(parts, query.slice(open + 1, close), open + 1, true);
+        cursor = close + 1;
+    }
+    return parts;
+}
+function pushPart(parts, text, start, quoted) {
+    if (text.trim().length > 0)
+        parts.push({ text, start, quoted });
+}
+function rebaseQueryToken(token, originalBase, normalizedBase, position) {
+    return {
+        ...token,
+        normalizedStart: token.normalizedStart + normalizedBase,
+        normalizedEnd: token.normalizedEnd + normalizedBase,
+        start: token.start + originalBase,
+        end: token.end + originalBase,
+        position,
+    };
+}

package/dist/analyzer.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/analyzer.test.node.js

@@ -0,0 +1,183 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { performance } from 'node:perf_hooks';
+import { MAX_SEARCH_QUERY_LENGTH } from '@byline/core';
+import { describe, expect, it } from 'vitest';
+import { createPortableSearchAnalyzer, resolveMatching } from './analyzer.js';
+import { extractIdentifierSpans } from './identifiers.js';
+import { detectSearchLocale, resolveSearchLocale } from './locale.js';
+import { normalizeForSearch } from './normalize.js';
+describe('portable search analysis', () => {
+    it('normalizes compatibility forms while preserving original UTF-16 ranges', () => {
+        const normalized = normalizeForSearch('A ffi Ｂ');
+        expect(normalized.value).toBe('a ffi b');
+        expect(normalized.originalRange(2, 5)).toEqual({ start: 2, end: 3 });
+        const analyzed = createPortableSearchAnalyzer().analyzeText({ text: 'A ffi Ｂ' });
+        expect(analyzed.exactTokens.map(({ value, start, end }) => ({ value, start, end }))).toEqual([
+            { value: 'a', start: 0, end: 1 },
+            { value: 'ffi', start: 2, end: 3 },
+            { value: 'b', start: 4, end: 5 },
+        ]);
+    });
+    it('prefers a valid declared locale and otherwise falls back through script detection', () => {
+        expect(resolveSearchLocale('ภาษาไทย', 'en-US')).toBe('en-US');
+        expect(resolveSearchLocale('ภาษาไทย', 'not_a_locale')).toBe('th');
+        expect(detectSearchLocale('한국어')).toBe('ko');
+        expect(detectSearchLocale('日本語です')).toBe('ja');
+        expect(detectSearchLocale('数据库')).toBe('zh');
+        expect(detectSearchLocale('数据库', 'ja')).toBe('ja');
+    });
+    it('extracts protected identifiers before word segmentation', () => {
+        const analyzed = createPortableSearchAnalyzer().analyzeText({
+            text: 'See HTTPS://Example.com/a). mail Tést@Example.com SKU-42 v1.2.3 C++ C# Node.js @ผู้ใช้ #หัวข้อ',
+        });
+        expect(analyzed.identifierTokens.map(({ identifierKind, value }) => ({
+            identifierKind,
+            value,
+        }))).toEqual([
+            { identifierKind: 'url', value: 'https://example.com/a' },
+            { identifierKind: 'email', value: 'tést@example.com' },
+            { identifierKind: 'sku', value: 'sku-42' },
+            { identifierKind: 'version', value: 'v1.2.3' },
+            { identifierKind: 'technical', value: 'c++' },
+            { identifierKind: 'technical', value: 'c#' },
+            { identifierKind: 'technical', value: 'node.js' },
+            { identifierKind: 'mention', value: '@ผู้ใช้' },
+            { identifierKind: 'hashtag', value: '#หัวข้อ' },
+        ]);
+        expect(analyzed.exactTokens.map((token) => token.value)).toContain('see');
+        expect(analyzed.exactTokens.map((token) => token.value)).toContain('mail');
+        expect(analyzed.exactTokens.map((token) => token.value)).not.toContain('example');
+        expect(analyzed.exactTokens.map((token) => token.value)).not.toContain('tést');
+    });
+    it('retains SKU and version constituents at one logical source position', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const analyzed = analyzer.analyzeText({
+            text: 'COVID-19 cases utf-8 encoding Section 1.2',
+        });
+        const positions = [...analyzed.exactTokens, ...analyzed.identifierTokens].map(({ kind, value, position }) => ({ kind, value, position }));
+        expect(positions).toEqual(expect.arrayContaining([
+            { kind: 'exact', value: 'covid', position: 0 },
+            { kind: 'exact', value: '19', position: 0 },
+            { kind: 'identifier', value: 'covid-19', position: 0 },
+            { kind: 'exact', value: 'cases', position: 1 },
+            { kind: 'exact', value: 'utf', position: 2 },
+            { kind: 'exact', value: '8', position: 2 },
+            { kind: 'identifier', value: 'utf-8', position: 2 },
+            { kind: 'exact', value: 'encoding', position: 3 },
+            { kind: 'exact', value: 'section', position: 4 },
+            { kind: 'exact', value: '1.2', position: 5 },
+            { kind: 'identifier', value: '1.2', position: 5 },
+        ]));
+    });
+    it('uses a complete identifier as one query concept', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        const cases = [
+            ['covid-19', 1],
+            ['covid-19 vaccine', 2],
+            ['utf-8 encoding', 2],
+            ['Section 1.2', 2],
+        ];
+        for (const [query, conceptCount] of cases) {
+            expect(analyzer.analyzeQuery({ query }).concepts, query).toHaveLength(conceptCount);
+        }
+        const quoted = analyzer.analyzeQuery({ query: '"covid-19 cases"' });
+        expect(quoted.concepts[0]?.identifierTokens.map((token) => token.value)).toEqual(['covid-19']);
+        expect(quoted.phrases).toEqual([{ conceptIndexes: [0, 1], explicit: true }]);
+    });
+    it('keeps adversarial identifier extraction below the quadratic baseline', () => {
+        extractIdentifierSpans('数'.repeat(1_000));
+        for (const character of ['数', 'a']) {
+            const shortStarted = performance.now();
+            extractIdentifierSpans(character.repeat(5_000));
+            const shortDuration = performance.now() - shortStarted;
+            const longStarted = performance.now();
+            extractIdentifierSpans(character.repeat(40_000));
+            const longDuration = performance.now() - longStarted;
+            // CI runs package suites concurrently, so this is a regression guard,
+            // not a production latency target. The old unbounded email rule took
+            // more than 10 seconds and grew roughly 50–60x for this 8x input.
+            expect(longDuration).toBeLessThan(Math.max(2_500, shortDuration * 32));
+        }
+    });
+    it('analyzes adversarial unbroken text within the unit-test timeout', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        for (const text of ['数'.repeat(40_000), 'a'.repeat(40_000)]) {
+            expect(analyzer.analyzeText({ text }).normalized).toHaveLength(40_000);
+        }
+    });
+    it('rejects an over-long query before analysis', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        expect(() => analyzer.analyzeQuery({ query: 'a'.repeat(MAX_SEARCH_QUERY_LENGTH) })).not.toThrow();
+        expect(() => analyzer.analyzeQuery({ query: 'a'.repeat(MAX_SEARCH_QUERY_LENGTH + 1) })).toThrow(RangeError);
+    });
+    it('emits ordered overlapping Han bigrams without replacing exact terms', () => {
+        const analyzed = createPortableSearchAnalyzer().analyzeText({ text: '数据库' });
+        expect(analyzed.exactTokens.map((token) => token.value).join('')).toBe('数据库');
+        expect(analyzed.gramTokens.map((token) => token.value)).toEqual(['数据', '据库']);
+        expect(analyzed.gramTokens.map((token) => token.position)).toEqual([0, 0]);
+    });
+    it('groups exact and expanded variants by concept and retains quoted phrase intent', () => {
+        const expander = {
+            fingerprint: 'english-test1',
+            supports: (locale) => locale.startsWith('en'),
+            expand: (token) => (token.value === 'running' ? [{ kind: 'stem', value: 'run' }] : []),
+        };
+        const analyzer = createPortableSearchAnalyzer({ expanders: [expander] });
+        const plan = analyzer.analyzeQuery({
+            query: '"Running Fast" Node.js',
+            locale: 'en',
+            matching: { operator: 'any', minimumShouldMatch: 2 },
+        });
+        expect(plan.matching).toEqual({
+            operator: 'any',
+            phrase: 'auto',
+            minimumShouldMatch: 2,
+        });
+        expect(plan.concepts).toHaveLength(3);
+        expect(plan.concepts[0]?.exactTokens[0]?.value).toBe('running');
+        expect(plan.concepts[0]?.stemTokens[0]?.value).toBe('run');
+        expect(plan.concepts[2]?.identifierTokens[0]?.value).toBe('node.js');
+        expect(plan.phrases).toEqual([{ conceptIndexes: [0, 1], explicit: true }]);
+        expect(plan.concepts[0]?.exactTokens[0]).toMatchObject({ start: 1, end: 8, position: 0 });
+        expect(plan.concepts[2]?.identifierTokens[0]).toMatchObject({
+            start: 15,
+            end: 22,
+            position: 2,
+        });
+        expect(plan.analyzerFingerprint).toContain('english-test1');
+    });
+    it('can require or disable phrase semantics explicitly', () => {
+        const analyzer = createPortableSearchAnalyzer();
+        expect(analyzer.analyzeQuery({
+            query: 'forest restoration',
+            matching: { phrase: 'required' },
+        }).phrases).toEqual([{ conceptIndexes: [0, 1], explicit: false }]);
+        expect(analyzer.analyzeQuery({
+            query: '"forest restoration"',
+            matching: { phrase: 'off' },
+        }).phrases).toEqual([]);
+    });
+    it('fingerprints every option that can change indexed terms', () => {
+        const baseline = createPortableSearchAnalyzer().fingerprint;
+        const locale = createPortableSearchAnalyzer({ defaultLocale: 'th' }).fingerprint;
+        const han = createPortableSearchAnalyzer({ hanLocale: 'ja' }).fingerprint;
+        const grams = createPortableSearchAnalyzer({ hanBigrams: false }).fingerprint;
+        expect(new Set([baseline, locale, han, grams]).size).toBe(4);
+        expect(baseline).toContain(`icu${process.versions.icu}`);
+    });
+});
+describe('matching policy', () => {
+    it('applies conservative defaults', () => {
+        expect(resolveMatching(undefined)).toEqual({ operator: 'all', phrase: 'auto' });
+    });
+    it('rejects invalid minimum-should-match policies', () => {
+        expect(() => resolveMatching({ operator: 'any', minimumShouldMatch: 0 })).toThrow(RangeError);
+        expect(() => resolveMatching({ operator: 'all', minimumShouldMatch: 1 })).toThrow(TypeError);
+    });
+});

package/dist/highlight.d.ts

@@ -0,0 +1,30 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import type { PortableQueryPlan, PortableSearchAnalyzer } from './types.js';
+export interface PortableHighlightOptions {
+    /** Maximum independently selected fragments. Defaults to 2. */
+    maxFragments?: number;
+    /** Maximum analyzed source terms in each snippet. Defaults to 24. */
+    maxWords?: number;
+}
+export interface HighlightPortableTextInput extends PortableHighlightOptions {
+    /** Original stored text from which source-preserving fragments are selected. */
+    text: string;
+    /** Query plan produced by the same analyzer used for the index. */
+    plan: PortableQueryPlan;
+    /** Analyzer used to recover source offsets from the stored text. */
+    analyzer: PortableSearchAnalyzer;
+}
+/**
+ * Build a backend-neutral matched snippet from portable logical-token offsets.
+ *
+ * The returned string contains `<mark>…</mark>` delimiters. Consumers must
+ * parse those delimiters and render the surrounding source text as text; they
+ * must not inject the complete result as trusted HTML.
+ */
+export declare function highlightPortableText({ text, plan, analyzer, maxFragments, maxWords, }: HighlightPortableTextInput): string | undefined;