@byline/search-analysis 4.9.0

package/dist/sql-token-codec.test.node.js

@@ -0,0 +1,38 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { describe, expect, it } from 'vitest';
+import { encodeSqlToken } from './sql-token-codec.js';
+describe('SQL token codec', () => {
+    it.each([
+        ['exact', 'a'],
+        ['exact', 'ฐานข้อมูล'],
+        ['gram', '数据'],
+        ['identifier', 'https://example.com/a'],
+    ])('encodes a %s token as parser-safe lowercase ASCII', (kind, value) => {
+        const encoded = encodeSqlToken({ kind, value });
+        expect(encoded).toMatch(/^[a-z0-9]+$/);
+        expect(encoded.length).toBeGreaterThan(3);
+    });
+    it('keeps logical token classes physically distinct', () => {
+        const exact = encodeSqlToken({ kind: 'exact', value: 'database' });
+        const stem = encodeSqlToken({ kind: 'stem', value: 'database' });
+        const gram = encodeSqlToken({ kind: 'gram', value: 'database' });
+        expect(new Set([exact, stem, gram]).size).toBe(3);
+    });
+    it('hashes long terms deterministically within the configured bound', () => {
+        const token = { kind: 'identifier', value: `https://example.com/${'x'.repeat(500)}` };
+        const first = encodeSqlToken(token, { maxLength: 32 });
+        expect(first).toHaveLength(32);
+        expect(encodeSqlToken(token, { maxLength: 32 })).toBe(first);
+        expect(encodeSqlToken({ ...token, value: `${token.value}y` }, { maxLength: 32 })).not.toBe(first);
+    });
+    it('rejects empty values and unsafe length limits', () => {
+        expect(() => encodeSqlToken({ kind: 'exact', value: '' })).toThrow(TypeError);
+        expect(() => encodeSqlToken({ kind: 'exact', value: 'term' }, { maxLength: 15 })).toThrow(RangeError);
+    });
+});

package/dist/types.d.ts

@@ -0,0 +1,125 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import type { SearchMatching, SearchPhraseMode, SearchTermOperator } from '@byline/core';
+/** Portable logical token classes. Physical backend representations are separate. */
+export type LogicalTokenKind = 'exact' | 'stem' | 'lemma' | 'normalized' | 'identifier' | 'gram';
+/** Domain category attached to an identifier token for diagnostics and tuning. */
+export type SearchIdentifierKind = 'url' | 'email' | 'mention' | 'hashtag' | 'sku' | 'version' | 'technical';
+/**
+ * One logical term produced by portable analysis. All offsets use UTF-16
+ * string indexes, matching `Intl.Segmenter` and JavaScript `slice`.
+ */
+export interface LogicalToken {
+    kind: LogicalTokenKind;
+    value: string;
+    /** Offset into normalized search text. */
+    normalizedStart: number;
+    /** Exclusive offset into normalized search text. */
+    normalizedEnd: number;
+    /** Offset into the original, unmodified input. */
+    start: number;
+    /** Exclusive offset into the original, unmodified input. */
+    end: number;
+    /** Source-order position; expansions retain their source token's position. */
+    position: number;
+    locale: string;
+    identifierKind?: SearchIdentifierKind;
+}
+/** Result of analyzing one original text value. */
+export interface AnalyzedText {
+    original: string;
+    normalized: string;
+    locale: string;
+    tokens: LogicalToken[];
+    exactTokens: LogicalToken[];
+    derivedTokens: LogicalToken[];
+    identifierTokens: LogicalToken[];
+    gramTokens: LogicalToken[];
+    analyzerFingerprint: string;
+}
+/** A variant emitted by an optional language-specific token expander. */
+export interface SearchTokenExpansion {
+    kind: 'stem' | 'lemma' | 'normalized';
+    value: string;
+}
+/**
+ * Plug-in point for stemming, lemmatization, or language-specific normalized
+ * variants. Expansion augments exact terms; it never replaces them.
+ */
+export interface SearchTokenExpander {
+    /** Stable family/version component included in the analyzer fingerprint. */
+    readonly fingerprint: string;
+    supports(locale: string): boolean;
+    expand(token: LogicalToken): readonly SearchTokenExpansion[];
+}
+/** One user concept with its recall alternatives kept together. */
+export interface QueryConcept {
+    index: number;
+    position: number;
+    exactTokens: LogicalToken[];
+    stemTokens: LogicalToken[];
+    lemmaTokens: LogicalToken[];
+    normalizedTokens: LogicalToken[];
+    identifierTokens: LogicalToken[];
+    /**
+     * Ordered grams that can act as low-weight fallback for this concept.
+     * They are a sequence, not independent OR alternatives.
+     */
+    gramTokens: LogicalToken[];
+}
+/** Ordered concept indexes that form a phrase constraint or boost. */
+export interface QueryPhrase {
+    conceptIndexes: number[];
+    explicit: boolean;
+}
+/** Resolved matching policy with analyzer defaults applied. */
+export interface ResolvedSearchMatching {
+    operator: SearchTermOperator;
+    phrase: SearchPhraseMode;
+    minimumShouldMatch?: number;
+}
+/**
+ * Backend-neutral lexical query plan. Adapters translate concept alternatives,
+ * phrase order, and gram fallback into native query syntax.
+ */
+export interface PortableQueryPlan {
+    original: string;
+    normalized: string;
+    locale: string;
+    concepts: QueryConcept[];
+    phrases: QueryPhrase[];
+    /** Ordered fallback grams, including grams that cross word boundaries. */
+    gramSequences: LogicalToken[][];
+    matching: ResolvedSearchMatching;
+    analyzerFingerprint: string;
+}
+export interface AnalyzeTextInput {
+    text: string;
+    /** Declared field/document locale. Invalid or unsupported values fall back safely. */
+    locale?: string;
+}
+export interface AnalyzeQueryInput {
+    query: string;
+    locale?: string;
+    matching?: SearchMatching;
+}
+export interface PortableSearchAnalyzer {
+    readonly fingerprint: string;
+    analyzeText(input: AnalyzeTextInput): AnalyzedText;
+    analyzeQuery(input: AnalyzeQueryInput): PortableQueryPlan;
+}
+export interface PortableSearchAnalyzerOptions {
+    /** Fallback after declared locale and script detection. Defaults to `en`. */
+    defaultLocale?: string;
+    /** Locale chosen for otherwise ambiguous Han-only text. Defaults to `zh`. */
+    hanLocale?: 'zh' | 'ja';
+    /** Emit overlapping Han bigrams. Defaults to true. */
+    hanBigrams?: boolean;
+    /** Optional exact-preserving language expansion stages. */
+    expanders?: readonly SearchTokenExpander[];
+}

package/dist/types.js

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@byline/search-analysis",
+  "private": false,
+  "license": "MPL-2.0",
+  "version": "4.9.0",
+  "engines": {
+    "node": ">=20.9.0"
+  },
+  "description": "Portable multilingual term analysis, query planning, and highlighting for Byline CMS search providers",
+  "keywords": [
+    "cms",
+    "headless cms",
+    "search",
+    "multilingual",
+    "tokenization"
+  ],
+  "homepage": "https://github.com/Byline-CMS/bylinecms.dev",
+  "bugs": {
+    "url": "https://github.com/Byline-CMS/bylinecms.dev/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Byline-CMS/bylinecms.dev.git",
+    "directory": "packages/search-analysis"
+  },
+  "type": "module",
+  "main": "dist/index.js",
+  "index": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@byline/core": "4.9.0"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.5.4",
+    "@types/node": "^26.1.1",
+    "chokidar": "^5.0.0",
+    "chokidar-cli": "^3.0.0",
+    "npm-run-all": "^4.1.5",
+    "tsc-alias": "^1.9.1",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "scripts": {
+    "dev": "chokidar 'src/**/*' -c 'npm-run-all build'",
+    "build": "tsc -p tsconfig.json && tsc-alias",
+    "clean": "node scripts/clean.js node_modules dist build .turbo",
+    "lint": "biome check --write --unsafe --diagnostic-level=error",
+    "test": "vitest run --mode=node",
+    "test:watch": "vitest --mode=node",
+    "typecheck": "tsc --noEmit"
+  }
+}