@byline/search-mysql 4.9.0

package/dist/portable-index-document.js

@@ -0,0 +1,157 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { encodeSqlToken, } from '@byline/search-analysis';
+const WEIGHT_CLASSES = ['A', 'B', 'C', 'D'];
+const STREAM_BOUNDARY = 'bylinefulltextboundary';
+/**
+ * Encode portable logical terms into text streams consumed by MySQL FULLTEXT.
+ * Separate exact, expansion-kind, and Han-gram streams preserve adjacency for
+ * phrase matching without letting phrases cross weight classes or token
+ * streams.
+ */
+export function buildPortableMySqlIndexDocument(row, analyzer) {
+    const matchingStreams = [];
+    const tokensByWeight = {
+        A: [],
+        B: [],
+        C: [],
+        D: [],
+    };
+    for (const sourceWeight of WEIGHT_CLASSES) {
+        const analyzed = analyzer.analyzeText({
+            text: row.weighted[sourceWeight],
+            locale: row.locale,
+        });
+        matchingStreams.push(...serializeMatchingStreams(analyzed.tokens));
+        for (const token of analyzed.tokens) {
+            tokensByWeight[tokenWeight(token, sourceWeight)].push(token);
+        }
+    }
+    const weighted = {
+        A: serializeStreams(tokensByWeight.A),
+        B: serializeStreams(tokensByWeight.B),
+        C: serializeStreams(tokensByWeight.C),
+        D: serializeStreams(tokensByWeight.D),
+    };
+    return {
+        searchText: matchingStreams.join(` ${STREAM_BOUNDARY} `),
+        weighted,
+        analyzerFingerprint: analyzer.fingerprint,
+    };
+}
+function tokenWeight(token, sourceWeight) {
+    if (token.kind === 'gram')
+        return 'D';
+    if (token.kind === 'exact' || token.kind === 'identifier')
+        return sourceWeight;
+    return lowerWeight(sourceWeight);
+}
+function lowerWeight(weight) {
+    switch (weight) {
+        case 'A':
+            return 'B';
+        case 'B':
+            return 'C';
+        case 'C':
+        case 'D':
+            return 'D';
+    }
+}
+function serializeStreams(tokens) {
+    const streams = [];
+    const source = tokens
+        .filter((token) => token.kind === 'exact' || token.kind === 'identifier')
+        .toSorted(compareTokens);
+    if (source.length > 0)
+        streams.push(source);
+    for (const kind of ['stem', 'lemma', 'normalized']) {
+        const derived = tokens.filter((token) => token.kind === kind).toSorted(compareTokens);
+        if (derived.length > 0)
+            streams.push(derived);
+    }
+    const grams = tokens.filter((token) => token.kind === 'gram').toSorted(compareTokens);
+    if (grams.length > 0)
+        streams.push(grams);
+    return streams
+        .map((stream) => stream.map((token) => encodeSqlToken(token)).join(' '))
+        .join(` ${STREAM_BOUNDARY} `);
+}
+/**
+ * Preserve source-token adjacency for matching while making each expansion
+ * kind phrase-capable. A derived stream falls back to the exact/identifier
+ * token at positions where that kind produced no alternative, so a query such
+ * as `"runs restoration"` can match indexed `"running restoration"` through
+ * the stem `run` without losing the unchanged neighboring term.
+ */
+function serializeMatchingStreams(tokens) {
+    const source = tokens
+        .filter((token) => token.kind === 'exact' || token.kind === 'identifier')
+        .toSorted(compareTokens);
+    const sourceGroups = groupTokensByPosition(source);
+    const sourceStreams = buildSourceStreams(sourceGroups);
+    const streams = [...sourceStreams];
+    const primarySource = sourceGroups.map(preferredSourceToken);
+    for (const kind of ['stem', 'lemma', 'normalized']) {
+        const derived = tokens.filter((token) => token.kind === kind);
+        if (derived.length === 0)
+            continue;
+        const byPosition = new Map();
+        for (const token of derived.toSorted(compareTokens)) {
+            const positioned = byPosition.get(token.position);
+            if (positioned == null)
+                byPosition.set(token.position, [token]);
+            else
+                positioned.push(token);
+        }
+        streams.push(serializeTokenStream(primarySource.map((fallback) => byPosition.get(fallback.position)?.[0] ?? fallback)));
+    }
+    const grams = tokens.filter((token) => token.kind === 'gram').toSorted(compareTokens);
+    if (grams.length > 0)
+        streams.push(serializeTokenStream(grams));
+    return [...new Set(streams.filter((stream) => stream.length > 0))];
+}
+function buildSourceStreams(groups) {
+    if (groups.length === 0)
+        return [];
+    const streams = [serializeTokenStream(groups.map(preferredSourceToken))];
+    const exactVariantCount = Math.max(0, ...groups.map((group) => group.filter((token) => token.kind === 'exact').length));
+    for (let variant = 0; variant < exactVariantCount; variant++) {
+        streams.push(serializeTokenStream(groups.map((group) => {
+            const exact = group.filter((token) => token.kind === 'exact');
+            return exact[variant] ?? exact[0] ?? preferredSourceToken(group);
+        })));
+    }
+    return [...new Set(streams)];
+}
+function preferredSourceToken(group) {
+    const token = group.find((candidate) => candidate.kind === 'identifier') ??
+        group.find((candidate) => candidate.kind === 'exact');
+    if (token == null)
+        throw new TypeError('Portable source position has no searchable token');
+    return token;
+}
+function groupTokensByPosition(tokens) {
+    const groups = new Map();
+    for (const token of tokens) {
+        const group = groups.get(token.position);
+        if (group == null)
+            groups.set(token.position, [token]);
+        else
+            group.push(token);
+    }
+    return [...groups.values()];
+}
+function serializeTokenStream(tokens) {
+    return tokens.map((token) => encodeSqlToken(token)).join(' ');
+}
+function compareTokens(left, right) {
+    return (left.position - right.position ||
+        left.normalizedStart - right.normalizedStart ||
+        left.normalizedEnd - right.normalizedEnd ||
+        left.value.localeCompare(right.value));
+}

package/dist/portable-index-document.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/portable-index-document.test.node.js

@@ -0,0 +1,67 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { createPortableSearchAnalyzer, encodeSqlToken, } from '@byline/search-analysis';
+import { describe, expect, it } from 'vitest';
+import { buildPortableMySqlIndexDocument } from './portable-index-document.js';
+const expander = {
+    fingerprint: 'english-test1',
+    supports: (locale) => locale.startsWith('en'),
+    expand: (token) => (token.value === 'running' ? [{ kind: 'stem', value: 'run' }] : []),
+};
+function row(overrides = {}) {
+    return {
+        collectionPath: 'reports',
+        documentId: 'report-1',
+        locale: 'en',
+        status: 'published',
+        zones: ['library'],
+        title: 'Running database',
+        path: 'running-database',
+        body: 'Running 数据库',
+        weighted: { A: 'Running 数据库', B: '', C: '', D: '' },
+        facets: {},
+        filters: {},
+        updatedAt: '2026-07-26T00:00:00.000Z',
+        ...overrides,
+    };
+}
+describe('buildPortableMySqlIndexDocument', () => {
+    it('keeps exact terms at source weight and lowers derived variants', () => {
+        const analyzer = createPortableSearchAnalyzer({ expanders: [expander] });
+        const document = buildPortableMySqlIndexDocument(row(), analyzer);
+        const exact = encodeSqlToken({ kind: 'exact', value: 'running' });
+        const stem = encodeSqlToken({ kind: 'stem', value: 'run' });
+        expect(document.weighted.A).toContain(exact);
+        expect(document.weighted.A).not.toContain(stem);
+        expect(document.weighted.B).toContain(stem);
+        expect(document.analyzerFingerprint).toBe(analyzer.fingerprint);
+    });
+    it('always assigns Han grams to the lightest weight class', () => {
+        const document = buildPortableMySqlIndexDocument(row(), createPortableSearchAnalyzer());
+        expect(document.weighted.D).toContain(encodeSqlToken({ kind: 'gram', value: '数据' }));
+        expect(document.weighted.D).toContain(encodeSqlToken({ kind: 'gram', value: '据库' }));
+    });
+    it('inserts unsearchable boundaries between independent streams', () => {
+        const document = buildPortableMySqlIndexDocument(row({ weighted: { A: 'alpha', B: 'beta', C: '', D: '' } }), createPortableSearchAnalyzer());
+        expect(document.searchText).toContain('bylinefulltextboundary');
+    });
+    it('uses exact fallbacks beside derived terms in matching streams', () => {
+        const analyzer = createPortableSearchAnalyzer({ expanders: [expander] });
+        const document = buildPortableMySqlIndexDocument(row({ weighted: { A: 'running restoration', B: '', C: '', D: '' } }), analyzer);
+        const stem = encodeSqlToken({ kind: 'stem', value: 'run' });
+        const exactNeighbor = encodeSqlToken({ kind: 'exact', value: 'restoration' });
+        expect(document.searchText).toContain(`${stem} ${exactNeighbor}`);
+    });
+    it('writes identifier and constituent phrase streams at one logical position', () => {
+        const document = buildPortableMySqlIndexDocument(row({ weighted: { A: 'COVID-19 cases', B: '', C: '', D: '' } }), createPortableSearchAnalyzer());
+        const cases = encodeSqlToken({ kind: 'exact', value: 'cases' });
+        expect(document.searchText).toContain(`${encodeSqlToken({ kind: 'identifier', value: 'covid-19' })} ${cases}`);
+        expect(document.searchText).toContain(`${encodeSqlToken({ kind: 'exact', value: 'covid' })} ${cases}`);
+        expect(document.searchText).toContain(`${encodeSqlToken({ kind: 'exact', value: '19' })} ${cases}`);
+    });
+});

package/dist/portable-query.d.ts

@@ -0,0 +1,22 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { type PortableQueryPlan } from '@byline/search-analysis';
+export interface PortableMySqlQuery {
+    /** Boolean-mode query per source concept; terms inside one query are ORs. */
+    conceptQueries: string[];
+    /** Alternative quoted Boolean-mode queries for each required phrase. */
+    phraseQueries: string[][];
+    /** Ordered Han-gram fallback phrases. */
+    gramQueries: string[];
+    /** Flat optional-term query used only for weighted relevance scoring. */
+    rankingQuery: string;
+    operator: 'all' | 'any';
+    minimumShouldMatch?: number;
+}
+/** Translate a portable plan into parser-safe MySQL Boolean-mode inputs. */
+export declare function buildPortableMySqlQuery(plan: PortableQueryPlan): PortableMySqlQuery;

package/dist/portable-query.js

@@ -0,0 +1,87 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { encodeSqlToken, } from '@byline/search-analysis';
+/** Translate a portable plan into parser-safe MySQL Boolean-mode inputs. */
+export function buildPortableMySqlQuery(plan) {
+    const conceptTerms = plan.concepts.map(conceptAlternatives);
+    if (conceptTerms.some((terms) => terms.length === 0)) {
+        throw new TypeError('Portable query plan contains a concept with no searchable terms');
+    }
+    const phraseQueries = plan.phrases.map((phrase) => {
+        const concepts = phrase.conceptIndexes.map((index) => plan.concepts[index]);
+        if (concepts.some((concept) => concept == null)) {
+            throw new RangeError('Portable query phrase references an unknown concept');
+        }
+        return phraseStreamVariants(concepts);
+    });
+    const gramQueries = plan.gramSequences
+        .map((sequence) => quotedPhrase(orderedTerms(sequence)))
+        .filter((query) => query.length > 0);
+    const rankingQuery = [
+        ...conceptTerms.flat(),
+        ...plan.gramSequences.flatMap((sequence) => orderedTerms(sequence)),
+    ]
+        .filter((term, index, terms) => terms.indexOf(term) === index)
+        .join(' ');
+    return {
+        conceptQueries: conceptTerms.map((terms) => terms.join(' ')),
+        phraseQueries,
+        gramQueries,
+        rankingQuery,
+        operator: plan.matching.operator,
+        ...(plan.matching.minimumShouldMatch != null
+            ? { minimumShouldMatch: plan.matching.minimumShouldMatch }
+            : {}),
+    };
+}
+function conceptAlternatives(concept) {
+    const terms = [
+        ...concept.exactTokens,
+        ...concept.stemTokens,
+        ...concept.lemmaTokens,
+        ...concept.normalizedTokens,
+        ...concept.identifierTokens,
+    ].map((token) => encodeSqlToken(token));
+    const gramPhrase = quotedPhrase(orderedTerms(concept.gramTokens));
+    if (gramPhrase.length > 0)
+        terms.push(gramPhrase);
+    return [...new Set(terms)];
+}
+/**
+ * Mirror the index's phrase-capable source and expansion-kind streams. Mixed
+ * expansion kinds never coexist in one indexed stream, so their Cartesian
+ * product would only produce impossible phrases.
+ */
+function phraseStreamVariants(concepts) {
+    const source = concepts.map(sourceToken);
+    const variants = [source];
+    for (const kind of ['stemTokens', 'lemmaTokens', 'normalizedTokens']) {
+        if (!concepts.some((concept) => concept[kind].length > 0))
+            continue;
+        variants.push(concepts.map((concept, index) => concept[kind][0] ?? source[index]));
+    }
+    return [
+        ...new Set(variants
+            .map((tokens) => quotedPhrase(tokens.map((token) => encodeSqlToken(token))))
+            .filter((query) => query.length > 0)),
+    ];
+}
+function sourceToken(concept) {
+    const token = concept.identifierTokens[0] ?? concept.exactTokens[0];
+    if (token == null)
+        throw new TypeError('Portable query concept has no source token');
+    return token;
+}
+function orderedTerms(tokens) {
+    return tokens
+        .toSorted((left, right) => left.normalizedStart - right.normalizedStart || left.normalizedEnd - right.normalizedEnd)
+        .map((token) => encodeSqlToken(token));
+}
+function quotedPhrase(terms) {
+    return terms.length > 0 ? `"${terms.join(' ')}"` : '';
+}

package/dist/portable-query.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/portable-query.test.node.js

@@ -0,0 +1,68 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { createPortableSearchAnalyzer, encodeSqlToken, } from '@byline/search-analysis';
+import { describe, expect, it } from 'vitest';
+import { buildPortableMySqlQuery } from './portable-query.js';
+const expander = {
+    fingerprint: 'english-test1',
+    supports: (locale) => locale.startsWith('en'),
+    expand: (token) => (token.value === 'running' ? [{ kind: 'stem', value: 'run' }] : []),
+};
+describe('buildPortableMySqlQuery', () => {
+    it('keeps alternatives grouped by source concept', () => {
+        const analyzer = createPortableSearchAnalyzer({ expanders: [expander] });
+        const translated = buildPortableMySqlQuery(analyzer.analyzeQuery({ query: 'running restoration', locale: 'en' }));
+        expect(translated.conceptQueries).toHaveLength(2);
+        expect(translated.conceptQueries[0]).toContain(encodeSqlToken({ kind: 'exact', value: 'running' }));
+        expect(translated.conceptQueries[0]).toContain(encodeSqlToken({ kind: 'stem', value: 'run' }));
+        expect(translated.operator).toBe('all');
+    });
+    it('preserves phrase order and minimum-should-match intent', () => {
+        const translated = buildPortableMySqlQuery(createPortableSearchAnalyzer().analyzeQuery({
+            query: '"forest restoration" database',
+            matching: { operator: 'any', minimumShouldMatch: 2 },
+        }));
+        expect(translated.conceptQueries).toHaveLength(3);
+        expect(translated.phraseQueries[0]?.[0]).toMatch(/^"[a-z0-9]+ [a-z0-9]+"$/);
+        expect(translated.minimumShouldMatch).toBe(2);
+    });
+    it('emits only phrase variants that mirror physical index streams', () => {
+        const phraseExpander = {
+            fingerprint: 'phrase-test1',
+            supports: (locale) => locale.startsWith('en'),
+            expand: (token) => token.value === 'running'
+                ? [{ kind: 'stem', value: 'run' }]
+                : token.value === 'restoration'
+                    ? [{ kind: 'stem', value: 'restore' }]
+                    : [],
+        };
+        const translated = buildPortableMySqlQuery(createPortableSearchAnalyzer({ expanders: [phraseExpander] }).analyzeQuery({
+            query: '"running restoration"',
+            locale: 'en',
+        }));
+        expect(translated.phraseQueries[0]).toEqual([
+            `"${encodeSqlToken({ kind: 'exact', value: 'running' })} ${encodeSqlToken({
+                kind: 'exact',
+                value: 'restoration',
+            })}"`,
+            `"${encodeSqlToken({ kind: 'stem', value: 'run' })} ${encodeSqlToken({
+                kind: 'stem',
+                value: 'restore',
+            })}"`,
+        ]);
+    });
+    it('returns an empty ranking query for punctuation-only input', () => {
+        const translated = buildPortableMySqlQuery(createPortableSearchAnalyzer().analyzeQuery({ query: '— -- !!!' }));
+        expect(translated).toMatchObject({
+            conceptQueries: [],
+            phraseQueries: [],
+            gramQueries: [],
+            rankingQuery: '',
+        });
+    });
+});

package/migrations/0001_init.sql

@@ -0,0 +1,41 @@
+-- @byline/search-mysql — 0001_init
+--
+-- Disposable portable full-text index. One row per
+-- (collection_path, document_id, locale). Every searchable logical token is
+-- encoded by @byline/search-analysis before MySQL's parser sees it, avoiding
+-- language-specific stopword and minimum-token divergence.
+
+CREATE TABLE IF NOT EXISTS byline_search_documents (
+  collection_path      varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
+  document_id          varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
+  locale               varchar(35)  CHARACTER SET ascii   COLLATE ascii_bin   NOT NULL,
+  status               varchar(64)  CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
+  zones                json                                                   NOT NULL,
+  title                text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci  NOT NULL,
+  path                 text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin,
+  body                 longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
+  search_text          longtext CHARACTER SET ascii COLLATE ascii_bin         NOT NULL,
+  search_a             longtext CHARACTER SET ascii COLLATE ascii_bin         NOT NULL,
+  search_b             longtext CHARACTER SET ascii COLLATE ascii_bin         NOT NULL,
+  search_c             longtext CHARACTER SET ascii COLLATE ascii_bin         NOT NULL,
+  search_d             longtext CHARACTER SET ascii COLLATE ascii_bin         NOT NULL,
+  analyzer_fingerprint varchar(512) CHARACTER SET ascii COLLATE ascii_bin      NOT NULL,
+  facets               json                                                   NOT NULL,
+  filters              json                                                   NOT NULL,
+  updated_at           datetime(6)                                            NOT NULL,
+  PRIMARY KEY (collection_path, document_id, locale),
+  KEY byline_search_documents_collection_idx (collection_path, status),
+  FULLTEXT KEY byline_search_documents_text_idx (search_text),
+  FULLTEXT KEY byline_search_documents_a_idx (search_a),
+  FULLTEXT KEY byline_search_documents_b_idx (search_b),
+  FULLTEXT KEY byline_search_documents_c_idx (search_c),
+  FULLTEXT KEY byline_search_documents_d_idx (search_d)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS byline_search_index_metadata (
+  collection_path      varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
+  analyzer_fingerprint varchar(512) CHARACTER SET ascii COLLATE ascii_bin      NOT NULL,
+  zones                json                                                   NOT NULL,
+  updated_at           datetime(6)                                            NOT NULL,
+  PRIMARY KEY (collection_path)
+) ENGINE=InnoDB;

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@byline/search-mysql",
+  "private": false,
+  "license": "MPL-2.0",
+  "version": "4.9.0",
+  "engines": {
+    "node": ">=20.9.0"
+  },
+  "description": "Byline CMS built-in MySQL full-text search provider",
+  "keywords": [
+    "cms",
+    "headless cms",
+    "content management",
+    "search",
+    "mysql",
+    "full-text search"
+  ],
+  "homepage": "https://github.com/Byline-CMS/bylinecms.dev",
+  "bugs": {
+    "url": "https://github.com/Byline-CMS/bylinecms.dev/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Byline-CMS/bylinecms.dev.git",
+    "directory": "packages/search-mysql"
+  },
+  "type": "module",
+  "main": "dist/index.js",
+  "index": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "migrations"
+  ],
+  "dependencies": {
+    "@byline/core": "4.9.0",
+    "@byline/search-analysis": "4.9.0"
+  },
+  "peerDependencies": {
+    "mysql2": "^3.23.1"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.5.4",
+    "@types/node": "^26.1.1",
+    "chokidar": "^5.0.0",
+    "chokidar-cli": "^3.0.0",
+    "dotenv": "^17.4.2",
+    "mysql2": "^3.23.1",
+    "npm-run-all": "^4.1.5",
+    "tsc-alias": "^1.9.1",
+    "tsx": "^4.23.1",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10",
+    "@byline/search-conformance": "0.0.2"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "dev": "chokidar 'src/**/*' -c 'npm-run-all build'",
+    "build": "tsc -p tsconfig.json && tsc-alias",
+    "clean": "node scripts/clean.js node_modules dist build .turbo",
+    "lint": "biome check --write --unsafe --diagnostic-level=error",
+    "test": "vitest run --mode=node",
+    "test:integration": "vitest run --mode=integration",
+    "test:watch": "vitest --mode=node",
+    "typecheck": "tsc --noEmit"
+  }
+}