@world-engines/project-host 0.1.0-alpha.0

package/dist/chinese-lexical.js

@@ -0,0 +1,555 @@
+import { createHash } from "node:crypto";
+import OpenCC from "opencc-js/t2cn";
+export const LEGACY_CHINESE_LEXICAL_PROFILE_VERSION = "chinese-lexical/v1";
+export const LEGACY_CHINESE_LEXICAL_PROFILE_V2_VERSION = "chinese-lexical/v2";
+export const LEGACY_CHINESE_LEXICAL_PROFILE_V3_VERSION = "chinese-lexical/v3";
+export const CHINESE_LEXICAL_PROFILE_VERSION = "chinese-lexical/v4";
+const LEGACY_CHINESE_LEXICAL_STRATEGY_V2_VERSION = "icu-word+han-bigram+entity-overlay/v1";
+const LEGACY_CHINESE_LEXICAL_STRATEGY_V3_VERSION = "icu-word+in-word-han-bigram+entity-overlay/v2";
+const CHINESE_LEXICAL_STRATEGY_VERSION = "icu-word+in-word-han-bigram+entity-overlay-bigram/v3";
+const OPENCC_VERSION = "1.4.2";
+const OPENCC_DICTIONARY_IDENTITY = "opencc-js/t2cn@1.4.2:t-to-cn";
+const MIN_CORPUS_TERM_FREQUENCY = 2;
+const MIN_HAN_TERM_CODE_POINTS = 2;
+const MAX_HAN_TERM_CODE_POINTS = 8;
+const MAX_LATIN_NUMERIC_TOKEN_CODE_POINTS = 64;
+const MAX_PROFILE_DICTIONARY_TERMS = 100_000;
+const MAX_SOURCE_TEXTS = 100_000;
+const MAX_SOURCE_CODE_UNITS = 10_000_000;
+const MAX_ENTITY_LEXICON_ENTRIES = 100_000;
+const MAX_ENTITY_LEXICON_ENTRY_CODE_UNITS = 512;
+const MAX_LEXICALIZE_INPUT_CODE_UNITS = 1_000_000;
+const MAX_LEXICALIZE_OUTPUT_TERMS = 1_000_000;
+const CHINESE_QUERY_STOP_TERMS = new Set(["的", "了", "在", "是", "吗", "呢", "与", "和", "跟", "向", "从", "由", "把", "被", "为", "到",
+    "什么", "哪里", "哪儿", "哪个", "哪些", "谁", "是否", "的吗", "怎么", "如何", "为何", "为什么"]);
+export class ChineseLexicalError extends Error {
+    code;
+    name = "ChineseLexicalError";
+    constructor(code) {
+        super(`chinese_lexical:${code}`);
+        this.code = code;
+    }
+}
+const legacyTries = new WeakMap();
+const overlayTries = new WeakMap();
+const hanCharacter = /^\p{Script=Han}$/u;
+const latinOrNumberCharacter = /^(?:\p{Script=Latin}|\p{Number})$/u;
+const sha256Pattern = /^[a-f\d]{64}$/u;
+const normalizeToSimplified = OpenCC.Converter({ from: "t", to: "cn" });
+function fail(code) {
+    throw new ChineseLexicalError(code);
+}
+function sha256(parts) {
+    const hash = createHash("sha256");
+    for (const part of parts) {
+        hash.update(String(Buffer.byteLength(part, "utf8")), "utf8");
+        hash.update(":", "utf8");
+        hash.update(part, "utf8");
+        hash.update(";", "utf8");
+    }
+    return hash.digest("hex");
+}
+function multisetHash(kind, values) {
+    const entries = values.map((value) => sha256([kind, value])).sort(compareText);
+    return sha256([kind, ...entries]);
+}
+function compareText(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function isHanTerm(value) {
+    const characters = [...value];
+    return characters.length >= MIN_HAN_TERM_CODE_POINTS
+        && characters.length <= MAX_HAN_TERM_CODE_POINTS
+        && characters.every((character) => hanCharacter.test(character));
+}
+function validateStringArray(value, maximumEntries, maximumCodeUnits) {
+    if (!Array.isArray(value) || value.length > maximumEntries
+        || value.some((entry) => typeof entry !== "string" || entry.length > maximumCodeUnits))
+        fail("invalid_input");
+    return value;
+}
+function legacyRuntimeIdentity(segmenter) {
+    return Object.freeze({
+        engine: "Intl.Segmenter",
+        locale: segmenter.resolvedOptions().locale,
+        granularity: "word",
+        node: process.versions.node,
+        icu: process.versions.icu ?? "unknown",
+        unicode: process.versions.unicode ?? "unknown",
+        cldr: process.versions.cldr ?? "unknown",
+    });
+}
+function normalizedRuntimeIdentity(segmenter, strategy) {
+    return Object.freeze({
+        ...legacyRuntimeIdentity(segmenter),
+        strategy,
+        normalization: Object.freeze({
+            engine: "opencc-js",
+            version: OPENCC_VERSION,
+            mode: "t-to-cn",
+            dictionaryIdentity: OPENCC_DICTIONARY_IDENTITY,
+        }),
+    });
+}
+function identityString(identity) {
+    return JSON.stringify(identity);
+}
+function addEntityLexiconTerms(segmenter, value, destination) {
+    const trimmed = value.trim();
+    if (isHanTerm(trimmed))
+        destination.add(trimmed);
+    for (const item of segmenter.segment(trimmed)) {
+        if (item.isWordLike && isHanTerm(item.segment))
+            destination.add(item.segment);
+    }
+}
+/**
+ * 按 v1 规则从已校验的 source 重建 checkpoint 派生 profile。
+ *
+ * 保留这个具名入口，迁移代码不得把 checkpoint 中未经重算的 dictionary/profileHash
+ * 直接当成可信派生数据。
+ */
+export function buildLegacyChineseLexicalIndexProfileV1(texts, options = {}) {
+    const sources = validateStringArray(texts, MAX_SOURCE_TEXTS, MAX_SOURCE_CODE_UNITS);
+    if (sources.length === 0)
+        fail("invalid_input");
+    const sourceCodeUnits = sources.reduce((sum, value) => sum + value.length, 0);
+    if (!Number.isSafeInteger(sourceCodeUnits) || sourceCodeUnits > MAX_SOURCE_CODE_UNITS)
+        fail("resource_limit");
+    const entityLabels = validateStringArray(options.entityLabels ?? [], MAX_ENTITY_LEXICON_ENTRIES, MAX_ENTITY_LEXICON_ENTRY_CODE_UNITS);
+    const aliases = validateStringArray(options.aliases ?? [], MAX_ENTITY_LEXICON_ENTRIES, MAX_ENTITY_LEXICON_ENTRY_CODE_UNITS);
+    if (entityLabels.length + aliases.length > MAX_ENTITY_LEXICON_ENTRIES)
+        fail("resource_limit");
+    const segmenter = new Intl.Segmenter("zh", { granularity: "word" });
+    const counts = new Map();
+    for (const text of sources) {
+        for (const item of segmenter.segment(text)) {
+            if (item.isWordLike && isHanTerm(item.segment))
+                counts.set(item.segment, (counts.get(item.segment) ?? 0) + 1);
+        }
+    }
+    const corpusTerms = new Set([...counts].filter(([, count]) => count >= MIN_CORPUS_TERM_FREQUENCY).map(([term]) => term));
+    const entityTerms = new Set();
+    for (const value of [...entityLabels, ...aliases])
+        addEntityLexiconTerms(segmenter, value, entityTerms);
+    const dictionary = [...new Set([...corpusTerms, ...entityTerms])].sort(compareText);
+    if (dictionary.length > MAX_PROFILE_DICTIONARY_TERMS)
+        fail("resource_limit");
+    const identity = legacyRuntimeIdentity(segmenter);
+    const corpusHash = multisetHash("source-text", sources);
+    const entityLexiconHash = multisetHash("entity-lexicon", [...entityLabels, ...aliases]);
+    const profileHash = sha256([
+        LEGACY_CHINESE_LEXICAL_PROFILE_VERSION,
+        identityString(identity),
+        corpusHash,
+        entityLexiconHash,
+        ...dictionary,
+    ]);
+    return Object.freeze({
+        schemaVersion: 1,
+        version: LEGACY_CHINESE_LEXICAL_PROFILE_VERSION,
+        runtimeIdentity: identity,
+        corpusHash,
+        entityLexiconHash,
+        profileHash,
+        sourceCodeUnits,
+        dictionary: Object.freeze(dictionary),
+        dictionaryStats: Object.freeze({
+            corpusTerms: corpusTerms.size,
+            entityLexiconTerms: entityTerms.size,
+            totalTerms: dictionary.length,
+        }),
+        limits: Object.freeze({
+            minCorpusTermFrequency: MIN_CORPUS_TERM_FREQUENCY,
+            minHanTermCodePoints: MIN_HAN_TERM_CODE_POINTS,
+            maxHanTermCodePoints: MAX_HAN_TERM_CODE_POINTS,
+            maxLatinNumericTokenCodePoints: MAX_LATIN_NUMERIC_TOKEN_CODE_POINTS,
+            maxDictionaryTerms: MAX_PROFILE_DICTIONARY_TERMS,
+            maxLexicalizeInputCodeUnits: MAX_LEXICALIZE_INPUT_CODE_UNITS,
+            maxLexicalizeOutputTerms: MAX_LEXICALIZE_OUTPUT_TERMS,
+        }),
+        simplifiedTraditionalConversion: "none",
+    });
+}
+function buildNormalizedChineseLexicalIndexProfile(texts, options, schemaVersion) {
+    const sources = validateStringArray(texts, MAX_SOURCE_TEXTS, MAX_SOURCE_CODE_UNITS);
+    if (sources.length === 0)
+        fail("invalid_input");
+    const sourceCodeUnits = sources.reduce((sum, value) => sum + value.length, 0);
+    if (!Number.isSafeInteger(sourceCodeUnits) || sourceCodeUnits > MAX_SOURCE_CODE_UNITS)
+        fail("resource_limit");
+    const entityLabels = validateStringArray(options.entityLabels ?? [], MAX_ENTITY_LEXICON_ENTRIES, MAX_ENTITY_LEXICON_ENTRY_CODE_UNITS);
+    const aliases = validateStringArray(options.aliases ?? [], MAX_ENTITY_LEXICON_ENTRIES, MAX_ENTITY_LEXICON_ENTRY_CODE_UNITS);
+    if (entityLabels.length + aliases.length > MAX_ENTITY_LEXICON_ENTRIES)
+        fail("resource_limit");
+    const normalizedSources = sources.map((source) => normalizeToSimplified(source));
+    const rawEntityLexicon = [...entityLabels, ...aliases];
+    const normalizedEntityLexicon = rawEntityLexicon.map((entry) => normalizeToSimplified(entry));
+    const dictionary = [...new Set(normalizedEntityLexicon.map((entry) => entry.trim()).filter(isHanTerm))].sort(compareText);
+    if (dictionary.length > MAX_PROFILE_DICTIONARY_TERMS)
+        fail("resource_limit");
+    const version = schemaVersion === 2
+        ? LEGACY_CHINESE_LEXICAL_PROFILE_V2_VERSION
+        : schemaVersion === 3 ? LEGACY_CHINESE_LEXICAL_PROFILE_V3_VERSION : CHINESE_LEXICAL_PROFILE_VERSION;
+    const strategy = schemaVersion === 2
+        ? LEGACY_CHINESE_LEXICAL_STRATEGY_V2_VERSION
+        : schemaVersion === 3 ? LEGACY_CHINESE_LEXICAL_STRATEGY_V3_VERSION : CHINESE_LEXICAL_STRATEGY_VERSION;
+    const identity = normalizedRuntimeIdentity(new Intl.Segmenter("zh", { granularity: "word" }), strategy);
+    const rawCorpusHash = multisetHash("raw-source-text", sources);
+    const corpusHash = multisetHash("normalized-source-text", normalizedSources);
+    const rawEntityLexiconHash = multisetHash("raw-entity-lexicon", rawEntityLexicon);
+    const entityLexiconHash = multisetHash("normalized-entity-lexicon", normalizedEntityLexicon);
+    const dictionaryHash = sha256(["entity-overlay", ...dictionary]);
+    const profileHash = sha256([
+        version,
+        identityString(identity),
+        rawCorpusHash,
+        corpusHash,
+        rawEntityLexiconHash,
+        entityLexiconHash,
+        dictionaryHash,
+    ]);
+    return Object.freeze({
+        schemaVersion,
+        version,
+        runtimeIdentity: identity,
+        rawCorpusHash,
+        corpusHash,
+        rawEntityLexiconHash,
+        entityLexiconHash,
+        dictionaryHash,
+        profileHash,
+        sourceCodeUnits,
+        dictionary: Object.freeze(dictionary),
+        dictionaryStats: Object.freeze({ entityLexiconTerms: dictionary.length, totalTerms: dictionary.length }),
+        limits: Object.freeze({
+            minHanTermCodePoints: MIN_HAN_TERM_CODE_POINTS,
+            maxHanTermCodePoints: MAX_HAN_TERM_CODE_POINTS,
+            maxLatinNumericTokenCodePoints: MAX_LATIN_NUMERIC_TOKEN_CODE_POINTS,
+            maxDictionaryTerms: MAX_PROFILE_DICTIONARY_TERMS,
+            maxLexicalizeInputCodeUnits: MAX_LEXICALIZE_INPUT_CODE_UNITS,
+            maxLexicalizeOutputTerms: MAX_LEXICALIZE_OUTPUT_TERMS,
+        }),
+        simplifiedTraditionalConversion: "opencc-t-to-cn",
+    });
+}
+/** 按旧 v2 hash/strategy 原样重建，供已校验 checkpoint 的显式迁移使用。 */
+export function buildLegacyChineseLexicalIndexProfileV2(texts, options = {}) {
+    return buildNormalizedChineseLexicalIndexProfile(texts, options, 2);
+}
+/** 按旧 v3 hash/strategy 原样重建，供已校验 checkpoint 的显式迁移使用。 */
+export function buildLegacyChineseLexicalIndexProfileV3(texts, options = {}) {
+    return buildNormalizedChineseLexicalIndexProfile(texts, options, 3);
+}
+export function buildChineseLexicalIndexProfile(texts, options = {}) {
+    return buildNormalizedChineseLexicalIndexProfile(texts, options, 4);
+}
+function validateLegacyProfile(profile) {
+    const runtime = profile?.runtimeIdentity;
+    if (profile?.schemaVersion !== 1 || profile.version !== LEGACY_CHINESE_LEXICAL_PROFILE_VERSION
+        || profile.simplifiedTraditionalConversion !== "none"
+        || !sha256Pattern.test(profile.corpusHash) || !sha256Pattern.test(profile.entityLexiconHash)
+        || !sha256Pattern.test(profile.profileHash)
+        || runtime?.engine !== "Intl.Segmenter" || runtime.granularity !== "word"
+        || typeof runtime.locale !== "string" || runtime.locale.length === 0
+        || !Array.isArray(profile.dictionary) || profile.dictionary.length > MAX_PROFILE_DICTIONARY_TERMS
+        || profile.dictionary.some((term, index) => !isHanTerm(term)
+            || (index > 0 && compareText(profile.dictionary[index - 1], term) >= 0)))
+        fail("profile_invalid");
+    const expectedProfileHash = sha256([
+        LEGACY_CHINESE_LEXICAL_PROFILE_VERSION,
+        identityString(runtime),
+        profile.corpusHash,
+        profile.entityLexiconHash,
+        ...profile.dictionary,
+    ]);
+    if (profile.profileHash !== expectedProfileHash)
+        fail("profile_invalid");
+}
+function buildTrie(dictionary) {
+    const root = { terminal: false, children: new Map() };
+    for (const term of dictionary) {
+        let node = root;
+        for (const character of term) {
+            let child = node.children.get(character);
+            if (child === undefined) {
+                child = { terminal: false, children: new Map() };
+                node.children.set(character, child);
+            }
+            node = child;
+        }
+        node.terminal = true;
+    }
+    return root;
+}
+function legacyTrie(profile) {
+    const cached = legacyTries.get(profile);
+    if (cached !== undefined)
+        return cached;
+    validateLegacyProfile(profile);
+    const root = buildTrie(profile.dictionary);
+    legacyTries.set(profile, root);
+    return root;
+}
+function validateNormalizedProfile(profile) {
+    const runtime = profile?.runtimeIdentity;
+    const normalization = runtime?.normalization;
+    const expectedVersion = profile?.schemaVersion === 2
+        ? LEGACY_CHINESE_LEXICAL_PROFILE_V2_VERSION
+        : profile?.schemaVersion === 3 ? LEGACY_CHINESE_LEXICAL_PROFILE_V3_VERSION : CHINESE_LEXICAL_PROFILE_VERSION;
+    const expectedStrategy = profile?.schemaVersion === 2
+        ? LEGACY_CHINESE_LEXICAL_STRATEGY_V2_VERSION
+        : profile?.schemaVersion === 3 ? LEGACY_CHINESE_LEXICAL_STRATEGY_V3_VERSION : CHINESE_LEXICAL_STRATEGY_VERSION;
+    if ((profile?.schemaVersion !== 2 && profile?.schemaVersion !== 3 && profile?.schemaVersion !== 4)
+        || profile.version !== expectedVersion
+        || profile.simplifiedTraditionalConversion !== "opencc-t-to-cn"
+        || !sha256Pattern.test(profile.rawCorpusHash) || !sha256Pattern.test(profile.corpusHash)
+        || !sha256Pattern.test(profile.rawEntityLexiconHash) || !sha256Pattern.test(profile.entityLexiconHash)
+        || !sha256Pattern.test(profile.dictionaryHash) || !sha256Pattern.test(profile.profileHash)
+        || runtime?.engine !== "Intl.Segmenter" || runtime.granularity !== "word"
+        || runtime.strategy !== expectedStrategy
+        || typeof runtime.locale !== "string" || runtime.locale.length === 0
+        || normalization?.engine !== "opencc-js" || normalization.version !== OPENCC_VERSION
+        || normalization.mode !== "t-to-cn" || normalization.dictionaryIdentity !== OPENCC_DICTIONARY_IDENTITY
+        || !Array.isArray(profile.dictionary) || profile.dictionary.length > MAX_PROFILE_DICTIONARY_TERMS
+        || profile.dictionary.some((term, index) => !isHanTerm(term)
+            || term !== normalizeToSimplified(term)
+            || (index > 0 && compareText(profile.dictionary[index - 1], term) >= 0)))
+        fail("profile_invalid");
+    const currentRuntime = normalizedRuntimeIdentity(new Intl.Segmenter("zh", { granularity: "word" }), expectedStrategy);
+    if (identityString(runtime) !== identityString(currentRuntime))
+        fail("profile_invalid");
+    const expectedDictionaryHash = sha256(["entity-overlay", ...profile.dictionary]);
+    const expectedProfileHash = sha256([
+        expectedVersion,
+        identityString(runtime),
+        profile.rawCorpusHash,
+        profile.corpusHash,
+        profile.rawEntityLexiconHash,
+        profile.entityLexiconHash,
+        expectedDictionaryHash,
+    ]);
+    if (profile.dictionaryHash !== expectedDictionaryHash || profile.profileHash !== expectedProfileHash)
+        fail("profile_invalid");
+}
+function overlayTrie(profile) {
+    const cached = overlayTries.get(profile);
+    if (cached !== undefined)
+        return cached;
+    validateNormalizedProfile(profile);
+    const root = buildTrie(profile.dictionary);
+    overlayTries.set(profile, root);
+    return root;
+}
+function emitBounded(tokens, token) {
+    tokens.push(token);
+    if (tokens.length > MAX_LEXICALIZE_OUTPUT_TERMS)
+        fail("resource_limit");
+}
+/**
+ * 生成供 FTS 派生字段与 query planner 共用的确定性 terms。
+ * 不修改原文、不做简繁转换；profile 首次使用时构建一次 trie，后续按对象 identity 复用。
+ */
+export function legacyChineseLexicalTermsV1(text, profile) {
+    if (typeof text !== "string")
+        fail("invalid_input");
+    if (text.length > MAX_LEXICALIZE_INPUT_CODE_UNITS)
+        fail("resource_limit");
+    const root = legacyTrie(profile);
+    const characters = [...text];
+    const tokens = [];
+    let cursor = 0;
+    while (cursor < characters.length) {
+        const current = characters[cursor];
+        if (hanCharacter.test(current)) {
+            let node = root;
+            let bestEnd = -1;
+            let scan = cursor;
+            while (scan < characters.length && hanCharacter.test(characters[scan])) {
+                node = node.children.get(characters[scan]);
+                if (node === undefined)
+                    break;
+                scan += 1;
+                if (node.terminal)
+                    bestEnd = scan;
+            }
+            if (bestEnd > cursor) {
+                emitBounded(tokens, characters.slice(cursor, bestEnd).join(""));
+                cursor = bestEnd;
+            }
+            else {
+                emitBounded(tokens, current);
+                cursor += 1;
+            }
+            continue;
+        }
+        if (latinOrNumberCharacter.test(current)) {
+            const start = cursor;
+            while (cursor < characters.length && latinOrNumberCharacter.test(characters[cursor]))
+                cursor += 1;
+            for (let offset = start; offset < cursor; offset += MAX_LATIN_NUMERIC_TOKEN_CODE_POINTS) {
+                emitBounded(tokens, characters.slice(offset, Math.min(cursor, offset + MAX_LATIN_NUMERIC_TOKEN_CODE_POINTS)).join(""));
+            }
+            continue;
+        }
+        cursor += 1;
+    }
+    return Object.freeze(tokens);
+}
+function emitHanBigrams(characters, tokens) {
+    let cursor = 0;
+    while (cursor < characters.length) {
+        if (!hanCharacter.test(characters[cursor])) {
+            cursor += 1;
+            continue;
+        }
+        const start = cursor;
+        while (cursor < characters.length && hanCharacter.test(characters[cursor]))
+            cursor += 1;
+        if (cursor - start === 1) {
+            emitBounded(tokens, characters[start]);
+            continue;
+        }
+        for (let offset = start; offset + 1 < cursor; offset += 1) {
+            emitBounded(tokens, characters.slice(offset, offset + 2).join(""));
+        }
+    }
+}
+function emitInWordHanBigrams(characters, tokens) {
+    if (characters.length <= 2)
+        return;
+    for (let offset = 0; offset + 1 < characters.length; offset += 1) {
+        emitBounded(tokens, characters.slice(offset, offset + 2).join(""));
+    }
+}
+function emitLatinNumericRuns(characters, tokens) {
+    let cursor = 0;
+    while (cursor < characters.length) {
+        if (!latinOrNumberCharacter.test(characters[cursor])) {
+            cursor += 1;
+            continue;
+        }
+        const start = cursor;
+        while (cursor < characters.length && latinOrNumberCharacter.test(characters[cursor]))
+            cursor += 1;
+        for (let offset = start; offset < cursor; offset += MAX_LATIN_NUMERIC_TOKEN_CODE_POINTS) {
+            emitBounded(tokens, characters.slice(offset, Math.min(cursor, offset + MAX_LATIN_NUMERIC_TOKEN_CODE_POINTS)).join(""));
+        }
+    }
+}
+function emitEntityOverlay(characters, root, tokens, includeInternalBigrams) {
+    const alreadyEmitted = new Set(tokens);
+    const overlayEmitted = new Set();
+    for (let cursor = 0; cursor < characters.length; cursor += 1) {
+        if (!hanCharacter.test(characters[cursor]))
+            continue;
+        let node = root;
+        const maximumEnd = Math.min(characters.length, cursor + MAX_HAN_TERM_CODE_POINTS);
+        for (let scan = cursor; scan < maximumEnd && hanCharacter.test(characters[scan]); scan += 1) {
+            node = node.children.get(characters[scan]);
+            if (node === undefined)
+                break;
+            if (!node.terminal)
+                continue;
+            const term = characters.slice(cursor, scan + 1).join("");
+            if (!alreadyEmitted.has(term) && !overlayEmitted.has(term)) {
+                emitBounded(tokens, term);
+                overlayEmitted.add(term);
+            }
+            if (includeInternalBigrams && scan - cursor + 1 > 2) {
+                for (let offset = cursor; offset < scan; offset += 1) {
+                    const bigram = characters.slice(offset, offset + 2).join("");
+                    if (!alreadyEmitted.has(bigram) && !overlayEmitted.has(bigram)) {
+                        emitBounded(tokens, bigram);
+                        overlayEmitted.add(bigram);
+                    }
+                }
+            }
+        }
+    }
+}
+function prepareNormalizedTerms(text, profile) {
+    if (typeof text !== "string")
+        fail("invalid_input");
+    if (text.length > MAX_LEXICALIZE_INPUT_CODE_UNITS)
+        fail("resource_limit");
+    const root = overlayTrie(profile);
+    const normalizedText = normalizeToSimplified(text);
+    const characters = [...normalizedText];
+    return { root, normalizedText, characters };
+}
+/** 按旧 v2 full-run bigram 规则生成 terms，供 checkpoint 验证和显式迁移使用。 */
+export function legacyChineseLexicalTermsV2(text, profile) {
+    const { root, normalizedText, characters } = prepareNormalizedTerms(text, profile);
+    const tokens = [];
+    const segmenter = new Intl.Segmenter(profile.runtimeIdentity.locale, { granularity: "word" });
+    for (const item of segmenter.segment(normalizedText)) {
+        if (!item.isWordLike)
+            continue;
+        const wordCharacters = [...item.segment];
+        if (wordCharacters.length > 0 && wordCharacters.every((character) => hanCharacter.test(character))) {
+            emitBounded(tokens, item.segment);
+        }
+    }
+    emitHanBigrams(characters, tokens);
+    emitLatinNumericRuns(characters, tokens);
+    emitEntityOverlay(characters, root, tokens, false);
+    return Object.freeze(tokens);
+}
+/** 按旧 v3 词内 bigram 规则生成 terms，供 checkpoint 验证和显式迁移使用。 */
+export function legacyChineseLexicalTermsV3(text, profile) {
+    const { root, normalizedText, characters } = prepareNormalizedTerms(text, profile);
+    const tokens = [];
+    const segmenter = new Intl.Segmenter(profile.runtimeIdentity.locale, { granularity: "word" });
+    for (const item of segmenter.segment(normalizedText)) {
+        if (!item.isWordLike)
+            continue;
+        const wordCharacters = [...item.segment];
+        if (wordCharacters.length > 0 && wordCharacters.every((character) => hanCharacter.test(character))) {
+            emitBounded(tokens, item.segment);
+            emitInWordHanBigrams(wordCharacters, tokens);
+        }
+    }
+    emitLatinNumericRuns(characters, tokens);
+    emitEntityOverlay(characters, root, tokens, false);
+    return Object.freeze(tokens);
+}
+/**
+ * 只生成 derived lexical terms。OpenCC 转换不回写原文，caller 持有的 text/provenance/offset 保持不变。
+ */
+export function chineseLexicalTerms(text, profile) {
+    if (profile?.schemaVersion === 1)
+        return legacyChineseLexicalTermsV1(text, profile);
+    if (profile?.schemaVersion === 2)
+        return legacyChineseLexicalTermsV2(text, profile);
+    if (profile?.schemaVersion === 3)
+        return legacyChineseLexicalTermsV3(text, profile);
+    const { root, normalizedText, characters } = prepareNormalizedTerms(text, profile);
+    const tokens = [];
+    const segmenter = new Intl.Segmenter(profile.runtimeIdentity.locale, { granularity: "word" });
+    for (const item of segmenter.segment(normalizedText)) {
+        if (!item.isWordLike)
+            continue;
+        const wordCharacters = [...item.segment];
+        if (wordCharacters.length > 0 && wordCharacters.every((character) => hanCharacter.test(character))) {
+            emitBounded(tokens, item.segment);
+            emitInWordHanBigrams(wordCharacters, tokens);
+        }
+    }
+    emitLatinNumericRuns(characters, tokens);
+    emitEntityOverlay(characters, root, tokens, true);
+    return Object.freeze(tokens);
+}
+/** 仅用于 FTS derived field；调用方必须另外保存原始 text/provenance/offset。 */
+export function lexicalize(text, profile) {
+    return chineseLexicalTerms(text, profile).join(" ");
+}
+/**
+ * FTS query 专用投影：只移除封闭集合中的中文疑问功能词，保留实体、否定、时间和谓词。
+ * corpus 仍使用完整 lexicalize，不能借 stop-term 规则改写原文或推断语义。
+ */
+export function lexicalizeQuery(text, profile) {
+    const terms = chineseLexicalTerms(text, profile);
+    const filtered = terms.filter((term) => !CHINESE_QUERY_STOP_TERMS.has(term));
+    return (filtered.length === 0 ? terms : filtered).join(" ");
+}

package/dist/chromium-preview-browser-driver.d.ts

@@ -0,0 +1,43 @@
+import type { PreviewBrowserDriver, PreviewBrowserInspectCapture } from "./preview-browser-adapter.js";
+import type { DesktopAuthoringCapabilities } from "./desktop-authoring-runtime.js";
+export interface ChromiumPreviewBrowserDriverOptions {
+    readonly executablePath?: string;
+    readonly tempRoot?: string;
+    readonly headless?: boolean;
+    /** 可选真实 Host 页面；提供时 DOM/交互只在其 sandbox Preview frame context 中执行。 */
+    readonly hostPageUrl?: string;
+    readonly previewFrameSelector?: string;
+    readonly activatePreviewSelector?: string;
+    readonly expectedProjectId?: string;
+    /** 首次 inspect 截图前等待 ChatPlay ready 与可绘制 View 的最长时限。 */
+    readonly renderReadyTimeoutMs?: number;
+}
+export interface ChromiumPreviewBrowserHarness {
+    readonly driver: ChromiumPreviewBrowserDriver;
+    readonly desktopAuthoringCapabilities: Pick<DesktopAuthoringCapabilities, "preview_inspect" | "preview_interact">;
+    close(): Promise<void>;
+}
+export interface PreviewBrowserHarnessRuntime {
+    inspect(): import("./preview-runtime.js").PreviewSession | null;
+}
+export interface CreateChromiumPreviewBrowserHarnessOptions extends ChromiumPreviewBrowserDriverOptions {
+    readonly startTimeoutMs?: number;
+}
+/** 系统 Chrome/Edge 的真实 CDP driver；默认 headless，不弹窗，也不注入 cookie 或作者能力。 */
+export declare class ChromiumPreviewBrowserDriver implements PreviewBrowserDriver {
+    #private;
+    constructor(options?: ChromiumPreviewBrowserDriverOptions);
+    inspect(request: Parameters<PreviewBrowserDriver["inspect"]>[0]): Promise<PreviewBrowserInspectCapture>;
+    interact(request: Parameters<PreviewBrowserDriver["interact"]>[0]): Promise<Readonly<{
+        url: string;
+    }>>;
+    /** 仅供开发 harness 验证正式高层 SDK；不暴露任意 RPC/eval。 */
+    sendChatPlayUserMessage(sessionId: string, previewUrl: string, text: string): Promise<Readonly<{
+        messageId?: string;
+        turnId?: string;
+    }>>;
+    listChatPlayMessages(sessionId: string, previewUrl: string): Promise<readonly Readonly<Record<string, unknown>>[]>;
+    close(): Promise<void>;
+}
+/** ProjectHost/CLI 的窄接线入口：CLI 只需把返回 capability 传入 LocalProjectHostOptions。 */
+export declare function createChromiumPreviewBrowserHarness(runtime: PreviewBrowserHarnessRuntime, options?: CreateChromiumPreviewBrowserHarnessOptions): Promise<ChromiumPreviewBrowserHarness>;