@world-engines/project-host 0.1.0-alpha.0

package/dist/chinese-lexical.d.ts

@@ -0,0 +1,152 @@
+export declare const LEGACY_CHINESE_LEXICAL_PROFILE_VERSION: "chinese-lexical/v1";
+export declare const LEGACY_CHINESE_LEXICAL_PROFILE_V2_VERSION: "chinese-lexical/v2";
+export declare const LEGACY_CHINESE_LEXICAL_PROFILE_V3_VERSION: "chinese-lexical/v3";
+export declare const CHINESE_LEXICAL_PROFILE_VERSION: "chinese-lexical/v4";
+declare const LEGACY_CHINESE_LEXICAL_STRATEGY_V2_VERSION: "icu-word+han-bigram+entity-overlay/v1";
+declare const LEGACY_CHINESE_LEXICAL_STRATEGY_V3_VERSION: "icu-word+in-word-han-bigram+entity-overlay/v2";
+declare const CHINESE_LEXICAL_STRATEGY_VERSION: "icu-word+in-word-han-bigram+entity-overlay-bigram/v3";
+declare const OPENCC_VERSION: "1.4.2";
+declare const OPENCC_DICTIONARY_IDENTITY: "opencc-js/t2cn@1.4.2:t-to-cn";
+export type ChineseLexicalErrorCode = "invalid_input" | "resource_limit" | "profile_invalid";
+export declare class ChineseLexicalError extends Error {
+    readonly code: ChineseLexicalErrorCode;
+    readonly name = "ChineseLexicalError";
+    constructor(code: ChineseLexicalErrorCode);
+}
+export interface LegacyChineseLexicalRuntimeIdentityV1 {
+    readonly engine: "Intl.Segmenter";
+    readonly locale: string;
+    readonly granularity: "word";
+    readonly node: string;
+    readonly icu: string;
+    readonly unicode: string;
+    readonly cldr: string;
+}
+export interface LegacyChineseLexicalIndexProfileV1 {
+    readonly schemaVersion: 1;
+    readonly version: typeof LEGACY_CHINESE_LEXICAL_PROFILE_VERSION;
+    readonly runtimeIdentity: LegacyChineseLexicalRuntimeIdentityV1;
+    /** 原始 source texts 的顺序无关 multiset hash；不对正文做 trim/NFC/简繁转换。 */
+    readonly corpusHash: string;
+    readonly entityLexiconHash: string;
+    readonly profileHash: string;
+    /** UTF-16 code unit 数，用于显式约束 profile 的构建成本。 */
+    readonly sourceCodeUnits: number;
+    readonly dictionary: readonly string[];
+    readonly dictionaryStats: Readonly<{
+        corpusTerms: number;
+        entityLexiconTerms: number;
+        totalTerms: number;
+    }>;
+    readonly limits: Readonly<{
+        minCorpusTermFrequency: 2;
+        minHanTermCodePoints: 2;
+        maxHanTermCodePoints: 8;
+        maxLatinNumericTokenCodePoints: 64;
+        maxDictionaryTerms: 100_000;
+        maxLexicalizeInputCodeUnits: 1_000_000;
+        maxLexicalizeOutputTerms: 1_000_000;
+    }>;
+    /** 此算法保持码点原样，明确不承担简繁转换。 */
+    readonly simplifiedTraditionalConversion: "none";
+}
+export interface LegacyChineseLexicalRuntimeIdentityV2 {
+    readonly engine: "Intl.Segmenter";
+    readonly locale: string;
+    readonly granularity: "word";
+    readonly node: string;
+    readonly icu: string;
+    readonly unicode: string;
+    readonly cldr: string;
+    readonly strategy: typeof LEGACY_CHINESE_LEXICAL_STRATEGY_V2_VERSION;
+    readonly normalization: Readonly<{
+        engine: "opencc-js";
+        version: typeof OPENCC_VERSION;
+        mode: "t-to-cn";
+        dictionaryIdentity: typeof OPENCC_DICTIONARY_IDENTITY;
+    }>;
+}
+export interface LegacyChineseLexicalIndexProfileV2 {
+    readonly schemaVersion: 2;
+    readonly version: typeof LEGACY_CHINESE_LEXICAL_PROFILE_V2_VERSION;
+    readonly runtimeIdentity: LegacyChineseLexicalRuntimeIdentityV2;
+    /** 原始 source texts 的顺序无关 multiset hash。 */
+    readonly rawCorpusHash: string;
+    /** t→cn 派生 source texts 的顺序无关 multiset hash。 */
+    readonly corpusHash: string;
+    readonly rawEntityLexiconHash: string;
+    readonly entityLexiconHash: string;
+    /** 只读实体 overlay 的内容 hash；不注入 OpenCC 或 Intl 的全局状态。 */
+    readonly dictionaryHash: string;
+    readonly profileHash: string;
+    readonly sourceCodeUnits: number;
+    readonly dictionary: readonly string[];
+    readonly dictionaryStats: Readonly<{
+        entityLexiconTerms: number;
+        totalTerms: number;
+    }>;
+    readonly limits: Readonly<{
+        minHanTermCodePoints: 2;
+        maxHanTermCodePoints: 8;
+        maxLatinNumericTokenCodePoints: 64;
+        maxDictionaryTerms: 100_000;
+        maxLexicalizeInputCodeUnits: 1_000_000;
+        maxLexicalizeOutputTerms: 1_000_000;
+    }>;
+    readonly simplifiedTraditionalConversion: "opencc-t-to-cn";
+}
+export interface LegacyChineseLexicalRuntimeIdentityV3 extends Omit<LegacyChineseLexicalRuntimeIdentityV2, "strategy"> {
+    readonly strategy: typeof LEGACY_CHINESE_LEXICAL_STRATEGY_V3_VERSION;
+}
+export interface LegacyChineseLexicalIndexProfileV3 extends Omit<LegacyChineseLexicalIndexProfileV2, "schemaVersion" | "version" | "runtimeIdentity"> {
+    readonly schemaVersion: 3;
+    readonly version: typeof LEGACY_CHINESE_LEXICAL_PROFILE_V3_VERSION;
+    readonly runtimeIdentity: LegacyChineseLexicalRuntimeIdentityV3;
+}
+export interface ChineseLexicalRuntimeIdentity extends Omit<LegacyChineseLexicalRuntimeIdentityV2, "strategy"> {
+    readonly strategy: typeof CHINESE_LEXICAL_STRATEGY_VERSION;
+}
+export interface ChineseLexicalIndexProfile extends Omit<LegacyChineseLexicalIndexProfileV2, "schemaVersion" | "version" | "runtimeIdentity"> {
+    readonly schemaVersion: 4;
+    readonly version: typeof CHINESE_LEXICAL_PROFILE_VERSION;
+    readonly runtimeIdentity: ChineseLexicalRuntimeIdentity;
+}
+type NormalizedChineseLexicalIndexProfile = ChineseLexicalIndexProfile | LegacyChineseLexicalIndexProfileV3 | LegacyChineseLexicalIndexProfileV2;
+export type AnyChineseLexicalIndexProfile = NormalizedChineseLexicalIndexProfile | LegacyChineseLexicalIndexProfileV1;
+export type ChineseLexicalProfileOptions = Readonly<{
+    entityLabels?: readonly string[];
+    aliases?: readonly string[];
+}>;
+/**
+ * 按 v1 规则从已校验的 source 重建 checkpoint 派生 profile。
+ *
+ * 保留这个具名入口，迁移代码不得把 checkpoint 中未经重算的 dictionary/profileHash
+ * 直接当成可信派生数据。
+ */
+export declare function buildLegacyChineseLexicalIndexProfileV1(texts: readonly string[], options?: ChineseLexicalProfileOptions): LegacyChineseLexicalIndexProfileV1;
+/** 按旧 v2 hash/strategy 原样重建，供已校验 checkpoint 的显式迁移使用。 */
+export declare function buildLegacyChineseLexicalIndexProfileV2(texts: readonly string[], options?: ChineseLexicalProfileOptions): LegacyChineseLexicalIndexProfileV2;
+/** 按旧 v3 hash/strategy 原样重建，供已校验 checkpoint 的显式迁移使用。 */
+export declare function buildLegacyChineseLexicalIndexProfileV3(texts: readonly string[], options?: ChineseLexicalProfileOptions): LegacyChineseLexicalIndexProfileV3;
+export declare function buildChineseLexicalIndexProfile(texts: readonly string[], options?: ChineseLexicalProfileOptions): ChineseLexicalIndexProfile;
+/**
+ * 生成供 FTS 派生字段与 query planner 共用的确定性 terms。
+ * 不修改原文、不做简繁转换；profile 首次使用时构建一次 trie，后续按对象 identity 复用。
+ */
+export declare function legacyChineseLexicalTermsV1(text: string, profile: LegacyChineseLexicalIndexProfileV1): readonly string[];
+/** 按旧 v2 full-run bigram 规则生成 terms，供 checkpoint 验证和显式迁移使用。 */
+export declare function legacyChineseLexicalTermsV2(text: string, profile: LegacyChineseLexicalIndexProfileV2): readonly string[];
+/** 按旧 v3 词内 bigram 规则生成 terms，供 checkpoint 验证和显式迁移使用。 */
+export declare function legacyChineseLexicalTermsV3(text: string, profile: LegacyChineseLexicalIndexProfileV3): readonly string[];
+/**
+ * 只生成 derived lexical terms。OpenCC 转换不回写原文，caller 持有的 text/provenance/offset 保持不变。
+ */
+export declare function chineseLexicalTerms(text: string, profile: AnyChineseLexicalIndexProfile): readonly string[];
+/** 仅用于 FTS derived field；调用方必须另外保存原始 text/provenance/offset。 */
+export declare function lexicalize(text: string, profile: AnyChineseLexicalIndexProfile): string;
+/**
+ * FTS query 专用投影：只移除封闭集合中的中文疑问功能词，保留实体、否定、时间和谓词。
+ * corpus 仍使用完整 lexicalize，不能借 stop-term 规则改写原文或推断语义。
+ */
+export declare function lexicalizeQuery(text: string, profile: AnyChineseLexicalIndexProfile): string;
+export {};