@wei840222/qmd 2026.8.23

package/dist/search/cjk-analyzer.js

@@ -0,0 +1,158 @@
+import { createJiebaUnavailableCapability, loadJiebaCapability, loadJiebaCapabilitySync, } from "./jieba-loader.js";
+const LEGACY_CJK_RUN_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+const DIRECT_CJK_PATTERN = /^(?:\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana}|\p{Script=Hangul})$/u;
+const CJK_SCRIPT_EXTENSIONS_PATTERN = /^(?:\p{Script_Extensions=Han}|\p{Script_Extensions=Hiragana}|\p{Script_Extensions=Katakana}|\p{Script_Extensions=Hangul})$/u;
+const DIRECT_HAN_PATTERN = /^\p{Script=Han}$/u;
+const HAN_SCRIPT_EXTENSIONS_PATTERN = /^\p{Script_Extensions=Han}$/u;
+const DIRECT_KANA_OR_HANGUL_PATTERN = /^(?:\p{Script=Hiragana}|\p{Script=Katakana}|\p{Script=Hangul})$/u;
+const KANA_OR_HANGUL_SCRIPT_EXTENSIONS_PATTERN = /^(?:\p{Script_Extensions=Hiragana}|\p{Script_Extensions=Katakana}|\p{Script_Extensions=Hangul})$/u;
+const LETTER_OR_MARK_PATTERN = /^[\p{L}\p{M}]$/u;
+const LINE_BREAK_PATTERN = /[\u000b\u000c\r\n\u0085\u2028\u2029]/u;
+const SENTENCE_TERMINAL_PATTERN = /\p{Sentence_Terminal}/u;
+const IDENTIFIER_DOT_PATTERN = /^[.\u2024\ufe52\uff0e]$/u;
+const IDENTIFIER_CHARACTER_PATTERN = /[\p{L}\p{M}\p{N}_-]/u;
+const SEARCHABLE_TOKEN_PATTERN = /[\p{L}\p{N}]/u;
+export const CJK_ANALYZER_POLICY_VERSIONS = Object.freeze({
+    analyzer: "cjk-analyzer-v3",
+    char: "legacy-script-runs-v1",
+    wordBoundary: "unicode-sentence-terminal-v3",
+    wordEligibility: "direct-script-and-shared-letter-mark-v2",
+    bigram: "direct-script-and-shared-letter-mark-v2",
+});
+function serializeCharSignal(text) {
+    return text.replace(LEGACY_CJK_RUN_PATTERN, run => ` ${Array.from(run).join(" ")} `);
+}
+function belongsToScript(codePoint, directPattern, scriptExtensionsPattern) {
+    return directPattern.test(codePoint)
+        || (LETTER_OR_MARK_PATTERN.test(codePoint) && scriptExtensionsPattern.test(codePoint));
+}
+function isCjkCodePoint(codePoint) {
+    return belongsToScript(codePoint, DIRECT_CJK_PATTERN, CJK_SCRIPT_EXTENSIONS_PATTERN);
+}
+export function containsCjk(text) {
+    return Array.from(text).some(isCjkCodePoint);
+}
+function hasScriptCodePoint(text, directPattern, scriptExtensionsPattern) {
+    return Array.from(text).some(codePoint => belongsToScript(codePoint, directPattern, scriptExtensionsPattern));
+}
+function serializeBigramSignal(text) {
+    const tokens = [];
+    let previous;
+    for (const codePoint of text) {
+        if (!isCjkCodePoint(codePoint)) {
+            previous = undefined;
+            continue;
+        }
+        if (previous !== undefined)
+            tokens.push(previous + codePoint);
+        previous = codePoint;
+    }
+    return tokens.join(" ");
+}
+function isIdentifierDot(codePoints, index) {
+    const codePoint = codePoints[index];
+    if (codePoint === undefined || !IDENTIFIER_DOT_PATTERN.test(codePoint))
+        return false;
+    const previous = codePoints[index - 1];
+    const next = codePoints[index + 1];
+    return previous !== undefined
+        && next !== undefined
+        && IDENTIFIER_CHARACTER_PATTERN.test(previous)
+        && IDENTIFIER_CHARACTER_PATTERN.test(next);
+}
+function isSegmentBoundary(codePoints, index) {
+    const codePoint = codePoints[index];
+    if (LINE_BREAK_PATTERN.test(codePoint))
+        return true;
+    return SENTENCE_TERMINAL_PATTERN.test(codePoint) && !isIdentifierDot(codePoints, index);
+}
+function getJiebaSegments(text) {
+    const segments = [];
+    const codePoints = Array.from(text);
+    let current = "";
+    const flush = () => {
+        const segment = current.trim();
+        const hasHan = hasScriptCodePoint(segment, DIRECT_HAN_PATTERN, HAN_SCRIPT_EXTENSIONS_PATTERN);
+        const hasKanaOrHangul = hasScriptCodePoint(segment, DIRECT_KANA_OR_HANGUL_PATTERN, KANA_OR_HANGUL_SCRIPT_EXTENSIONS_PATTERN);
+        if (hasHan && !hasKanaOrHangul) {
+            segments.push(segment);
+        }
+        current = "";
+    };
+    for (let index = 0; index < codePoints.length; index++) {
+        if (isSegmentBoundary(codePoints, index))
+            flush();
+        else
+            current += codePoints[index];
+    }
+    flush();
+    return segments;
+}
+function serializeWordSignal(segments, capability) {
+    const tokens = [];
+    for (const segment of segments) {
+        const segmentTokens = capability.cut(segment, false);
+        if (!Array.isArray(segmentTokens))
+            throw new TypeError("Invalid jieba token stream");
+        for (const token of segmentTokens) {
+            if (typeof token !== "string")
+                throw new TypeError("Invalid jieba token");
+            const trimmed = token.trim();
+            if (trimmed && SEARCHABLE_TOKEN_PATTERN.test(trimmed))
+                tokens.push(trimmed);
+        }
+    }
+    return tokens.join(" ");
+}
+function unavailableWordCapability() {
+    const { diagnostic } = createJiebaUnavailableCapability();
+    return Object.freeze({ available: false, diagnostic });
+}
+function failedWordCapability() {
+    const diagnostic = Object.freeze({
+        code: "CJK_ANALYZER_FAILED",
+        message: "Chinese word segmentation failed while analyzing indexed content.",
+        runtime: `${process.platform}-${process.arch}`,
+        remediation: "Retry the operation; if it continues to fail, verify the dictionary and @node-rs/jieba runtime compatibility.",
+    });
+    return Object.freeze({ available: false, diagnostic });
+}
+export function analyzeCjkWithCapability(text, capability) {
+    const char = serializeCharSignal(text);
+    const bigram = serializeBigramSignal(text);
+    if (!capability.available) {
+        return { char, word: "", bigram, wordCapability: unavailableWordCapability() };
+    }
+    try {
+        const word = serializeWordSignal(getJiebaSegments(text), capability);
+        return {
+            char,
+            word,
+            bigram,
+            wordCapability: Object.freeze({ available: true }),
+        };
+    }
+    catch {
+        return { char, word: "", bigram, wordCapability: failedWordCapability() };
+    }
+}
+export async function analyzeCjk(text, loadCapability = loadJiebaCapability) {
+    try {
+        return analyzeCjkWithCapability(text, await loadCapability());
+    }
+    catch {
+        const char = serializeCharSignal(text);
+        const bigram = serializeBigramSignal(text);
+        return { char, word: "", bigram, wordCapability: unavailableWordCapability() };
+    }
+}
+export function analyzeCjkSync(text, loadCapability = loadJiebaCapabilitySync) {
+    try {
+        return analyzeCjkWithCapability(text, loadCapability());
+    }
+    catch {
+        const char = serializeCharSignal(text);
+        const bigram = serializeBigramSignal(text);
+        return { char, word: "", bigram, wordCapability: unavailableWordCapability() };
+    }
+}

package/dist/search/cjk-index.d.ts

@@ -0,0 +1,104 @@
+import { type CjkWordDiagnostic, type JiebaCapabilityLoader } from "./cjk-analyzer.js";
+import { type Database } from "../db.js";
+export interface CjkAnalyzerFingerprintSource {
+    analyzerVersion: string;
+    charPolicy: string;
+    wordBoundaryPolicy: string;
+    wordEligibilityPolicy: string;
+    bigramPolicy: string;
+    jiebaPackage: string;
+    jiebaVersion: string;
+    jiebaHmm: string;
+    dictionaryVersion: string;
+    dictionarySha256: string;
+    userDictionarySha256?: string;
+}
+export declare const CJK_ANALYZER_FINGERPRINT_SOURCE: Readonly<CjkAnalyzerFingerprintSource>;
+export declare function computeCjkAnalyzerFingerprint(source: CjkAnalyzerFingerprintSource): string;
+export declare function getCjkAnalyzerFingerprint(userDictContent?: Uint8Array | string): string;
+export type CjkLexicalIndexStatus = "empty" | "building" | "ready" | "unavailable" | "dirty";
+export interface CjkLexicalIndexState {
+    status: CjkLexicalIndexStatus;
+    generation: number;
+    analyzerFingerprint: string | null;
+    wordCapability: "unknown" | "available" | "unavailable";
+    diagnosticCode: string | null;
+    activeBuildId: string | null;
+    publishedBuildId: string | null;
+    dirtySinceMutationSeq: number | null;
+    updatedAt: number;
+}
+export type CjkIndexBuildPhase = {
+    phase: "snapshot-complete";
+    buildId: string;
+    baseMutationSeq: number;
+} | {
+    phase: "catchup-page-complete";
+    buildId: string;
+    targetMutationSeq: number;
+    appliedMutationSeq: number;
+    documentCount: number;
+} | {
+    phase: "catchup-complete";
+    buildId: string;
+    targetMutationSeq: number;
+    documentCount: number;
+} | {
+    phase: "published";
+    buildId: string;
+    generation: number;
+};
+export type CjkIndexBuildResult = {
+    status: "ready";
+    buildId: string;
+    generation: number;
+    fingerprint: string;
+    reused: boolean;
+} | {
+    status: "busy";
+    buildId: string;
+} | {
+    status: "unavailable";
+    diagnostic: CjkWordDiagnostic;
+    publishedBuildId: string | null;
+};
+export interface CjkIndexBuildOptions {
+    force?: boolean;
+    loadCapability?: JiebaCapabilityLoader;
+    leaseDurationMs?: number;
+    onPhase?: (event: CjkIndexBuildPhase) => void | Promise<void>;
+}
+/**
+ * Return the kernel process start-time token used to distinguish a live build
+ * owner from an unrelated process that later reused the same PID. Linux exposes
+ * this as field 22 in /proc/<pid>/stat. Other platforms conservatively return
+ * null, preserving legacy PID-only cleanup rather than risking a live owner.
+ */
+export declare function getProcessStartToken(pid?: number): string | null;
+/**
+ * Reclaim a bounded number of atomically registered retired FTS table pairs.
+ * Each pair is dropped and its tombstone cleared in one transaction, so a
+ * crash leaves either a fully discoverable pair or a fully reclaimed pair.
+ */
+export declare function cleanupRetiredCjkIndexTables(db: Database, options?: {
+    limit?: number;
+}): string[];
+export declare function cleanupExpiredCjkIndexBuilds(db: Database, options?: {
+    nowMs?: number;
+}): string[];
+export declare function initializeCjkLexicalIndexSchema(db: Database): void;
+export declare function getCjkLexicalIndexState(db: Database): CjkLexicalIndexState;
+/**
+ * Repair the char-only fallback rows affected by raw document writes.
+ * Word/bigram rows and the dirty state intentionally remain gated until a
+ * complete analyzer rebuild publishes the journal head.
+ */
+export declare function repairDirtyCjkCharFallback(db: Database): number;
+/**
+ * Commit a document/collection mutation and all published CJK lexical signals
+ * at one SQLite transaction boundary. If the published index is not safe to
+ * update (missing, rebuilding, stale fingerprint), raw triggers remove stale
+ * word/bigram rows and leave an explicit dirty marker for maintenance rebuild.
+ */
+export declare function runCjkSynchronizedMutation<T>(db: Database, mutation: () => T): T;
+export declare function rebuildCjkLexicalIndex(dbPath: string, options?: CjkIndexBuildOptions): Promise<CjkIndexBuildResult>;