@biaoo/tiangong-wiki 0.3.3 → 0.3.5

package/README.md

@@ -89,10 +89,24 @@ That means commands still work best from inside a workspace, but they can also r
 ```bash
 tiangong-wiki find --type concept --status active     # structured query
 tiangong-wiki fts "Bayesian"                          # full-text search
+tiangong-wiki rebuild-fts --check                     # inspect FTS drift / metadata
+tiangong-wiki rebuild-fts                             # rebuild FTS index explicitly
 tiangong-wiki search "convergence conditions"         # semantic search
 tiangong-wiki graph bayes-theorem --depth 2           # graph traversal
 ```
 
+`wiki.config.json` now supports:
+
+```json
+{
+  "fts": {
+    "tokenizer": "simple"
+  }
+}
+```
+
+`simple` is now the default. Set `tokenizer` to `default` only if you want the legacy `Intl.Segmenter`-based FTS behavior instead of the bundled `wangfenjin/simple` SQLite extension.
+
 ```bash
 tiangong-wiki daemon start                            # start the daemon in the background
 tiangong-wiki dashboard                               # open dashboard in browser

package/README.zh-CN.md

@@ -89,10 +89,24 @@ tiangong-wiki sync                                    # 索引 Markdown 文件
 ```bash
 tiangong-wiki find --type concept --status active     # 结构化查询
 tiangong-wiki fts "贝叶斯"                             # 全文搜索
+tiangong-wiki rebuild-fts --check                     # 检查 FTS 漂移 / 元数据
+tiangong-wiki rebuild-fts                             # 显式重建 FTS 索引
 tiangong-wiki search "优化算法的收敛条件"                # 语义搜索
 tiangong-wiki graph bayes-theorem --depth 2           # 图遍历
 ```
 
+`wiki.config.json` 现在支持：
+
+```json
+{
+  "fts": {
+    "tokenizer": "simple"
+  }
+}
+```
+
+现在默认就是 `simple`。只有在你想退回到基于 `Intl.Segmenter` 的旧 FTS 行为时，才需要把 `tokenizer` 显式设为 `default`。
+
 ```bash
 tiangong-wiki daemon start                            # 后台启动 daemon
 tiangong-wiki dashboard                               # 在浏览器中打开仪表盘

package/assets/wiki.config.default.json

@@ -1,5 +1,8 @@
 {
   "schemaVersion": 1,
+  "fts": {
+    "tokenizer": "simple"
+  },
   "customColumns": {},
   "defaultSummaryFields": [
     "title",

package/dist/commands/rebuild-fts.js

@@ -0,0 +1,41 @@
+import { executeServerBackedOperation, requestDaemonJson } from "../daemon/client.js";
+import { buildCliWriteActor } from "../daemon/write-actor.js";
+import { rebuildFtsCommand } from "../operations/write.js";
+import { writeJson } from "../utils/output.js";
+export function registerRebuildFtsCommand(program) {
+    program
+        .command("rebuild-fts")
+        .description("Validate or rebuild the SQLite FTS index")
+        .option("--mode <mode>", "Override tokenizer mode for this command (default|simple)")
+        .option("--check", "Only inspect FTS drift and metadata without rebuilding")
+        .action(async (options) => {
+        const result = await executeServerBackedOperation({
+            kind: options.check === true ? "read" : "write",
+            local: () => rebuildFtsCommand(process.env, {
+                mode: options.mode ?? undefined,
+                check: options.check === true,
+            }),
+            remote: (endpoint) => options.check === true
+                ? requestDaemonJson({
+                    endpoint,
+                    method: "GET",
+                    path: "/fts/rebuild",
+                    query: {
+                        check: true,
+                        mode: options.mode ?? undefined,
+                    },
+                })
+                : requestDaemonJson({
+                    endpoint,
+                    method: "POST",
+                    path: "/fts/rebuild",
+                    body: {
+                        actor: buildCliWriteActor(process.env),
+                        mode: options.mode ?? undefined,
+                    },
+                    timeoutMs: 310_000,
+                }),
+        });
+        writeJson(result);
+    });
+}

package/dist/core/config.js

@@ -16,6 +16,15 @@ function ensureStringArray(value, label) {
     }
     return value;
 }
+function ensureFtsTokenizerMode(value, label) {
+    if (value === undefined) {
+        return "simple";
+    }
+    if (value === "default" || value === "simple") {
+        return value;
+    }
+    throw new AppError(`${label} must be "default" or "simple"`, "config");
+}
 function ensureVaultFileTypes(value, label) {
     const normalized = ensureStringArray(value, label).map((item, index) => {
         const fileType = item.trim().replace(/^\./, "").toLowerCase();
@@ -81,6 +90,9 @@ export function loadConfig(configPath) {
     assertCondition(Number.isInteger(raw.schemaVersion), "schemaVersion must be an integer", "config");
     const baseConfig = {
         schemaVersion: Number(raw.schemaVersion),
+        fts: {
+            tokenizer: ensureFtsTokenizerMode(ensureObject(raw.fts ?? {}, "fts").tokenizer, "fts.tokenizer"),
+        },
         customColumns: ensureColumnMap(raw.customColumns ?? {}, "customColumns"),
         defaultSummaryFields: ensureStringArray(raw.defaultSummaryFields ?? [], "defaultSummaryFields"),
         vaultFileTypes: ensureVaultFileTypes(raw.vaultFileTypes ?? DEFAULT_VAULT_FILE_TYPES, "vaultFileTypes"),

package/dist/core/db.js

@@ -1,9 +1,9 @@
 import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
+import { buildFtsRow, createFtsTable, ftsTableMatchesMode, FTS_INDEX_VERSION, isLegacyExternalContentFts, } from "./fts.js";
+import { loadSqliteExtensions } from "./sqlite-extensions.js";
 import { AppError } from "../utils/errors.js";
-import { segmentForFts } from "../utils/segmenter.js";
+import { toOffsetIso } from "../utils/time.js";
 export const SCHEMA_VERSION = "1";
-const FTS_INDEX_VERSION = "2";
 function tableExists(db, tableName) {
     const row = db
         .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?")
@@ -28,45 +28,6 @@ function ensureTableColumns(db, tableName, definitions) {
         }
     }
 }
-function createFtsTable(db) {
-    db.exec(`
-    CREATE VIRTUAL TABLE pages_fts USING fts5(
-      title,
-      tags,
-      summary_text
-    );
-  `);
-}
-function normalizeTagsForFts(rawTags) {
-    if (!rawTags) {
-        return "";
-    }
-    try {
-        const parsed = JSON.parse(rawTags);
-        if (Array.isArray(parsed)) {
-            return parsed
-                .map((value) => String(value).trim())
-                .filter(Boolean)
-                .join(" ");
-        }
-    }
-    catch {
-        // Fall back to the stored value if legacy data is not valid JSON.
-    }
-    return rawTags;
-}
-function buildFtsRow(row) {
-    return {
-        rowid: row.rowid,
-        title: segmentForFts(row.title),
-        tags: segmentForFts(normalizeTagsForFts(row.tags)),
-        summary_text: segmentForFts(row.summaryText ?? ""),
-    };
-}
-function isLegacyExternalContentFts(db) {
-    const sql = getTableSql(db, "pages_fts");
-    return typeof sql === "string" && /content\s*=\s*'pages'/i.test(sql);
-}
 function ensureBaseTables(db, embeddingDimensions) {
     db.exec(`
     CREATE TABLE IF NOT EXISTS pages (
@@ -240,7 +201,7 @@ export function setMetaValues(db, values) {
     });
     transaction(values);
 }
-export function rebuildFts(db) {
+export function rebuildFts(db, config, extensionVersion) {
     if (!tableExists(db, "pages_fts")) {
         return;
     }
@@ -250,27 +211,83 @@ export function rebuildFts(db) {
     const transaction = db.transaction(() => {
         clearStatement.run();
         for (const row of rows) {
-            insertStatement.run(buildFtsRow(row));
+            insertStatement.run(buildFtsRow(row, config.fts.tokenizer));
         }
     });
     transaction();
+    setMetaValues(db, {
+        fts_index_version: FTS_INDEX_VERSION,
+        fts_tokenizer_mode: config.fts.tokenizer,
+        fts_extension_version: extensionVersion,
+        fts_last_rebuild_at: toOffsetIso(),
+    });
 }
-function ensureFtsTable(db) {
+export function inspectFtsIndex(db, config, extensionVersion) {
     const hasTable = tableExists(db, "pages_fts");
-    const storedFtsIndexVersion = getMeta(db, "fts_index_version");
-    const needsRecreate = !hasTable || isLegacyExternalContentFts(db);
-    const needsRebuild = needsRecreate || storedFtsIndexVersion !== FTS_INDEX_VERSION;
-    if (needsRecreate && hasTable) {
-        db.exec("DROP TABLE pages_fts");
+    const tableSql = hasTable ? getTableSql(db, "pages_fts") : null;
+    const storedIndexVersion = getMeta(db, "fts_index_version");
+    const storedTokenizerMode = getMeta(db, "fts_tokenizer_mode");
+    const storedExtensionVersion = getMeta(db, "fts_extension_version");
+    const lastRebuildAt = getMeta(db, "fts_last_rebuild_at");
+    const rowCount = hasTable
+        ? (db.prepare("SELECT COUNT(*) AS count FROM pages_fts").get()?.count ?? 0)
+        : 0;
+    const problems = [];
+    const needsRecreate = !hasTable || isLegacyExternalContentFts(tableSql) || !ftsTableMatchesMode(tableSql, config.fts.tokenizer);
+    if (!hasTable) {
+        problems.push("pages_fts table is missing.");
+    }
+    else if (isLegacyExternalContentFts(tableSql)) {
+        problems.push("pages_fts uses the legacy external-content schema.");
+    }
+    else if (!ftsTableMatchesMode(tableSql, config.fts.tokenizer)) {
+        problems.push(`pages_fts tokenizer schema does not match configured mode ${config.fts.tokenizer}.`);
+    }
+    let needsRebuild = needsRecreate;
+    if (storedIndexVersion !== FTS_INDEX_VERSION) {
+        problems.push(storedIndexVersion === null
+            ? "fts_index_version metadata is missing."
+            : `fts_index_version mismatch: expected ${FTS_INDEX_VERSION}, found ${storedIndexVersion}.`);
+        needsRebuild = true;
+    }
+    if (storedTokenizerMode !== config.fts.tokenizer) {
+        problems.push(storedTokenizerMode === null
+            ? "fts_tokenizer_mode metadata is missing."
+            : `fts_tokenizer_mode mismatch: expected ${config.fts.tokenizer}, found ${storedTokenizerMode}.`);
+        needsRebuild = true;
+    }
+    if (storedExtensionVersion !== extensionVersion) {
+        if (!(storedExtensionVersion === null && extensionVersion === null)) {
+            problems.push(storedExtensionVersion === null
+                ? `fts_extension_version metadata is missing for configured mode ${config.fts.tokenizer}.`
+                : `fts_extension_version mismatch: expected ${extensionVersion ?? "null"}, found ${storedExtensionVersion}.`);
+            needsRebuild = true;
+        }
     }
-    if (needsRecreate) {
-        createFtsTable(db);
+    return {
+        mode: config.fts.tokenizer,
+        hasTable,
+        rowCount,
+        expectedIndexVersion: FTS_INDEX_VERSION,
+        storedIndexVersion,
+        storedTokenizerMode,
+        storedExtensionVersion,
+        lastRebuildAt,
+        needsRecreate,
+        needsRebuild,
+        problems,
+    };
+}
+function ensureFtsTable(db, config, extensionVersion) {
+    const inspection = inspectFtsIndex(db, config, extensionVersion);
+    if (inspection.needsRecreate && inspection.hasTable) {
+        db.exec("DROP TABLE pages_fts");
     }
-    if (needsRebuild) {
-        rebuildFts(db);
+    if (inspection.needsRecreate) {
+        createFtsTable(db, config.fts.tokenizer);
     }
-    if (needsRebuild || storedFtsIndexVersion !== FTS_INDEX_VERSION) {
-        setMeta(db, "fts_index_version", FTS_INDEX_VERSION);
+    if (inspection.needsRebuild) {
+        rebuildFts(db, config, extensionVersion);
     }
 }
 export function resetVectorTable(db, embeddingDimensions) {
@@ -283,7 +300,7 @@ export function resetVectorTable(db, embeddingDimensions) {
     );
   `);
 }
-export function clearAllIndexedData(db) {
+export function clearAllIndexedData(db, config, extensionVersion) {
     db.exec(`
     DELETE FROM edges;
     DELETE FROM pages;
@@ -295,17 +312,21 @@ export function clearAllIndexedData(db) {
         db.exec("DELETE FROM vec_pages");
     }
     if (tableExists(db, "pages_fts")) {
-        rebuildFts(db);
+        rebuildFts(db, config, extensionVersion);
     }
     db.prepare("DELETE FROM sync_meta WHERE key IN ('last_sync_at', 'last_sync_id', 'last_full_rebuild_at', 'embedding_profile')").run();
 }
-export function openDb(dbPath, config, embeddingDimensions) {
+export function openDb(dbPath, config, embeddingDimensions, packageRoot, options = {}) {
     const db = new Database(dbPath);
     db.pragma("journal_mode = WAL");
     db.pragma("foreign_keys = ON");
-    sqliteVec.load(db);
+    const extensionResult = loadSqliteExtensions(db, config, packageRoot);
+    const ftsMetadataExtensionVersion = config.fts.tokenizer === "simple" ? extensionResult.loadedSimpleVersion : null;
     ensureBaseTables(db, embeddingDimensions);
-    ensureFtsTable(db);
+    const initialFtsInspection = inspectFtsIndex(db, config, ftsMetadataExtensionVersion);
+    if (options.ensureFts !== false) {
+        ensureFtsTable(db, config, ftsMetadataExtensionVersion);
+    }
     const vectorDimensions = getVectorTableDimensions(db);
     const vectorDimensionsChanged = vectorDimensions !== null && vectorDimensions !== embeddingDimensions;
     const storedSchemaVersion = getMeta(db, "schema_version");
@@ -320,5 +341,13 @@ export function openDb(dbPath, config, embeddingDimensions) {
         schema_version: SCHEMA_VERSION,
         ...(storedConfigVersion === null ? { config_version: config.configVersion } : {}),
     });
-    return { db, configChanged, vectorDimensions, vectorDimensionsChanged };
+    return {
+        db,
+        configChanged,
+        vectorDimensions,
+        vectorDimensionsChanged,
+        ftsExtensionVersion: ftsMetadataExtensionVersion,
+        simpleExtensionPath: extensionResult.simpleExtensionPath,
+        initialFtsInspection,
+    };
 }

package/dist/core/fts.js

@@ -0,0 +1,68 @@
+import { normalizeFtsQuery, segmentForFts } from "../utils/segmenter.js";
+export const FTS_INDEX_VERSION = "3";
+export function createFtsTable(db, mode) {
+    db.exec(`
+    CREATE VIRTUAL TABLE pages_fts USING fts5(
+      title,
+      tags,
+      summary_text
+      ${mode === "simple" ? ", tokenize = 'simple'" : ""}
+    );
+  `);
+}
+export function normalizeTagsForFts(rawTags) {
+    if (!rawTags) {
+        return "";
+    }
+    try {
+        const parsed = JSON.parse(rawTags);
+        if (Array.isArray(parsed)) {
+            return parsed
+                .map((value) => String(value).trim())
+                .filter(Boolean)
+                .join(" ");
+        }
+    }
+    catch {
+        // Fall back to the stored value if legacy data is not valid JSON.
+    }
+    return rawTags;
+}
+export function buildFtsRow(row, mode) {
+    const tags = normalizeTagsForFts(row.tags);
+    if (mode === "simple") {
+        return {
+            rowid: row.rowid,
+            title: row.title,
+            tags,
+            summary_text: row.summaryText ?? "",
+        };
+    }
+    return {
+        rowid: row.rowid,
+        title: segmentForFts(row.title),
+        tags: segmentForFts(tags),
+        summary_text: segmentForFts(row.summaryText ?? ""),
+    };
+}
+export function buildFtsQueryPlan(query, mode) {
+    if (mode === "simple") {
+        return {
+            whereClause: "pages_fts MATCH simple_query(?)",
+            params: [query.trim()],
+        };
+    }
+    return {
+        whereClause: "pages_fts MATCH ?",
+        params: [normalizeFtsQuery(query)],
+    };
+}
+export function isLegacyExternalContentFts(sql) {
+    return typeof sql === "string" && /content\s*=\s*'pages'/i.test(sql);
+}
+export function isSimpleTokenizerSql(sql) {
+    return typeof sql === "string" && /tokenize\s*=\s*'simple'/i.test(sql);
+}
+export function ftsTableMatchesMode(sql, mode) {
+    return mode === "simple" ? isSimpleTokenizerSql(sql) : !isSimpleTokenizerSql(sql);
+}

package/dist/core/indexer.js

@@ -145,7 +145,7 @@ function buildUpdateStatement(config) {
     ];
     return `UPDATE pages SET ${columns.map((column) => `${column} = @${column}`).join(", ")} WHERE id = @id`;
 }
-export function applyChanges(db, changes, wikiPath, config) {
+export function applyChanges(db, changes, wikiPath, config, ftsExtensionVersion) {
     const parseResults = [...changes.added, ...changes.modified].map((entry) => ({
         entry,
         result: parsePage(entry.filePath, wikiPath, config),
@@ -184,6 +184,15 @@ export function applyChanges(db, changes, wikiPath, config) {
     const summaryChangedIds = [];
     const hasContentChanges = parsedEntries.length > 0 || changes.deleted.length > 0;
     const transaction = db.transaction(() => {
+        for (const page of changes.deleted) {
+            const existing = selectPageRowid.get(page.id);
+            deleteEdgesBySourcePage.run(page.id);
+            if (existing) {
+                deleteVecRow.run(BigInt(existing.rowid));
+            }
+            deletePage.run(page.id);
+            deleted.push(page.id);
+        }
         for (const { entry, parsed } of parsedEntries) {
             const existing = selectExistingPage.get(entry.id);
             const isInsert = !existing;
@@ -217,17 +226,8 @@ export function applyChanges(db, changes, wikiPath, config) {
                 });
             }
         }
-        for (const page of changes.deleted) {
-            const existing = selectPageRowid.get(page.id);
-            deleteEdgesBySourcePage.run(page.id);
-            if (existing) {
-                deleteVecRow.run(BigInt(existing.rowid));
-            }
-            deletePage.run(page.id);
-            deleted.push(page.id);
-        }
         if (hasContentChanges) {
-            rebuildFts(db);
+            rebuildFts(db, config, ftsExtensionVersion);
         }
     });
     transaction();

package/dist/core/runtime.js

@@ -15,6 +15,6 @@ export function loadRuntimeConfig(env = process.env) {
 export function openRuntimeDb(env = process.env) {
     const { paths, config } = loadRuntimeConfig(env);
     const embeddingClient = EmbeddingClient.fromEnv(env);
-    const { db, vectorDimensions, vectorDimensionsChanged } = openDb(paths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimensionFromEnv(env));
+    const { db, vectorDimensions, vectorDimensionsChanged } = openDb(paths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimensionFromEnv(env), paths.packageRoot);
     return { db, paths, config, embeddingClient, vectorDimensions, vectorDimensionsChanged };
 }

package/dist/core/sqlite-extensions.js

@@ -0,0 +1,78 @@
+import path from "node:path";
+import * as sqliteVec from "sqlite-vec";
+import { AppError } from "../utils/errors.js";
+import { pathExistsSync } from "../utils/fs.js";
+import { isSimpleTokenizerSql } from "./fts.js";
+import { getPackageRoot } from "./paths.js";
+export const BUNDLED_SIMPLE_EXTENSION_VERSION = "v0.7.1";
+const SIMPLE_ASSET_MAP = {
+    darwin: {
+        arm64: "assets/sqlite-extensions/darwin-arm64/libsimple.dylib",
+        x64: "assets/sqlite-extensions/darwin-x64/libsimple.dylib",
+    },
+    linux: {
+        x64: "assets/sqlite-extensions/linux-x64/libsimple.so",
+    },
+    win32: {
+        x64: "assets/sqlite-extensions/win32-x64/simple.dll",
+    },
+};
+function getExistingFtsSql(db) {
+    const row = db
+        .prepare("SELECT sql FROM sqlite_master WHERE type IN ('table', 'view') AND name = 'pages_fts'")
+        .get();
+    return row?.sql ?? null;
+}
+function resolveBundledSimpleExtensionPath(packageRoot) {
+    const byArch = SIMPLE_ASSET_MAP[process.platform];
+    if (!byArch) {
+        throw new AppError(`Bundled simple extension is not available for platform ${process.platform}-${process.arch}.`, "config", {
+            platform: process.platform,
+            arch: process.arch,
+        });
+    }
+    const relativePath = byArch[process.arch];
+    if (!relativePath) {
+        throw new AppError(`Bundled simple extension is not available for platform ${process.platform}-${process.arch}.`, "config", {
+            platform: process.platform,
+            arch: process.arch,
+        });
+    }
+    const extensionPath = path.join(packageRoot, relativePath);
+    if (!pathExistsSync(extensionPath)) {
+        throw new AppError(`Bundled simple extension not found: ${extensionPath}`, "runtime", {
+            platform: process.platform,
+            arch: process.arch,
+            extensionPath,
+            version: BUNDLED_SIMPLE_EXTENSION_VERSION,
+        });
+    }
+    return extensionPath;
+}
+export function loadSqliteExtensions(db, config, packageRoot) {
+    sqliteVec.load(db);
+    const shouldLoadSimple = config.fts.tokenizer === "simple" || isSimpleTokenizerSql(getExistingFtsSql(db));
+    if (!shouldLoadSimple) {
+        return {
+            simpleLoaded: false,
+            loadedSimpleVersion: null,
+            simpleExtensionPath: null,
+        };
+    }
+    const simpleExtensionPath = resolveBundledSimpleExtensionPath(packageRoot ?? getPackageRoot());
+    try {
+        db.loadExtension(simpleExtensionPath);
+    }
+    catch (error) {
+        throw new AppError(`Failed to load bundled simple extension: ${simpleExtensionPath}`, "runtime", {
+            extensionPath: simpleExtensionPath,
+            version: BUNDLED_SIMPLE_EXTENSION_VERSION,
+            cause: error instanceof Error ? error.message : String(error),
+        });
+    }
+    return {
+        simpleLoaded: true,
+        loadedSimpleVersion: BUNDLED_SIMPLE_EXTENSION_VERSION,
+        simpleExtensionPath,
+    };
+}

package/dist/core/sync.js

@@ -78,7 +78,7 @@ export async function syncWorkspace(options = {}) {
     }
     const config = loadConfig(runtimePaths.configPath);
     const embeddingClient = EmbeddingClient.fromEnv(env);
-    const { db, configChanged, vectorDimensionsChanged } = openDb(runtimePaths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimension(env));
+    const { db, configChanged, vectorDimensionsChanged, ftsExtensionVersion } = openDb(runtimePaths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimension(env), runtimePaths.packageRoot);
     try {
         let mode = options.targetPaths && options.targetPaths.length > 0 && !options.force ? "path" : "full";
         let upgradedToFullSync = false;
@@ -93,7 +93,7 @@ export async function syncWorkspace(options = {}) {
             throw new AppError("Embedding profile changed, cannot skip embedding.", "config");
         }
         if (options.force) {
-            clearAllIndexedData(db);
+            clearAllIndexedData(db, config, ftsExtensionVersion);
             mode = "full";
         }
         const changes = mode === "path"
@@ -106,7 +106,7 @@ export async function syncWorkspace(options = {}) {
             changes.unchanged.length === 0) {
             throw new AppError(`No page matched the requested --path value(s).`, "not_found");
         }
-        const applyResult = applyChanges(db, changes, runtimePaths.wikiPath, config);
+        const applyResult = applyChanges(db, changes, runtimePaths.wikiPath, config, ftsExtensionVersion);
         if (applyResult.parseErrors.length > 0) {
             throw new AppError("Failed to parse one or more wiki pages during sync.", "runtime", {
                 parseErrors: applyResult.parseErrors,
@@ -195,7 +195,7 @@ export async function embedPendingPages(env = process.env) {
     if (!embeddingClient) {
         return;
     }
-    const { db } = openDb(runtimePaths.dbPath, config, embeddingClient.settings.dimensions);
+    const { db } = openDb(runtimePaths.dbPath, config, embeddingClient.settings.dimensions, runtimePaths.packageRoot);
     try {
         const targets = getEmbeddingTargets(db, false, [], []);
         const result = await embedPages(db, embeddingClient, targets);

package/dist/core/vault-processing.js

@@ -815,7 +815,7 @@ async function processClaimedQueueItem(input) {
 export function getVaultQueueSnapshot(env = process.env, status) {
     const paths = resolveRuntimePaths(env);
     const config = loadConfig(paths.configPath);
-    const { db } = openDb(paths.dbPath, config, Number.parseInt(env.EMBEDDING_DIMENSIONS ?? "384", 10) || 384);
+    const { db } = openDb(paths.dbPath, config, Number.parseInt(env.EMBEDDING_DIMENSIONS ?? "384", 10) || 384, paths.packageRoot);
     try {
         const items = fetchQueueItemsByStatus(db, status);
         const counts = db.prepare(`
@@ -843,7 +843,7 @@ export function getVaultQueueSnapshot(env = process.env, status) {
 export function getVaultQueueItem(env = process.env, fileId) {
     const paths = resolveRuntimePaths(env);
     const config = loadConfig(paths.configPath);
-    const { db } = openDb(paths.dbPath, config, Number.parseInt(env.EMBEDDING_DIMENSIONS ?? "384", 10) || 384);
+    const { db } = openDb(paths.dbPath, config, Number.parseInt(env.EMBEDDING_DIMENSIONS ?? "384", 10) || 384, paths.packageRoot);
     try {
         return fetchQueueItemByFileId(db, fileId);
     }
@@ -865,7 +865,7 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
     }
     const paths = resolveRuntimePaths(env);
     const config = loadConfig(paths.configPath);
-    const { db } = openDb(paths.dbPath, config, Number.parseInt(env.EMBEDDING_DIMENSIONS ?? "384", 10) || 384);
+    const { db } = openDb(paths.dbPath, config, Number.parseInt(env.EMBEDDING_DIMENSIONS ?? "384", 10) || 384, paths.packageRoot);
     try {
         const result = {
             enabled: true,

package/dist/daemon/server.js

@@ -15,7 +15,7 @@ import { getDashboardGraphOverview, getDashboardLintSummary, getDashboardPageDet
 import { diffVaultFiles, findPages, ftsSearchPages, getPageInfo, getVaultQueue, getWikiStat, listPages, listVaultFiles, renderLintResult, runLint, searchPages, traverseGraph, } from "../operations/query.js";
 import { createTemplate, listTemplates, listTypes, recommendTypes, showTemplate, showType, } from "../operations/type-template.js";
 import { runTemplateLint } from "../operations/template-lint.js";
-import { createPage, runSync, runSyncCommand, updatePage } from "../operations/write.js";
+import { createPage, rebuildFtsCommand, runSync, runSyncCommand, updatePage } from "../operations/write.js";
 import { AppError, asAppError } from "../utils/errors.js";
 import { pathExistsSync } from "../utils/fs.js";
 import { addSeconds, toOffsetIso } from "../utils/time.js";
@@ -614,6 +614,28 @@ export async function runDaemonServer(options) {
             });
         }
     };
+    const runRebuildFtsTransaction = async (actor, input) => {
+        try {
+            const result = rebuildFtsCommand(env, input);
+            return await finalizeJournaledWrite(actor, {
+                operation: "rebuild-fts",
+                resourceId: "pages_fts",
+                revisionBefore: null,
+                revisionAfter: null,
+                result,
+            });
+        }
+        catch (error) {
+            const appError = asAppError(error);
+            return recordSyncFailureAndThrow(actor, {
+                operation: "rebuild-fts",
+                resourceId: "pages_fts",
+                revisionBefore: null,
+                revisionAfter: null,
+                error: appError,
+            });
+        }
+    };
     const runCycleTransaction = async (actor, task) => {
         try {
             const result = await runCycleTask(task);
@@ -743,6 +765,22 @@ export async function runDaemonServer(options) {
                 writeJsonResponse(response, 200, result);
                 return;
             }
+            if (method === "POST" && pathname === "/fts/rebuild") {
+                const body = await readJsonBody(request);
+                const actor = resolveWriteActor(request, body, buildCliWriteActor(env));
+                const result = await enqueueWriteTask("rebuild-fts", () => runRebuildFtsTransaction(actor, {
+                    mode: body.mode === "simple" ? "simple" : body.mode === "default" ? "default" : undefined,
+                }), {
+                    summarizeResult: (payload) => ({
+                        rebuilt: payload.rebuilt,
+                        mode: payload.mode,
+                        rowCount: payload.rowCount,
+                        needsRebuild: payload.needsRebuild,
+                    }),
+                });
+                writeJsonResponse(response, 200, result);
+                return;
+            }
             if (method === "GET" && pathname === "/write-queue/summary") {
                 writeJsonResponse(response, 200, writeQueue.getSummary());
                 return;
@@ -885,6 +923,17 @@ export async function runDaemonServer(options) {
                 }));
                 return;
             }
+            if (method === "GET" && pathname === "/fts/rebuild") {
+                writeJsonResponse(response, 200, rebuildFtsCommand(env, {
+                    mode: url.searchParams.get("mode") === "simple"
+                        ? "simple"
+                        : url.searchParams.get("mode") === "default"
+                            ? "default"
+                            : undefined,
+                    check: url.searchParams.get("check") === "true",
+                }));
+                return;
+            }
             if (method === "GET" && pathname === "/search") {
                 writeJsonResponse(response, 200, await searchPages(env, {
                     query: url.searchParams.get("query") ?? "",

package/dist/index.js

@@ -16,6 +16,7 @@ import { registerInitCommand } from "./commands/init.js";
 import { registerLintCommand } from "./commands/lint.js";
 import { registerListCommand } from "./commands/list.js";
 import { registerPageInfoCommand } from "./commands/page-info.js";
+import { registerRebuildFtsCommand } from "./commands/rebuild-fts.js";
 import { registerSearchCommand } from "./commands/search.js";
 import { registerSetupCommand } from "./commands/setup.js";
 import { registerSkillCommand } from "./commands/skill.js";
@@ -80,6 +81,7 @@ function buildProgram() {
     registerFindCommand(program, runtimeConfig);
     registerSearchCommand(program);
     registerFtsCommand(program);
+    registerRebuildFtsCommand(program);
     registerGraphCommand(program);
     registerPageInfoCommand(program);
     registerListCommand(program);

package/dist/operations/query.js

@@ -1,5 +1,6 @@
 import path from "node:path";
 import { getMeta } from "../core/db.js";
+import { buildFtsQueryPlan } from "../core/fts.js";
 import { parsePage } from "../core/frontmatter.js";
 import { normalizePageId, resolvePagePath } from "../core/paths.js";
 import { compactPageSummary } from "../core/presenters.js";
@@ -10,7 +11,6 @@ import { getVaultQueueSnapshot } from "../core/vault-processing.js";
 import { camelToSnake } from "../utils/case.js";
 import { AppError } from "../utils/errors.js";
 import { listFilesRecursiveSync, pathExistsSync } from "../utils/fs.js";
-import { normalizeFtsQuery } from "../utils/segmenter.js";
 function parsePositiveLimit(value, label, fallback) {
     const normalized = value ?? fallback;
     const limit = Number.parseInt(String(normalized), 10);
@@ -306,18 +306,18 @@ export function ftsSearchPages(env = process.env, options) {
     const { db, config } = openRuntimeDb(env);
     try {
         const limit = parsePositiveLimit(options.limit, "--limit", 20);
-        const normalizedQuery = normalizeFtsQuery(options.query);
+        const queryPlan = buildFtsQueryPlan(options.query, config.fts.tokenizer);
         const rows = db
             .prepare(`
           SELECT ${listPageColumns(config).map((column) => `pages.${column}`).join(", ")}, bm25(pages_fts) AS rank
           FROM pages_fts
           JOIN pages ON pages.rowid = pages_fts.rowid
-          WHERE pages_fts MATCH ?
+          WHERE ${queryPlan.whereClause}
           ${options.type ? "AND pages.page_type = ?" : ""}
           ORDER BY rank
           LIMIT ?
         `)
-            .all(...(options.type ? [normalizedQuery, options.type, limit] : [normalizedQuery, limit]));
+            .all(...(options.type ? [...queryPlan.params, options.type, limit] : [...queryPlan.params, limit]));
         return rows.map((row) => ({
             ...compactPageSummary(mapPageRow(row, config), config),
             summaryText: row.summary_text,

package/dist/operations/write.js

@@ -1,18 +1,33 @@
 import { getTemplate } from "../core/config.js";
+import { inspectFtsIndex, openDb, rebuildFts as rebuildFtsIndex } from "../core/db.js";
+import { createFtsTable, FTS_INDEX_VERSION } from "../core/fts.js";
 import { resolveAgentSettings } from "../core/paths.js";
 import { normalizePageId } from "../core/paths.js";
 import { createPageFromTemplate } from "../core/page-files.js";
 import { updatePageById } from "../core/page-files.js";
 import { readCanonicalPageSourceById } from "../core/page-source.js";
-import { loadRuntimeConfig } from "../core/runtime.js";
+import { getEmbeddingDimensionFromEnv, loadRuntimeConfig } from "../core/runtime.js";
 import { openRuntimeDb } from "../core/runtime.js";
 import { syncWorkspace } from "../core/sync.js";
 import { getVaultQueueItem, processVaultQueueBatch } from "../core/vault-processing.js";
 import { selectPageById } from "../core/query.js";
 import { AppError, asAppError } from "../utils/errors.js";
+import { pathExistsSync } from "../utils/fs.js";
 function isPlainObject(value) {
     return Object.prototype.toString.call(value) === "[object Object]";
 }
+function applyFtsModeOverride(config, mode) {
+    if (!mode || config.fts.tokenizer === mode) {
+        return config;
+    }
+    return {
+        ...config,
+        fts: {
+            ...config.fts,
+            tokenizer: mode,
+        },
+    };
+}
 function assertValidSyncCommandOptions(options) {
     if (options.vaultFileId && !options.process) {
         throw new AppError("--vault-file requires --process.", "config");
@@ -132,6 +147,64 @@ export async function runSyncCommand(env = process.env, options = {}) {
         }),
     };
 }
+export function rebuildFtsCommand(env = process.env, options = {}) {
+    const { paths, config } = loadRuntimeConfig(env);
+    const effectiveConfig = applyFtsModeOverride(config, options.mode);
+    if (!pathExistsSync(paths.dbPath)) {
+        return {
+            checked: true,
+            rebuilt: false,
+            mode: effectiveConfig.fts.tokenizer,
+            rowCount: 0,
+            expectedIndexVersion: FTS_INDEX_VERSION,
+            storedIndexVersion: null,
+            storedTokenizerMode: null,
+            storedExtensionVersion: null,
+            extensionVersion: null,
+            simpleExtensionPath: null,
+            lastRebuildAt: null,
+            needsRecreate: true,
+            needsRebuild: true,
+            problems: [`Wiki database does not exist yet: ${paths.dbPath}`],
+        };
+    }
+    const { db, ftsExtensionVersion, simpleExtensionPath, initialFtsInspection } = openDb(paths.dbPath, effectiveConfig, getEmbeddingDimensionFromEnv(env), paths.packageRoot, {
+        ensureFts: false,
+    });
+    try {
+        if (!options.check) {
+            if (initialFtsInspection.needsRecreate) {
+                if (initialFtsInspection.hasTable) {
+                    db.exec("DROP TABLE pages_fts");
+                }
+                createFtsTable(db, effectiveConfig.fts.tokenizer);
+            }
+            rebuildFtsIndex(db, effectiveConfig, ftsExtensionVersion);
+        }
+        const inspection = options.check
+            ? initialFtsInspection
+            : inspectFtsIndex(db, effectiveConfig, ftsExtensionVersion);
+        return {
+            checked: true,
+            rebuilt: options.check ? false : true,
+            mode: effectiveConfig.fts.tokenizer,
+            rowCount: inspection.rowCount,
+            expectedIndexVersion: inspection.expectedIndexVersion,
+            storedIndexVersion: inspection.storedIndexVersion,
+            storedTokenizerMode: inspection.storedTokenizerMode,
+            storedExtensionVersion: inspection.storedExtensionVersion,
+            extensionVersion: ftsExtensionVersion,
+            simpleExtensionPath,
+            lastRebuildAt: inspection.lastRebuildAt,
+            needsRecreate: inspection.needsRecreate,
+            needsRebuild: inspection.needsRebuild,
+            problems: inspection.problems,
+        };
+    }
+    finally {
+        db.close();
+    }
+}
 export async function createPage(env = process.env, options) {
     const { paths, config } = loadRuntimeConfig(env);
     getTemplate(config, options.type);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@biaoo/tiangong-wiki",
-  "version": "0.3.3",
+  "version": "0.3.5",
   "description": "Local-first wiki index and query engine for Markdown knowledge pages (Tiangong Wiki).",
   "type": "module",
   "publishConfig": {

package/references/cli-interface.md

@@ -35,6 +35,7 @@ Global workspace resolution priority:
 | `find` | Query pages by structured metadata filters |
 | `search` | Semantic search over page summary embeddings |
 | `fts` | Full-text search over title, tags, and summary text |
+| `rebuild-fts` | Inspect or rebuild the SQLite FTS index |
 | `graph` | Traverse the knowledge graph from a root node |
 | `page-info` | Show full metadata and edges for a single page |
 | `list` | List wiki pages |
@@ -158,6 +159,31 @@ tiangong-wiki fts <query> [--type <pageType>] [--limit <n>]
 
 Full-text search against the `pages_fts` table (title, tags, summary_text). Default limit: 20.
 
+`wiki.config.json` can set:
+
+```json
+{
+  "fts": {
+    "tokenizer": "simple"
+  }
+}
+```
+
+`simple` is the default. Set `tokenizer` to `default` only if you need the legacy `Intl.Segmenter`-based FTS behavior.
+
+### rebuild-fts
+
+```
+tiangong-wiki rebuild-fts [--mode <default|simple>] [--check]
+```
+
+Inspects or rebuilds the `pages_fts` table and its metadata. Useful after changing `wiki.config.json` FTS tokenizer mode, repairing drift, or forcing a clean rebuild.
+
+Options:
+
+- `--mode <default|simple>` — temporarily override the configured tokenizer mode for this command
+- `--check` — report drift and metadata without rebuilding
+
 ### graph
 
 ```