@biaoo/tiangong-wiki 0.3.2 → 0.3.4

package/README.md

@@ -89,10 +89,24 @@ That means commands still work best from inside a workspace, but they can also r
 ```bash
 tiangong-wiki find --type concept --status active     # structured query
 tiangong-wiki fts "Bayesian"                          # full-text search
+tiangong-wiki rebuild-fts --check                     # inspect FTS drift / metadata
+tiangong-wiki rebuild-fts                             # rebuild FTS index explicitly
 tiangong-wiki search "convergence conditions"         # semantic search
 tiangong-wiki graph bayes-theorem --depth 2           # graph traversal
 ```
 
+`wiki.config.json` now supports:
+
+```json
+{
+  "fts": {
+    "tokenizer": "simple"
+  }
+}
+```
+
+`simple` is now the default. Set `tokenizer` to `default` only if you want the legacy `Intl.Segmenter`-based FTS behavior instead of the bundled `wangfenjin/simple` SQLite extension.
+
 ```bash
 tiangong-wiki daemon start                            # start the daemon in the background
 tiangong-wiki dashboard                               # open dashboard in browser

package/README.zh-CN.md

@@ -89,10 +89,24 @@ tiangong-wiki sync                                    # 索引 Markdown 文件
 ```bash
 tiangong-wiki find --type concept --status active     # 结构化查询
 tiangong-wiki fts "贝叶斯"                             # 全文搜索
+tiangong-wiki rebuild-fts --check                     # 检查 FTS 漂移 / 元数据
+tiangong-wiki rebuild-fts                             # 显式重建 FTS 索引
 tiangong-wiki search "优化算法的收敛条件"                # 语义搜索
 tiangong-wiki graph bayes-theorem --depth 2           # 图遍历
 ```
 
+`wiki.config.json` 现在支持：
+
+```json
+{
+  "fts": {
+    "tokenizer": "simple"
+  }
+}
+```
+
+现在默认就是 `simple`。只有在你想退回到基于 `Intl.Segmenter` 的旧 FTS 行为时，才需要把 `tokenizer` 显式设为 `default`。
+
 ```bash
 tiangong-wiki daemon start                            # 后台启动 daemon
 tiangong-wiki dashboard                               # 在浏览器中打开仪表盘

package/assets/wiki.config.default.json

@@ -1,5 +1,8 @@
 {
   "schemaVersion": 1,
+  "fts": {
+    "tokenizer": "simple"
+  },
   "customColumns": {},
   "defaultSummaryFields": [
     "title",

package/dist/commands/rebuild-fts.js

@@ -0,0 +1,41 @@
+import { executeServerBackedOperation, requestDaemonJson } from "../daemon/client.js";
+import { buildCliWriteActor } from "../daemon/write-actor.js";
+import { rebuildFtsCommand } from "../operations/write.js";
+import { writeJson } from "../utils/output.js";
+export function registerRebuildFtsCommand(program) {
+    program
+        .command("rebuild-fts")
+        .description("Validate or rebuild the SQLite FTS index")
+        .option("--mode <mode>", "Override tokenizer mode for this command (default|simple)")
+        .option("--check", "Only inspect FTS drift and metadata without rebuilding")
+        .action(async (options) => {
+        const result = await executeServerBackedOperation({
+            kind: options.check === true ? "read" : "write",
+            local: () => rebuildFtsCommand(process.env, {
+                mode: options.mode ?? undefined,
+                check: options.check === true,
+            }),
+            remote: (endpoint) => options.check === true
+                ? requestDaemonJson({
+                    endpoint,
+                    method: "GET",
+                    path: "/fts/rebuild",
+                    query: {
+                        check: true,
+                        mode: options.mode ?? undefined,
+                    },
+                })
+                : requestDaemonJson({
+                    endpoint,
+                    method: "POST",
+                    path: "/fts/rebuild",
+                    body: {
+                        actor: buildCliWriteActor(process.env),
+                        mode: options.mode ?? undefined,
+                    },
+                    timeoutMs: 310_000,
+                }),
+        });
+        writeJson(result);
+    });
+}

package/dist/core/config.js

@@ -16,6 +16,15 @@ function ensureStringArray(value, label) {
     }
     return value;
 }
+function ensureFtsTokenizerMode(value, label) {
+    if (value === undefined) {
+        return "simple";
+    }
+    if (value === "default" || value === "simple") {
+        return value;
+    }
+    throw new AppError(`${label} must be "default" or "simple"`, "config");
+}
 function ensureVaultFileTypes(value, label) {
     const normalized = ensureStringArray(value, label).map((item, index) => {
         const fileType = item.trim().replace(/^\./, "").toLowerCase();
@@ -81,6 +90,9 @@ export function loadConfig(configPath) {
     assertCondition(Number.isInteger(raw.schemaVersion), "schemaVersion must be an integer", "config");
     const baseConfig = {
         schemaVersion: Number(raw.schemaVersion),
+        fts: {
+            tokenizer: ensureFtsTokenizerMode(ensureObject(raw.fts ?? {}, "fts").tokenizer, "fts.tokenizer"),
+        },
         customColumns: ensureColumnMap(raw.customColumns ?? {}, "customColumns"),
         defaultSummaryFields: ensureStringArray(raw.defaultSummaryFields ?? [], "defaultSummaryFields"),
         vaultFileTypes: ensureVaultFileTypes(raw.vaultFileTypes ?? DEFAULT_VAULT_FILE_TYPES, "vaultFileTypes"),

package/dist/core/db.js

@@ -1,9 +1,9 @@
 import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
+import { buildFtsRow, createFtsTable, ftsTableMatchesMode, FTS_INDEX_VERSION, isLegacyExternalContentFts, } from "./fts.js";
+import { loadSqliteExtensions } from "./sqlite-extensions.js";
 import { AppError } from "../utils/errors.js";
-import { segmentForFts } from "../utils/segmenter.js";
+import { toOffsetIso } from "../utils/time.js";
 export const SCHEMA_VERSION = "1";
-const FTS_INDEX_VERSION = "2";
 function tableExists(db, tableName) {
     const row = db
         .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?")
@@ -28,45 +28,6 @@ function ensureTableColumns(db, tableName, definitions) {
         }
     }
 }
-function createFtsTable(db) {
-    db.exec(`
-    CREATE VIRTUAL TABLE pages_fts USING fts5(
-      title,
-      tags,
-      summary_text
-    );
-  `);
-}
-function normalizeTagsForFts(rawTags) {
-    if (!rawTags) {
-        return "";
-    }
-    try {
-        const parsed = JSON.parse(rawTags);
-        if (Array.isArray(parsed)) {
-            return parsed
-                .map((value) => String(value).trim())
-                .filter(Boolean)
-                .join(" ");
-        }
-    }
-    catch {
-        // Fall back to the stored value if legacy data is not valid JSON.
-    }
-    return rawTags;
-}
-function buildFtsRow(row) {
-    return {
-        rowid: row.rowid,
-        title: segmentForFts(row.title),
-        tags: segmentForFts(normalizeTagsForFts(row.tags)),
-        summary_text: segmentForFts(row.summaryText ?? ""),
-    };
-}
-function isLegacyExternalContentFts(db) {
-    const sql = getTableSql(db, "pages_fts");
-    return typeof sql === "string" && /content\s*=\s*'pages'/i.test(sql);
-}
 function ensureBaseTables(db, embeddingDimensions) {
     db.exec(`
     CREATE TABLE IF NOT EXISTS pages (
@@ -177,6 +138,7 @@ function ensureBaseTables(db, embeddingDimensions) {
         decision: "TEXT",
         result_manifest_path: "TEXT",
         last_error_at: "TEXT",
+        last_error_code: "TEXT",
         retry_after: "TEXT",
         created_page_ids: "TEXT",
         updated_page_ids: "TEXT",
@@ -239,7 +201,7 @@ export function setMetaValues(db, values) {
     });
     transaction(values);
 }
-export function rebuildFts(db) {
+export function rebuildFts(db, config, extensionVersion) {
     if (!tableExists(db, "pages_fts")) {
         return;
     }
@@ -249,27 +211,83 @@ export function rebuildFts(db) {
     const transaction = db.transaction(() => {
         clearStatement.run();
         for (const row of rows) {
-            insertStatement.run(buildFtsRow(row));
+            insertStatement.run(buildFtsRow(row, config.fts.tokenizer));
         }
     });
     transaction();
+    setMetaValues(db, {
+        fts_index_version: FTS_INDEX_VERSION,
+        fts_tokenizer_mode: config.fts.tokenizer,
+        fts_extension_version: extensionVersion,
+        fts_last_rebuild_at: toOffsetIso(),
+    });
 }
-function ensureFtsTable(db) {
+export function inspectFtsIndex(db, config, extensionVersion) {
     const hasTable = tableExists(db, "pages_fts");
-    const storedFtsIndexVersion = getMeta(db, "fts_index_version");
-    const needsRecreate = !hasTable || isLegacyExternalContentFts(db);
-    const needsRebuild = needsRecreate || storedFtsIndexVersion !== FTS_INDEX_VERSION;
-    if (needsRecreate && hasTable) {
-        db.exec("DROP TABLE pages_fts");
+    const tableSql = hasTable ? getTableSql(db, "pages_fts") : null;
+    const storedIndexVersion = getMeta(db, "fts_index_version");
+    const storedTokenizerMode = getMeta(db, "fts_tokenizer_mode");
+    const storedExtensionVersion = getMeta(db, "fts_extension_version");
+    const lastRebuildAt = getMeta(db, "fts_last_rebuild_at");
+    const rowCount = hasTable
+        ? (db.prepare("SELECT COUNT(*) AS count FROM pages_fts").get()?.count ?? 0)
+        : 0;
+    const problems = [];
+    const needsRecreate = !hasTable || isLegacyExternalContentFts(tableSql) || !ftsTableMatchesMode(tableSql, config.fts.tokenizer);
+    if (!hasTable) {
+        problems.push("pages_fts table is missing.");
+    }
+    else if (isLegacyExternalContentFts(tableSql)) {
+        problems.push("pages_fts uses the legacy external-content schema.");
+    }
+    else if (!ftsTableMatchesMode(tableSql, config.fts.tokenizer)) {
+        problems.push(`pages_fts tokenizer schema does not match configured mode ${config.fts.tokenizer}.`);
+    }
+    let needsRebuild = needsRecreate;
+    if (storedIndexVersion !== FTS_INDEX_VERSION) {
+        problems.push(storedIndexVersion === null
+            ? "fts_index_version metadata is missing."
+            : `fts_index_version mismatch: expected ${FTS_INDEX_VERSION}, found ${storedIndexVersion}.`);
+        needsRebuild = true;
+    }
+    if (storedTokenizerMode !== config.fts.tokenizer) {
+        problems.push(storedTokenizerMode === null
+            ? "fts_tokenizer_mode metadata is missing."
+            : `fts_tokenizer_mode mismatch: expected ${config.fts.tokenizer}, found ${storedTokenizerMode}.`);
+        needsRebuild = true;
+    }
+    if (storedExtensionVersion !== extensionVersion) {
+        if (!(storedExtensionVersion === null && extensionVersion === null)) {
+            problems.push(storedExtensionVersion === null
+                ? `fts_extension_version metadata is missing for configured mode ${config.fts.tokenizer}.`
+                : `fts_extension_version mismatch: expected ${extensionVersion ?? "null"}, found ${storedExtensionVersion}.`);
+            needsRebuild = true;
+        }
     }
-    if (needsRecreate) {
-        createFtsTable(db);
+    return {
+        mode: config.fts.tokenizer,
+        hasTable,
+        rowCount,
+        expectedIndexVersion: FTS_INDEX_VERSION,
+        storedIndexVersion,
+        storedTokenizerMode,
+        storedExtensionVersion,
+        lastRebuildAt,
+        needsRecreate,
+        needsRebuild,
+        problems,
+    };
+}
+function ensureFtsTable(db, config, extensionVersion) {
+    const inspection = inspectFtsIndex(db, config, extensionVersion);
+    if (inspection.needsRecreate && inspection.hasTable) {
+        db.exec("DROP TABLE pages_fts");
     }
-    if (needsRebuild) {
-        rebuildFts(db);
+    if (inspection.needsRecreate) {
+        createFtsTable(db, config.fts.tokenizer);
     }
-    if (needsRebuild || storedFtsIndexVersion !== FTS_INDEX_VERSION) {
-        setMeta(db, "fts_index_version", FTS_INDEX_VERSION);
+    if (inspection.needsRebuild) {
+        rebuildFts(db, config, extensionVersion);
     }
 }
 export function resetVectorTable(db, embeddingDimensions) {
@@ -282,7 +300,7 @@ export function resetVectorTable(db, embeddingDimensions) {
     );
   `);
 }
-export function clearAllIndexedData(db) {
+export function clearAllIndexedData(db, config, extensionVersion) {
     db.exec(`
     DELETE FROM edges;
     DELETE FROM pages;
@@ -294,17 +312,21 @@ export function clearAllIndexedData(db) {
         db.exec("DELETE FROM vec_pages");
     }
     if (tableExists(db, "pages_fts")) {
-        rebuildFts(db);
+        rebuildFts(db, config, extensionVersion);
     }
     db.prepare("DELETE FROM sync_meta WHERE key IN ('last_sync_at', 'last_sync_id', 'last_full_rebuild_at', 'embedding_profile')").run();
 }
-export function openDb(dbPath, config, embeddingDimensions) {
+export function openDb(dbPath, config, embeddingDimensions, packageRoot, options = {}) {
     const db = new Database(dbPath);
     db.pragma("journal_mode = WAL");
     db.pragma("foreign_keys = ON");
-    sqliteVec.load(db);
+    const extensionResult = loadSqliteExtensions(db, config, packageRoot);
+    const ftsMetadataExtensionVersion = config.fts.tokenizer === "simple" ? extensionResult.loadedSimpleVersion : null;
     ensureBaseTables(db, embeddingDimensions);
-    ensureFtsTable(db);
+    const initialFtsInspection = inspectFtsIndex(db, config, ftsMetadataExtensionVersion);
+    if (options.ensureFts !== false) {
+        ensureFtsTable(db, config, ftsMetadataExtensionVersion);
+    }
     const vectorDimensions = getVectorTableDimensions(db);
     const vectorDimensionsChanged = vectorDimensions !== null && vectorDimensions !== embeddingDimensions;
     const storedSchemaVersion = getMeta(db, "schema_version");
@@ -319,5 +341,13 @@ export function openDb(dbPath, config, embeddingDimensions) {
         schema_version: SCHEMA_VERSION,
         ...(storedConfigVersion === null ? { config_version: config.configVersion } : {}),
     });
-    return { db, configChanged, vectorDimensions, vectorDimensionsChanged };
+    return {
+        db,
+        configChanged,
+        vectorDimensions,
+        vectorDimensionsChanged,
+        ftsExtensionVersion: ftsMetadataExtensionVersion,
+        simpleExtensionPath: extensionResult.simpleExtensionPath,
+        initialFtsInspection,
+    };
 }

package/dist/core/fts.js

@@ -0,0 +1,68 @@
+import { normalizeFtsQuery, segmentForFts } from "../utils/segmenter.js";
+export const FTS_INDEX_VERSION = "3";
+export function createFtsTable(db, mode) {
+    db.exec(`
+    CREATE VIRTUAL TABLE pages_fts USING fts5(
+      title,
+      tags,
+      summary_text
+      ${mode === "simple" ? ", tokenize = 'simple'" : ""}
+    );
+  `);
+}
+export function normalizeTagsForFts(rawTags) {
+    if (!rawTags) {
+        return "";
+    }
+    try {
+        const parsed = JSON.parse(rawTags);
+        if (Array.isArray(parsed)) {
+            return parsed
+                .map((value) => String(value).trim())
+                .filter(Boolean)
+                .join(" ");
+        }
+    }
+    catch {
+        // Fall back to the stored value if legacy data is not valid JSON.
+    }
+    return rawTags;
+}
+export function buildFtsRow(row, mode) {
+    const tags = normalizeTagsForFts(row.tags);
+    if (mode === "simple") {
+        return {
+            rowid: row.rowid,
+            title: row.title,
+            tags,
+            summary_text: row.summaryText ?? "",
+        };
+    }
+    return {
+        rowid: row.rowid,
+        title: segmentForFts(row.title),
+        tags: segmentForFts(tags),
+        summary_text: segmentForFts(row.summaryText ?? ""),
+    };
+}
+export function buildFtsQueryPlan(query, mode) {
+    if (mode === "simple") {
+        return {
+            whereClause: "pages_fts MATCH simple_query(?)",
+            params: [query.trim()],
+        };
+    }
+    return {
+        whereClause: "pages_fts MATCH ?",
+        params: [normalizeFtsQuery(query)],
+    };
+}
+export function isLegacyExternalContentFts(sql) {
+    return typeof sql === "string" && /content\s*=\s*'pages'/i.test(sql);
+}
+export function isSimpleTokenizerSql(sql) {
+    return typeof sql === "string" && /tokenize\s*=\s*'simple'/i.test(sql);
+}
+export function ftsTableMatchesMode(sql, mode) {
+    return mode === "simple" ? isSimpleTokenizerSql(sql) : !isSimpleTokenizerSql(sql);
+}

package/dist/core/indexer.js

@@ -145,7 +145,7 @@ function buildUpdateStatement(config) {
     ];
     return `UPDATE pages SET ${columns.map((column) => `${column} = @${column}`).join(", ")} WHERE id = @id`;
 }
-export function applyChanges(db, changes, wikiPath, config) {
+export function applyChanges(db, changes, wikiPath, config, ftsExtensionVersion) {
     const parseResults = [...changes.added, ...changes.modified].map((entry) => ({
         entry,
         result: parsePage(entry.filePath, wikiPath, config),
@@ -227,7 +227,7 @@ export function applyChanges(db, changes, wikiPath, config) {
             deleted.push(page.id);
         }
         if (hasContentChanges) {
-            rebuildFts(db);
+            rebuildFts(db, config, ftsExtensionVersion);
         }
     });
     transaction();

package/dist/core/runtime.js

@@ -15,6 +15,6 @@ export function loadRuntimeConfig(env = process.env) {
 export function openRuntimeDb(env = process.env) {
     const { paths, config } = loadRuntimeConfig(env);
     const embeddingClient = EmbeddingClient.fromEnv(env);
-    const { db, vectorDimensions, vectorDimensionsChanged } = openDb(paths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimensionFromEnv(env));
+    const { db, vectorDimensions, vectorDimensionsChanged } = openDb(paths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimensionFromEnv(env), paths.packageRoot);
     return { db, paths, config, embeddingClient, vectorDimensions, vectorDimensionsChanged };
 }

package/dist/core/sqlite-extensions.js

@@ -0,0 +1,78 @@
+import path from "node:path";
+import * as sqliteVec from "sqlite-vec";
+import { AppError } from "../utils/errors.js";
+import { pathExistsSync } from "../utils/fs.js";
+import { isSimpleTokenizerSql } from "./fts.js";
+import { getPackageRoot } from "./paths.js";
+export const BUNDLED_SIMPLE_EXTENSION_VERSION = "v0.7.1";
+const SIMPLE_ASSET_MAP = {
+    darwin: {
+        arm64: "assets/sqlite-extensions/darwin-arm64/libsimple.dylib",
+        x64: "assets/sqlite-extensions/darwin-x64/libsimple.dylib",
+    },
+    linux: {
+        x64: "assets/sqlite-extensions/linux-x64/libsimple.so",
+    },
+    win32: {
+        x64: "assets/sqlite-extensions/win32-x64/simple.dll",
+    },
+};
+function getExistingFtsSql(db) {
+    const row = db
+        .prepare("SELECT sql FROM sqlite_master WHERE type IN ('table', 'view') AND name = 'pages_fts'")
+        .get();
+    return row?.sql ?? null;
+}
+function resolveBundledSimpleExtensionPath(packageRoot) {
+    const byArch = SIMPLE_ASSET_MAP[process.platform];
+    if (!byArch) {
+        throw new AppError(`Bundled simple extension is not available for platform ${process.platform}-${process.arch}.`, "config", {
+            platform: process.platform,
+            arch: process.arch,
+        });
+    }
+    const relativePath = byArch[process.arch];
+    if (!relativePath) {
+        throw new AppError(`Bundled simple extension is not available for platform ${process.platform}-${process.arch}.`, "config", {
+            platform: process.platform,
+            arch: process.arch,
+        });
+    }
+    const extensionPath = path.join(packageRoot, relativePath);
+    if (!pathExistsSync(extensionPath)) {
+        throw new AppError(`Bundled simple extension not found: ${extensionPath}`, "runtime", {
+            platform: process.platform,
+            arch: process.arch,
+            extensionPath,
+            version: BUNDLED_SIMPLE_EXTENSION_VERSION,
+        });
+    }
+    return extensionPath;
+}
+export function loadSqliteExtensions(db, config, packageRoot) {
+    sqliteVec.load(db);
+    const shouldLoadSimple = config.fts.tokenizer === "simple" || isSimpleTokenizerSql(getExistingFtsSql(db));
+    if (!shouldLoadSimple) {
+        return {
+            simpleLoaded: false,
+            loadedSimpleVersion: null,
+            simpleExtensionPath: null,
+        };
+    }
+    const simpleExtensionPath = resolveBundledSimpleExtensionPath(packageRoot ?? getPackageRoot());
+    try {
+        db.loadExtension(simpleExtensionPath);
+    }
+    catch (error) {
+        throw new AppError(`Failed to load bundled simple extension: ${simpleExtensionPath}`, "runtime", {
+            extensionPath: simpleExtensionPath,
+            version: BUNDLED_SIMPLE_EXTENSION_VERSION,
+            cause: error instanceof Error ? error.message : String(error),
+        });
+    }
+    return {
+        simpleLoaded: true,
+        loadedSimpleVersion: BUNDLED_SIMPLE_EXTENSION_VERSION,
+        simpleExtensionPath,
+    };
+}

package/dist/core/sync.js

@@ -78,7 +78,7 @@ export async function syncWorkspace(options = {}) {
     }
     const config = loadConfig(runtimePaths.configPath);
     const embeddingClient = EmbeddingClient.fromEnv(env);
-    const { db, configChanged, vectorDimensionsChanged } = openDb(runtimePaths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimension(env));
+    const { db, configChanged, vectorDimensionsChanged, ftsExtensionVersion } = openDb(runtimePaths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimension(env), runtimePaths.packageRoot);
     try {
         let mode = options.targetPaths && options.targetPaths.length > 0 && !options.force ? "path" : "full";
         let upgradedToFullSync = false;
@@ -93,7 +93,7 @@ export async function syncWorkspace(options = {}) {
             throw new AppError("Embedding profile changed, cannot skip embedding.", "config");
         }
         if (options.force) {
-            clearAllIndexedData(db);
+            clearAllIndexedData(db, config, ftsExtensionVersion);
             mode = "full";
         }
         const changes = mode === "path"
@@ -106,7 +106,7 @@ export async function syncWorkspace(options = {}) {
             changes.unchanged.length === 0) {
             throw new AppError(`No page matched the requested --path value(s).`, "not_found");
         }
-        const applyResult = applyChanges(db, changes, runtimePaths.wikiPath, config);
+        const applyResult = applyChanges(db, changes, runtimePaths.wikiPath, config, ftsExtensionVersion);
         if (applyResult.parseErrors.length > 0) {
             throw new AppError("Failed to parse one or more wiki pages during sync.", "runtime", {
                 parseErrors: applyResult.parseErrors,
@@ -195,7 +195,7 @@ export async function embedPendingPages(env = process.env) {
     if (!embeddingClient) {
         return;
     }
-    const { db } = openDb(runtimePaths.dbPath, config, embeddingClient.settings.dimensions);
+    const { db } = openDb(runtimePaths.dbPath, config, embeddingClient.settings.dimensions, runtimePaths.packageRoot);
     try {
         const targets = getEmbeddingTargets(db, false, [], []);
         const result = await embedPages(db, embeddingClient, targets);