@biaoo/tiangong-wiki 0.2.0

package/dist/core/page-files.js

@@ -0,0 +1,91 @@
+import matter from "gray-matter";
+import path from "node:path";
+import { getTemplate, resolveTemplateFilePath } from "./config.js";
+import { AppError } from "../utils/errors.js";
+import { ensureDirSync, pathExistsSync, readTextFileSync, writeTextFileSync } from "../utils/fs.js";
+import { slugify } from "../utils/slug.js";
+import { toDateOnly } from "../utils/time.js";
+export const PAGE_TYPE_DIRS = {
+    concept: "concepts",
+    misconception: "misconceptions",
+    bridge: "bridges",
+    "source-summary": "source-summaries",
+    lesson: "lessons",
+    method: "methods",
+    person: "people",
+    achievement: "achievements",
+    resume: "resumes",
+    "research-note": "research-notes",
+    faq: "faqs",
+};
+function makeUniquePagePath(baseDir, slug) {
+    let candidate = path.join(baseDir, `${slug}.md`);
+    let counter = 2;
+    while (pathExistsSync(candidate)) {
+        candidate = path.join(baseDir, `${slug}-${counter}.md`);
+        counter += 1;
+    }
+    return candidate;
+}
+function normalizeFrontmatterArrays(patch, templateData) {
+    return {
+        sourceRefs: patch.sourceRefs ?? templateData.sourceRefs ?? [],
+        relatedPages: patch.relatedPages ?? templateData.relatedPages ?? [],
+        tags: patch.tags ?? templateData.tags ?? [],
+    };
+}
+export function createPageFromTemplate(paths, config, options) {
+    const { pageType, title } = options;
+    getTemplate(config, pageType);
+    const templatePath = resolveTemplateFilePath(config, paths.wikiRoot, pageType);
+    if (!pathExistsSync(templatePath)) {
+        throw new AppError(`Template file not found: ${templatePath}`, "not_found");
+    }
+    const template = matter(readTextFileSync(templatePath));
+    const slug = slugify(title);
+    const targetDir = path.join(paths.wikiPath, PAGE_TYPE_DIRS[pageType] ?? `${pageType}s`);
+    ensureDirSync(targetDir);
+    const filePath = makeUniquePagePath(targetDir, slug);
+    const pageId = path.relative(paths.wikiPath, filePath).split(path.sep).join("/");
+    const today = toDateOnly();
+    const patch = options.frontmatterPatch ?? {};
+    const templateData = (template.data ?? {});
+    const data = {
+        ...templateData,
+        ...patch,
+        pageType,
+        title,
+        nodeId: options.nodeId !== undefined
+            ? options.nodeId
+            : patch.nodeId !== undefined
+                ? patch.nodeId
+                : slug,
+        status: patch.status ?? templateData.status ?? "draft",
+        visibility: patch.visibility ?? templateData.visibility ?? "private",
+        ...normalizeFrontmatterArrays(patch, templateData),
+        createdAt: patch.createdAt ?? templateData.createdAt ?? today,
+        updatedAt: patch.updatedAt ?? templateData.updatedAt ?? today,
+    };
+    const output = matter.stringify(options.bodyMarkdown ?? template.content, data);
+    writeTextFileSync(filePath, output);
+    return { pageId, filePath };
+}
+export function updatePageById(paths, pageId, options) {
+    const filePath = path.join(paths.wikiPath, ...pageId.split("/"));
+    if (!pathExistsSync(filePath)) {
+        throw new AppError(`Page file not found: ${pageId}`, "not_found");
+    }
+    const existing = matter(readTextFileSync(filePath));
+    const patch = options.frontmatterPatch ?? {};
+    const currentData = (existing.data ?? {});
+    const today = toDateOnly();
+    const nextData = {
+        ...currentData,
+        ...patch,
+        ...normalizeFrontmatterArrays(patch, currentData),
+        updatedAt: patch.updatedAt ?? today,
+    };
+    const output = matter.stringify(options.bodyMarkdown ?? existing.content, nextData);
+    writeTextFileSync(filePath, output);
+    return { pageId, filePath };
+}

package/dist/core/paths.js

@@ -0,0 +1,161 @@
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+import { AppError } from "../utils/errors.js";
+const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
+const FALSE_VALUES = new Set(["0", "false", "no", "off"]);
+function parseBooleanFlag(label, rawValue, defaultValue) {
+    if (rawValue === undefined) {
+        return defaultValue;
+    }
+    const normalized = rawValue.trim().toLowerCase();
+    if (TRUE_VALUES.has(normalized)) {
+        return true;
+    }
+    if (FALSE_VALUES.has(normalized)) {
+        return false;
+    }
+    throw new AppError(`${label} must be a boolean value, got ${rawValue}`, "config");
+}
+function parseSyncInterval(raw) {
+    if (!raw) {
+        return 86_400;
+    }
+    const value = Number.parseInt(raw, 10);
+    if (!Number.isFinite(value) || value < 0) {
+        throw new AppError(`WIKI_SYNC_INTERVAL must be a non-negative integer, got ${raw}`, "config");
+    }
+    return value;
+}
+function parseNonNegativeInteger(raw, defaultValue, label) {
+    if (!raw) {
+        return defaultValue;
+    }
+    const value = Number.parseInt(raw, 10);
+    if (!Number.isFinite(value) || value < 0) {
+        throw new AppError(`${label} must be a non-negative integer, got ${raw}`, "config");
+    }
+    return value;
+}
+function parsePositiveInteger(raw, defaultValue, label) {
+    if (!raw) {
+        return defaultValue;
+    }
+    const value = Number.parseInt(raw, 10);
+    if (!Number.isFinite(value) || value <= 0) {
+        throw new AppError(`${label} must be a positive integer, got ${raw}`, "config");
+    }
+    return value;
+}
+function parseOptionalPort(raw, label) {
+    if (!raw || !raw.trim()) {
+        return null;
+    }
+    const value = Number.parseInt(raw, 10);
+    if (!Number.isFinite(value) || value < 1 || value > 65_535) {
+        throw new AppError(`${label} must be an integer between 1 and 65535, got ${raw}`, "config");
+    }
+    return value;
+}
+function normalizeOptionalUrl(rawValue) {
+    const value = rawValue?.trim();
+    if (!value) {
+        return null;
+    }
+    if (!/^https?:\/\//i.test(value)) {
+        throw new AppError(`WIKI_AGENT_BASE_URL must start with http:// or https://: ${rawValue}`, "config");
+    }
+    return value.replace(/\/+$/g, "");
+}
+function requireAbsolutePath(label, rawValue) {
+    if (!rawValue) {
+        throw new AppError(`${label} is required`, "config");
+    }
+    if (!path.isAbsolute(rawValue)) {
+        throw new AppError(`${label} must be an absolute path: ${rawValue}`, "config");
+    }
+    return path.resolve(rawValue);
+}
+export function parseVaultHashMode(raw) {
+    const value = (raw ?? "content").trim().toLowerCase();
+    if (value === "content" || value === "mtime") {
+        return value;
+    }
+    throw new AppError(`VAULT_HASH_MODE must be "content" or "mtime", got ${raw}`, "config");
+}
+export function parseWikiAgentBackend(raw) {
+    const value = (raw ?? "codex-workflow").trim().toLowerCase();
+    if (value === "codex-workflow") {
+        return value;
+    }
+    throw new AppError(`WIKI_AGENT_BACKEND must be "codex-workflow", got ${raw}`, "config");
+}
+export function resolveAgentSettings(env = process.env, options = {}) {
+    const enabled = parseBooleanFlag("WIKI_AGENT_ENABLED", env.WIKI_AGENT_ENABLED, false);
+    const baseUrl = normalizeOptionalUrl(env.WIKI_AGENT_BASE_URL);
+    const apiKey = env.WIKI_AGENT_API_KEY?.trim() || null;
+    const model = env.WIKI_AGENT_MODEL?.trim() || null;
+    const batchSize = parseNonNegativeInteger(env.WIKI_AGENT_BATCH_SIZE, 5, "WIKI_AGENT_BATCH_SIZE");
+    const workflowTimeoutSeconds = parsePositiveInteger(env.WIKI_WORKFLOW_TIMEOUT, 600, "WIKI_WORKFLOW_TIMEOUT");
+    const missing = [];
+    if (enabled) {
+        if (!apiKey) {
+            missing.push("WIKI_AGENT_API_KEY");
+        }
+        if (!model) {
+            missing.push("WIKI_AGENT_MODEL");
+        }
+    }
+    if (options.strict && enabled && missing.length > 0) {
+        throw new AppError(`WIKI_AGENT_ENABLED=true but missing required settings: ${missing.join(", ")}`, "config");
+    }
+    return {
+        enabled,
+        baseUrl,
+        apiKey,
+        model,
+        batchSize,
+        workflowTimeoutSeconds,
+        configured: enabled && missing.length === 0,
+        missing,
+    };
+}
+export function getPackageRoot() {
+    return path.resolve(fileURLToPath(new URL("../../", import.meta.url)));
+}
+export function resolveRuntimePaths(env = process.env) {
+    const wikiPath = requireAbsolutePath("WIKI_PATH", env.WIKI_PATH);
+    const wikiRoot = path.resolve(wikiPath, "..");
+    const vaultPath = path.resolve(env.VAULT_PATH ?? path.join(wikiRoot, "..", "vault"));
+    const dbPath = path.resolve(env.WIKI_DB_PATH ?? path.join(wikiRoot, "index.db"));
+    const configPath = path.resolve(env.WIKI_CONFIG_PATH ?? path.join(wikiRoot, "wiki.config.json"));
+    const templatesPath = path.resolve(env.WIKI_TEMPLATES_PATH ?? path.join(wikiRoot, "templates"));
+    return {
+        wikiPath,
+        wikiRoot,
+        vaultPath,
+        vaultHashMode: parseVaultHashMode(env.VAULT_HASH_MODE),
+        agentBackend: parseWikiAgentBackend(env.WIKI_AGENT_BACKEND),
+        dbPath,
+        configPath,
+        templatesPath,
+        queueArtifactsPath: path.join(wikiRoot, ".queue-artifacts"),
+        packageRoot: getPackageRoot(),
+        syncIntervalSeconds: parseSyncInterval(env.WIKI_SYNC_INTERVAL),
+        daemonHost: "127.0.0.1",
+        daemonPort: parseOptionalPort(env.WIKI_DAEMON_PORT, "WIKI_DAEMON_PORT"),
+        daemonPidPath: path.join(wikiRoot, ".wiki-daemon.pid"),
+        daemonLogPath: path.join(wikiRoot, ".wiki-daemon.log"),
+        daemonStatePath: path.join(wikiRoot, ".wiki-daemon.state.json"),
+    };
+}
+export function normalizePageId(inputPath, wikiPath) {
+    const resolved = path.isAbsolute(inputPath) ? inputPath : path.resolve(wikiPath, inputPath);
+    const relative = path.relative(wikiPath, resolved);
+    if (relative.startsWith("..") || path.isAbsolute(relative)) {
+        throw new AppError(`Path is outside wiki/pages: ${inputPath}`, "config");
+    }
+    return relative.split(path.sep).join("/");
+}
+export function resolvePagePath(pageId, wikiPath) {
+    return path.join(wikiPath, ...pageId.split("/"));
+}

package/dist/core/presenters.js

@@ -0,0 +1,23 @@
+export function compactPageSummary(page, config) {
+    const summary = {
+        id: page.id,
+        title: page.title,
+        pageType: page.pageType,
+        status: page.status,
+        visibility: page.visibility,
+        filePath: page.filePath,
+        tags: page.tags,
+        updatedAt: page.updatedAt,
+    };
+    if (page.nodeId) {
+        summary.nodeId = page.nodeId;
+    }
+    for (const columnName of config.allColumnNames) {
+        const camelName = columnName.replace(/_([a-z0-9])/g, (_, char) => char.toUpperCase());
+        const value = page[camelName];
+        if (value !== null && value !== undefined && value !== "") {
+            summary[camelName] = value;
+        }
+    }
+    return summary;
+}

package/dist/core/query.js

@@ -0,0 +1,58 @@
+import { snakeToCamel } from "../utils/case.js";
+function parseJsonField(value, fallback) {
+    if (typeof value !== "string" || value.trim() === "") {
+        return fallback;
+    }
+    return JSON.parse(value);
+}
+export function mapPageRow(row, config) {
+    const result = {
+        id: row.id,
+        nodeId: row.node_id,
+        title: row.title,
+        pageType: row.page_type,
+        status: row.status,
+        visibility: row.visibility,
+        tags: parseJsonField(row.tags, []),
+        extra: parseJsonField(row.extra, {}),
+        filePath: row.file_path,
+        contentHash: row.content_hash,
+        summaryText: row.summary_text,
+        embeddingStatus: row.embedding_status,
+        fileMtime: row.file_mtime,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+        indexedAt: row.indexed_at,
+    };
+    for (const columnName of config.allColumnNames) {
+        result[snakeToCamel(columnName)] = row[columnName];
+    }
+    return result;
+}
+export function listPageColumns(config) {
+    return [
+        "id",
+        "node_id",
+        "title",
+        "page_type",
+        "status",
+        "visibility",
+        "tags",
+        ...config.allColumnNames,
+        "extra",
+        "file_path",
+        "content_hash",
+        "summary_text",
+        "embedding_status",
+        "file_mtime",
+        "created_at",
+        "updated_at",
+        "indexed_at",
+    ];
+}
+export function selectPageById(db, config, pageId) {
+    const row = db
+        .prepare(`SELECT ${listPageColumns(config).join(", ")} FROM pages WHERE id = ?`)
+        .get(pageId);
+    return row ? mapPageRow(row, config) : null;
+}

package/dist/core/runtime.js

@@ -0,0 +1,20 @@
+import { loadConfig } from "./config.js";
+import { openDb } from "./db.js";
+import { EmbeddingClient } from "./embedding.js";
+import { resolveRuntimePaths } from "./paths.js";
+export function getEmbeddingDimensionFromEnv(env = process.env) {
+    const raw = env.EMBEDDING_DIMENSIONS ?? "384";
+    const value = Number.parseInt(raw, 10);
+    return Number.isFinite(value) && value > 0 ? value : 384;
+}
+export function loadRuntimeConfig(env = process.env) {
+    const paths = resolveRuntimePaths(env);
+    const config = loadConfig(paths.configPath);
+    return { paths, config };
+}
+export function openRuntimeDb(env = process.env) {
+    const { paths, config } = loadRuntimeConfig(env);
+    const embeddingClient = EmbeddingClient.fromEnv(env);
+    const { db } = openDb(paths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimensionFromEnv(env));
+    return { db, paths, config, embeddingClient };
+}

package/dist/core/sync.js

@@ -0,0 +1,235 @@
+import { loadConfig } from "./config.js";
+import { clearAllIndexedData, getMeta, openDb, resetVectorTable, setMetaValues, } from "./db.js";
+import { EmbeddingClient } from "./embedding.js";
+import { applyChanges, scanPages, scanSpecificPages } from "./indexer.js";
+import { resolveRuntimePaths } from "./paths.js";
+import { collectVaultFiles, syncVaultIndex } from "./vault.js";
+import { AppError } from "../utils/errors.js";
+import { pathExistsSync } from "../utils/fs.js";
+import { makeSyncId, toOffsetIso } from "../utils/time.js";
+function getEmbeddingDimension(env) {
+    const raw = env.EMBEDDING_DIMENSIONS ?? "384";
+    const value = Number.parseInt(raw, 10);
+    return Number.isFinite(value) && value > 0 ? value : 384;
+}
+function getEmbeddingTargets(db, embedAll, insertedIds, summaryChangedIds) {
+    const rows = db.prepare("SELECT rowid, id, summary_text AS summaryText, embedding_status AS embeddingStatus FROM pages").all();
+    if (embedAll) {
+        return rows.map(({ rowid, id, summaryText }) => ({ rowid, id, summaryText }));
+    }
+    const changedIds = new Set([...insertedIds, ...summaryChangedIds]);
+    return rows
+        .filter((row) => changedIds.has(row.id) || row.embeddingStatus === "pending" || row.embeddingStatus === "error")
+        .map(({ rowid, id, summaryText }) => ({ rowid, id, summaryText }));
+}
+async function embedPages(db, embeddingClient, targets) {
+    if (targets.length === 0) {
+        return { attempted: 0, succeeded: 0, failed: 0 };
+    }
+    const upsertVector = db.prepare("INSERT INTO vec_pages(page_rowid, page_id, embedding) VALUES (?, ?, ?)");
+    const deleteVector = db.prepare("DELETE FROM vec_pages WHERE page_rowid = ?");
+    const updateStatus = db.prepare("UPDATE pages SET embedding_status = ? WHERE id = ?");
+    let succeeded = 0;
+    let failed = 0;
+    for (let index = 0; index < targets.length; index += 50) {
+        const batch = targets.slice(index, index + 50);
+        try {
+            const embeddings = await embeddingClient.embedBatch(batch.map((item) => item.summaryText));
+            const transaction = db.transaction(() => {
+                for (let offset = 0; offset < batch.length; offset += 1) {
+                    deleteVector.run(BigInt(batch[offset].rowid));
+                    upsertVector.run(BigInt(batch[offset].rowid), batch[offset].id, new Float32Array(embeddings[offset]));
+                    updateStatus.run("done", batch[offset].id);
+                }
+            });
+            transaction();
+            succeeded += batch.length;
+        }
+        catch (error) {
+            const transaction = db.transaction(() => {
+                for (const item of batch) {
+                    updateStatus.run("error", item.id);
+                }
+            });
+            transaction();
+            failed += batch.length;
+            if (targets.length === batch.length) {
+                throw error;
+            }
+        }
+    }
+    return {
+        attempted: targets.length,
+        succeeded,
+        failed,
+    };
+}
+function countPendingEmbeddings(db) {
+    const row = db
+        .prepare("SELECT COUNT(*) AS count FROM pages WHERE embedding_status IN ('pending', 'error')")
+        .get();
+    return row.count;
+}
+export async function syncWorkspace(options = {}) {
+    const env = options.env ?? process.env;
+    const runtimePaths = resolveRuntimePaths(env);
+    if (!pathExistsSync(runtimePaths.wikiPath)) {
+        throw new AppError(`WIKI_PATH does not exist: ${runtimePaths.wikiPath}`, "config");
+    }
+    const config = loadConfig(runtimePaths.configPath);
+    const embeddingClient = EmbeddingClient.fromEnv(env);
+    const { db, configChanged } = openDb(runtimePaths.dbPath, config, embeddingClient?.settings.dimensions ?? getEmbeddingDimension(env));
+    try {
+        let mode = options.targetPaths && options.targetPaths.length > 0 && !options.force ? "path" : "full";
+        let upgradedToFullSync = false;
+        const storedEmbeddingProfile = embeddingClient ? getMeta(db, "embedding_profile") : null;
+        const profileChanged = Boolean(embeddingClient && storedEmbeddingProfile && storedEmbeddingProfile !== embeddingClient.profileHash);
+        if (mode === "path" && (configChanged || profileChanged)) {
+            mode = "full";
+            upgradedToFullSync = true;
+        }
+        if (profileChanged && options.skipEmbedding) {
+            throw new AppError("Embedding profile changed, cannot skip embedding.", "config");
+        }
+        if (options.force) {
+            clearAllIndexedData(db);
+            mode = "full";
+        }
+        const changes = mode === "path"
+            ? scanSpecificPages(db, runtimePaths.wikiPath, options.targetPaths ?? [], configChanged, false)
+            : scanPages(db, runtimePaths.wikiPath, configChanged, false);
+        if (mode === "path" &&
+            changes.added.length === 0 &&
+            changes.modified.length === 0 &&
+            changes.deleted.length === 0 &&
+            changes.unchanged.length === 0) {
+            throw new AppError(`No page matched the requested --path value(s).`, "not_found");
+        }
+        const applyResult = applyChanges(db, changes, runtimePaths.wikiPath, config);
+        if (applyResult.parseErrors.length > 0) {
+            throw new AppError("Failed to parse one or more wiki pages during sync.", "runtime", {
+                parseErrors: applyResult.parseErrors,
+            });
+        }
+        let embedAll = false;
+        if (embeddingClient && profileChanged) {
+            resetVectorTable(db, embeddingClient.settings.dimensions);
+            db.prepare("UPDATE pages SET embedding_status = 'pending'").run();
+            embedAll = true;
+        }
+        let embeddingAttempted = 0;
+        let embeddingSucceeded = 0;
+        let embeddingFailed = 0;
+        const skipEmbedding = options.skipEmbedding === true || embeddingClient === null;
+        if (embeddingClient && !skipEmbedding) {
+            const targets = getEmbeddingTargets(db, embedAll, applyResult.inserted, applyResult.summaryChangedIds);
+            const embeddingResult = await embedPages(db, embeddingClient, targets);
+            embeddingAttempted = embeddingResult.attempted;
+            embeddingSucceeded = embeddingResult.succeeded;
+            embeddingFailed = embeddingResult.failed;
+            const hasPending = countPendingEmbeddings(db) > 0;
+            if (!hasPending) {
+                setMetaValues(db, {
+                    embedding_profile: embeddingClient.profileHash,
+                });
+            }
+        }
+        let vaultFiles = 0;
+        let vaultChanges = 0;
+        let syncId = null;
+        let vaultQueue = {
+            pendingAdded: 0,
+            pendingReset: 0,
+            removed: 0,
+        };
+        if (mode === "full") {
+            syncId = makeSyncId();
+            const currentVaultFiles = await collectVaultFiles(runtimePaths.vaultPath, config.vaultFileTypes, env);
+            const vaultResult = syncVaultIndex(db, currentVaultFiles, syncId);
+            vaultFiles = vaultResult.files;
+            vaultChanges = vaultResult.changes.length;
+            vaultQueue = vaultResult.queue;
+        }
+        setMetaValues(db, {
+            config_version: config.configVersion,
+            last_sync_at: toOffsetIso(),
+            ...(mode === "full" ? { last_sync_id: syncId } : {}),
+            ...(options.force ? { last_full_rebuild_at: toOffsetIso() } : {}),
+        });
+        return {
+            mode,
+            upgradedToFullSync,
+            configChanged,
+            profileChanged,
+            inserted: applyResult.inserted.length,
+            updated: applyResult.updated.length,
+            deleted: applyResult.deleted.length,
+            unchanged: changes.unchanged.length,
+            summaryChanged: applyResult.summaryChangedIds.length,
+            embedding: {
+                enabled: embeddingClient !== null,
+                skipped: skipEmbedding,
+                attempted: embeddingAttempted,
+                succeeded: embeddingSucceeded,
+                failed: embeddingFailed,
+                embedAll,
+            },
+            vault: {
+                scanned: mode === "full",
+                files: vaultFiles,
+                changes: vaultChanges,
+                syncId,
+                queue: vaultQueue,
+            },
+        };
+    }
+    finally {
+        db.close();
+    }
+}
+export async function embedPendingPages(env = process.env) {
+    const runtimePaths = resolveRuntimePaths(env);
+    const config = loadConfig(runtimePaths.configPath);
+    const embeddingClient = EmbeddingClient.fromEnv(env);
+    if (!embeddingClient) {
+        return;
+    }
+    const { db } = openDb(runtimePaths.dbPath, config, embeddingClient.settings.dimensions);
+    try {
+        const targets = getEmbeddingTargets(db, false, [], []);
+        const result = await embedPages(db, embeddingClient, targets);
+        if (result.failed === 0 && countPendingEmbeddings(db) === 0) {
+            setMetaValues(db, { embedding_profile: embeddingClient.profileHash });
+        }
+    }
+    finally {
+        db.close();
+    }
+}
+export function readAllPages(db) {
+    const rows = db.prepare(`
+      SELECT
+        id,
+        node_id AS nodeId,
+        title,
+        page_type AS pageType,
+        status,
+        visibility,
+        tags,
+        extra,
+        file_path AS filePath,
+        content_hash AS contentHash,
+        summary_text AS summaryText,
+        embedding_status AS embeddingStatus,
+        file_mtime AS fileMtime,
+        created_at AS createdAt,
+        updated_at AS updatedAt,
+        indexed_at AS indexedAt
+      FROM pages
+    `).all();
+    return rows.map((row) => ({
+        ...row,
+        tags: row.tags ? JSON.parse(row.tags) : [],
+        extra: row.extra ? JSON.parse(row.extra) : {},
+    }));
+}