@biaoo/tiangong-wiki 0.2.0

package/dist/core/embedding.js

@@ -0,0 +1,104 @@
+import { sha256Text } from "../utils/fs.js";
+import { AppError } from "../utils/errors.js";
+export class EmbeddingClient {
+    settings;
+    constructor(settings) {
+        this.settings = settings;
+    }
+    static fromEnv(env = process.env) {
+        const baseUrl = env.EMBEDDING_BASE_URL ?? env.OPENROUTER_BASE_URL;
+        const apiKey = env.EMBEDDING_API_KEY ?? env.OPENROUTER_API_KEY;
+        const model = env.EMBEDDING_MODEL ?? env.OPENROUTER_EMBEDDING_MODEL;
+        const rawDimensions = env.EMBEDDING_DIMENSIONS ?? "384";
+        if (!baseUrl || !apiKey || !model) {
+            return null;
+        }
+        const dimensions = Number.parseInt(rawDimensions, 10);
+        if (!Number.isFinite(dimensions) || dimensions <= 0) {
+            throw new AppError(`Invalid EMBEDDING_DIMENSIONS: ${rawDimensions}`, "config");
+        }
+        return new EmbeddingClient({
+            baseUrl: baseUrl.replace(/\/+$/g, ""),
+            apiKey,
+            model,
+            dimensions,
+        });
+    }
+    get profileHash() {
+        return sha256Text(`${this.settings.baseUrl}:${this.settings.model}:${this.settings.dimensions}`);
+    }
+    async probe() {
+        await this.embedBatch(["tiangong-wiki-skill probe"]);
+    }
+    async embedBatch(inputs) {
+        if (inputs.length === 0) {
+            return [];
+        }
+        const response = await this.requestWithRetry({
+            model: this.settings.model,
+            input: inputs,
+            dimensions: this.settings.dimensions,
+        });
+        if (!response?.data || !Array.isArray(response.data)) {
+            throw new AppError("Embedding API returned an invalid response", "runtime");
+        }
+        const embeddings = response.data
+            .sort((left, right) => left.index - right.index)
+            .map((item) => item.embedding);
+        if (embeddings.length !== inputs.length) {
+            throw new AppError(`Embedding API returned ${embeddings.length} vectors for ${inputs.length} inputs`, "runtime");
+        }
+        const normalizedEmbeddings = embeddings.map((embedding) => {
+            if (embedding.length === this.settings.dimensions) {
+                return embedding;
+            }
+            if (embedding.length > this.settings.dimensions) {
+                return embedding.slice(0, this.settings.dimensions);
+            }
+            throw new AppError(`Embedding dimensions mismatch: expected ${this.settings.dimensions}, got ${embedding.length}`, "runtime");
+        });
+        for (const embedding of normalizedEmbeddings) {
+            if (embedding.length !== this.settings.dimensions) {
+                throw new AppError(`Embedding dimensions mismatch: expected ${this.settings.dimensions}, got ${embedding.length}`, "runtime");
+            }
+        }
+        return normalizedEmbeddings;
+    }
+    async embedAll(inputs, batchSize = 50) {
+        const results = [];
+        for (let index = 0; index < inputs.length; index += batchSize) {
+            const batch = inputs.slice(index, index + batchSize);
+            const embeddings = await this.embedBatch(batch);
+            results.push(...embeddings);
+        }
+        return results;
+    }
+    async requestWithRetry(payload) {
+        let lastError;
+        for (let attempt = 1; attempt <= 3; attempt += 1) {
+            try {
+                const response = await fetch(`${this.settings.baseUrl}/embeddings`, {
+                    method: "POST",
+                    headers: {
+                        "content-type": "application/json",
+                        authorization: `Bearer ${this.settings.apiKey}`,
+                    },
+                    body: JSON.stringify(payload),
+                    signal: AbortSignal.timeout(30_000),
+                });
+                if (!response.ok) {
+                    const body = await response.text();
+                    throw new AppError(`Embedding API request failed with status ${response.status}`, "runtime", { body });
+                }
+                return (await response.json());
+            }
+            catch (error) {
+                lastError = error;
+                if (attempt < 3) {
+                    await new Promise((resolve) => setTimeout(resolve, attempt * 250));
+                }
+            }
+        }
+        throw lastError instanceof AppError ? lastError : new AppError(String(lastError), "runtime");
+    }
+}

package/dist/core/frontmatter.js

@@ -0,0 +1,287 @@
+import matter from "gray-matter";
+import path from "node:path";
+import { getTemplate } from "./config.js";
+import { normalizePageId } from "./paths.js";
+import { camelToSnake, humanizeFieldName } from "../utils/case.js";
+import { toDateOnly } from "../utils/time.js";
+const FIXED_FIELDS = new Set([
+    "pageType",
+    "title",
+    "nodeId",
+    "status",
+    "visibility",
+    "tags",
+    "createdAt",
+    "updatedAt",
+]);
+function isPlainObject(value) {
+    return Object.prototype.toString.call(value) === "[object Object]";
+}
+function normalizeStringArray(value) {
+    if (value === undefined || value === null || value === "") {
+        return [];
+    }
+    if (Array.isArray(value)) {
+        return value
+            .map((item) => (item === null || item === undefined ? "" : String(item).trim()))
+            .filter(Boolean);
+    }
+    return [String(value).trim()].filter(Boolean);
+}
+function normalizeScalar(value) {
+    if (value instanceof Date) {
+        return value.toISOString().slice(0, 10);
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => normalizeScalar(item));
+    }
+    if (isPlainObject(value)) {
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, normalizeScalar(item)]));
+    }
+    if (typeof value === "string") {
+        return value.trim();
+    }
+    return value;
+}
+function normalizeColumnValue(value) {
+    const normalized = normalizeScalar(value);
+    if (typeof normalized === "boolean") {
+        return normalized ? 1 : 0;
+    }
+    if (Array.isArray(normalized) || isPlainObject(normalized)) {
+        return JSON.stringify(normalized);
+    }
+    return normalized;
+}
+export function normalizeDateField(value) {
+    if (value instanceof Date) {
+        return value.toISOString().slice(0, 10);
+    }
+    if (typeof value === "string" && value.trim()) {
+        const dateOnly = value.trim().slice(0, 10);
+        if (/^\d{4}-\d{2}-\d{2}$/.test(dateOnly)) {
+            return dateOnly;
+        }
+    }
+    return null;
+}
+function normalizeReference(value, resolve) {
+    const trimmed = value.trim();
+    if (resolve === "nodeId") {
+        return trimmed;
+    }
+    return trimmed
+        .replace(/^\.?\//, "")
+        .replace(/^pages\//, "")
+        .split(path.sep)
+        .join("/");
+}
+function extractFirstParagraph(body) {
+    const cleaned = body
+        .replace(/```[\s\S]*?```/g, "")
+        .split(/\r?\n\r?\n/)
+        .map((chunk) => chunk.trim())
+        .find((chunk) => chunk && !chunk.startsWith("#"));
+    if (!cleaned) {
+        return "";
+    }
+    return cleaned.replace(/\s+/g, " ").slice(0, 200);
+}
+function valueToSummaryLine(fieldName, value) {
+    if (value === undefined || value === null || value === "") {
+        return null;
+    }
+    if (Array.isArray(value) && value.length === 0) {
+        return null;
+    }
+    const label = humanizeFieldName(fieldName);
+    const content = Array.isArray(value)
+        ? value.map((item) => String(item)).join(", ")
+        : typeof value === "object"
+            ? JSON.stringify(value)
+            : String(value);
+    return `${label}: ${content}`;
+}
+function buildSummaryText(pageType, title, tags, data, config, templateFields, body) {
+    const lines = [`[${pageType}] ${title}`];
+    if (tags.length > 0) {
+        lines.push(`标签: ${tags.join(", ")}`);
+    }
+    for (const field of Object.keys(config.customColumns)) {
+        const line = valueToSummaryLine(field, data[field]);
+        if (line) {
+            lines.push(line);
+        }
+    }
+    const seen = new Set(["title", "tags"]);
+    for (const field of [...config.defaultSummaryFields, ...templateFields]) {
+        if (seen.has(field)) {
+            continue;
+        }
+        seen.add(field);
+        const line = valueToSummaryLine(field, data[field]);
+        if (line) {
+            lines.push(line);
+        }
+    }
+    const bodyPreview = extractFirstParagraph(body);
+    if (bodyPreview) {
+        lines.push("---", bodyPreview);
+    }
+    return lines.join("\n");
+}
+function buildEdges(page, data, config, pageType) {
+    const template = getTemplate(config, pageType);
+    const source = page.nodeId ?? page.id;
+    const rules = [
+        ...Object.entries(config.commonEdges),
+        ...Object.entries(template.edges),
+    ];
+    const edges = [];
+    for (const [field, rule] of rules) {
+        const rawValues = normalizeStringArray(data[field]);
+        const matcher = rule.match ? new RegExp(rule.match) : null;
+        for (const value of rawValues) {
+            if (matcher && !matcher.test(value)) {
+                continue;
+            }
+            const target = normalizeReference(value, rule.resolve);
+            if (!target) {
+                continue;
+            }
+            if (rule.resolve === "path" && target.startsWith("vault/")) {
+                continue;
+            }
+            edges.push({
+                source,
+                target,
+                edgeType: rule.edgeType,
+                sourcePage: page.id,
+                metadata: { field },
+            });
+        }
+    }
+    return edges;
+}
+function buildExtraAndColumns(data, config, pageType) {
+    const template = getTemplate(config, pageType);
+    const columnValues = {};
+    const extra = {};
+    const registeredFields = new Set([
+        ...FIXED_FIELDS,
+        ...Object.keys(config.customColumns),
+        ...Object.keys(template.columns),
+        ...Object.keys(config.commonEdges),
+        ...Object.keys(template.edges),
+    ]);
+    for (const [field, value] of Object.entries(data)) {
+        if (config.customColumns[field] || template.columns[field]) {
+            columnValues[camelToSnake(field)] = normalizeColumnValue(value);
+            continue;
+        }
+        if (FIXED_FIELDS.has(field)) {
+            continue;
+        }
+        extra[field] = normalizeScalar(value);
+    }
+    const unregisteredFields = Object.keys(data)
+        .filter((field) => !registeredFields.has(field))
+        .sort();
+    return { columnValues, extra, unregisteredFields };
+}
+export function parsePage(filePath, wikiPath, config) {
+    const pageId = normalizePageId(filePath, wikiPath);
+    const today = toDateOnly();
+    let parsedMatter;
+    try {
+        parsedMatter = matter.read(filePath);
+    }
+    catch (error) {
+        return {
+            ok: false,
+            error: {
+                filePath,
+                code: "yaml_parse_error",
+                message: `Failed to parse frontmatter for ${pageId}`,
+                details: error instanceof Error ? error.message : String(error),
+            },
+        };
+    }
+    if (!isPlainObject(parsedMatter.data)) {
+        return {
+            ok: false,
+            error: {
+                filePath,
+                code: "invalid_frontmatter",
+                message: `Frontmatter must be a YAML object for ${pageId}`,
+            },
+        };
+    }
+    const data = parsedMatter.data;
+    const pageType = typeof data.pageType === "string" ? data.pageType.trim() : "";
+    const title = typeof data.title === "string" ? data.title.trim() : "";
+    if (!pageType) {
+        return {
+            ok: false,
+            error: {
+                filePath,
+                code: "missing_page_type",
+                message: `Missing pageType in ${pageId}`,
+            },
+        };
+    }
+    if (!config.templates[pageType]) {
+        return {
+            ok: false,
+            error: {
+                filePath,
+                code: "unknown_page_type",
+                message: `Unknown pageType "${pageType}" in ${pageId}`,
+            },
+        };
+    }
+    if (!title) {
+        return {
+            ok: false,
+            error: {
+                filePath,
+                code: "missing_title",
+                message: `Missing title in ${pageId}`,
+            },
+        };
+    }
+    const tags = normalizeStringArray(data.tags);
+    const { columnValues, extra, unregisteredFields } = buildExtraAndColumns(data, config, pageType);
+    const page = {
+        id: pageId,
+        nodeId: typeof data.nodeId === "string" && data.nodeId.trim() ? data.nodeId.trim() : null,
+        title,
+        pageType,
+        status: typeof data.status === "string" && data.status.trim() ? data.status.trim() : "draft",
+        visibility: typeof data.visibility === "string" && data.visibility.trim() ? data.visibility.trim() : "private",
+        tags,
+        extra,
+        filePath: path.resolve(filePath),
+        contentHash: null,
+        summaryText: "",
+        embeddingStatus: "pending",
+        fileMtime: null,
+        createdAt: normalizeDateField(data.createdAt) ?? today,
+        updatedAt: normalizeDateField(data.updatedAt) ?? today,
+        indexedAt: null,
+    };
+    const template = getTemplate(config, pageType);
+    const summaryText = buildSummaryText(pageType, title, tags, data, config, template.summaryFields, parsedMatter.content);
+    page.summaryText = summaryText;
+    const edges = buildEdges(page, data, config, pageType);
+    const parsed = {
+        page,
+        columnValues,
+        edges,
+        summaryText,
+        body: parsedMatter.content,
+        rawData: data,
+        unregisteredFields,
+    };
+    return { ok: true, parsed };
+}

package/dist/core/indexer.js

@@ -0,0 +1,241 @@
+import { normalizeDateField, parsePage } from "./frontmatter.js";
+import { rebuildFts } from "./db.js";
+import { normalizePageId, resolvePagePath } from "./paths.js";
+import { fileStatSync, listFilesRecursiveSync, pathExistsSync, sha256FileSync } from "../utils/fs.js";
+import { toDateOnly, toOffsetIso } from "../utils/time.js";
+function getExistingPages(db) {
+    const rows = db
+        .prepare("SELECT id, file_path AS filePath, content_hash AS contentHash FROM pages")
+        .all();
+    return new Map(rows.map((row) => [row.id, { filePath: row.filePath, contentHash: row.contentHash }]));
+}
+function makeScanEntry(filePath, wikiPath) {
+    return {
+        id: normalizePageId(filePath, wikiPath),
+        filePath,
+        contentHash: sha256FileSync(filePath),
+        fileMtime: Number(new Date().getTime()),
+    };
+}
+function createScanEntry(filePath, wikiPath) {
+    const entry = makeScanEntry(filePath, wikiPath);
+    const stats = fileStatSync(filePath);
+    return { ...entry, fileMtime: stats.mtimeMs };
+}
+export function scanPages(db, wikiPath, configChanged, force = false) {
+    const existing = getExistingPages(db);
+    const currentEntries = listFilesRecursiveSync(wikiPath, ".md").map((filePath) => createScanEntry(filePath, wikiPath));
+    const seenIds = new Set();
+    const added = [];
+    const modified = [];
+    const unchanged = [];
+    for (const entry of currentEntries) {
+        seenIds.add(entry.id);
+        const existingEntry = existing.get(entry.id);
+        if (!existingEntry) {
+            added.push(entry);
+            continue;
+        }
+        if (force || configChanged || existingEntry.contentHash !== entry.contentHash) {
+            modified.push(entry);
+            continue;
+        }
+        unchanged.push(entry);
+    }
+    const deleted = [...existing.entries()]
+        .filter(([pageId]) => !seenIds.has(pageId))
+        .map(([id, value]) => ({ id, filePath: value.filePath }))
+        .sort((left, right) => left.id.localeCompare(right.id));
+    return { added, modified, deleted, unchanged };
+}
+export function scanSpecificPages(db, wikiPath, pageIdsOrPaths, configChanged, force = false) {
+    const existing = getExistingPages(db);
+    const requestedIds = [...new Set(pageIdsOrPaths.map((value) => normalizePageId(value, wikiPath)))];
+    const added = [];
+    const modified = [];
+    const unchanged = [];
+    const deleted = [];
+    for (const pageId of requestedIds) {
+        const filePath = resolvePagePath(pageId, wikiPath);
+        const existingEntry = existing.get(pageId);
+        if (!pathExistsSync(filePath)) {
+            if (existingEntry) {
+                deleted.push({ id: pageId, filePath: existingEntry.filePath });
+            }
+            continue;
+        }
+        const entry = createScanEntry(filePath, wikiPath);
+        if (!existingEntry) {
+            added.push(entry);
+            continue;
+        }
+        if (force || configChanged || existingEntry.contentHash !== entry.contentHash) {
+            modified.push(entry);
+            continue;
+        }
+        unchanged.push(entry);
+    }
+    return { added, modified, deleted, unchanged };
+}
+function serializeRow(parsed, entry, config, previousEmbeddingStatus, summaryChanged) {
+    const row = {
+        id: parsed.page.id,
+        node_id: parsed.page.nodeId,
+        title: parsed.page.title,
+        page_type: parsed.page.pageType,
+        status: parsed.page.status,
+        visibility: parsed.page.visibility,
+        tags: JSON.stringify(parsed.page.tags),
+        extra: JSON.stringify(parsed.page.extra),
+        file_path: parsed.page.filePath,
+        content_hash: entry.contentHash,
+        summary_text: parsed.summaryText,
+        embedding_status: summaryChanged || !previousEmbeddingStatus ? "pending" : previousEmbeddingStatus,
+        file_mtime: entry.fileMtime,
+        created_at: parsed.page.createdAt,
+        updated_at: parsed.page.updatedAt,
+        indexed_at: toOffsetIso(),
+    };
+    for (const columnName of config.allColumnNames) {
+        row[columnName] = parsed.columnValues[columnName] ?? null;
+    }
+    return row;
+}
+function buildInsertStatement(config) {
+    const columns = [
+        "id",
+        "node_id",
+        "title",
+        "page_type",
+        "status",
+        "visibility",
+        "tags",
+        ...config.allColumnNames,
+        "extra",
+        "file_path",
+        "content_hash",
+        "summary_text",
+        "embedding_status",
+        "file_mtime",
+        "created_at",
+        "updated_at",
+        "indexed_at",
+    ];
+    const placeholders = columns.map((column) => `@${column}`);
+    return `INSERT INTO pages (${columns.join(", ")}) VALUES (${placeholders.join(", ")})`;
+}
+function buildUpdateStatement(config) {
+    const columns = [
+        "node_id",
+        "title",
+        "page_type",
+        "status",
+        "visibility",
+        "tags",
+        ...config.allColumnNames,
+        "extra",
+        "file_path",
+        "content_hash",
+        "summary_text",
+        "embedding_status",
+        "file_mtime",
+        "created_at",
+        "updated_at",
+        "indexed_at",
+    ];
+    return `UPDATE pages SET ${columns.map((column) => `${column} = @${column}`).join(", ")} WHERE id = @id`;
+}
+export function applyChanges(db, changes, wikiPath, config) {
+    const parseResults = [...changes.added, ...changes.modified].map((entry) => ({
+        entry,
+        result: parsePage(entry.filePath, wikiPath, config),
+    }));
+    const parseErrors = parseResults
+        .filter((item) => !item.result.ok)
+        .map((item) => (item.result.ok ? null : item.result.error))
+        .filter((item) => item !== null);
+    if (parseErrors.length > 0) {
+        return {
+            inserted: [],
+            updated: [],
+            deleted: [],
+            summaryChangedIds: [],
+            parseErrors,
+        };
+    }
+    const parsedEntries = parseResults.map((item) => ({
+        entry: item.entry,
+        parsed: item.result.parsed,
+    }));
+    const insertStatement = db.prepare(buildInsertStatement(config));
+    const updateStatement = db.prepare(buildUpdateStatement(config));
+    const selectExistingPage = db.prepare("SELECT rowid, summary_text AS summaryText, embedding_status AS embeddingStatus, created_at AS createdAt FROM pages WHERE id = ?");
+    const deleteEdgesBySourcePage = db.prepare("DELETE FROM edges WHERE source_page = ?");
+    const insertEdge = db.prepare(`
+      INSERT OR REPLACE INTO edges(source, target, edge_type, source_page, metadata)
+      VALUES(@source, @target, @edge_type, @source_page, @metadata)
+    `);
+    const deletePage = db.prepare("DELETE FROM pages WHERE id = ?");
+    const deleteVecRow = db.prepare("DELETE FROM vec_pages WHERE page_rowid = ?");
+    const selectPageRowid = db.prepare("SELECT rowid FROM pages WHERE id = ?");
+    const inserted = [];
+    const updated = [];
+    const deleted = [];
+    const summaryChangedIds = [];
+    const hasContentChanges = parsedEntries.length > 0 || changes.deleted.length > 0;
+    const transaction = db.transaction(() => {
+        for (const { entry, parsed } of parsedEntries) {
+            const existing = selectExistingPage.get(entry.id);
+            const isInsert = !existing;
+            const summaryChanged = isInsert || existing.summaryText !== parsed.summaryText;
+            const today = toDateOnly();
+            parsed.page.createdAt =
+                normalizeDateField(parsed.rawData.createdAt) ?? existing?.createdAt ?? today;
+            parsed.page.updatedAt = isInsert
+                ? normalizeDateField(parsed.rawData.updatedAt) ?? today
+                : today;
+            const row = serializeRow(parsed, entry, config, existing?.embeddingStatus ?? null, summaryChanged);
+            if (isInsert) {
+                insertStatement.run(row);
+                inserted.push(entry.id);
+            }
+            else {
+                updateStatement.run(row);
+                updated.push(entry.id);
+            }
+            if (summaryChanged) {
+                summaryChangedIds.push(entry.id);
+            }
+            deleteEdgesBySourcePage.run(entry.id);
+            for (const edge of parsed.edges) {
+                insertEdge.run({
+                    source: edge.source,
+                    target: edge.target,
+                    edge_type: edge.edgeType,
+                    source_page: edge.sourcePage,
+                    metadata: JSON.stringify(edge.metadata),
+                });
+            }
+        }
+        for (const page of changes.deleted) {
+            const existing = selectPageRowid.get(page.id);
+            deleteEdgesBySourcePage.run(page.id);
+            if (existing) {
+                deleteVecRow.run(BigInt(existing.rowid));
+            }
+            deletePage.run(page.id);
+            deleted.push(page.id);
+        }
+        if (hasContentChanges) {
+            rebuildFts(db);
+        }
+    });
+    transaction();
+    return {
+        inserted,
+        updated,
+        deleted,
+        summaryChangedIds,
+        parseErrors: [],
+    };
+}