@biaoo/tiangong-wiki 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +167 -0
- package/README.zh-CN.md +167 -0
- package/SKILL.md +116 -0
- package/agents/openai.yaml +4 -0
- package/assets/config.example.env +18 -0
- package/assets/templates/achievement.md +32 -0
- package/assets/templates/bridge.md +33 -0
- package/assets/templates/concept.md +47 -0
- package/assets/templates/faq.md +31 -0
- package/assets/templates/lesson.md +31 -0
- package/assets/templates/method.md +31 -0
- package/assets/templates/misconception.md +35 -0
- package/assets/templates/person.md +31 -0
- package/assets/templates/research-note.md +34 -0
- package/assets/templates/resume.md +34 -0
- package/assets/templates/source-summary.md +35 -0
- package/assets/vllm/qwen3_5_openai_developer.jinja +182 -0
- package/assets/wiki.config.default.json +193 -0
- package/dist/commands/check-config.js +77 -0
- package/dist/commands/create.js +32 -0
- package/dist/commands/daemon.js +186 -0
- package/dist/commands/dashboard.js +112 -0
- package/dist/commands/doctor.js +22 -0
- package/dist/commands/export-graph.js +28 -0
- package/dist/commands/export-index.js +31 -0
- package/dist/commands/find.js +36 -0
- package/dist/commands/fts.js +32 -0
- package/dist/commands/graph.js +35 -0
- package/dist/commands/init.js +48 -0
- package/dist/commands/lint.js +35 -0
- package/dist/commands/list.js +28 -0
- package/dist/commands/page-info.js +24 -0
- package/dist/commands/search.js +32 -0
- package/dist/commands/setup.js +15 -0
- package/dist/commands/stat.js +20 -0
- package/dist/commands/sync.js +38 -0
- package/dist/commands/template.js +71 -0
- package/dist/commands/type.js +88 -0
- package/dist/commands/vault.js +64 -0
- package/dist/core/agent.js +201 -0
- package/dist/core/cli-env.js +129 -0
- package/dist/core/codex-workflow.js +233 -0
- package/dist/core/config.js +126 -0
- package/dist/core/db.js +292 -0
- package/dist/core/embedding.js +104 -0
- package/dist/core/frontmatter.js +287 -0
- package/dist/core/indexer.js +241 -0
- package/dist/core/onboarding.js +967 -0
- package/dist/core/page-files.js +91 -0
- package/dist/core/paths.js +161 -0
- package/dist/core/presenters.js +23 -0
- package/dist/core/query.js +58 -0
- package/dist/core/runtime.js +20 -0
- package/dist/core/sync.js +235 -0
- package/dist/core/synology.js +412 -0
- package/dist/core/template-evolution.js +38 -0
- package/dist/core/vault-processing.js +742 -0
- package/dist/core/vault.js +594 -0
- package/dist/core/workflow-context.js +188 -0
- package/dist/core/workflow-result.js +162 -0
- package/dist/core/workspace-bootstrap.js +30 -0
- package/dist/core/workspace-skills.js +220 -0
- package/dist/daemon/client.js +147 -0
- package/dist/daemon/server.js +807 -0
- package/dist/daemon/state.js +53 -0
- package/dist/dashboard/assets/index-1FgAUZ28.css +1 -0
- package/dist/dashboard/assets/index-6A0PWT4X.js +154 -0
- package/dist/dashboard/assets/jetbrains-mono-cyrillic-400-normal-BEIGL1Tu.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-cyrillic-400-normal-ugxPyKxw.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-cyrillic-500-normal-DJqRU3vO.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-cyrillic-500-normal-DmUKJPL_.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-cyrillic-700-normal-BWTpRfYl.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-cyrillic-700-normal-CEoEElIJ.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-greek-400-normal-B9oWc5Lo.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-greek-400-normal-C190GLew.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-greek-500-normal-D7SFKleX.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-greek-500-normal-JpySY46c.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-greek-700-normal-C6CZE3T8.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-greek-700-normal-DEigVDxa.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-400-normal-6-qcROiO.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-400-normal-V6pRDFza.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-500-normal-BWZEU5yA.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-500-normal-CJOVTJB7.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-700-normal-BYuf6tUa.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-700-normal-D3wTyLJW.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-ext-400-normal-Bc8Ftmh3.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-ext-400-normal-fXTG6kC5.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-ext-500-normal-Cut-4mMH.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-ext-500-normal-ckzbgY84.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-ext-700-normal-CZipNAKV.woff2 +0 -0
- package/dist/dashboard/assets/jetbrains-mono-latin-ext-700-normal-CxPITLHs.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-vietnamese-400-normal-CqNFfHCs.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-vietnamese-500-normal-DNRqzVM1.woff +0 -0
- package/dist/dashboard/assets/jetbrains-mono-vietnamese-700-normal-BDLVIk2r.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-400-normal-BnQMeOim.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-400-normal-CJ-V5oYT.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-500-normal-CNSSEhBt.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-500-normal-lFbtlQH6.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-700-normal-CwsQ-cCU.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-700-normal-RjhwGPKo.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-ext-400-normal-CfP_5XZW.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-ext-400-normal-DRPE3kg4.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-ext-500-normal-3dgZTiw9.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-ext-500-normal-DUe3BAxM.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-ext-700-normal-BQnZhY3m.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-latin-ext-700-normal-HVCqSBdx.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-vietnamese-400-normal-B7xT_GF5.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-vietnamese-400-normal-BIWiOVfw.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-vietnamese-500-normal-BTqKIpxg.woff +0 -0
- package/dist/dashboard/assets/space-grotesk-vietnamese-500-normal-BmEvtly_.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-vietnamese-700-normal-DMty7AZE.woff2 +0 -0
- package/dist/dashboard/assets/space-grotesk-vietnamese-700-normal-Duxec5Rn.woff +0 -0
- package/dist/dashboard/index.html +18 -0
- package/dist/index.js +86 -0
- package/dist/operations/dashboard.js +1231 -0
- package/dist/operations/export.js +110 -0
- package/dist/operations/query.js +649 -0
- package/dist/operations/type-template.js +210 -0
- package/dist/operations/write.js +143 -0
- package/dist/types/config.js +1 -0
- package/dist/types/page.js +1 -0
- package/dist/utils/case.js +22 -0
- package/dist/utils/errors.js +26 -0
- package/dist/utils/fs.js +77 -0
- package/dist/utils/output.js +33 -0
- package/dist/utils/process.js +60 -0
- package/dist/utils/segmenter.js +24 -0
- package/dist/utils/slug.js +10 -0
- package/dist/utils/time.js +24 -0
- package/package.json +64 -0
- package/references/cli-interface.md +312 -0
- package/references/env.md +122 -0
- package/references/template-design-guide.md +271 -0
- package/references/vault-to-wiki-instruction.md +110 -0
- package/references/wiki-maintenance-instruction.md +190 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { Codex } from "@openai/codex-sdk";
|
|
3
|
+
import { readWorkflowResult } from "./workflow-result.js";
|
|
4
|
+
import { readTextFileSync, writeTextFileSync } from "../utils/fs.js";
|
|
5
|
+
import { AppError } from "../utils/errors.js";
|
|
6
|
+
export const CODEX_WORKFLOW_VERSION = "2026-04-07";
|
|
7
|
+
function normalizeEnv(input) {
|
|
8
|
+
const normalized = {};
|
|
9
|
+
for (const [key, value] of Object.entries({
|
|
10
|
+
...process.env,
|
|
11
|
+
...input.env,
|
|
12
|
+
...(input.env?.WIKI_AGENT_API_KEY && !input.env.OPENAI_API_KEY ? { OPENAI_API_KEY: input.env.WIKI_AGENT_API_KEY } : {}),
|
|
13
|
+
})) {
|
|
14
|
+
if (typeof value === "string") {
|
|
15
|
+
normalized[key] = value;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
normalized.PATH = [input.skillArtifactsPath, normalized.PATH].filter(Boolean).join(path.delimiter);
|
|
19
|
+
return normalized;
|
|
20
|
+
}
|
|
21
|
+
function createCodexClient(input) {
|
|
22
|
+
const env = normalizeEnv(input);
|
|
23
|
+
const baseUrl = input.env?.WIKI_AGENT_BASE_URL?.trim();
|
|
24
|
+
const apiKey = input.env?.WIKI_AGENT_API_KEY?.trim();
|
|
25
|
+
const options = {
|
|
26
|
+
apiKey: apiKey || undefined,
|
|
27
|
+
env,
|
|
28
|
+
};
|
|
29
|
+
if (baseUrl) {
|
|
30
|
+
// Define a custom model_provider to override any global ~/.codex/config.toml settings.
|
|
31
|
+
// The SDK's `baseUrl` option maps to `openai_base_url` which gets overridden by
|
|
32
|
+
// model_provider; using `config` directly avoids this precedence issue.
|
|
33
|
+
options.config = {
|
|
34
|
+
model_provider: "tiangong-wiki-agent",
|
|
35
|
+
model_providers: {
|
|
36
|
+
"tiangong-wiki-agent": {
|
|
37
|
+
name: "tiangong-wiki-agent",
|
|
38
|
+
base_url: baseUrl,
|
|
39
|
+
wire_api: "responses",
|
|
40
|
+
experimental_bearer_token: apiKey || "",
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
return new Codex(options);
|
|
46
|
+
}
|
|
47
|
+
function persistWorkflowThreadId(queueItemPath, threadId) {
|
|
48
|
+
try {
|
|
49
|
+
const raw = readTextFileSync(queueItemPath);
|
|
50
|
+
const parsed = raw.trim() ? JSON.parse(raw) : {};
|
|
51
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
52
|
+
throw new Error("queue-item.json must contain an object");
|
|
53
|
+
}
|
|
54
|
+
writeTextFileSync(queueItemPath, `${JSON.stringify({ ...parsed, threadId }, null, 2)}\n`);
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
throw new AppError("Failed to persist workflow thread id into queue-item.json", "runtime", {
|
|
58
|
+
queueItemPath,
|
|
59
|
+
threadId,
|
|
60
|
+
cause: error instanceof Error ? error.message : String(error),
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async function runThread(thread, input) {
|
|
65
|
+
let activeThreadId = thread.id ?? null;
|
|
66
|
+
let emittedThreadId = null;
|
|
67
|
+
const emitThreadStarted = (threadId) => {
|
|
68
|
+
if (threadId === emittedThreadId) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
emittedThreadId = threadId;
|
|
72
|
+
input.onThreadStarted?.(threadId);
|
|
73
|
+
};
|
|
74
|
+
if (activeThreadId) {
|
|
75
|
+
persistWorkflowThreadId(input.queueItemPath, activeThreadId);
|
|
76
|
+
emitThreadStarted(activeThreadId);
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
const streamed = await thread.runStreamed(input.promptText, input.signal ? { signal: input.signal } : undefined);
|
|
80
|
+
for await (const event of streamed.events) {
|
|
81
|
+
if (event.type === "thread.started") {
|
|
82
|
+
activeThreadId = event.thread_id;
|
|
83
|
+
persistWorkflowThreadId(input.queueItemPath, activeThreadId);
|
|
84
|
+
emitThreadStarted(activeThreadId);
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (event.type === "turn.failed") {
|
|
88
|
+
throw new AppError("Codex workflow turn failed", "runtime", {
|
|
89
|
+
cause: event.error.message,
|
|
90
|
+
threadId: activeThreadId,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
if (event.type === "error") {
|
|
94
|
+
throw new AppError("Codex workflow stream failed", "runtime", {
|
|
95
|
+
cause: event.message,
|
|
96
|
+
threadId: activeThreadId,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
if (error instanceof AppError) {
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
106
|
+
throw new AppError("Codex workflow turn failed", "runtime", {
|
|
107
|
+
cause: message,
|
|
108
|
+
threadId: activeThreadId,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
if (!activeThreadId && thread.id) {
|
|
112
|
+
activeThreadId = thread.id;
|
|
113
|
+
}
|
|
114
|
+
if (!activeThreadId) {
|
|
115
|
+
throw new AppError("Codex workflow did not provide a thread id", "runtime");
|
|
116
|
+
}
|
|
117
|
+
persistWorkflowThreadId(input.queueItemPath, activeThreadId);
|
|
118
|
+
return activeThreadId;
|
|
119
|
+
}
|
|
120
|
+
export class CodexSdkWorkflowRunner {
|
|
121
|
+
// The SDK can only continue a thread by sending a new input, so queue retries
|
|
122
|
+
// must not automatically resume real workflow threads inline.
|
|
123
|
+
async startWorkflow(input) {
|
|
124
|
+
const codex = createCodexClient(input);
|
|
125
|
+
const thread = codex.startThread({
|
|
126
|
+
model: input.model ?? undefined,
|
|
127
|
+
modelReasoningEffort: "low",
|
|
128
|
+
workingDirectory: input.workspaceRoot,
|
|
129
|
+
skipGitRepoCheck: true,
|
|
130
|
+
sandboxMode: "workspace-write",
|
|
131
|
+
networkAccessEnabled: true,
|
|
132
|
+
approvalPolicy: "never",
|
|
133
|
+
webSearchMode: "disabled",
|
|
134
|
+
additionalDirectories: [input.packageRoot, input.skillArtifactsPath],
|
|
135
|
+
});
|
|
136
|
+
const threadId = await runThread(thread, input);
|
|
137
|
+
return { threadId, mode: "start" };
|
|
138
|
+
}
|
|
139
|
+
async resumeWorkflow(threadId, input) {
|
|
140
|
+
const codex = createCodexClient(input);
|
|
141
|
+
const thread = codex.resumeThread(threadId, {
|
|
142
|
+
model: input.model ?? undefined,
|
|
143
|
+
modelReasoningEffort: "low",
|
|
144
|
+
workingDirectory: input.workspaceRoot,
|
|
145
|
+
skipGitRepoCheck: true,
|
|
146
|
+
sandboxMode: "workspace-write",
|
|
147
|
+
networkAccessEnabled: true,
|
|
148
|
+
approvalPolicy: "never",
|
|
149
|
+
webSearchMode: "disabled",
|
|
150
|
+
additionalDirectories: [input.packageRoot, input.skillArtifactsPath],
|
|
151
|
+
});
|
|
152
|
+
const resumedThreadId = await runThread(thread, input);
|
|
153
|
+
return { threadId: resumedThreadId, mode: "resume" };
|
|
154
|
+
}
|
|
155
|
+
async collectResult(handle, input) {
|
|
156
|
+
const manifest = readWorkflowResult(input.resultPath);
|
|
157
|
+
if (manifest.threadId !== handle.threadId) {
|
|
158
|
+
throw new AppError(`Workflow result threadId mismatch: expected ${handle.threadId}, got ${manifest.threadId}`, "runtime");
|
|
159
|
+
}
|
|
160
|
+
return manifest;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
export class FakeCodexWorkflowRunner {
|
|
164
|
+
handler;
|
|
165
|
+
calls = [];
|
|
166
|
+
counter = 0;
|
|
167
|
+
constructor(handler) {
|
|
168
|
+
this.handler = handler;
|
|
169
|
+
}
|
|
170
|
+
async startWorkflow(input) {
|
|
171
|
+
const threadId = `fake-thread-${++this.counter}`;
|
|
172
|
+
input.onThreadStarted?.(threadId);
|
|
173
|
+
this.calls.push({ mode: "start", queueItemId: input.queueItemId, threadId });
|
|
174
|
+
const manifest = await this.handler({
|
|
175
|
+
mode: "start",
|
|
176
|
+
queueItemId: input.queueItemId,
|
|
177
|
+
threadId,
|
|
178
|
+
input,
|
|
179
|
+
});
|
|
180
|
+
writeTextFileSync(input.resultPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
181
|
+
return { threadId, mode: "start" };
|
|
182
|
+
}
|
|
183
|
+
async resumeWorkflow(threadId, input) {
|
|
184
|
+
input.onThreadStarted?.(threadId);
|
|
185
|
+
this.calls.push({ mode: "resume", queueItemId: input.queueItemId, threadId });
|
|
186
|
+
const manifest = await this.handler({
|
|
187
|
+
mode: "resume",
|
|
188
|
+
queueItemId: input.queueItemId,
|
|
189
|
+
threadId,
|
|
190
|
+
input,
|
|
191
|
+
});
|
|
192
|
+
writeTextFileSync(input.resultPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
193
|
+
return { threadId, mode: "resume" };
|
|
194
|
+
}
|
|
195
|
+
async collectResult(_handle, input) {
|
|
196
|
+
return readWorkflowResult(input.resultPath);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
async function delay(ms) {
|
|
200
|
+
if (ms <= 0) {
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
204
|
+
}
|
|
205
|
+
function createSkipOnlyTestWorkflowRunner(options) {
|
|
206
|
+
const delayMs = Math.max(0, options.delayMs ?? 0);
|
|
207
|
+
return new FakeCodexWorkflowRunner(async ({ threadId }) => {
|
|
208
|
+
await delay(delayMs);
|
|
209
|
+
return {
|
|
210
|
+
status: "skipped",
|
|
211
|
+
decision: "skip",
|
|
212
|
+
reason: `Skipped by WIKI_TEST_FAKE_WORKFLOW_MODE=${options.mode}.`,
|
|
213
|
+
threadId,
|
|
214
|
+
skillsUsed: ["tiangong-wiki-skill"],
|
|
215
|
+
createdPageIds: [],
|
|
216
|
+
updatedPageIds: [],
|
|
217
|
+
appliedTypeNames: [],
|
|
218
|
+
proposedTypes: [],
|
|
219
|
+
actions: [],
|
|
220
|
+
lint: [],
|
|
221
|
+
};
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
export function createDefaultWorkflowRunner(env = process.env) {
|
|
225
|
+
if (env.WIKI_TEST_FAKE_WORKFLOW_MODE === "skip") {
|
|
226
|
+
return createSkipOnlyTestWorkflowRunner({ mode: "skip" });
|
|
227
|
+
}
|
|
228
|
+
if (env.WIKI_TEST_FAKE_WORKFLOW_MODE === "delay-skip") {
|
|
229
|
+
const delayMs = Number.parseInt(env.WIKI_TEST_FAKE_WORKFLOW_DELAY_MS ?? "0", 10) || 0;
|
|
230
|
+
return createSkipOnlyTestWorkflowRunner({ delayMs, mode: "delay-skip" });
|
|
231
|
+
}
|
|
232
|
+
return new CodexSdkWorkflowRunner();
|
|
233
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { AppError, assertCondition } from "../utils/errors.js";
|
|
3
|
+
import { camelToSnake } from "../utils/case.js";
|
|
4
|
+
import { pathExistsSync, readTextFileSync, sha256Text } from "../utils/fs.js";
|
|
5
|
+
const ALLOWED_SQLITE_TYPES = new Set(["text", "integer", "real", "numeric", "blob"]);
|
|
6
|
+
export const DEFAULT_VAULT_FILE_TYPES = ["md", "txt", "pdf", "docx", "pptx", "xlsx", "csv", "json", "yaml", "yml"];
|
|
7
|
+
function ensureObject(value, label) {
|
|
8
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
9
|
+
throw new AppError(`${label} must be an object`, "config");
|
|
10
|
+
}
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
function ensureStringArray(value, label) {
|
|
14
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
15
|
+
throw new AppError(`${label} must be a string array`, "config");
|
|
16
|
+
}
|
|
17
|
+
return value;
|
|
18
|
+
}
|
|
19
|
+
function ensureVaultFileTypes(value, label) {
|
|
20
|
+
const normalized = ensureStringArray(value, label).map((item, index) => {
|
|
21
|
+
const fileType = item.trim().replace(/^\./, "").toLowerCase();
|
|
22
|
+
if (!fileType) {
|
|
23
|
+
throw new AppError(`${label}[${index}] must not be empty`, "config");
|
|
24
|
+
}
|
|
25
|
+
return fileType;
|
|
26
|
+
});
|
|
27
|
+
return [...new Set(normalized)];
|
|
28
|
+
}
|
|
29
|
+
function ensureColumnMap(value, label) {
|
|
30
|
+
const objectValue = ensureObject(value, label);
|
|
31
|
+
const entries = Object.entries(objectValue).map(([key, rawType]) => {
|
|
32
|
+
if (typeof rawType !== "string" || !ALLOWED_SQLITE_TYPES.has(rawType)) {
|
|
33
|
+
throw new AppError(`${label}.${key} must be a valid SQLite column type`, "config");
|
|
34
|
+
}
|
|
35
|
+
return [key, rawType];
|
|
36
|
+
});
|
|
37
|
+
return Object.fromEntries(entries);
|
|
38
|
+
}
|
|
39
|
+
function ensureEdgeRule(value, label) {
|
|
40
|
+
const objectValue = ensureObject(value, label);
|
|
41
|
+
assertCondition(typeof objectValue.edgeType === "string" && objectValue.edgeType, `${label}.edgeType is required`, "config");
|
|
42
|
+
assertCondition(objectValue.resolve === "nodeId" || objectValue.resolve === "path", `${label}.resolve must be "nodeId" or "path"`, "config");
|
|
43
|
+
if (objectValue.match !== undefined && typeof objectValue.match !== "string") {
|
|
44
|
+
throw new AppError(`${label}.match must be a string when provided`, "config");
|
|
45
|
+
}
|
|
46
|
+
return {
|
|
47
|
+
edgeType: objectValue.edgeType,
|
|
48
|
+
resolve: objectValue.resolve,
|
|
49
|
+
...(objectValue.match ? { match: objectValue.match } : {}),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
function ensureEdgeMap(value, label) {
|
|
53
|
+
const objectValue = ensureObject(value, label);
|
|
54
|
+
return Object.fromEntries(Object.entries(objectValue).map(([field, rule]) => [field, ensureEdgeRule(rule, `${label}.${field}`)]));
|
|
55
|
+
}
|
|
56
|
+
function ensureTemplateConfig(value, label) {
|
|
57
|
+
const objectValue = ensureObject(value, label);
|
|
58
|
+
assertCondition(typeof objectValue.file === "string" && objectValue.file, `${label}.file is required`, "config");
|
|
59
|
+
return {
|
|
60
|
+
file: objectValue.file,
|
|
61
|
+
columns: ensureColumnMap(objectValue.columns ?? {}, `${label}.columns`),
|
|
62
|
+
edges: ensureEdgeMap(objectValue.edges ?? {}, `${label}.edges`),
|
|
63
|
+
summaryFields: ensureStringArray(objectValue.summaryFields ?? [], `${label}.summaryFields`),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export function loadConfig(configPath) {
|
|
67
|
+
if (!pathExistsSync(configPath)) {
|
|
68
|
+
throw new AppError(`Config file not found: ${configPath}`, "config");
|
|
69
|
+
}
|
|
70
|
+
let parsedJson;
|
|
71
|
+
const rawContent = readTextFileSync(configPath);
|
|
72
|
+
try {
|
|
73
|
+
parsedJson = JSON.parse(rawContent);
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
throw new AppError(`Failed to parse config JSON: ${configPath}`, "config", {
|
|
77
|
+
cause: error instanceof Error ? error.message : String(error),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
const raw = ensureObject(parsedJson, "wiki.config.json");
|
|
81
|
+
assertCondition(Number.isInteger(raw.schemaVersion), "schemaVersion must be an integer", "config");
|
|
82
|
+
const baseConfig = {
|
|
83
|
+
schemaVersion: Number(raw.schemaVersion),
|
|
84
|
+
customColumns: ensureColumnMap(raw.customColumns ?? {}, "customColumns"),
|
|
85
|
+
defaultSummaryFields: ensureStringArray(raw.defaultSummaryFields ?? [], "defaultSummaryFields"),
|
|
86
|
+
vaultFileTypes: ensureVaultFileTypes(raw.vaultFileTypes ?? DEFAULT_VAULT_FILE_TYPES, "vaultFileTypes"),
|
|
87
|
+
commonEdges: ensureEdgeMap(raw.commonEdges ?? {}, "commonEdges"),
|
|
88
|
+
templates: Object.fromEntries(Object.entries(ensureObject(raw.templates, "templates")).map(([pageType, template]) => [
|
|
89
|
+
pageType,
|
|
90
|
+
ensureTemplateConfig(template, `templates.${pageType}`),
|
|
91
|
+
])),
|
|
92
|
+
};
|
|
93
|
+
assertCondition(Object.keys(baseConfig.templates).length > 0, "templates must not be empty", "config");
|
|
94
|
+
const allColumnDefinitions = {};
|
|
95
|
+
for (const [field, type] of Object.entries(baseConfig.customColumns)) {
|
|
96
|
+
allColumnDefinitions[camelToSnake(field)] = type;
|
|
97
|
+
}
|
|
98
|
+
for (const [pageType, template] of Object.entries(baseConfig.templates)) {
|
|
99
|
+
assertCondition(template.file.endsWith(".md"), `templates.${pageType}.file must point to a .md file`, "config");
|
|
100
|
+
for (const [field, type] of Object.entries(template.columns)) {
|
|
101
|
+
const columnName = camelToSnake(field);
|
|
102
|
+
if (allColumnDefinitions[columnName] && allColumnDefinitions[columnName] !== type) {
|
|
103
|
+
throw new AppError(`Column ${field} is declared with conflicting types`, "config");
|
|
104
|
+
}
|
|
105
|
+
allColumnDefinitions[columnName] = type;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
...baseConfig,
|
|
110
|
+
configPath: path.resolve(configPath),
|
|
111
|
+
configVersion: sha256Text(rawContent),
|
|
112
|
+
allColumnDefinitions,
|
|
113
|
+
allColumnNames: Object.keys(allColumnDefinitions).sort(),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
export function getTemplate(config, pageType) {
|
|
117
|
+
const template = config.templates[pageType];
|
|
118
|
+
if (!template) {
|
|
119
|
+
throw new AppError(`Unknown pageType: ${pageType}`, "config");
|
|
120
|
+
}
|
|
121
|
+
return template;
|
|
122
|
+
}
|
|
123
|
+
export function resolveTemplateFilePath(config, wikiRoot, pageType) {
|
|
124
|
+
const template = getTemplate(config, pageType);
|
|
125
|
+
return path.resolve(wikiRoot, template.file);
|
|
126
|
+
}
|
package/dist/core/db.js
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import * as sqliteVec from "sqlite-vec";
|
|
3
|
+
import { AppError } from "../utils/errors.js";
|
|
4
|
+
import { segmentForFts } from "../utils/segmenter.js";
|
|
5
|
+
export const SCHEMA_VERSION = "1";
|
|
6
|
+
const FTS_INDEX_VERSION = "2";
|
|
7
|
+
function tableExists(db, tableName) {
|
|
8
|
+
const row = db
|
|
9
|
+
.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?")
|
|
10
|
+
.get(tableName);
|
|
11
|
+
return Boolean(row?.name);
|
|
12
|
+
}
|
|
13
|
+
function getTableSql(db, tableName) {
|
|
14
|
+
const row = db
|
|
15
|
+
.prepare("SELECT sql FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?")
|
|
16
|
+
.get(tableName);
|
|
17
|
+
return row?.sql ?? null;
|
|
18
|
+
}
|
|
19
|
+
function getExistingTableColumns(db, tableName) {
|
|
20
|
+
const rows = db.prepare(`PRAGMA table_info(${tableName})`).all();
|
|
21
|
+
return new Set(rows.map((row) => row.name));
|
|
22
|
+
}
|
|
23
|
+
function ensureTableColumns(db, tableName, definitions) {
|
|
24
|
+
const existingColumns = getExistingTableColumns(db, tableName);
|
|
25
|
+
for (const [columnName, definition] of Object.entries(definitions)) {
|
|
26
|
+
if (!existingColumns.has(columnName)) {
|
|
27
|
+
db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${definition}`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function createFtsTable(db) {
|
|
32
|
+
db.exec(`
|
|
33
|
+
CREATE VIRTUAL TABLE pages_fts USING fts5(
|
|
34
|
+
title,
|
|
35
|
+
tags,
|
|
36
|
+
summary_text
|
|
37
|
+
);
|
|
38
|
+
`);
|
|
39
|
+
}
|
|
40
|
+
function normalizeTagsForFts(rawTags) {
|
|
41
|
+
if (!rawTags) {
|
|
42
|
+
return "";
|
|
43
|
+
}
|
|
44
|
+
try {
|
|
45
|
+
const parsed = JSON.parse(rawTags);
|
|
46
|
+
if (Array.isArray(parsed)) {
|
|
47
|
+
return parsed
|
|
48
|
+
.map((value) => String(value).trim())
|
|
49
|
+
.filter(Boolean)
|
|
50
|
+
.join(" ");
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// Fall back to the stored value if legacy data is not valid JSON.
|
|
55
|
+
}
|
|
56
|
+
return rawTags;
|
|
57
|
+
}
|
|
58
|
+
function buildFtsRow(row) {
|
|
59
|
+
return {
|
|
60
|
+
rowid: row.rowid,
|
|
61
|
+
title: segmentForFts(row.title),
|
|
62
|
+
tags: segmentForFts(normalizeTagsForFts(row.tags)),
|
|
63
|
+
summary_text: segmentForFts(row.summaryText ?? ""),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function isLegacyExternalContentFts(db) {
|
|
67
|
+
const sql = getTableSql(db, "pages_fts");
|
|
68
|
+
return typeof sql === "string" && /content\s*=\s*'pages'/i.test(sql);
|
|
69
|
+
}
|
|
70
|
+
function ensureBaseTables(db, embeddingDimensions) {
|
|
71
|
+
db.exec(`
|
|
72
|
+
CREATE TABLE IF NOT EXISTS pages (
|
|
73
|
+
id TEXT PRIMARY KEY,
|
|
74
|
+
node_id TEXT UNIQUE,
|
|
75
|
+
title TEXT NOT NULL,
|
|
76
|
+
page_type TEXT NOT NULL,
|
|
77
|
+
status TEXT DEFAULT 'draft',
|
|
78
|
+
visibility TEXT DEFAULT 'private',
|
|
79
|
+
tags TEXT,
|
|
80
|
+
extra TEXT,
|
|
81
|
+
file_path TEXT NOT NULL,
|
|
82
|
+
content_hash TEXT,
|
|
83
|
+
summary_text TEXT,
|
|
84
|
+
embedding_status TEXT DEFAULT 'pending',
|
|
85
|
+
file_mtime REAL,
|
|
86
|
+
created_at TEXT,
|
|
87
|
+
updated_at TEXT,
|
|
88
|
+
indexed_at TEXT
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
CREATE INDEX IF NOT EXISTS idx_pages_type ON pages(page_type);
|
|
92
|
+
CREATE INDEX IF NOT EXISTS idx_pages_status ON pages(status);
|
|
93
|
+
CREATE INDEX IF NOT EXISTS idx_pages_node ON pages(node_id);
|
|
94
|
+
|
|
95
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
96
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
97
|
+
source TEXT NOT NULL,
|
|
98
|
+
target TEXT NOT NULL,
|
|
99
|
+
edge_type TEXT NOT NULL,
|
|
100
|
+
source_page TEXT,
|
|
101
|
+
metadata TEXT,
|
|
102
|
+
UNIQUE(source, target, edge_type, source_page)
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source);
|
|
106
|
+
CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target);
|
|
107
|
+
CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(edge_type);
|
|
108
|
+
|
|
109
|
+
CREATE TABLE IF NOT EXISTS vault_files (
|
|
110
|
+
id TEXT PRIMARY KEY,
|
|
111
|
+
file_name TEXT NOT NULL,
|
|
112
|
+
file_ext TEXT,
|
|
113
|
+
source_type TEXT,
|
|
114
|
+
file_size INTEGER,
|
|
115
|
+
file_path TEXT NOT NULL,
|
|
116
|
+
content_hash TEXT,
|
|
117
|
+
file_mtime REAL,
|
|
118
|
+
indexed_at TEXT
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
CREATE TABLE IF NOT EXISTS vault_changelog (
|
|
122
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
123
|
+
file_id TEXT NOT NULL,
|
|
124
|
+
action TEXT NOT NULL,
|
|
125
|
+
detected_at TEXT NOT NULL,
|
|
126
|
+
sync_id TEXT NOT NULL
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
CREATE INDEX IF NOT EXISTS idx_vchangelog_sync ON vault_changelog(sync_id);
|
|
130
|
+
CREATE INDEX IF NOT EXISTS idx_vchangelog_time ON vault_changelog(detected_at);
|
|
131
|
+
|
|
132
|
+
CREATE TABLE IF NOT EXISTS vault_processing_queue (
|
|
133
|
+
file_id TEXT PRIMARY KEY,
|
|
134
|
+
status TEXT DEFAULT 'pending',
|
|
135
|
+
priority INTEGER DEFAULT 0,
|
|
136
|
+
queued_at TEXT NOT NULL,
|
|
137
|
+
claimed_at TEXT,
|
|
138
|
+
started_at TEXT,
|
|
139
|
+
processed_at TEXT,
|
|
140
|
+
result_page_id TEXT,
|
|
141
|
+
error_message TEXT,
|
|
142
|
+
attempts INTEGER DEFAULT 0
|
|
143
|
+
);
|
|
144
|
+
|
|
145
|
+
CREATE INDEX IF NOT EXISTS idx_vpq_status ON vault_processing_queue(status);
|
|
146
|
+
CREATE INDEX IF NOT EXISTS idx_vpq_priority ON vault_processing_queue(priority DESC, queued_at ASC);
|
|
147
|
+
|
|
148
|
+
CREATE TABLE IF NOT EXISTS sync_meta (
|
|
149
|
+
key TEXT PRIMARY KEY,
|
|
150
|
+
value TEXT
|
|
151
|
+
);
|
|
152
|
+
`);
|
|
153
|
+
ensureTableColumns(db, "vault_processing_queue", {
|
|
154
|
+
claimed_at: "TEXT",
|
|
155
|
+
started_at: "TEXT",
|
|
156
|
+
thread_id: "TEXT",
|
|
157
|
+
workflow_version: "TEXT",
|
|
158
|
+
decision: "TEXT",
|
|
159
|
+
result_manifest_path: "TEXT",
|
|
160
|
+
last_error_at: "TEXT",
|
|
161
|
+
retry_after: "TEXT",
|
|
162
|
+
created_page_ids: "TEXT",
|
|
163
|
+
updated_page_ids: "TEXT",
|
|
164
|
+
applied_type_names: "TEXT",
|
|
165
|
+
proposed_type_names: "TEXT",
|
|
166
|
+
skills_used: "TEXT",
|
|
167
|
+
});
|
|
168
|
+
if (!tableExists(db, "vec_pages")) {
|
|
169
|
+
db.exec(`
|
|
170
|
+
CREATE VIRTUAL TABLE vec_pages USING vec0(
|
|
171
|
+
page_rowid INTEGER PRIMARY KEY,
|
|
172
|
+
page_id TEXT,
|
|
173
|
+
embedding float[${embeddingDimensions}]
|
|
174
|
+
);
|
|
175
|
+
`);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function getExistingColumns(db) {
|
|
179
|
+
return getExistingTableColumns(db, "pages");
|
|
180
|
+
}
|
|
181
|
+
function ensureDynamicColumns(db, config) {
|
|
182
|
+
const existingColumns = getExistingColumns(db);
|
|
183
|
+
for (const [columnName, columnType] of Object.entries(config.allColumnDefinitions)) {
|
|
184
|
+
if (!existingColumns.has(columnName)) {
|
|
185
|
+
db.exec(`ALTER TABLE pages ADD COLUMN ${columnName} ${columnType.toUpperCase()}`);
|
|
186
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_${columnName} ON pages(${columnName})`);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
export function getMeta(db, key) {
|
|
191
|
+
const row = db.prepare("SELECT value FROM sync_meta WHERE key = ?").get(key);
|
|
192
|
+
return row?.value ?? null;
|
|
193
|
+
}
|
|
194
|
+
export function setMeta(db, key, value) {
|
|
195
|
+
if (value === null) {
|
|
196
|
+
db.prepare("DELETE FROM sync_meta WHERE key = ?").run(key);
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
db.prepare(`
|
|
200
|
+
INSERT INTO sync_meta(key, value)
|
|
201
|
+
VALUES(@key, @value)
|
|
202
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
203
|
+
`).run({ key, value });
|
|
204
|
+
}
|
|
205
|
+
export function setMetaValues(db, values) {
|
|
206
|
+
const transaction = db.transaction((payload) => {
|
|
207
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
208
|
+
setMeta(db, key, value);
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
transaction(values);
|
|
212
|
+
}
|
|
213
|
+
export function rebuildFts(db) {
|
|
214
|
+
if (!tableExists(db, "pages_fts")) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
const rows = db.prepare("SELECT rowid, title, tags, summary_text AS summaryText FROM pages ORDER BY rowid").all();
|
|
218
|
+
const clearStatement = db.prepare("DELETE FROM pages_fts");
|
|
219
|
+
const insertStatement = db.prepare("INSERT INTO pages_fts(rowid, title, tags, summary_text) VALUES (@rowid, @title, @tags, @summary_text)");
|
|
220
|
+
const transaction = db.transaction(() => {
|
|
221
|
+
clearStatement.run();
|
|
222
|
+
for (const row of rows) {
|
|
223
|
+
insertStatement.run(buildFtsRow(row));
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
transaction();
|
|
227
|
+
}
|
|
228
|
+
function ensureFtsTable(db) {
|
|
229
|
+
const hasTable = tableExists(db, "pages_fts");
|
|
230
|
+
const storedFtsIndexVersion = getMeta(db, "fts_index_version");
|
|
231
|
+
const needsRecreate = !hasTable || isLegacyExternalContentFts(db);
|
|
232
|
+
const needsRebuild = needsRecreate || storedFtsIndexVersion !== FTS_INDEX_VERSION;
|
|
233
|
+
if (needsRecreate && hasTable) {
|
|
234
|
+
db.exec("DROP TABLE pages_fts");
|
|
235
|
+
}
|
|
236
|
+
if (needsRecreate) {
|
|
237
|
+
createFtsTable(db);
|
|
238
|
+
}
|
|
239
|
+
if (needsRebuild) {
|
|
240
|
+
rebuildFts(db);
|
|
241
|
+
}
|
|
242
|
+
if (needsRebuild || storedFtsIndexVersion !== FTS_INDEX_VERSION) {
|
|
243
|
+
setMeta(db, "fts_index_version", FTS_INDEX_VERSION);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
export function resetVectorTable(db, embeddingDimensions) {
|
|
247
|
+
db.exec("DROP TABLE IF EXISTS vec_pages");
|
|
248
|
+
db.exec(`
|
|
249
|
+
CREATE VIRTUAL TABLE vec_pages USING vec0(
|
|
250
|
+
page_rowid INTEGER PRIMARY KEY,
|
|
251
|
+
page_id TEXT,
|
|
252
|
+
embedding float[${embeddingDimensions}]
|
|
253
|
+
);
|
|
254
|
+
`);
|
|
255
|
+
}
|
|
256
|
+
export function clearAllIndexedData(db) {
|
|
257
|
+
db.exec(`
|
|
258
|
+
DELETE FROM edges;
|
|
259
|
+
DELETE FROM pages;
|
|
260
|
+
DELETE FROM vault_files;
|
|
261
|
+
DELETE FROM vault_changelog;
|
|
262
|
+
DELETE FROM vault_processing_queue;
|
|
263
|
+
`);
|
|
264
|
+
if (tableExists(db, "vec_pages")) {
|
|
265
|
+
db.exec("DELETE FROM vec_pages");
|
|
266
|
+
}
|
|
267
|
+
if (tableExists(db, "pages_fts")) {
|
|
268
|
+
rebuildFts(db);
|
|
269
|
+
}
|
|
270
|
+
db.prepare("DELETE FROM sync_meta WHERE key IN ('last_sync_at', 'last_sync_id', 'last_full_rebuild_at', 'embedding_profile')").run();
|
|
271
|
+
}
|
|
272
|
+
export function openDb(dbPath, config, embeddingDimensions) {
|
|
273
|
+
const db = new Database(dbPath);
|
|
274
|
+
db.pragma("journal_mode = WAL");
|
|
275
|
+
db.pragma("foreign_keys = ON");
|
|
276
|
+
sqliteVec.load(db);
|
|
277
|
+
ensureBaseTables(db, embeddingDimensions);
|
|
278
|
+
ensureFtsTable(db);
|
|
279
|
+
const storedSchemaVersion = getMeta(db, "schema_version");
|
|
280
|
+
if (storedSchemaVersion && storedSchemaVersion !== SCHEMA_VERSION) {
|
|
281
|
+
db.close();
|
|
282
|
+
throw new AppError(`Schema version mismatch: expected ${SCHEMA_VERSION}, found ${storedSchemaVersion}. Run tiangong-wiki init --force.`, "config");
|
|
283
|
+
}
|
|
284
|
+
ensureDynamicColumns(db, config);
|
|
285
|
+
const storedConfigVersion = getMeta(db, "config_version");
|
|
286
|
+
const configChanged = Boolean(storedConfigVersion && storedConfigVersion !== config.configVersion);
|
|
287
|
+
setMetaValues(db, {
|
|
288
|
+
schema_version: SCHEMA_VERSION,
|
|
289
|
+
...(storedConfigVersion === null ? { config_version: config.configVersion } : {}),
|
|
290
|
+
});
|
|
291
|
+
return { db, configChanged };
|
|
292
|
+
}
|