forge-openclaw-plugin 0.2.19 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -2
- package/dist/assets/board-_C6oMy5w.js +6 -0
- package/dist/assets/{board-8L3uX7_O.js.map → board-_C6oMy5w.js.map} +1 -1
- package/dist/assets/index-B4A6TooJ.js +63 -0
- package/dist/assets/index-B4A6TooJ.js.map +1 -0
- package/dist/assets/index-D6Xs_2mo.css +1 -0
- package/dist/assets/{motion-1GAqqi8M.js → motion-D4sZgCHd.js} +2 -2
- package/dist/assets/{motion-1GAqqi8M.js.map → motion-D4sZgCHd.js.map} +1 -1
- package/dist/assets/{table-DBGlgRjk.js → table-BWzTaky1.js} +2 -2
- package/dist/assets/{table-DBGlgRjk.js.map → table-BWzTaky1.js.map} +1 -1
- package/dist/assets/{ui-iTluWjC4.js → ui-BzK4azQb.js} +7 -7
- package/dist/assets/{ui-iTluWjC4.js.map → ui-BzK4azQb.js.map} +1 -1
- package/dist/assets/vendor-DT3pnAKJ.css +1 -0
- package/dist/assets/vendor-De38P6YR.js +729 -0
- package/dist/assets/vendor-De38P6YR.js.map +1 -0
- package/dist/assets/viz-C6hfyqzu.js +34 -0
- package/dist/assets/viz-C6hfyqzu.js.map +1 -0
- package/dist/index.html +9 -9
- package/dist/openclaw/parity.d.ts +1 -1
- package/dist/openclaw/parity.js +29 -2
- package/dist/openclaw/routes.js +207 -24
- package/dist/openclaw/tools.js +324 -35
- package/dist/server/app.js +2080 -92
- package/dist/server/db.js +3 -0
- package/dist/server/health.js +1284 -0
- package/dist/server/managers/platform/background-job-manager.js +138 -2
- package/dist/server/managers/platform/llm-manager.js +126 -0
- package/dist/server/managers/platform/openai-responses-provider.js +773 -0
- package/dist/server/managers/runtime.js +6 -1
- package/dist/server/openapi.js +718 -0
- package/dist/server/preferences-seeds.js +409 -0
- package/dist/server/preferences-types.js +368 -0
- package/dist/server/psyche-types.js +42 -18
- package/dist/server/repositories/activity-events.js +53 -4
- package/dist/server/repositories/calendar.js +89 -15
- package/dist/server/repositories/collaboration.js +8 -3
- package/dist/server/repositories/diagnostic-logs.js +243 -0
- package/dist/server/repositories/entity-ownership.js +92 -0
- package/dist/server/repositories/goals.js +7 -2
- package/dist/server/repositories/habits.js +122 -16
- package/dist/server/repositories/notes.js +119 -41
- package/dist/server/repositories/preferences.js +1765 -0
- package/dist/server/repositories/projects.js +18 -7
- package/dist/server/repositories/psyche.js +84 -27
- package/dist/server/repositories/rewards.js +112 -4
- package/dist/server/repositories/strategies.js +450 -0
- package/dist/server/repositories/tags.js +11 -6
- package/dist/server/repositories/task-runs.js +10 -2
- package/dist/server/repositories/tasks.js +99 -17
- package/dist/server/repositories/users.js +417 -0
- package/dist/server/repositories/wiki-memory.js +3366 -0
- package/dist/server/services/context.js +20 -18
- package/dist/server/services/dashboard.js +29 -6
- package/dist/server/services/entity-crud.js +21 -3
- package/dist/server/services/insights.js +9 -7
- package/dist/server/services/projects.js +2 -1
- package/dist/server/services/psyche.js +10 -9
- package/dist/server/types.js +594 -30
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/server/migrations/015_multi_user_and_strategies.sql +244 -0
- package/server/migrations/016_health_companion.sql +158 -0
- package/server/migrations/016_strategy_contracts_and_user_graph.sql +22 -0
- package/server/migrations/017_preferences.sql +131 -0
- package/server/migrations/018_preference_catalogs.sql +31 -0
- package/server/migrations/019_wiki_memory.sql +255 -0
- package/server/migrations/020_wiki_page_hierarchy.sql +11 -0
- package/server/migrations/021_hide_evidence_from_wiki_index.sql +3 -0
- package/server/migrations/022_wiki_ingest_background.sql +85 -0
- package/server/migrations/023_diagnostic_logs.sql +28 -0
- package/skills/forge-openclaw/SKILL.md +126 -34
- package/skills/forge-openclaw/entity_conversation_playbooks.md +337 -0
- package/skills/forge-openclaw/psyche_entity_playbooks.md +404 -0
- package/dist/assets/board-8L3uX7_O.js +0 -6
- package/dist/assets/index-Cj1IBH_w.js +0 -36
- package/dist/assets/index-Cj1IBH_w.js.map +0 -1
- package/dist/assets/index-DQT6EbuS.css +0 -1
- package/dist/assets/vendor-BvM2F9Dp.js +0 -503
- package/dist/assets/vendor-BvM2F9Dp.js.map +0 -1
- package/dist/assets/vendor-CRS-psbw.css +0 -1
- package/dist/assets/viz-CNeunkfu.js +0 -34
- package/dist/assets/viz-CNeunkfu.js.map +0 -1
|
@@ -0,0 +1,3366 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import AdmZip from "adm-zip";
|
|
3
|
+
import { existsSync, mkdirSync, readdirSync, unlinkSync, rmSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
import { resolveDataDir, getDatabase } from "../db.js";
|
|
8
|
+
import { decorateOwnedEntity } from "./entity-ownership.js";
|
|
9
|
+
import { createNoteLinkSchema, crudEntityTypeSchema, noteKindSchema, noteSchema as persistedNoteSchema, wikiSearchModeSchema, wikiSpaceVisibilitySchema } from "../types.js";
|
|
10
|
+
import { deleteEncryptedSecret, readEncryptedSecret, storeEncryptedSecret } from "./calendar.js";
|
|
11
|
+
import { recordDiagnosticLog } from "./diagnostic-logs.js";
|
|
12
|
+
const wikiSpaceSchema = z.object({
|
|
13
|
+
id: z.string(),
|
|
14
|
+
slug: z.string(),
|
|
15
|
+
label: z.string(),
|
|
16
|
+
description: z.string(),
|
|
17
|
+
ownerUserId: z.string().nullable(),
|
|
18
|
+
visibility: wikiSpaceVisibilitySchema,
|
|
19
|
+
createdAt: z.string(),
|
|
20
|
+
updatedAt: z.string()
|
|
21
|
+
});
|
|
22
|
+
const wikiLinkEdgeSchema = z.object({
|
|
23
|
+
sourceNoteId: z.string(),
|
|
24
|
+
targetType: z.enum(["page", "entity", "unresolved"]),
|
|
25
|
+
targetNoteId: z.string().nullable(),
|
|
26
|
+
targetEntityType: crudEntityTypeSchema.nullable(),
|
|
27
|
+
targetEntityId: z.string().nullable(),
|
|
28
|
+
label: z.string(),
|
|
29
|
+
rawTarget: z.string(),
|
|
30
|
+
isEmbed: z.boolean(),
|
|
31
|
+
createdAt: z.string(),
|
|
32
|
+
updatedAt: z.string()
|
|
33
|
+
});
|
|
34
|
+
const wikiMediaAssetSchema = z.object({
|
|
35
|
+
id: z.string(),
|
|
36
|
+
spaceId: z.string(),
|
|
37
|
+
noteId: z.string().nullable(),
|
|
38
|
+
label: z.string(),
|
|
39
|
+
mimeType: z.string(),
|
|
40
|
+
fileName: z.string(),
|
|
41
|
+
filePath: z.string(),
|
|
42
|
+
sizeBytes: z.number().int().nonnegative(),
|
|
43
|
+
checksum: z.string(),
|
|
44
|
+
transcriptNoteId: z.string().nullable(),
|
|
45
|
+
metadata: z.record(z.string(), z.unknown()),
|
|
46
|
+
createdAt: z.string(),
|
|
47
|
+
updatedAt: z.string()
|
|
48
|
+
});
|
|
49
|
+
const wikiLlmProfileSchema = z.object({
|
|
50
|
+
id: z.string(),
|
|
51
|
+
label: z.string(),
|
|
52
|
+
provider: z.string(),
|
|
53
|
+
baseUrl: z.string(),
|
|
54
|
+
model: z.string(),
|
|
55
|
+
secretId: z.string().nullable(),
|
|
56
|
+
systemPrompt: z.string(),
|
|
57
|
+
enabled: z.boolean(),
|
|
58
|
+
metadata: z.record(z.string(), z.unknown()),
|
|
59
|
+
createdAt: z.string(),
|
|
60
|
+
updatedAt: z.string()
|
|
61
|
+
});
|
|
62
|
+
const wikiEmbeddingProfileSchema = z.object({
|
|
63
|
+
id: z.string(),
|
|
64
|
+
label: z.string(),
|
|
65
|
+
provider: z.string(),
|
|
66
|
+
baseUrl: z.string(),
|
|
67
|
+
model: z.string(),
|
|
68
|
+
secretId: z.string().nullable(),
|
|
69
|
+
dimensions: z.number().int().positive().nullable(),
|
|
70
|
+
chunkSize: z.number().int().positive(),
|
|
71
|
+
chunkOverlap: z.number().int().nonnegative(),
|
|
72
|
+
enabled: z.boolean(),
|
|
73
|
+
metadata: z.record(z.string(), z.unknown()),
|
|
74
|
+
createdAt: z.string(),
|
|
75
|
+
updatedAt: z.string()
|
|
76
|
+
});
|
|
77
|
+
const wikiSettingsPayloadSchema = z.object({
|
|
78
|
+
spaces: z.array(wikiSpaceSchema),
|
|
79
|
+
llmProfiles: z.array(wikiLlmProfileSchema),
|
|
80
|
+
embeddingProfiles: z.array(wikiEmbeddingProfileSchema)
|
|
81
|
+
});
|
|
82
|
+
const wikiPageTreeNodeSchema = z.lazy(() => z.object({
|
|
83
|
+
page: persistedNoteSchema,
|
|
84
|
+
children: z.array(wikiPageTreeNodeSchema)
|
|
85
|
+
}));
|
|
86
|
+
const wikiHealthPayloadSchema = z.object({
|
|
87
|
+
space: wikiSpaceSchema,
|
|
88
|
+
indexPath: z.string(),
|
|
89
|
+
rawDirectoryPath: z.string(),
|
|
90
|
+
pageCount: z.number().int().nonnegative(),
|
|
91
|
+
wikiPageCount: z.number().int().nonnegative(),
|
|
92
|
+
evidencePageCount: z.number().int().nonnegative(),
|
|
93
|
+
assetCount: z.number().int().nonnegative(),
|
|
94
|
+
rawSourceCount: z.number().int().nonnegative(),
|
|
95
|
+
unresolvedLinks: z.array(z.object({
|
|
96
|
+
sourceNoteId: z.string(),
|
|
97
|
+
sourceSlug: z.string(),
|
|
98
|
+
sourceTitle: z.string(),
|
|
99
|
+
rawTarget: z.string(),
|
|
100
|
+
updatedAt: z.string()
|
|
101
|
+
})),
|
|
102
|
+
orphanPages: z.array(z.object({
|
|
103
|
+
id: z.string(),
|
|
104
|
+
slug: z.string(),
|
|
105
|
+
title: z.string(),
|
|
106
|
+
kind: noteKindSchema,
|
|
107
|
+
updatedAt: z.string()
|
|
108
|
+
})),
|
|
109
|
+
missingSummaries: z.array(z.object({
|
|
110
|
+
id: z.string(),
|
|
111
|
+
slug: z.string(),
|
|
112
|
+
title: z.string(),
|
|
113
|
+
updatedAt: z.string()
|
|
114
|
+
})),
|
|
115
|
+
enabledEmbeddingProfiles: z.array(z.object({
|
|
116
|
+
id: z.string(),
|
|
117
|
+
label: z.string(),
|
|
118
|
+
model: z.string()
|
|
119
|
+
})),
|
|
120
|
+
enabledLlmProfiles: z.array(z.object({
|
|
121
|
+
id: z.string(),
|
|
122
|
+
label: z.string(),
|
|
123
|
+
model: z.string()
|
|
124
|
+
}))
|
|
125
|
+
});
|
|
126
|
+
const wikiIngestJobLogSchema = z.object({
|
|
127
|
+
id: z.string(),
|
|
128
|
+
level: z.string(),
|
|
129
|
+
message: z.string(),
|
|
130
|
+
metadata: z.record(z.string(), z.unknown()),
|
|
131
|
+
createdAt: z.string()
|
|
132
|
+
});
|
|
133
|
+
const wikiIngestJobAssetSchema = z.object({
|
|
134
|
+
id: z.string(),
|
|
135
|
+
status: z.string(),
|
|
136
|
+
sourceKind: z.string(),
|
|
137
|
+
sourceLocator: z.string(),
|
|
138
|
+
fileName: z.string(),
|
|
139
|
+
mimeType: z.string(),
|
|
140
|
+
filePath: z.string(),
|
|
141
|
+
sizeBytes: z.number().int().nonnegative(),
|
|
142
|
+
checksum: z.string(),
|
|
143
|
+
metadata: z.record(z.string(), z.unknown()),
|
|
144
|
+
createdAt: z.string(),
|
|
145
|
+
updatedAt: z.string()
|
|
146
|
+
});
|
|
147
|
+
const wikiIngestJobCandidateSchema = z.object({
|
|
148
|
+
id: z.string(),
|
|
149
|
+
sourceAssetId: z.string().nullable(),
|
|
150
|
+
candidateType: z.string(),
|
|
151
|
+
status: z.string(),
|
|
152
|
+
title: z.string(),
|
|
153
|
+
summary: z.string(),
|
|
154
|
+
targetKey: z.string(),
|
|
155
|
+
payload: z.record(z.string(), z.unknown()),
|
|
156
|
+
publishedNoteId: z.string().nullable(),
|
|
157
|
+
publishedEntityType: z.string().nullable(),
|
|
158
|
+
publishedEntityId: z.string().nullable(),
|
|
159
|
+
createdAt: z.string(),
|
|
160
|
+
updatedAt: z.string()
|
|
161
|
+
});
|
|
162
|
+
const wikiIngestJobPayloadSchema = z.object({
|
|
163
|
+
job: z.object({
|
|
164
|
+
id: z.string(),
|
|
165
|
+
spaceId: z.string(),
|
|
166
|
+
llmProfileId: z.string().nullable(),
|
|
167
|
+
status: z.string(),
|
|
168
|
+
phase: z.string(),
|
|
169
|
+
progressPercent: z.number().int().nonnegative(),
|
|
170
|
+
totalFiles: z.number().int().nonnegative(),
|
|
171
|
+
processedFiles: z.number().int().nonnegative(),
|
|
172
|
+
createdPageCount: z.number().int().nonnegative(),
|
|
173
|
+
createdEntityCount: z.number().int().nonnegative(),
|
|
174
|
+
acceptedCount: z.number().int().nonnegative(),
|
|
175
|
+
rejectedCount: z.number().int().nonnegative(),
|
|
176
|
+
latestMessage: z.string(),
|
|
177
|
+
sourceKind: z.string(),
|
|
178
|
+
sourceLocator: z.string(),
|
|
179
|
+
mimeType: z.string(),
|
|
180
|
+
titleHint: z.string(),
|
|
181
|
+
summary: z.string(),
|
|
182
|
+
pageNoteId: z.string().nullable(),
|
|
183
|
+
createdByActor: z.string().nullable(),
|
|
184
|
+
errorMessage: z.string(),
|
|
185
|
+
createdAt: z.string(),
|
|
186
|
+
updatedAt: z.string(),
|
|
187
|
+
completedAt: z.string().nullable()
|
|
188
|
+
}),
|
|
189
|
+
items: z.array(z.object({
|
|
190
|
+
id: z.string(),
|
|
191
|
+
itemType: z.string(),
|
|
192
|
+
status: z.string(),
|
|
193
|
+
noteId: z.string().nullable(),
|
|
194
|
+
mediaAssetId: z.string().nullable(),
|
|
195
|
+
payload: z.record(z.string(), z.unknown()),
|
|
196
|
+
createdAt: z.string(),
|
|
197
|
+
updatedAt: z.string()
|
|
198
|
+
})),
|
|
199
|
+
logs: z.array(wikiIngestJobLogSchema),
|
|
200
|
+
assets: z.array(wikiIngestJobAssetSchema),
|
|
201
|
+
candidates: z.array(wikiIngestJobCandidateSchema)
|
|
202
|
+
});
|
|
203
|
+
const listWikiIngestJobsQuerySchema = z.object({
|
|
204
|
+
spaceId: z.string().trim().optional(),
|
|
205
|
+
limit: z.coerce.number().int().positive().max(200).default(20)
|
|
206
|
+
});
|
|
207
|
+
export const reviewWikiIngestJobSchema = z.object({
|
|
208
|
+
decisions: z
|
|
209
|
+
.array(z
|
|
210
|
+
.object({
|
|
211
|
+
candidateId: z.string().trim().min(1),
|
|
212
|
+
keep: z.boolean().optional(),
|
|
213
|
+
action: z
|
|
214
|
+
.enum(["keep", "discard", "map_existing", "merge_existing"])
|
|
215
|
+
.optional(),
|
|
216
|
+
mappedEntityType: crudEntityTypeSchema.optional(),
|
|
217
|
+
mappedEntityId: z.string().trim().min(1).optional(),
|
|
218
|
+
targetNoteId: z.string().trim().min(1).optional()
|
|
219
|
+
})
|
|
220
|
+
.superRefine((value, context) => {
|
|
221
|
+
if (value.action === "map_existing") {
|
|
222
|
+
if (!value.mappedEntityType) {
|
|
223
|
+
context.addIssue({
|
|
224
|
+
code: z.ZodIssueCode.custom,
|
|
225
|
+
path: ["mappedEntityType"],
|
|
226
|
+
message: "mappedEntityType is required when action is map_existing"
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
if (!value.mappedEntityId) {
|
|
230
|
+
context.addIssue({
|
|
231
|
+
code: z.ZodIssueCode.custom,
|
|
232
|
+
path: ["mappedEntityId"],
|
|
233
|
+
message: "mappedEntityId is required when action is map_existing"
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
if (value.action === "merge_existing" && !value.targetNoteId) {
|
|
238
|
+
context.addIssue({
|
|
239
|
+
code: z.ZodIssueCode.custom,
|
|
240
|
+
path: ["targetNoteId"],
|
|
241
|
+
message: "targetNoteId is required when action is merge_existing"
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
if (value.action === undefined && value.keep === undefined) {
|
|
245
|
+
context.addIssue({
|
|
246
|
+
code: z.ZodIssueCode.custom,
|
|
247
|
+
path: ["action"],
|
|
248
|
+
message: "Either keep or action is required"
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
}))
|
|
252
|
+
.min(1)
|
|
253
|
+
});
|
|
254
|
+
export const createWikiSpaceSchema = z.object({
|
|
255
|
+
label: z.string().trim().min(1),
|
|
256
|
+
slug: z.string().trim().optional(),
|
|
257
|
+
description: z.string().trim().default(""),
|
|
258
|
+
ownerUserId: z.string().trim().nullable().optional(),
|
|
259
|
+
visibility: wikiSpaceVisibilitySchema.default("personal")
|
|
260
|
+
});
|
|
261
|
+
const wikiLlmReasoningEffortSchema = z.enum([
|
|
262
|
+
"none",
|
|
263
|
+
"low",
|
|
264
|
+
"medium",
|
|
265
|
+
"high",
|
|
266
|
+
"xhigh"
|
|
267
|
+
]);
|
|
268
|
+
const wikiLlmVerbositySchema = z.enum(["low", "medium", "high"]);
|
|
269
|
+
export const upsertWikiLlmProfileSchema = z.object({
|
|
270
|
+
id: z.string().trim().optional(),
|
|
271
|
+
label: z.string().trim().min(1),
|
|
272
|
+
provider: z.string().trim().min(1).default("openai-responses"),
|
|
273
|
+
baseUrl: z.string().trim().default("https://api.openai.com/v1"),
|
|
274
|
+
model: z.string().trim().min(1),
|
|
275
|
+
apiKey: z.string().trim().optional(),
|
|
276
|
+
systemPrompt: z.string().trim().default(""),
|
|
277
|
+
reasoningEffort: wikiLlmReasoningEffortSchema.optional(),
|
|
278
|
+
verbosity: wikiLlmVerbositySchema.optional(),
|
|
279
|
+
enabled: z.boolean().default(true),
|
|
280
|
+
metadata: z.record(z.string(), z.unknown()).default({})
|
|
281
|
+
});
|
|
282
|
+
export const testWikiLlmProfileSchema = z.object({
|
|
283
|
+
profileId: z.string().trim().optional(),
|
|
284
|
+
provider: z.string().trim().min(1).default("openai-responses"),
|
|
285
|
+
baseUrl: z.string().trim().default("https://api.openai.com/v1"),
|
|
286
|
+
model: z.string().trim().min(1),
|
|
287
|
+
apiKey: z.string().trim().optional(),
|
|
288
|
+
reasoningEffort: wikiLlmReasoningEffortSchema.optional(),
|
|
289
|
+
verbosity: wikiLlmVerbositySchema.optional()
|
|
290
|
+
});
|
|
291
|
+
export const upsertWikiEmbeddingProfileSchema = z.object({
|
|
292
|
+
id: z.string().trim().optional(),
|
|
293
|
+
label: z.string().trim().min(1),
|
|
294
|
+
provider: z.string().trim().min(1).default("openai-compatible"),
|
|
295
|
+
baseUrl: z.string().trim().default("https://api.openai.com/v1"),
|
|
296
|
+
model: z.string().trim().min(1).default("text-embedding-3-small"),
|
|
297
|
+
dimensions: z.number().int().positive().nullable().optional(),
|
|
298
|
+
chunkSize: z.number().int().positive().default(1200),
|
|
299
|
+
chunkOverlap: z.number().int().nonnegative().default(200),
|
|
300
|
+
apiKey: z.string().trim().optional(),
|
|
301
|
+
enabled: z.boolean().default(true),
|
|
302
|
+
metadata: z.record(z.string(), z.unknown()).default({})
|
|
303
|
+
});
|
|
304
|
+
export const wikiSearchQuerySchema = z.object({
|
|
305
|
+
spaceId: z.string().trim().optional(),
|
|
306
|
+
kind: noteKindSchema.optional(),
|
|
307
|
+
mode: wikiSearchModeSchema.default("hybrid"),
|
|
308
|
+
query: z.string().trim().default(""),
|
|
309
|
+
profileId: z.string().trim().optional(),
|
|
310
|
+
linkedEntity: z
|
|
311
|
+
.object({
|
|
312
|
+
entityType: crudEntityTypeSchema,
|
|
313
|
+
entityId: z.string().trim().min(1)
|
|
314
|
+
})
|
|
315
|
+
.optional(),
|
|
316
|
+
limit: z.coerce.number().int().positive().max(50).default(20)
|
|
317
|
+
});
|
|
318
|
+
export const syncWikiVaultSchema = z.object({
|
|
319
|
+
spaceId: z.string().trim().optional()
|
|
320
|
+
});
|
|
321
|
+
export const reindexWikiEmbeddingsSchema = z.object({
|
|
322
|
+
spaceId: z.string().trim().optional(),
|
|
323
|
+
profileId: z.string().trim().optional()
|
|
324
|
+
});
|
|
325
|
+
export const createWikiIngestJobSchema = z.object({
|
|
326
|
+
spaceId: z.string().trim().optional(),
|
|
327
|
+
titleHint: z.string().trim().default(""),
|
|
328
|
+
sourceKind: z.enum(["raw_text", "local_path", "url"]),
|
|
329
|
+
sourceText: z.string().default(""),
|
|
330
|
+
sourcePath: z.string().trim().optional(),
|
|
331
|
+
sourceUrl: z.string().trim().optional(),
|
|
332
|
+
mimeType: z.string().trim().default(""),
|
|
333
|
+
llmProfileId: z.string().trim().optional(),
|
|
334
|
+
parseStrategy: z.enum(["auto", "text_only", "multimodal"]).default("auto"),
|
|
335
|
+
entityProposalMode: z.enum(["none", "suggest"]).default("suggest"),
|
|
336
|
+
userId: z.string().trim().nullable().optional(),
|
|
337
|
+
createAsKind: noteKindSchema.default("wiki"),
|
|
338
|
+
linkedEntityHints: z.array(createNoteLinkSchema).default([])
|
|
339
|
+
});
|
|
340
|
+
function nowIso() {
|
|
341
|
+
return new Date().toISOString();
|
|
342
|
+
}
|
|
343
|
+
const WIKI_STARTER_PAGES = [
|
|
344
|
+
{
|
|
345
|
+
slug: "index",
|
|
346
|
+
title: "Home",
|
|
347
|
+
parentSlug: null,
|
|
348
|
+
indexOrder: 0,
|
|
349
|
+
summary: "Top-level home page for this wiki space."
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
slug: "people",
|
|
353
|
+
title: "People",
|
|
354
|
+
parentSlug: "index",
|
|
355
|
+
indexOrder: 10,
|
|
356
|
+
summary: "People, collaborators, and relationship context."
|
|
357
|
+
},
|
|
358
|
+
{
|
|
359
|
+
slug: "projects",
|
|
360
|
+
title: "Projects",
|
|
361
|
+
parentSlug: "index",
|
|
362
|
+
indexOrder: 20,
|
|
363
|
+
summary: "Active projects, initiatives, and workstreams."
|
|
364
|
+
},
|
|
365
|
+
{
|
|
366
|
+
slug: "concepts",
|
|
367
|
+
title: "Concepts",
|
|
368
|
+
parentSlug: "index",
|
|
369
|
+
indexOrder: 30,
|
|
370
|
+
summary: "Ideas, themes, philosophies, and operating concepts."
|
|
371
|
+
},
|
|
372
|
+
{
|
|
373
|
+
slug: "sources",
|
|
374
|
+
title: "Sources",
|
|
375
|
+
parentSlug: "index",
|
|
376
|
+
indexOrder: 40,
|
|
377
|
+
summary: "Raw materials, references, imports, and citations."
|
|
378
|
+
},
|
|
379
|
+
{
|
|
380
|
+
slug: "chronicle",
|
|
381
|
+
title: "Chronicle",
|
|
382
|
+
parentSlug: "index",
|
|
383
|
+
indexOrder: 50,
|
|
384
|
+
summary: "Timeline-style notes, field logs, and ongoing narrative."
|
|
385
|
+
}
|
|
386
|
+
];
|
|
387
|
+
function normalizeAnchorKey(anchorKey) {
|
|
388
|
+
return anchorKey.trim().length > 0 ? anchorKey.trim() : null;
|
|
389
|
+
}
|
|
390
|
+
function normalizeTags(tags) {
|
|
391
|
+
if (!tags) {
|
|
392
|
+
return [];
|
|
393
|
+
}
|
|
394
|
+
const seen = new Set();
|
|
395
|
+
return tags
|
|
396
|
+
.map((tag) => tag.trim())
|
|
397
|
+
.filter(Boolean)
|
|
398
|
+
.filter((tag) => {
|
|
399
|
+
const normalized = tag.toLowerCase();
|
|
400
|
+
if (seen.has(normalized)) {
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
seen.add(normalized);
|
|
404
|
+
return true;
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
function normalizeAliases(aliases) {
|
|
408
|
+
if (!aliases) {
|
|
409
|
+
return [];
|
|
410
|
+
}
|
|
411
|
+
const seen = new Set();
|
|
412
|
+
return aliases
|
|
413
|
+
.map((alias) => alias.trim())
|
|
414
|
+
.filter(Boolean)
|
|
415
|
+
.filter((alias) => {
|
|
416
|
+
const normalized = alias.toLowerCase();
|
|
417
|
+
if (seen.has(normalized)) {
|
|
418
|
+
return false;
|
|
419
|
+
}
|
|
420
|
+
seen.add(normalized);
|
|
421
|
+
return true;
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
function parseJsonRecord(raw) {
|
|
425
|
+
if (!raw) {
|
|
426
|
+
return {};
|
|
427
|
+
}
|
|
428
|
+
try {
|
|
429
|
+
const parsed = JSON.parse(raw);
|
|
430
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
|
431
|
+
? parsed
|
|
432
|
+
: {};
|
|
433
|
+
}
|
|
434
|
+
catch {
|
|
435
|
+
return {};
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
function parseJsonStringArray(raw) {
|
|
439
|
+
if (!raw) {
|
|
440
|
+
return [];
|
|
441
|
+
}
|
|
442
|
+
try {
|
|
443
|
+
const parsed = JSON.parse(raw);
|
|
444
|
+
return Array.isArray(parsed)
|
|
445
|
+
? parsed.filter((entry) => typeof entry === "string")
|
|
446
|
+
: [];
|
|
447
|
+
}
|
|
448
|
+
catch {
|
|
449
|
+
return [];
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
function readStringRecordValue(record, key) {
|
|
453
|
+
const value = record[key];
|
|
454
|
+
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
|
455
|
+
}
|
|
456
|
+
function listLinkRowsForNotes(noteIds) {
|
|
457
|
+
if (noteIds.length === 0) {
|
|
458
|
+
return [];
|
|
459
|
+
}
|
|
460
|
+
const placeholders = noteIds.map(() => "?").join(", ");
|
|
461
|
+
return getDatabase()
|
|
462
|
+
.prepare(`SELECT note_id, entity_type, entity_id, anchor_key, created_at
|
|
463
|
+
FROM note_links
|
|
464
|
+
WHERE note_id IN (${placeholders})
|
|
465
|
+
ORDER BY created_at ASC`)
|
|
466
|
+
.all(...noteIds);
|
|
467
|
+
}
|
|
468
|
+
function mapLinks(rows) {
|
|
469
|
+
return rows.map((row) => ({
|
|
470
|
+
entityType: row.entity_type,
|
|
471
|
+
entityId: row.entity_id,
|
|
472
|
+
anchorKey: normalizeAnchorKey(row.anchor_key)
|
|
473
|
+
}));
|
|
474
|
+
}
|
|
475
|
+
function mapNoteRow(row, linkRows) {
|
|
476
|
+
return persistedNoteSchema.parse(decorateOwnedEntity("note", {
|
|
477
|
+
id: row.id,
|
|
478
|
+
kind: row.kind,
|
|
479
|
+
title: row.title,
|
|
480
|
+
slug: row.slug,
|
|
481
|
+
spaceId: row.space_id,
|
|
482
|
+
parentSlug: row.parent_slug,
|
|
483
|
+
indexOrder: row.index_order,
|
|
484
|
+
showInIndex: row.show_in_index === 1,
|
|
485
|
+
aliases: normalizeAliases(parseJsonStringArray(row.aliases_json)),
|
|
486
|
+
summary: row.summary,
|
|
487
|
+
contentMarkdown: row.content_markdown,
|
|
488
|
+
contentPlain: row.content_plain,
|
|
489
|
+
author: row.author,
|
|
490
|
+
source: row.source,
|
|
491
|
+
sourcePath: row.source_path,
|
|
492
|
+
frontmatter: parseJsonRecord(row.frontmatter_json),
|
|
493
|
+
revisionHash: row.revision_hash,
|
|
494
|
+
lastSyncedAt: row.last_synced_at,
|
|
495
|
+
createdAt: row.created_at,
|
|
496
|
+
updatedAt: row.updated_at,
|
|
497
|
+
links: mapLinks(linkRows),
|
|
498
|
+
tags: normalizeTags(parseJsonStringArray(row.tags_json)),
|
|
499
|
+
destroyAt: row.destroy_at
|
|
500
|
+
}));
|
|
501
|
+
}
|
|
502
|
+
function getNoteRows(whereClause = "", params = []) {
|
|
503
|
+
return getDatabase()
|
|
504
|
+
.prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
|
|
505
|
+
tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
|
|
506
|
+
FROM notes
|
|
507
|
+
${whereClause}
|
|
508
|
+
ORDER BY updated_at DESC`)
|
|
509
|
+
.all(...params);
|
|
510
|
+
}
|
|
511
|
+
function getNoteByIdRaw(noteId) {
|
|
512
|
+
return getDatabase()
|
|
513
|
+
.prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
|
|
514
|
+
tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
|
|
515
|
+
FROM notes
|
|
516
|
+
WHERE id = ?`)
|
|
517
|
+
.get(noteId);
|
|
518
|
+
}
|
|
519
|
+
function getNoteBySlugRaw(spaceId, slug, exceptNoteId) {
|
|
520
|
+
const row = getDatabase()
|
|
521
|
+
.prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
|
|
522
|
+
tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
|
|
523
|
+
FROM notes
|
|
524
|
+
WHERE space_id = ?
|
|
525
|
+
AND lower(slug) = lower(?)
|
|
526
|
+
${exceptNoteId ? "AND id != ?" : ""}
|
|
527
|
+
LIMIT 1`)
|
|
528
|
+
.get(...(exceptNoteId ? [spaceId, slug, exceptNoteId] : [spaceId, slug]));
|
|
529
|
+
return row;
|
|
530
|
+
}
|
|
531
|
+
function buildContentPlain(markdown) {
|
|
532
|
+
return markdown
|
|
533
|
+
.replace(/^---[\s\S]*?---\s*/m, "")
|
|
534
|
+
.replace(/```[\s\S]*?```/g, (block) => block.replace(/```/g, "").trim())
|
|
535
|
+
.replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1")
|
|
536
|
+
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
|
537
|
+
.replace(/\[\[([^\]|]+)\|?([^\]]*)\]\]/g, (_match, left, right) => (right || left).trim())
|
|
538
|
+
.replace(/`([^`]+)`/g, "$1")
|
|
539
|
+
.replace(/\*\*([^*]+)\*\*/g, "$1")
|
|
540
|
+
.replace(/\*([^*]+)\*/g, "$1")
|
|
541
|
+
.replace(/^>\s?/gm, "")
|
|
542
|
+
.replace(/^#{1,6}\s+/gm, "")
|
|
543
|
+
.replace(/^[-*+]\s+/gm, "")
|
|
544
|
+
.replace(/^\d+\.\s+/gm, "")
|
|
545
|
+
.replace(/\r/g, "")
|
|
546
|
+
.trim();
|
|
547
|
+
}
|
|
548
|
+
function inferTitle(markdown, fallback) {
|
|
549
|
+
const headingMatch = markdown.match(/^#{1,6}\s+(.+)$/m);
|
|
550
|
+
if (headingMatch?.[1]?.trim()) {
|
|
551
|
+
return headingMatch[1].trim().slice(0, 160);
|
|
552
|
+
}
|
|
553
|
+
const plain = buildContentPlain(markdown);
|
|
554
|
+
if (plain.trim()) {
|
|
555
|
+
return plain.trim().split("\n")[0].slice(0, 160);
|
|
556
|
+
}
|
|
557
|
+
return fallback;
|
|
558
|
+
}
|
|
559
|
+
function inferSummary(markdown) {
|
|
560
|
+
const plain = buildContentPlain(markdown).replace(/\s+/g, " ").trim();
|
|
561
|
+
return plain.slice(0, 240);
|
|
562
|
+
}
|
|
563
|
+
function stripLeadingHeading(markdown, title) {
|
|
564
|
+
const normalizedTitle = title.trim().toLowerCase();
|
|
565
|
+
const trimmed = markdown.trim();
|
|
566
|
+
const match = trimmed.match(/^#\s+(.+?)\n+/);
|
|
567
|
+
if (!match) {
|
|
568
|
+
return trimmed;
|
|
569
|
+
}
|
|
570
|
+
const heading = match[1]?.trim().toLowerCase() ?? "";
|
|
571
|
+
if (heading !== normalizedTitle) {
|
|
572
|
+
return trimmed;
|
|
573
|
+
}
|
|
574
|
+
return trimmed.slice(match[0].length).trim();
|
|
575
|
+
}
|
|
576
|
+
function mergeWikiPageContent(targetMarkdown, incoming) {
|
|
577
|
+
const mergedBody = stripLeadingHeading(incoming.markdown, incoming.title) ||
|
|
578
|
+
incoming.markdown.trim();
|
|
579
|
+
return [
|
|
580
|
+
targetMarkdown.trim(),
|
|
581
|
+
"",
|
|
582
|
+
`## ${incoming.title}`,
|
|
583
|
+
"",
|
|
584
|
+
mergedBody
|
|
585
|
+
]
|
|
586
|
+
.filter((part) => part.trim().length > 0)
|
|
587
|
+
.join("\n");
|
|
588
|
+
}
|
|
589
|
+
function slugify(value) {
|
|
590
|
+
const normalized = value
|
|
591
|
+
.toLowerCase()
|
|
592
|
+
.normalize("NFKD")
|
|
593
|
+
.replace(/[^\w\s-]/g, "")
|
|
594
|
+
.trim()
|
|
595
|
+
.replace(/[\s_]+/g, "-")
|
|
596
|
+
.replace(/-+/g, "-");
|
|
597
|
+
return normalized || `page-${randomUUID().replaceAll("-", "").slice(0, 8)}`;
|
|
598
|
+
}
|
|
599
|
+
function buildUniqueSlug(spaceId, requestedSlug, noteId) {
|
|
600
|
+
const base = slugify(requestedSlug);
|
|
601
|
+
let candidate = base;
|
|
602
|
+
let suffix = 2;
|
|
603
|
+
while (getNoteBySlugRaw(spaceId, candidate, noteId)) {
|
|
604
|
+
candidate = `${base}-${suffix}`;
|
|
605
|
+
suffix += 1;
|
|
606
|
+
}
|
|
607
|
+
return candidate;
|
|
608
|
+
}
|
|
609
|
+
function mapWikiSpace(row) {
|
|
610
|
+
return wikiSpaceSchema.parse({
|
|
611
|
+
id: row.id,
|
|
612
|
+
slug: row.slug,
|
|
613
|
+
label: row.label,
|
|
614
|
+
description: row.description,
|
|
615
|
+
ownerUserId: row.owner_user_id,
|
|
616
|
+
visibility: row.visibility,
|
|
617
|
+
createdAt: row.created_at,
|
|
618
|
+
updatedAt: row.updated_at
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
function getWikiRootDir() {
|
|
622
|
+
return path.join(resolveDataDir(), "wiki");
|
|
623
|
+
}
|
|
624
|
+
function getSpaceStorageDir(space) {
|
|
625
|
+
if (space.visibility === "shared") {
|
|
626
|
+
return path.join(getWikiRootDir(), "shared", space.slug);
|
|
627
|
+
}
|
|
628
|
+
return path.join(getWikiRootDir(), "users", space.ownerUserId ?? space.slug);
|
|
629
|
+
}
|
|
630
|
+
function getSpaceIndexPath(space) {
|
|
631
|
+
return path.join(getSpaceStorageDir(space), "index.md");
|
|
632
|
+
}
|
|
633
|
+
function getSpaceRawDir(space) {
|
|
634
|
+
return path.join(getSpaceStorageDir(space), "raw");
|
|
635
|
+
}
|
|
636
|
+
function getNoteStoragePath(note, space) {
|
|
637
|
+
const directory = note.kind === "wiki" ? "pages" : "evidence";
|
|
638
|
+
return path.join(getSpaceStorageDir(space), directory, `${note.slug}.md`);
|
|
639
|
+
}
|
|
640
|
+
function buildNoteFrontmatter(note) {
|
|
641
|
+
return {
|
|
642
|
+
id: note.id,
|
|
643
|
+
kind: note.kind,
|
|
644
|
+
title: note.title,
|
|
645
|
+
slug: note.slug,
|
|
646
|
+
spaceId: note.spaceId,
|
|
647
|
+
parentSlug: note.parentSlug,
|
|
648
|
+
indexOrder: note.indexOrder,
|
|
649
|
+
showInIndex: note.showInIndex,
|
|
650
|
+
aliases: note.aliases,
|
|
651
|
+
summary: note.summary,
|
|
652
|
+
tags: note.tags ?? [],
|
|
653
|
+
linkedEntities: note.links.map((link) => ({
|
|
654
|
+
entityType: link.entityType,
|
|
655
|
+
entityId: link.entityId,
|
|
656
|
+
anchorKey: link.anchorKey
|
|
657
|
+
})),
|
|
658
|
+
createdAt: note.createdAt,
|
|
659
|
+
updatedAt: note.updatedAt,
|
|
660
|
+
lastSyncedAt: note.lastSyncedAt,
|
|
661
|
+
author: note.author
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
function stringifyFrontmatterValue(value) {
|
|
665
|
+
if (typeof value === "string") {
|
|
666
|
+
return JSON.stringify(value);
|
|
667
|
+
}
|
|
668
|
+
return JSON.stringify(value);
|
|
669
|
+
}
|
|
670
|
+
function renderFrontmatter(frontmatter) {
|
|
671
|
+
const lines = ["---"];
|
|
672
|
+
for (const [key, value] of Object.entries(frontmatter)) {
|
|
673
|
+
lines.push(`${key}: ${stringifyFrontmatterValue(value)}`);
|
|
674
|
+
}
|
|
675
|
+
lines.push("---", "");
|
|
676
|
+
return lines.join("\n");
|
|
677
|
+
}
|
|
678
|
+
function hashContent(value) {
|
|
679
|
+
return createHash("sha256").update(value).digest("hex");
|
|
680
|
+
}
|
|
681
|
+
function hashBuffer(value) {
|
|
682
|
+
return createHash("sha256").update(value).digest("hex");
|
|
683
|
+
}
|
|
684
|
+
function buildLinkedEntityTokens(note) {
|
|
685
|
+
return note.links
|
|
686
|
+
.map((link) => `${link.entityType}:${link.entityId}`)
|
|
687
|
+
.join(" ");
|
|
688
|
+
}
|
|
689
|
+
function buildWikiFtsQuery(query) {
|
|
690
|
+
const tokens = query
|
|
691
|
+
.trim()
|
|
692
|
+
.split(/\s+/)
|
|
693
|
+
.map((token) => token.replace(/["*']/g, "").trim())
|
|
694
|
+
.filter(Boolean);
|
|
695
|
+
if (tokens.length === 0) {
|
|
696
|
+
return null;
|
|
697
|
+
}
|
|
698
|
+
return tokens.map((token) => `${token}*`).join(" AND ");
|
|
699
|
+
}
|
|
700
|
+
function deleteWikiSearchRow(noteId) {
|
|
701
|
+
getDatabase()
|
|
702
|
+
.prepare(`DELETE FROM wiki_pages_fts WHERE note_id = ?`)
|
|
703
|
+
.run(noteId);
|
|
704
|
+
}
|
|
705
|
+
function chunkHeadingAware(markdown, chunkSize, chunkOverlap) {
|
|
706
|
+
const stripped = markdown
|
|
707
|
+
.replace(/^---[\s\S]*?---\s*/m, "")
|
|
708
|
+
.replace(/\r/g, "");
|
|
709
|
+
const sections = [];
|
|
710
|
+
const lines = stripped.split("\n");
|
|
711
|
+
let currentHeading = "Document";
|
|
712
|
+
let currentLines = [];
|
|
713
|
+
const flush = () => {
|
|
714
|
+
const text = currentLines.join("\n").trim();
|
|
715
|
+
if (text) {
|
|
716
|
+
sections.push({ headingPath: currentHeading, text });
|
|
717
|
+
}
|
|
718
|
+
currentLines = [];
|
|
719
|
+
};
|
|
720
|
+
for (const line of lines) {
|
|
721
|
+
const match = line.match(/^(#{1,6})\s+(.+)$/);
|
|
722
|
+
if (match) {
|
|
723
|
+
flush();
|
|
724
|
+
currentHeading = match[2].trim();
|
|
725
|
+
continue;
|
|
726
|
+
}
|
|
727
|
+
currentLines.push(line);
|
|
728
|
+
}
|
|
729
|
+
flush();
|
|
730
|
+
const chunks = [];
|
|
731
|
+
sections.forEach((section, sectionIndex) => {
|
|
732
|
+
const content = section.text.replace(/\s+/g, " ").trim();
|
|
733
|
+
if (!content) {
|
|
734
|
+
return;
|
|
735
|
+
}
|
|
736
|
+
if (content.length <= chunkSize) {
|
|
737
|
+
chunks.push({
|
|
738
|
+
key: `${sectionIndex}-0`,
|
|
739
|
+
headingPath: section.headingPath,
|
|
740
|
+
contentText: content
|
|
741
|
+
});
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
let offset = 0;
|
|
745
|
+
let partIndex = 0;
|
|
746
|
+
while (offset < content.length) {
|
|
747
|
+
const slice = content.slice(offset, offset + chunkSize).trim();
|
|
748
|
+
if (slice) {
|
|
749
|
+
chunks.push({
|
|
750
|
+
key: `${sectionIndex}-${partIndex}`,
|
|
751
|
+
headingPath: section.headingPath,
|
|
752
|
+
contentText: slice
|
|
753
|
+
});
|
|
754
|
+
}
|
|
755
|
+
if (offset + chunkSize >= content.length) {
|
|
756
|
+
break;
|
|
757
|
+
}
|
|
758
|
+
offset += Math.max(1, chunkSize - chunkOverlap);
|
|
759
|
+
partIndex += 1;
|
|
760
|
+
}
|
|
761
|
+
});
|
|
762
|
+
return chunks;
|
|
763
|
+
}
|
|
764
|
+
function cosineSimilarity(left, right) {
|
|
765
|
+
if (left.length !== right.length || left.length === 0) {
|
|
766
|
+
return 0;
|
|
767
|
+
}
|
|
768
|
+
let dot = 0;
|
|
769
|
+
let leftNorm = 0;
|
|
770
|
+
let rightNorm = 0;
|
|
771
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
772
|
+
dot += left[index] * right[index];
|
|
773
|
+
leftNorm += left[index] * left[index];
|
|
774
|
+
rightNorm += right[index] * right[index];
|
|
775
|
+
}
|
|
776
|
+
if (leftNorm === 0 || rightNorm === 0) {
|
|
777
|
+
return 0;
|
|
778
|
+
}
|
|
779
|
+
return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
|
|
780
|
+
}
|
|
781
|
+
async function getFetchedContent(sourceKind, options) {
|
|
782
|
+
if (sourceKind === "raw_text") {
|
|
783
|
+
return {
|
|
784
|
+
locator: "raw_text",
|
|
785
|
+
contentText: options.sourceText,
|
|
786
|
+
mimeType: options.mimeType || "text/plain",
|
|
787
|
+
fileName: "inline.txt",
|
|
788
|
+
binary: null
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
if (sourceKind === "local_path") {
|
|
792
|
+
const filePath = options.sourcePath?.trim();
|
|
793
|
+
if (!filePath) {
|
|
794
|
+
throw new Error("sourcePath is required for local_path ingest.");
|
|
795
|
+
}
|
|
796
|
+
const payload = await readFile(filePath);
|
|
797
|
+
const fileName = path.basename(filePath);
|
|
798
|
+
const mimeType = options.mimeType?.trim() || inferMimeTypeFromPath(fileName);
|
|
799
|
+
return {
|
|
800
|
+
locator: filePath,
|
|
801
|
+
contentText: mimeType.startsWith("text/") ||
|
|
802
|
+
mimeType === "application/json" ||
|
|
803
|
+
mimeType === "text/markdown"
|
|
804
|
+
? payload.toString("utf8")
|
|
805
|
+
: "",
|
|
806
|
+
mimeType,
|
|
807
|
+
fileName,
|
|
808
|
+
binary: payload
|
|
809
|
+
};
|
|
810
|
+
}
|
|
811
|
+
const sourceUrl = options.sourceUrl?.trim();
|
|
812
|
+
if (!sourceUrl) {
|
|
813
|
+
throw new Error("sourceUrl is required for url ingest.");
|
|
814
|
+
}
|
|
815
|
+
const response = await fetch(sourceUrl);
|
|
816
|
+
if (!response.ok) {
|
|
817
|
+
throw new Error(`Could not fetch ${sourceUrl}: ${response.status}`);
|
|
818
|
+
}
|
|
819
|
+
const mimeType = options.mimeType?.trim() ||
|
|
820
|
+
response.headers.get("content-type")?.split(";")[0]?.trim() ||
|
|
821
|
+
"application/octet-stream";
|
|
822
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
823
|
+
const binary = Buffer.from(arrayBuffer);
|
|
824
|
+
const fileName = sourceUrl.split("/").pop()?.split("?")[0]?.trim() || "remote-source.bin";
|
|
825
|
+
return {
|
|
826
|
+
locator: sourceUrl,
|
|
827
|
+
contentText: mimeType.startsWith("text/") ||
|
|
828
|
+
mimeType === "application/json" ||
|
|
829
|
+
mimeType === "text/markdown"
|
|
830
|
+
? binary.toString("utf8")
|
|
831
|
+
: "",
|
|
832
|
+
mimeType,
|
|
833
|
+
fileName,
|
|
834
|
+
binary
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
function inferExtensionFromMimeType(mimeType) {
|
|
838
|
+
switch (mimeType) {
|
|
839
|
+
case "text/markdown":
|
|
840
|
+
return ".md";
|
|
841
|
+
case "text/plain":
|
|
842
|
+
return ".txt";
|
|
843
|
+
case "text/html":
|
|
844
|
+
return ".html";
|
|
845
|
+
case "application/json":
|
|
846
|
+
return ".json";
|
|
847
|
+
case "application/pdf":
|
|
848
|
+
return ".pdf";
|
|
849
|
+
case "image/png":
|
|
850
|
+
return ".png";
|
|
851
|
+
case "image/jpeg":
|
|
852
|
+
return ".jpg";
|
|
853
|
+
case "image/gif":
|
|
854
|
+
return ".gif";
|
|
855
|
+
case "image/webp":
|
|
856
|
+
return ".webp";
|
|
857
|
+
case "audio/mpeg":
|
|
858
|
+
return ".mp3";
|
|
859
|
+
case "audio/wav":
|
|
860
|
+
return ".wav";
|
|
861
|
+
case "audio/mp4":
|
|
862
|
+
return ".m4a";
|
|
863
|
+
case "video/mp4":
|
|
864
|
+
return ".mp4";
|
|
865
|
+
case "video/quicktime":
|
|
866
|
+
return ".mov";
|
|
867
|
+
default:
|
|
868
|
+
return "";
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
function inferMimeTypeFromPath(fileName) {
|
|
872
|
+
const extension = path.extname(fileName).toLowerCase();
|
|
873
|
+
switch (extension) {
|
|
874
|
+
case ".md":
|
|
875
|
+
case ".markdown":
|
|
876
|
+
return "text/markdown";
|
|
877
|
+
case ".txt":
|
|
878
|
+
return "text/plain";
|
|
879
|
+
case ".html":
|
|
880
|
+
case ".htm":
|
|
881
|
+
return "text/html";
|
|
882
|
+
case ".json":
|
|
883
|
+
return "application/json";
|
|
884
|
+
case ".png":
|
|
885
|
+
return "image/png";
|
|
886
|
+
case ".jpg":
|
|
887
|
+
case ".jpeg":
|
|
888
|
+
return "image/jpeg";
|
|
889
|
+
case ".gif":
|
|
890
|
+
return "image/gif";
|
|
891
|
+
case ".webp":
|
|
892
|
+
return "image/webp";
|
|
893
|
+
case ".mp3":
|
|
894
|
+
return "audio/mpeg";
|
|
895
|
+
case ".wav":
|
|
896
|
+
return "audio/wav";
|
|
897
|
+
case ".m4a":
|
|
898
|
+
return "audio/mp4";
|
|
899
|
+
case ".mp4":
|
|
900
|
+
return "video/mp4";
|
|
901
|
+
case ".mov":
|
|
902
|
+
return "video/quicktime";
|
|
903
|
+
case ".pdf":
|
|
904
|
+
return "application/pdf";
|
|
905
|
+
default:
|
|
906
|
+
return "application/octet-stream";
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
function sanitizeFileName(value) {
|
|
910
|
+
return value.replace(/[^a-zA-Z0-9._-]+/g, "-");
|
|
911
|
+
}
|
|
912
|
+
async function readSecretApiKey(secretId, secrets) {
|
|
913
|
+
if (!secretId) {
|
|
914
|
+
return null;
|
|
915
|
+
}
|
|
916
|
+
const cipherText = readEncryptedSecret(secretId);
|
|
917
|
+
if (!cipherText) {
|
|
918
|
+
return null;
|
|
919
|
+
}
|
|
920
|
+
const payload = secrets.openJson(cipherText);
|
|
921
|
+
return payload.apiKey || null;
|
|
922
|
+
}
|
|
923
|
+
async function compileTextWithLlm(profile, secrets, input) {
|
|
924
|
+
const apiKey = await readSecretApiKey(profile.secretId, secrets);
|
|
925
|
+
if (!apiKey) {
|
|
926
|
+
return null;
|
|
927
|
+
}
|
|
928
|
+
const prompt = [
|
|
929
|
+
"You compile user-provided source material into a local wiki page.",
|
|
930
|
+
"Return JSON with keys title, summary, markdown, tags, entityProposals, pageUpdateSuggestions, articleCandidates.",
|
|
931
|
+
"The markdown should be concise, structured, and agent-readable.",
|
|
932
|
+
"entityProposals should be an array of objects with entityType, title, summary, rationale, confidence, and suggestedFields.",
|
|
933
|
+
"pageUpdateSuggestions should be an array of objects with targetSlug, rationale, and patchSummary.",
|
|
934
|
+
"articleCandidates should be an array of objects with title, slug, rationale, and summary.",
|
|
935
|
+
profile.systemPrompt.trim()
|
|
936
|
+
]
|
|
937
|
+
.filter(Boolean)
|
|
938
|
+
.join("\n");
|
|
939
|
+
const response = await fetch(`${profile.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
940
|
+
method: "POST",
|
|
941
|
+
headers: {
|
|
942
|
+
"content-type": "application/json",
|
|
943
|
+
authorization: `Bearer ${apiKey}`
|
|
944
|
+
},
|
|
945
|
+
body: JSON.stringify({
|
|
946
|
+
model: profile.model,
|
|
947
|
+
temperature: 0.2,
|
|
948
|
+
response_format: { type: "json_object" },
|
|
949
|
+
messages: [
|
|
950
|
+
{
|
|
951
|
+
role: "system",
|
|
952
|
+
content: prompt
|
|
953
|
+
},
|
|
954
|
+
{
|
|
955
|
+
role: "user",
|
|
956
|
+
content: `Title hint: ${input.titleHint || "none"}\nMime type: ${input.mimeType}\n\nSource:\n${input.rawText.slice(0, 24_000)}`
|
|
957
|
+
}
|
|
958
|
+
]
|
|
959
|
+
})
|
|
960
|
+
});
|
|
961
|
+
if (!response.ok) {
|
|
962
|
+
throw new Error(`LLM compilation failed: ${response.status}`);
|
|
963
|
+
}
|
|
964
|
+
const payload = (await response.json());
|
|
965
|
+
const content = payload.choices?.[0]?.message?.content;
|
|
966
|
+
if (!content) {
|
|
967
|
+
return null;
|
|
968
|
+
}
|
|
969
|
+
try {
|
|
970
|
+
const parsed = JSON.parse(content);
|
|
971
|
+
return {
|
|
972
|
+
title: parsed.title?.trim() || input.titleHint || "Imported source",
|
|
973
|
+
summary: parsed.summary?.trim() || "",
|
|
974
|
+
markdown: parsed.markdown?.trim() || input.rawText.trim(),
|
|
975
|
+
tags: normalizeTags(parsed.tags),
|
|
976
|
+
entityProposals: Array.isArray(parsed.entityProposals)
|
|
977
|
+
? parsed.entityProposals.filter((entry) => entry !== null && typeof entry === "object")
|
|
978
|
+
: [],
|
|
979
|
+
pageUpdateSuggestions: Array.isArray(parsed.pageUpdateSuggestions)
|
|
980
|
+
? parsed.pageUpdateSuggestions.filter((entry) => entry !== null && typeof entry === "object")
|
|
981
|
+
: [],
|
|
982
|
+
articleCandidates: Array.isArray(parsed.articleCandidates)
|
|
983
|
+
? parsed.articleCandidates.filter((entry) => entry !== null && typeof entry === "object")
|
|
984
|
+
: []
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
catch {
|
|
988
|
+
return null;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
async function compileImageWithLlm(profile, secrets, input) {
|
|
992
|
+
const apiKey = await readSecretApiKey(profile.secretId, secrets);
|
|
993
|
+
if (!apiKey) {
|
|
994
|
+
return null;
|
|
995
|
+
}
|
|
996
|
+
const prompt = [
|
|
997
|
+
"You compile a user-provided image into a local wiki page.",
|
|
998
|
+
"Return JSON with keys title, summary, markdown, tags, entityProposals, pageUpdateSuggestions, articleCandidates.",
|
|
999
|
+
"Describe the image, capture useful extracted text when visible, and keep the markdown structured for an agent memory wiki.",
|
|
1000
|
+
"entityProposals should be an array of objects with entityType, title, summary, rationale, confidence, and suggestedFields.",
|
|
1001
|
+
"pageUpdateSuggestions should be an array of objects with targetSlug, rationale, and patchSummary.",
|
|
1002
|
+
"articleCandidates should be an array of objects with title, slug, rationale, and summary.",
|
|
1003
|
+
profile.systemPrompt.trim()
|
|
1004
|
+
]
|
|
1005
|
+
.filter(Boolean)
|
|
1006
|
+
.join("\n");
|
|
1007
|
+
const response = await fetch(`${profile.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
1008
|
+
method: "POST",
|
|
1009
|
+
headers: {
|
|
1010
|
+
"content-type": "application/json",
|
|
1011
|
+
authorization: `Bearer ${apiKey}`
|
|
1012
|
+
},
|
|
1013
|
+
body: JSON.stringify({
|
|
1014
|
+
model: profile.model,
|
|
1015
|
+
temperature: 0.2,
|
|
1016
|
+
response_format: { type: "json_object" },
|
|
1017
|
+
messages: [
|
|
1018
|
+
{
|
|
1019
|
+
role: "system",
|
|
1020
|
+
content: prompt
|
|
1021
|
+
},
|
|
1022
|
+
{
|
|
1023
|
+
role: "user",
|
|
1024
|
+
content: [
|
|
1025
|
+
{
|
|
1026
|
+
type: "text",
|
|
1027
|
+
text: `Title hint: ${input.titleHint || "none"}\nMime type: ${input.mimeType}`
|
|
1028
|
+
},
|
|
1029
|
+
{
|
|
1030
|
+
type: "image_url",
|
|
1031
|
+
image_url: {
|
|
1032
|
+
url: `data:${input.mimeType};base64,${input.binary.toString("base64")}`,
|
|
1033
|
+
detail: "low"
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
]
|
|
1037
|
+
}
|
|
1038
|
+
]
|
|
1039
|
+
})
|
|
1040
|
+
});
|
|
1041
|
+
if (!response.ok) {
|
|
1042
|
+
throw new Error(`LLM image compilation failed: ${response.status}`);
|
|
1043
|
+
}
|
|
1044
|
+
const payload = (await response.json());
|
|
1045
|
+
const content = payload.choices?.[0]?.message?.content;
|
|
1046
|
+
if (!content) {
|
|
1047
|
+
return null;
|
|
1048
|
+
}
|
|
1049
|
+
try {
|
|
1050
|
+
const parsed = JSON.parse(content);
|
|
1051
|
+
return {
|
|
1052
|
+
title: parsed.title?.trim() || input.titleHint || "Imported image",
|
|
1053
|
+
summary: parsed.summary?.trim() || "",
|
|
1054
|
+
markdown: parsed.markdown?.trim() ||
|
|
1055
|
+
`# ${parsed.title?.trim() || input.titleHint || "Imported image"}\n\nImage imported into the wiki vault.\n`,
|
|
1056
|
+
tags: normalizeTags(parsed.tags),
|
|
1057
|
+
entityProposals: Array.isArray(parsed.entityProposals)
|
|
1058
|
+
? parsed.entityProposals.filter((entry) => entry !== null && typeof entry === "object")
|
|
1059
|
+
: [],
|
|
1060
|
+
pageUpdateSuggestions: Array.isArray(parsed.pageUpdateSuggestions)
|
|
1061
|
+
? parsed.pageUpdateSuggestions.filter((entry) => entry !== null && typeof entry === "object")
|
|
1062
|
+
: [],
|
|
1063
|
+
articleCandidates: Array.isArray(parsed.articleCandidates)
|
|
1064
|
+
? parsed.articleCandidates.filter((entry) => entry !== null && typeof entry === "object")
|
|
1065
|
+
: []
|
|
1066
|
+
};
|
|
1067
|
+
}
|
|
1068
|
+
catch {
|
|
1069
|
+
return null;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
async function compileSourceWithLlm(profile, secrets, input) {
|
|
1073
|
+
if (input.rawText.trim()) {
|
|
1074
|
+
return compileTextWithLlm(profile, secrets, {
|
|
1075
|
+
titleHint: input.titleHint,
|
|
1076
|
+
rawText: input.rawText,
|
|
1077
|
+
mimeType: input.mimeType
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
if (input.binary &&
|
|
1081
|
+
input.parseStrategy !== "text_only" &&
|
|
1082
|
+
input.mimeType.startsWith("image/")) {
|
|
1083
|
+
return compileImageWithLlm(profile, secrets, {
|
|
1084
|
+
titleHint: input.titleHint,
|
|
1085
|
+
binary: input.binary,
|
|
1086
|
+
mimeType: input.mimeType
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
return null;
|
|
1090
|
+
}
|
|
1091
|
+
async function embedTexts(profile, secrets, inputs) {
|
|
1092
|
+
const apiKey = await readSecretApiKey(profile.secretId, secrets);
|
|
1093
|
+
if (!apiKey || inputs.length === 0) {
|
|
1094
|
+
return [];
|
|
1095
|
+
}
|
|
1096
|
+
const response = await fetch(`${profile.baseUrl.replace(/\/$/, "")}/embeddings`, {
|
|
1097
|
+
method: "POST",
|
|
1098
|
+
headers: {
|
|
1099
|
+
"content-type": "application/json",
|
|
1100
|
+
authorization: `Bearer ${apiKey}`
|
|
1101
|
+
},
|
|
1102
|
+
body: JSON.stringify({
|
|
1103
|
+
model: profile.model,
|
|
1104
|
+
input: inputs,
|
|
1105
|
+
...(profile.dimensions ? { dimensions: profile.dimensions } : {})
|
|
1106
|
+
})
|
|
1107
|
+
});
|
|
1108
|
+
if (!response.ok) {
|
|
1109
|
+
throw new Error(`Embedding request failed: ${response.status}`);
|
|
1110
|
+
}
|
|
1111
|
+
const payload = (await response.json());
|
|
1112
|
+
return (payload.data?.map((entry) => Array.isArray(entry.embedding) ? entry.embedding : []) ?? []);
|
|
1113
|
+
}
|
|
1114
|
+
function findExistingSpaceByOwner(ownerUserId) {
|
|
1115
|
+
const row = getDatabase()
|
|
1116
|
+
.prepare(`SELECT id, slug, label, description, owner_user_id, visibility, created_at, updated_at
|
|
1117
|
+
FROM wiki_spaces
|
|
1118
|
+
WHERE owner_user_id = ?
|
|
1119
|
+
ORDER BY created_at ASC
|
|
1120
|
+
LIMIT 1`)
|
|
1121
|
+
.get(ownerUserId);
|
|
1122
|
+
return row ? mapWikiSpace(row) : null;
|
|
1123
|
+
}
|
|
1124
|
+
function getWikiSpaceById(spaceId) {
|
|
1125
|
+
const row = getDatabase()
|
|
1126
|
+
.prepare(`SELECT id, slug, label, description, owner_user_id, visibility, created_at, updated_at
|
|
1127
|
+
FROM wiki_spaces
|
|
1128
|
+
WHERE id = ?`)
|
|
1129
|
+
.get(spaceId);
|
|
1130
|
+
return row ? mapWikiSpace(row) : null;
|
|
1131
|
+
}
|
|
1132
|
+
function ensureSharedWikiSpace() {
|
|
1133
|
+
const existing = getWikiSpaceById("wiki_space_shared");
|
|
1134
|
+
if (existing) {
|
|
1135
|
+
ensureWikiSpaceSeedPages(existing.id);
|
|
1136
|
+
return existing;
|
|
1137
|
+
}
|
|
1138
|
+
const now = nowIso();
|
|
1139
|
+
getDatabase()
|
|
1140
|
+
.prepare(`INSERT INTO wiki_spaces (id, slug, label, description, owner_user_id, visibility, created_at, updated_at)
|
|
1141
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1142
|
+
.run("wiki_space_shared", "shared", "Shared Forge Memory", "Shared wiki space for file-backed Forge knowledge.", null, "shared", now, now);
|
|
1143
|
+
const space = getWikiSpaceById("wiki_space_shared");
|
|
1144
|
+
ensureWikiSpaceSeedPages(space.id);
|
|
1145
|
+
return space;
|
|
1146
|
+
}
|
|
1147
|
+
function ensurePersonalWikiSpace(userId) {
|
|
1148
|
+
const existing = findExistingSpaceByOwner(userId);
|
|
1149
|
+
if (existing) {
|
|
1150
|
+
ensureWikiSpaceSeedPages(existing.id);
|
|
1151
|
+
return existing;
|
|
1152
|
+
}
|
|
1153
|
+
const now = nowIso();
|
|
1154
|
+
const id = `wiki_space_user_${slugify(userId)}`;
|
|
1155
|
+
const slug = `user-${slugify(userId)}`;
|
|
1156
|
+
getDatabase()
|
|
1157
|
+
.prepare(`INSERT INTO wiki_spaces (id, slug, label, description, owner_user_id, visibility, created_at, updated_at)
|
|
1158
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1159
|
+
.run(id, slug, `${userId} Wiki`, "Personal Forge wiki space.", userId, "personal", now, now);
|
|
1160
|
+
const space = getWikiSpaceById(id);
|
|
1161
|
+
ensureWikiSpaceSeedPages(space.id);
|
|
1162
|
+
return space;
|
|
1163
|
+
}
|
|
1164
|
+
function buildStarterPageMarkdown(page, space) {
|
|
1165
|
+
if (page.slug === "index") {
|
|
1166
|
+
return [
|
|
1167
|
+
`# ${space.label}`,
|
|
1168
|
+
"",
|
|
1169
|
+
"This wiki is the explicit memory surface for Forge.",
|
|
1170
|
+
"",
|
|
1171
|
+
"Use it to maintain durable context, connect pages to Forge entities, and keep knowledge readable for both humans and agents.",
|
|
1172
|
+
"",
|
|
1173
|
+
"## Starting Points",
|
|
1174
|
+
"",
|
|
1175
|
+
"- [[people]]",
|
|
1176
|
+
"- [[projects]]",
|
|
1177
|
+
"- [[concepts]]",
|
|
1178
|
+
"- [[sources]]",
|
|
1179
|
+
"- [[chronicle]]",
|
|
1180
|
+
""
|
|
1181
|
+
].join("\n");
|
|
1182
|
+
}
|
|
1183
|
+
return [
|
|
1184
|
+
`# ${page.title}`,
|
|
1185
|
+
"",
|
|
1186
|
+
page.summary,
|
|
1187
|
+
"",
|
|
1188
|
+
`Return to [[index|Home]].`,
|
|
1189
|
+
""
|
|
1190
|
+
].join("\n");
|
|
1191
|
+
}
|
|
1192
|
+
function insertSeedNote(space, seed) {
|
|
1193
|
+
const now = nowIso();
|
|
1194
|
+
const noteId = `note_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
1195
|
+
const markdown = buildStarterPageMarkdown(seed, space);
|
|
1196
|
+
const contentPlain = buildContentPlain(markdown);
|
|
1197
|
+
const note = persistedNoteSchema.parse({
|
|
1198
|
+
id: noteId,
|
|
1199
|
+
kind: "wiki",
|
|
1200
|
+
title: seed.title,
|
|
1201
|
+
slug: seed.slug,
|
|
1202
|
+
spaceId: space.id,
|
|
1203
|
+
parentSlug: seed.parentSlug,
|
|
1204
|
+
indexOrder: seed.indexOrder,
|
|
1205
|
+
showInIndex: true,
|
|
1206
|
+
aliases: [],
|
|
1207
|
+
summary: seed.summary,
|
|
1208
|
+
contentMarkdown: markdown,
|
|
1209
|
+
contentPlain,
|
|
1210
|
+
author: null,
|
|
1211
|
+
source: "system",
|
|
1212
|
+
sourcePath: "",
|
|
1213
|
+
frontmatter: {},
|
|
1214
|
+
revisionHash: "",
|
|
1215
|
+
lastSyncedAt: null,
|
|
1216
|
+
createdAt: now,
|
|
1217
|
+
updatedAt: now,
|
|
1218
|
+
links: [],
|
|
1219
|
+
tags: []
|
|
1220
|
+
});
|
|
1221
|
+
getDatabase()
|
|
1222
|
+
.prepare(`INSERT INTO notes (
|
|
1223
|
+
id, kind, title, slug, space_id, parent_slug, index_order, show_in_index, aliases_json, summary, content_markdown, content_plain, author, source, tags_json, destroy_at,
|
|
1224
|
+
source_path, frontmatter_json, revision_hash, last_synced_at, created_at, updated_at
|
|
1225
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1226
|
+
.run(note.id, note.kind, note.title, note.slug, note.spaceId, note.parentSlug, note.indexOrder, note.showInIndex ? 1 : 0, JSON.stringify(note.aliases), note.summary, note.contentMarkdown, note.contentPlain, note.author, note.source, JSON.stringify(note.tags), null, "", "{}", "", null, now, now);
|
|
1227
|
+
return note;
|
|
1228
|
+
}
|
|
1229
|
+
function ensureWikiSpaceSeedPages(spaceId) {
|
|
1230
|
+
const space = getWikiSpaceById(spaceId);
|
|
1231
|
+
if (!space) {
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
const existingSlugs = new Set(getNoteRows("WHERE space_id = ?", [spaceId]).map((row) => row.slug.toLowerCase()));
|
|
1235
|
+
let inserted = false;
|
|
1236
|
+
const insertedNotes = [];
|
|
1237
|
+
for (const seed of WIKI_STARTER_PAGES) {
|
|
1238
|
+
if (existingSlugs.has(seed.slug)) {
|
|
1239
|
+
continue;
|
|
1240
|
+
}
|
|
1241
|
+
insertedNotes.push(insertSeedNote(space, seed));
|
|
1242
|
+
existingSlugs.add(seed.slug);
|
|
1243
|
+
inserted = true;
|
|
1244
|
+
}
|
|
1245
|
+
if (inserted) {
|
|
1246
|
+
for (const note of insertedNotes) {
|
|
1247
|
+
syncNoteWikiArtifacts(note);
|
|
1248
|
+
}
|
|
1249
|
+
syncWikiSpaceIndex(spaceId);
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
function resolveSpaceId(spaceId, userId) {
|
|
1253
|
+
if (spaceId?.trim()) {
|
|
1254
|
+
const existing = getWikiSpaceById(spaceId.trim());
|
|
1255
|
+
if (existing) {
|
|
1256
|
+
return existing.id;
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
if (userId?.trim()) {
|
|
1260
|
+
return ensurePersonalWikiSpace(userId.trim()).id;
|
|
1261
|
+
}
|
|
1262
|
+
return ensureSharedWikiSpace().id;
|
|
1263
|
+
}
|
|
1264
|
+
export function prepareNoteWikiFields(input) {
|
|
1265
|
+
const kind = input.kind ?? input.existing?.kind ?? "evidence";
|
|
1266
|
+
const spaceId = resolveSpaceId(input.spaceId ?? input.existing?.spaceId, input.userId);
|
|
1267
|
+
const title = input.title?.trim() ||
|
|
1268
|
+
input.existing?.title?.trim() ||
|
|
1269
|
+
inferTitle(input.contentMarkdown, kind === "wiki" ? "Untitled wiki page" : "Untitled note");
|
|
1270
|
+
const slug = buildUniqueSlug(spaceId, input.slug?.trim() || input.existing?.slug || title, input.id);
|
|
1271
|
+
return {
|
|
1272
|
+
kind,
|
|
1273
|
+
title,
|
|
1274
|
+
slug,
|
|
1275
|
+
spaceId,
|
|
1276
|
+
parentSlug: input.parentSlug === undefined
|
|
1277
|
+
? (input.existing?.parentSlug ?? null)
|
|
1278
|
+
: input.parentSlug?.trim() || null,
|
|
1279
|
+
indexOrder: input.indexOrder ?? input.existing?.indexOrder ?? 0,
|
|
1280
|
+
showInIndex: input.showInIndex ?? input.existing?.showInIndex ?? kind === "wiki",
|
|
1281
|
+
aliases: normalizeAliases(input.aliases ?? input.existing?.aliases),
|
|
1282
|
+
summary: input.summary?.trim() ||
|
|
1283
|
+
input.existing?.summary?.trim() ||
|
|
1284
|
+
inferSummary(input.contentMarkdown)
|
|
1285
|
+
};
|
|
1286
|
+
}
|
|
1287
|
+
export function syncNoteWikiArtifacts(note) {
|
|
1288
|
+
const space = getWikiSpaceById(note.spaceId) ?? ensureSharedWikiSpace();
|
|
1289
|
+
const filePath = getNoteStoragePath(note, space);
|
|
1290
|
+
const frontmatter = buildNoteFrontmatter(note);
|
|
1291
|
+
const payload = `${renderFrontmatter(frontmatter)}${note.contentMarkdown.trim()}\n`;
|
|
1292
|
+
const revisionHash = hashContent(payload);
|
|
1293
|
+
mkdirSync(path.dirname(filePath), { recursive: true });
|
|
1294
|
+
if (note.sourcePath && note.sourcePath !== filePath) {
|
|
1295
|
+
if (existsSync(note.sourcePath)) {
|
|
1296
|
+
rmSync(note.sourcePath, { force: true });
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
writeFileSync(filePath, payload, "utf8");
|
|
1300
|
+
const now = nowIso();
|
|
1301
|
+
getDatabase()
|
|
1302
|
+
.prepare(`UPDATE notes
|
|
1303
|
+
SET source_path = ?, frontmatter_json = ?, revision_hash = ?, last_synced_at = ?
|
|
1304
|
+
WHERE id = ?`)
|
|
1305
|
+
.run(filePath, JSON.stringify(frontmatter), revisionHash, now, note.id);
|
|
1306
|
+
upsertWikiSearchRow({
|
|
1307
|
+
...note,
|
|
1308
|
+
sourcePath: filePath,
|
|
1309
|
+
frontmatter,
|
|
1310
|
+
revisionHash,
|
|
1311
|
+
lastSyncedAt: now
|
|
1312
|
+
});
|
|
1313
|
+
rebuildWikiLinkEdges({
|
|
1314
|
+
...note,
|
|
1315
|
+
sourcePath: filePath,
|
|
1316
|
+
frontmatter,
|
|
1317
|
+
revisionHash,
|
|
1318
|
+
lastSyncedAt: now
|
|
1319
|
+
});
|
|
1320
|
+
syncWikiSpaceIndex(space.id);
|
|
1321
|
+
}
|
|
1322
|
+
export function deleteNoteWikiArtifacts(note) {
|
|
1323
|
+
if (note.sourcePath && existsSync(note.sourcePath)) {
|
|
1324
|
+
rmSync(note.sourcePath, { force: true });
|
|
1325
|
+
}
|
|
1326
|
+
deleteWikiSearchRow(note.id);
|
|
1327
|
+
getDatabase()
|
|
1328
|
+
.prepare(`DELETE FROM wiki_link_edges WHERE source_note_id = ?`)
|
|
1329
|
+
.run(note.id);
|
|
1330
|
+
getDatabase()
|
|
1331
|
+
.prepare(`DELETE FROM wiki_embedding_chunks WHERE note_id = ?`)
|
|
1332
|
+
.run(note.id);
|
|
1333
|
+
getDatabase()
|
|
1334
|
+
.prepare(`DELETE FROM wiki_media_assets WHERE note_id = ? OR transcript_note_id = ?`)
|
|
1335
|
+
.run(note.id, note.id);
|
|
1336
|
+
syncWikiSpaceIndex(note.spaceId);
|
|
1337
|
+
}
|
|
1338
|
+
function buildWikiIndexMarkdown(space, pages) {
|
|
1339
|
+
const wikiPages = [...pages]
|
|
1340
|
+
.filter((page) => page.kind === "wiki")
|
|
1341
|
+
.sort((left, right) => left.parentSlug === right.parentSlug
|
|
1342
|
+
? left.indexOrder - right.indexOrder ||
|
|
1343
|
+
left.title.localeCompare(right.title)
|
|
1344
|
+
: (left.parentSlug ?? "").localeCompare(right.parentSlug ?? ""));
|
|
1345
|
+
const evidencePages = [...pages]
|
|
1346
|
+
.filter((page) => page.kind === "evidence")
|
|
1347
|
+
.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
|
|
1348
|
+
const lines = [
|
|
1349
|
+
`# ${space.label}`,
|
|
1350
|
+
"",
|
|
1351
|
+
"Explicit Forge wiki index generated from the local vault.",
|
|
1352
|
+
"",
|
|
1353
|
+
"## How To Use",
|
|
1354
|
+
"",
|
|
1355
|
+
"- Start here when an agent needs a crawlable catalog of the space.",
|
|
1356
|
+
"- `pages/` contains durable wiki articles.",
|
|
1357
|
+
"- `evidence/` contains shorter notes and work traces.",
|
|
1358
|
+
"- `raw/` contains imported source material for future recompilation.",
|
|
1359
|
+
"",
|
|
1360
|
+
`Generated at ${nowIso()}.`,
|
|
1361
|
+
"",
|
|
1362
|
+
"## Wiki Index",
|
|
1363
|
+
""
|
|
1364
|
+
];
|
|
1365
|
+
if (wikiPages.length === 0) {
|
|
1366
|
+
lines.push("_No wiki pages yet._", "");
|
|
1367
|
+
}
|
|
1368
|
+
else {
|
|
1369
|
+
for (const page of wikiPages) {
|
|
1370
|
+
const depth = page.parentSlug ? 1 : 0;
|
|
1371
|
+
const prefix = `${" ".repeat(depth)}- `;
|
|
1372
|
+
lines.push(`${prefix}[[${page.slug}]]${page.summary ? ` - ${page.summary}` : ""}`);
|
|
1373
|
+
}
|
|
1374
|
+
lines.push("");
|
|
1375
|
+
}
|
|
1376
|
+
lines.push("## Evidence Pages", "");
|
|
1377
|
+
if (evidencePages.length === 0) {
|
|
1378
|
+
lines.push("_No evidence pages yet._", "");
|
|
1379
|
+
}
|
|
1380
|
+
else {
|
|
1381
|
+
for (const page of evidencePages.slice(0, 200)) {
|
|
1382
|
+
lines.push(`- [[${page.slug}]]${page.summary ? ` - ${page.summary}` : ""}`);
|
|
1383
|
+
}
|
|
1384
|
+
lines.push("");
|
|
1385
|
+
}
|
|
1386
|
+
return `${lines.join("\n")}\n`;
|
|
1387
|
+
}
|
|
1388
|
+
function syncWikiSpaceIndex(spaceId) {
|
|
1389
|
+
const space = getWikiSpaceById(spaceId) ?? ensureSharedWikiSpace();
|
|
1390
|
+
const rootDir = getSpaceStorageDir(space);
|
|
1391
|
+
mkdirSync(path.join(rootDir, "pages"), { recursive: true });
|
|
1392
|
+
mkdirSync(path.join(rootDir, "evidence"), { recursive: true });
|
|
1393
|
+
mkdirSync(path.join(rootDir, "assets"), { recursive: true });
|
|
1394
|
+
mkdirSync(path.join(rootDir, "raw"), { recursive: true });
|
|
1395
|
+
writeFileSync(getSpaceIndexPath(space), buildWikiIndexMarkdown(space, listWikiPages({ spaceId, limit: 10_000 })), "utf8");
|
|
1396
|
+
}
|
|
1397
|
+
function upsertWikiSearchRow(note) {
|
|
1398
|
+
deleteWikiSearchRow(note.id);
|
|
1399
|
+
getDatabase()
|
|
1400
|
+
.prepare(`INSERT INTO wiki_pages_fts (note_id, title, slug, aliases, summary, content_plain, linked_entities)
|
|
1401
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
1402
|
+
.run(note.id, note.title, note.slug, JSON.stringify(note.aliases ?? []), note.summary ?? "", note.contentPlain, buildLinkedEntityTokens(note));
|
|
1403
|
+
}
|
|
1404
|
+
function rebuildWikiLinkEdges(note) {
|
|
1405
|
+
const now = nowIso();
|
|
1406
|
+
getDatabase()
|
|
1407
|
+
.prepare(`DELETE FROM wiki_link_edges WHERE source_note_id = ?`)
|
|
1408
|
+
.run(note.id);
|
|
1409
|
+
const matches = [...note.contentMarkdown.matchAll(/(!)?\[\[([^[\]]+)\]\]/g)];
|
|
1410
|
+
const insert = getDatabase().prepare(`INSERT INTO wiki_link_edges (
|
|
1411
|
+
source_note_id, target_type, target_note_id, target_entity_type, target_entity_id, label, raw_target, is_embed, created_at, updated_at
|
|
1412
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
1413
|
+
for (const match of matches) {
|
|
1414
|
+
const isEmbed = Boolean(match[1]);
|
|
1415
|
+
const token = (match[2] ?? "").trim();
|
|
1416
|
+
if (!token) {
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
const [left, right] = token.split("|");
|
|
1420
|
+
const label = right?.trim() || left.trim();
|
|
1421
|
+
if (left.startsWith("forge:")) {
|
|
1422
|
+
const parts = left.split(":");
|
|
1423
|
+
const entityType = parts[1];
|
|
1424
|
+
const entityId = parts.slice(2).join(":");
|
|
1425
|
+
const parsedEntityType = crudEntityTypeSchema.safeParse(entityType);
|
|
1426
|
+
if (parsedEntityType.success && entityId.trim()) {
|
|
1427
|
+
insert.run(note.id, "entity", null, parsedEntityType.data, entityId.trim(), label, left, isEmbed ? 1 : 0, now, now);
|
|
1428
|
+
continue;
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
const targetNote = getNoteBySlugRaw(note.spaceId, left.trim(), note.id);
|
|
1432
|
+
if (targetNote) {
|
|
1433
|
+
insert.run(note.id, "page", targetNote.id, null, null, label, left.trim(), isEmbed ? 1 : 0, now, now);
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
insert.run(note.id, "unresolved", null, null, null, label, left.trim(), isEmbed ? 1 : 0, now, now);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
function loadNotesByIds(noteIds) {
|
|
1440
|
+
if (noteIds.length === 0) {
|
|
1441
|
+
return [];
|
|
1442
|
+
}
|
|
1443
|
+
const placeholders = noteIds.map(() => "?").join(", ");
|
|
1444
|
+
const rows = getDatabase()
|
|
1445
|
+
.prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
|
|
1446
|
+
tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
|
|
1447
|
+
FROM notes
|
|
1448
|
+
WHERE id IN (${placeholders})`)
|
|
1449
|
+
.all(...noteIds);
|
|
1450
|
+
const links = listLinkRowsForNotes(noteIds);
|
|
1451
|
+
const linksByNoteId = new Map();
|
|
1452
|
+
for (const link of links) {
|
|
1453
|
+
const current = linksByNoteId.get(link.note_id) ?? [];
|
|
1454
|
+
current.push(link);
|
|
1455
|
+
linksByNoteId.set(link.note_id, current);
|
|
1456
|
+
}
|
|
1457
|
+
return rows.map((row) => mapNoteRow(row, linksByNoteId.get(row.id) ?? []));
|
|
1458
|
+
}
|
|
1459
|
+
function listAllNotes() {
|
|
1460
|
+
const rows = getNoteRows();
|
|
1461
|
+
const links = listLinkRowsForNotes(rows.map((row) => row.id));
|
|
1462
|
+
const linksByNoteId = new Map();
|
|
1463
|
+
for (const link of links) {
|
|
1464
|
+
const current = linksByNoteId.get(link.note_id) ?? [];
|
|
1465
|
+
current.push(link);
|
|
1466
|
+
linksByNoteId.set(link.note_id, current);
|
|
1467
|
+
}
|
|
1468
|
+
return rows.map((row) => mapNoteRow(row, linksByNoteId.get(row.id) ?? []));
|
|
1469
|
+
}
|
|
1470
|
+
export function listWikiSpaces() {
|
|
1471
|
+
ensureSharedWikiSpace();
|
|
1472
|
+
const rows = getDatabase()
|
|
1473
|
+
.prepare(`SELECT id, slug, label, description, owner_user_id, visibility, created_at, updated_at
|
|
1474
|
+
FROM wiki_spaces
|
|
1475
|
+
ORDER BY visibility ASC, updated_at DESC`)
|
|
1476
|
+
.all();
|
|
1477
|
+
const spaces = rows.map(mapWikiSpace);
|
|
1478
|
+
for (const space of spaces) {
|
|
1479
|
+
ensureWikiSpaceSeedPages(space.id);
|
|
1480
|
+
}
|
|
1481
|
+
return spaces;
|
|
1482
|
+
}
|
|
1483
|
+
export function createWikiSpace(input) {
|
|
1484
|
+
const parsed = createWikiSpaceSchema.parse(input);
|
|
1485
|
+
const now = nowIso();
|
|
1486
|
+
const id = `wiki_space_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
1487
|
+
const slug = slugify(parsed.slug || parsed.label);
|
|
1488
|
+
getDatabase()
|
|
1489
|
+
.prepare(`INSERT INTO wiki_spaces (id, slug, label, description, owner_user_id, visibility, created_at, updated_at)
|
|
1490
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1491
|
+
.run(id, slug, parsed.label, parsed.description, parsed.ownerUserId ?? null, parsed.visibility, now, now);
|
|
1492
|
+
ensureWikiSpaceSeedPages(id);
|
|
1493
|
+
return getWikiSpaceById(id);
|
|
1494
|
+
}
|
|
1495
|
+
function compareWikiPageOrder(left, right) {
|
|
1496
|
+
if ((left.parentSlug ?? "") !== (right.parentSlug ?? "")) {
|
|
1497
|
+
return (left.parentSlug ?? "").localeCompare(right.parentSlug ?? "");
|
|
1498
|
+
}
|
|
1499
|
+
if (left.indexOrder !== right.indexOrder) {
|
|
1500
|
+
return left.indexOrder - right.indexOrder;
|
|
1501
|
+
}
|
|
1502
|
+
return left.title.localeCompare(right.title);
|
|
1503
|
+
}
|
|
1504
|
+
export function listWikiPages(query) {
|
|
1505
|
+
const spaceId = resolveSpaceId(query.spaceId, null);
|
|
1506
|
+
ensureWikiSpaceSeedPages(spaceId);
|
|
1507
|
+
return listAllNotes()
|
|
1508
|
+
.filter((note) => note.spaceId === spaceId)
|
|
1509
|
+
.filter((note) => (query.kind ? note.kind === query.kind : true))
|
|
1510
|
+
.sort(compareWikiPageOrder)
|
|
1511
|
+
.slice(0, query.limit ?? 100);
|
|
1512
|
+
}
|
|
1513
|
+
export function listWikiPageTree(query) {
|
|
1514
|
+
const pages = listWikiPages({ ...query, limit: 10_000 }).filter((page) => page.kind === "wiki" && page.showInIndex);
|
|
1515
|
+
const childrenByParent = new Map();
|
|
1516
|
+
for (const page of pages) {
|
|
1517
|
+
const key = page.parentSlug ?? null;
|
|
1518
|
+
const current = childrenByParent.get(key) ?? [];
|
|
1519
|
+
current.push(page);
|
|
1520
|
+
childrenByParent.set(key, current);
|
|
1521
|
+
}
|
|
1522
|
+
const build = (parentSlug) => (childrenByParent.get(parentSlug) ?? [])
|
|
1523
|
+
.sort(compareWikiPageOrder)
|
|
1524
|
+
.map((page) => ({
|
|
1525
|
+
page,
|
|
1526
|
+
children: build(page.slug)
|
|
1527
|
+
}));
|
|
1528
|
+
return z.array(wikiPageTreeNodeSchema).parse(build(null));
|
|
1529
|
+
}
|
|
1530
|
+
export function getWikiHomePageDetail(input = {}) {
|
|
1531
|
+
const spaceId = resolveSpaceId(input.spaceId, null);
|
|
1532
|
+
ensureWikiSpaceSeedPages(spaceId);
|
|
1533
|
+
const home = getNoteBySlugRaw(spaceId, "index");
|
|
1534
|
+
if (!home) {
|
|
1535
|
+
return null;
|
|
1536
|
+
}
|
|
1537
|
+
return getWikiPageDetail(home.id);
|
|
1538
|
+
}
|
|
1539
|
+
export function getWikiPageDetailBySlug(input) {
|
|
1540
|
+
const spaceId = resolveSpaceId(input.spaceId, null);
|
|
1541
|
+
ensureWikiSpaceSeedPages(spaceId);
|
|
1542
|
+
const row = getNoteBySlugRaw(spaceId, input.slug.trim());
|
|
1543
|
+
if (!row) {
|
|
1544
|
+
return null;
|
|
1545
|
+
}
|
|
1546
|
+
return getWikiPageDetail(row.id);
|
|
1547
|
+
}
|
|
1548
|
+
export function getWikiPageDetail(noteId) {
|
|
1549
|
+
const row = getNoteByIdRaw(noteId);
|
|
1550
|
+
if (!row) {
|
|
1551
|
+
return null;
|
|
1552
|
+
}
|
|
1553
|
+
const note = mapNoteRow(row, listLinkRowsForNotes([row.id]));
|
|
1554
|
+
const backlinkRows = getDatabase()
|
|
1555
|
+
.prepare(`SELECT source_note_id, target_type, target_note_id, target_entity_type, target_entity_id, label, raw_target, is_embed, created_at, updated_at
|
|
1556
|
+
FROM wiki_link_edges
|
|
1557
|
+
WHERE target_note_id = ?
|
|
1558
|
+
ORDER BY updated_at DESC`)
|
|
1559
|
+
.all(noteId);
|
|
1560
|
+
const assets = getDatabase()
|
|
1561
|
+
.prepare(`SELECT id, space_id, note_id, label, mime_type, file_name, file_path, size_bytes, checksum, transcript_note_id, metadata_json, created_at, updated_at
|
|
1562
|
+
FROM wiki_media_assets
|
|
1563
|
+
WHERE note_id = ? OR transcript_note_id = ?
|
|
1564
|
+
ORDER BY updated_at DESC`)
|
|
1565
|
+
.all(noteId, noteId);
|
|
1566
|
+
const backlinkSourceNotes = loadNotesByIds(Array.from(new Set(backlinkRows.map((row) => row.source_note_id))));
|
|
1567
|
+
const backlinkSourceById = new Map(backlinkSourceNotes.map((entry) => [entry.id, entry]));
|
|
1568
|
+
return {
|
|
1569
|
+
page: note,
|
|
1570
|
+
backlinks: backlinkRows.map((row) => wikiLinkEdgeSchema.parse({
|
|
1571
|
+
sourceNoteId: row.source_note_id,
|
|
1572
|
+
targetType: row.target_type,
|
|
1573
|
+
targetNoteId: row.target_note_id,
|
|
1574
|
+
targetEntityType: row.target_entity_type,
|
|
1575
|
+
targetEntityId: row.target_entity_id,
|
|
1576
|
+
label: row.label,
|
|
1577
|
+
rawTarget: row.raw_target,
|
|
1578
|
+
isEmbed: row.is_embed === 1,
|
|
1579
|
+
createdAt: row.created_at,
|
|
1580
|
+
updatedAt: row.updated_at
|
|
1581
|
+
})),
|
|
1582
|
+
backlinkSourceNotes,
|
|
1583
|
+
assets: assets.map((row) => wikiMediaAssetSchema.parse({
|
|
1584
|
+
id: row.id,
|
|
1585
|
+
spaceId: row.space_id,
|
|
1586
|
+
noteId: row.note_id,
|
|
1587
|
+
label: row.label,
|
|
1588
|
+
mimeType: row.mime_type,
|
|
1589
|
+
fileName: row.file_name,
|
|
1590
|
+
filePath: row.file_path,
|
|
1591
|
+
sizeBytes: row.size_bytes,
|
|
1592
|
+
checksum: row.checksum,
|
|
1593
|
+
transcriptNoteId: row.transcript_note_id,
|
|
1594
|
+
metadata: parseJsonRecord(row.metadata_json),
|
|
1595
|
+
createdAt: row.created_at,
|
|
1596
|
+
updatedAt: row.updated_at
|
|
1597
|
+
})),
|
|
1598
|
+
backlinksBySourceId: Object.fromEntries(backlinkRows.map((row) => [
|
|
1599
|
+
row.source_note_id,
|
|
1600
|
+
backlinkSourceById.get(row.source_note_id) ?? null
|
|
1601
|
+
]))
|
|
1602
|
+
};
|
|
1603
|
+
}
|
|
1604
|
+
export async function syncWikiVaultFromDisk(input) {
|
|
1605
|
+
const parsed = syncWikiVaultSchema.parse(input);
|
|
1606
|
+
const spaces = parsed.spaceId
|
|
1607
|
+
? [getWikiSpaceById(parsed.spaceId)].filter((entry) => entry !== null)
|
|
1608
|
+
: listWikiSpaces();
|
|
1609
|
+
let updated = 0;
|
|
1610
|
+
for (const space of spaces) {
|
|
1611
|
+
for (const directoryName of ["pages", "evidence"]) {
|
|
1612
|
+
const directory = path.join(getSpaceStorageDir(space), directoryName);
|
|
1613
|
+
try {
|
|
1614
|
+
const entries = await readdir(directory);
|
|
1615
|
+
for (const entry of entries) {
|
|
1616
|
+
if (!entry.endsWith(".md")) {
|
|
1617
|
+
continue;
|
|
1618
|
+
}
|
|
1619
|
+
const filePath = path.join(directory, entry);
|
|
1620
|
+
const content = await readFile(filePath, "utf8");
|
|
1621
|
+
const parsedFile = parseFrontmatter(content);
|
|
1622
|
+
const noteId = typeof parsedFile.frontmatter.id === "string"
|
|
1623
|
+
? parsedFile.frontmatter.id
|
|
1624
|
+
: null;
|
|
1625
|
+
if (!noteId) {
|
|
1626
|
+
continue;
|
|
1627
|
+
}
|
|
1628
|
+
const existing = getNoteByIdRaw(noteId);
|
|
1629
|
+
if (!existing) {
|
|
1630
|
+
continue;
|
|
1631
|
+
}
|
|
1632
|
+
const markdown = parsedFile.body.trim();
|
|
1633
|
+
const contentPlain = buildContentPlain(markdown);
|
|
1634
|
+
const title = typeof parsedFile.frontmatter.title === "string"
|
|
1635
|
+
? parsedFile.frontmatter.title
|
|
1636
|
+
: inferTitle(markdown, existing.title);
|
|
1637
|
+
const aliases = normalizeAliases(Array.isArray(parsedFile.frontmatter.aliases)
|
|
1638
|
+
? parsedFile.frontmatter.aliases.filter((entry) => typeof entry === "string")
|
|
1639
|
+
: []);
|
|
1640
|
+
const summary = typeof parsedFile.frontmatter.summary === "string"
|
|
1641
|
+
? parsedFile.frontmatter.summary
|
|
1642
|
+
: inferSummary(markdown);
|
|
1643
|
+
const payload = `${renderFrontmatter(parsedFile.frontmatter)}${markdown}\n`;
|
|
1644
|
+
const revisionHash = hashContent(payload);
|
|
1645
|
+
const now = nowIso();
|
|
1646
|
+
getDatabase()
|
|
1647
|
+
.prepare(`UPDATE notes
|
|
1648
|
+
SET title = ?, slug = ?, kind = ?, space_id = ?, parent_slug = ?, index_order = ?, show_in_index = ?, aliases_json = ?, summary = ?, content_markdown = ?, content_plain = ?,
|
|
1649
|
+
source_path = ?, frontmatter_json = ?, revision_hash = ?, last_synced_at = ?, updated_at = ?
|
|
1650
|
+
WHERE id = ?`)
|
|
1651
|
+
.run(title, typeof parsedFile.frontmatter.slug === "string"
|
|
1652
|
+
? parsedFile.frontmatter.slug
|
|
1653
|
+
: existing.slug, directoryName === "pages" ? "wiki" : "evidence", space.id, typeof parsedFile.frontmatter.parentSlug === "string"
|
|
1654
|
+
? parsedFile.frontmatter.parentSlug
|
|
1655
|
+
: existing.parent_slug, typeof parsedFile.frontmatter.indexOrder === "number"
|
|
1656
|
+
? Math.trunc(parsedFile.frontmatter.indexOrder)
|
|
1657
|
+
: existing.index_order, parsedFile.frontmatter.showInIndex === false ? 0 : 1, JSON.stringify(aliases), summary, markdown, contentPlain, filePath, JSON.stringify(parsedFile.frontmatter), revisionHash, now, now, noteId);
|
|
1658
|
+
const note = mapNoteRow(getNoteByIdRaw(noteId), listLinkRowsForNotes([noteId]));
|
|
1659
|
+
upsertWikiSearchRow(note);
|
|
1660
|
+
rebuildWikiLinkEdges(note);
|
|
1661
|
+
updated += 1;
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
catch {
|
|
1665
|
+
continue;
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
syncWikiSpaceIndex(space.id);
|
|
1669
|
+
}
|
|
1670
|
+
return { updated };
|
|
1671
|
+
}
|
|
1672
|
+
function parseFrontmatter(markdown) {
|
|
1673
|
+
const match = markdown.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
|
1674
|
+
if (!match) {
|
|
1675
|
+
return { frontmatter: {}, body: markdown };
|
|
1676
|
+
}
|
|
1677
|
+
const frontmatter = {};
|
|
1678
|
+
for (const line of match[1].split("\n")) {
|
|
1679
|
+
const separatorIndex = line.indexOf(":");
|
|
1680
|
+
if (separatorIndex <= 0) {
|
|
1681
|
+
continue;
|
|
1682
|
+
}
|
|
1683
|
+
const key = line.slice(0, separatorIndex).trim();
|
|
1684
|
+
const rawValue = line.slice(separatorIndex + 1).trim();
|
|
1685
|
+
if (!key) {
|
|
1686
|
+
continue;
|
|
1687
|
+
}
|
|
1688
|
+
try {
|
|
1689
|
+
frontmatter[key] = JSON.parse(rawValue);
|
|
1690
|
+
}
|
|
1691
|
+
catch {
|
|
1692
|
+
frontmatter[key] = rawValue.replace(/^"(.*)"$/, "$1");
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
return { frontmatter, body: match[2] };
|
|
1696
|
+
}
|
|
1697
|
+
function findMatchingWikiNoteIds(query) {
|
|
1698
|
+
const ftsQuery = buildWikiFtsQuery(query);
|
|
1699
|
+
if (!ftsQuery) {
|
|
1700
|
+
return new Set();
|
|
1701
|
+
}
|
|
1702
|
+
const rows = getDatabase()
|
|
1703
|
+
.prepare(`SELECT note_id FROM wiki_pages_fts WHERE wiki_pages_fts MATCH ?`)
|
|
1704
|
+
.all(ftsQuery);
|
|
1705
|
+
return new Set(rows.map((row) => row.note_id));
|
|
1706
|
+
}
|
|
1707
|
+
export async function searchWikiPages(input, secrets) {
|
|
1708
|
+
const parsed = wikiSearchQuerySchema.parse(input);
|
|
1709
|
+
const pages = listAllNotes()
|
|
1710
|
+
.filter((page) => (parsed.spaceId ? page.spaceId === parsed.spaceId : true))
|
|
1711
|
+
.filter((page) => (parsed.kind ? page.kind === parsed.kind : true));
|
|
1712
|
+
const scores = new Map();
|
|
1713
|
+
const addScore = (noteId, value) => {
|
|
1714
|
+
scores.set(noteId, (scores.get(noteId) ?? 0) + value);
|
|
1715
|
+
};
|
|
1716
|
+
if (parsed.mode === "text" ||
|
|
1717
|
+
parsed.mode === "hybrid" ||
|
|
1718
|
+
parsed.mode === "entity") {
|
|
1719
|
+
if (parsed.query) {
|
|
1720
|
+
for (const noteId of findMatchingWikiNoteIds(parsed.query)) {
|
|
1721
|
+
addScore(noteId, 4);
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
if (parsed.linkedEntity) {
|
|
1726
|
+
for (const page of pages) {
|
|
1727
|
+
if (page.links.some((link) => link.entityType === parsed.linkedEntity?.entityType &&
|
|
1728
|
+
link.entityId === parsed.linkedEntity?.entityId)) {
|
|
1729
|
+
addScore(page.id, 6);
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
if (secrets &&
|
|
1734
|
+
parsed.query &&
|
|
1735
|
+
(parsed.mode === "semantic" || parsed.mode === "hybrid")) {
|
|
1736
|
+
const profile = listWikiEmbeddingProfiles().find((entry) => entry.enabled && (!parsed.profileId || entry.id === parsed.profileId)) ?? null;
|
|
1737
|
+
if (profile) {
|
|
1738
|
+
const [queryVector] = await embedTexts(profile, secrets, [parsed.query]);
|
|
1739
|
+
if (queryVector && queryVector.length > 0) {
|
|
1740
|
+
const chunkRows = getDatabase()
|
|
1741
|
+
.prepare(`SELECT note_id, vector_json
|
|
1742
|
+
FROM wiki_embedding_chunks
|
|
1743
|
+
WHERE profile_id = ?
|
|
1744
|
+
${parsed.spaceId ? "AND space_id = ?" : ""}
|
|
1745
|
+
ORDER BY updated_at DESC`)
|
|
1746
|
+
.all(...(parsed.spaceId ? [profile.id, parsed.spaceId] : [profile.id]));
|
|
1747
|
+
for (const row of chunkRows) {
|
|
1748
|
+
try {
|
|
1749
|
+
const score = cosineSimilarity(JSON.parse(row.vector_json), queryVector);
|
|
1750
|
+
if (score > 0) {
|
|
1751
|
+
addScore(row.note_id, score * 5);
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
catch {
|
|
1755
|
+
continue;
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
if (parsed.query) {
|
|
1762
|
+
const normalizedQuery = parsed.query.toLowerCase();
|
|
1763
|
+
for (const page of pages) {
|
|
1764
|
+
if (page.slug.toLowerCase() === normalizedQuery) {
|
|
1765
|
+
addScore(page.id, 12);
|
|
1766
|
+
}
|
|
1767
|
+
else if (page.title.toLowerCase() === normalizedQuery) {
|
|
1768
|
+
addScore(page.id, 10);
|
|
1769
|
+
}
|
|
1770
|
+
else if (page.title.toLowerCase().includes(normalizedQuery)) {
|
|
1771
|
+
addScore(page.id, 2);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
const ranked = [...pages]
|
|
1776
|
+
.filter((page) => {
|
|
1777
|
+
if (!parsed.query && !parsed.linkedEntity) {
|
|
1778
|
+
return true;
|
|
1779
|
+
}
|
|
1780
|
+
return scores.has(page.id);
|
|
1781
|
+
})
|
|
1782
|
+
.sort((left, right) => {
|
|
1783
|
+
const scoreDelta = (scores.get(right.id) ?? 0) - (scores.get(left.id) ?? 0);
|
|
1784
|
+
if (scoreDelta !== 0) {
|
|
1785
|
+
return scoreDelta;
|
|
1786
|
+
}
|
|
1787
|
+
return right.updatedAt.localeCompare(left.updatedAt);
|
|
1788
|
+
})
|
|
1789
|
+
.slice(0, parsed.limit);
|
|
1790
|
+
return {
|
|
1791
|
+
mode: parsed.mode,
|
|
1792
|
+
profileId: parsed.profileId ?? null,
|
|
1793
|
+
results: ranked.map((page) => ({
|
|
1794
|
+
page,
|
|
1795
|
+
score: Number((scores.get(page.id) ?? 0).toFixed(4))
|
|
1796
|
+
}))
|
|
1797
|
+
};
|
|
1798
|
+
}
|
|
1799
|
+
export function listWikiLlmProfiles() {
|
|
1800
|
+
const rows = getDatabase()
|
|
1801
|
+
.prepare(`SELECT id, label, provider, base_url, model, secret_id, system_prompt, enabled, metadata_json, created_at, updated_at
|
|
1802
|
+
FROM wiki_llm_profiles
|
|
1803
|
+
ORDER BY updated_at DESC`)
|
|
1804
|
+
.all();
|
|
1805
|
+
return rows.map((row) => wikiLlmProfileSchema.parse({
|
|
1806
|
+
id: row.id,
|
|
1807
|
+
label: row.label,
|
|
1808
|
+
provider: row.provider,
|
|
1809
|
+
baseUrl: row.base_url,
|
|
1810
|
+
model: row.model,
|
|
1811
|
+
secretId: row.secret_id,
|
|
1812
|
+
systemPrompt: row.system_prompt,
|
|
1813
|
+
enabled: row.enabled === 1,
|
|
1814
|
+
metadata: parseJsonRecord(row.metadata_json),
|
|
1815
|
+
createdAt: row.created_at,
|
|
1816
|
+
updatedAt: row.updated_at
|
|
1817
|
+
}));
|
|
1818
|
+
}
|
|
1819
|
+
export function listWikiEmbeddingProfiles() {
|
|
1820
|
+
const rows = getDatabase()
|
|
1821
|
+
.prepare(`SELECT id, label, provider, base_url, model, secret_id, dimensions, chunk_size, chunk_overlap, enabled, metadata_json, created_at, updated_at
|
|
1822
|
+
FROM wiki_embedding_profiles
|
|
1823
|
+
ORDER BY updated_at DESC`)
|
|
1824
|
+
.all();
|
|
1825
|
+
return rows.map((row) => wikiEmbeddingProfileSchema.parse({
|
|
1826
|
+
id: row.id,
|
|
1827
|
+
label: row.label,
|
|
1828
|
+
provider: row.provider,
|
|
1829
|
+
baseUrl: row.base_url,
|
|
1830
|
+
model: row.model,
|
|
1831
|
+
secretId: row.secret_id,
|
|
1832
|
+
dimensions: row.dimensions,
|
|
1833
|
+
chunkSize: row.chunk_size,
|
|
1834
|
+
chunkOverlap: row.chunk_overlap,
|
|
1835
|
+
enabled: row.enabled === 1,
|
|
1836
|
+
metadata: parseJsonRecord(row.metadata_json),
|
|
1837
|
+
createdAt: row.created_at,
|
|
1838
|
+
updatedAt: row.updated_at
|
|
1839
|
+
}));
|
|
1840
|
+
}
|
|
1841
|
+
export function getWikiSettingsPayload() {
|
|
1842
|
+
return wikiSettingsPayloadSchema.parse({
|
|
1843
|
+
spaces: listWikiSpaces(),
|
|
1844
|
+
llmProfiles: listWikiLlmProfiles(),
|
|
1845
|
+
embeddingProfiles: listWikiEmbeddingProfiles()
|
|
1846
|
+
});
|
|
1847
|
+
}
|
|
1848
|
+
export function getWikiHealth(input = {}) {
|
|
1849
|
+
const spaceId = resolveSpaceId(input.spaceId, null);
|
|
1850
|
+
const space = getWikiSpaceById(spaceId) ?? ensureSharedWikiSpace();
|
|
1851
|
+
const pages = listWikiPages({ spaceId, limit: 10_000 });
|
|
1852
|
+
const noteIds = pages.map((page) => page.id);
|
|
1853
|
+
const noteIdSet = new Set(noteIds);
|
|
1854
|
+
const rootDir = getSpaceStorageDir(space);
|
|
1855
|
+
const indexPath = getSpaceIndexPath(space);
|
|
1856
|
+
const rawDirectoryPath = getSpaceRawDir(space);
|
|
1857
|
+
const edgeRows = getDatabase()
|
|
1858
|
+
.prepare(`SELECT e.source_note_id, e.target_type, e.target_note_id, e.raw_target, e.updated_at, n.slug AS source_slug, n.title AS source_title
|
|
1859
|
+
FROM wiki_link_edges e
|
|
1860
|
+
JOIN notes n ON n.id = e.source_note_id
|
|
1861
|
+
WHERE n.space_id = ?
|
|
1862
|
+
ORDER BY e.updated_at DESC`)
|
|
1863
|
+
.all(spaceId);
|
|
1864
|
+
const backlinkCounts = new Map();
|
|
1865
|
+
const outboundCounts = new Map();
|
|
1866
|
+
const unresolvedLinks = edgeRows
|
|
1867
|
+
.filter((row) => row.target_type === "unresolved")
|
|
1868
|
+
.map((row) => ({
|
|
1869
|
+
sourceNoteId: row.source_note_id,
|
|
1870
|
+
sourceSlug: row.source_slug,
|
|
1871
|
+
sourceTitle: row.source_title,
|
|
1872
|
+
rawTarget: row.raw_target,
|
|
1873
|
+
updatedAt: row.updated_at
|
|
1874
|
+
}));
|
|
1875
|
+
for (const row of edgeRows) {
|
|
1876
|
+
outboundCounts.set(row.source_note_id, (outboundCounts.get(row.source_note_id) ?? 0) + 1);
|
|
1877
|
+
if (row.target_note_id && noteIdSet.has(row.target_note_id)) {
|
|
1878
|
+
backlinkCounts.set(row.target_note_id, (backlinkCounts.get(row.target_note_id) ?? 0) + 1);
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
let rawSourceCount = 0;
|
|
1882
|
+
try {
|
|
1883
|
+
rawSourceCount = readdirSync(rawDirectoryPath).length;
|
|
1884
|
+
}
|
|
1885
|
+
catch {
|
|
1886
|
+
rawSourceCount = 0;
|
|
1887
|
+
}
|
|
1888
|
+
const assetCount = getDatabase()
|
|
1889
|
+
.prepare(`SELECT COUNT(*) AS count
|
|
1890
|
+
FROM wiki_media_assets
|
|
1891
|
+
WHERE space_id = ?`)
|
|
1892
|
+
.get(spaceId).count;
|
|
1893
|
+
return wikiHealthPayloadSchema.parse({
|
|
1894
|
+
space,
|
|
1895
|
+
indexPath,
|
|
1896
|
+
rawDirectoryPath,
|
|
1897
|
+
pageCount: pages.length,
|
|
1898
|
+
wikiPageCount: pages.filter((page) => page.kind === "wiki").length,
|
|
1899
|
+
evidencePageCount: pages.filter((page) => page.kind === "evidence").length,
|
|
1900
|
+
assetCount,
|
|
1901
|
+
rawSourceCount,
|
|
1902
|
+
unresolvedLinks,
|
|
1903
|
+
orphanPages: pages
|
|
1904
|
+
.filter((page) => page.kind === "wiki")
|
|
1905
|
+
.filter((page) => (backlinkCounts.get(page.id) ?? 0) === 0 &&
|
|
1906
|
+
(outboundCounts.get(page.id) ?? 0) === 0)
|
|
1907
|
+
.map((page) => ({
|
|
1908
|
+
id: page.id,
|
|
1909
|
+
slug: page.slug,
|
|
1910
|
+
title: page.title,
|
|
1911
|
+
kind: page.kind,
|
|
1912
|
+
updatedAt: page.updatedAt
|
|
1913
|
+
}))
|
|
1914
|
+
.slice(0, 50),
|
|
1915
|
+
missingSummaries: pages
|
|
1916
|
+
.filter((page) => !page.summary.trim())
|
|
1917
|
+
.map((page) => ({
|
|
1918
|
+
id: page.id,
|
|
1919
|
+
slug: page.slug,
|
|
1920
|
+
title: page.title,
|
|
1921
|
+
updatedAt: page.updatedAt
|
|
1922
|
+
}))
|
|
1923
|
+
.slice(0, 50),
|
|
1924
|
+
enabledEmbeddingProfiles: listWikiEmbeddingProfiles()
|
|
1925
|
+
.filter((profile) => profile.enabled)
|
|
1926
|
+
.map((profile) => ({
|
|
1927
|
+
id: profile.id,
|
|
1928
|
+
label: profile.label,
|
|
1929
|
+
model: profile.model
|
|
1930
|
+
})),
|
|
1931
|
+
enabledLlmProfiles: listWikiLlmProfiles()
|
|
1932
|
+
.filter((profile) => profile.enabled)
|
|
1933
|
+
.map((profile) => ({
|
|
1934
|
+
id: profile.id,
|
|
1935
|
+
label: profile.label,
|
|
1936
|
+
model: profile.model
|
|
1937
|
+
}))
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
async function persistWikiRawSource(options) {
|
|
1941
|
+
const rawDir = getSpaceRawDir(options.space);
|
|
1942
|
+
await mkdir(rawDir, { recursive: true });
|
|
1943
|
+
const extension = path.extname(options.fetched.fileName) ||
|
|
1944
|
+
inferExtensionFromMimeType(options.fetched.mimeType) ||
|
|
1945
|
+
(options.fetched.contentText ? ".txt" : ".bin");
|
|
1946
|
+
const baseName = sanitizeFileName(path.basename(options.fetched.fileName, path.extname(options.fetched.fileName)) || options.jobId);
|
|
1947
|
+
const rawPath = path.join(rawDir, `${options.jobId}-${baseName}${extension}`);
|
|
1948
|
+
const payload = options.fetched.binary ?? Buffer.from(options.fetched.contentText, "utf8");
|
|
1949
|
+
await writeFile(rawPath, payload);
|
|
1950
|
+
return {
|
|
1951
|
+
filePath: rawPath,
|
|
1952
|
+
sizeBytes: payload.byteLength,
|
|
1953
|
+
checksum: hashBuffer(payload)
|
|
1954
|
+
};
|
|
1955
|
+
}
|
|
1956
|
+
export function upsertWikiLlmProfile(input, secrets) {
|
|
1957
|
+
const parsed = upsertWikiLlmProfileSchema.parse(input);
|
|
1958
|
+
const now = nowIso();
|
|
1959
|
+
const id = parsed.id?.trim() ||
|
|
1960
|
+
`wiki_llm_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
1961
|
+
let secretId = listWikiLlmProfiles().find((entry) => entry.id === id)?.secretId ?? null;
|
|
1962
|
+
if (parsed.apiKey?.trim()) {
|
|
1963
|
+
secretId =
|
|
1964
|
+
secretId ??
|
|
1965
|
+
`wiki_llm_secret_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
1966
|
+
storeEncryptedSecret(secretId, secrets.sealJson({ apiKey: parsed.apiKey.trim() }), `${parsed.label} wiki LLM profile`);
|
|
1967
|
+
}
|
|
1968
|
+
const metadata = {
|
|
1969
|
+
...parsed.metadata
|
|
1970
|
+
};
|
|
1971
|
+
if (parsed.reasoningEffort) {
|
|
1972
|
+
metadata.reasoningEffort = parsed.reasoningEffort;
|
|
1973
|
+
}
|
|
1974
|
+
else {
|
|
1975
|
+
delete metadata.reasoningEffort;
|
|
1976
|
+
}
|
|
1977
|
+
if (parsed.verbosity) {
|
|
1978
|
+
metadata.verbosity = parsed.verbosity;
|
|
1979
|
+
}
|
|
1980
|
+
else {
|
|
1981
|
+
delete metadata.verbosity;
|
|
1982
|
+
}
|
|
1983
|
+
getDatabase()
|
|
1984
|
+
.prepare(`INSERT INTO wiki_llm_profiles (id, label, provider, base_url, model, secret_id, system_prompt, enabled, metadata_json, created_at, updated_at)
|
|
1985
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1986
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1987
|
+
label = excluded.label,
|
|
1988
|
+
provider = excluded.provider,
|
|
1989
|
+
base_url = excluded.base_url,
|
|
1990
|
+
model = excluded.model,
|
|
1991
|
+
secret_id = excluded.secret_id,
|
|
1992
|
+
system_prompt = excluded.system_prompt,
|
|
1993
|
+
enabled = excluded.enabled,
|
|
1994
|
+
metadata_json = excluded.metadata_json,
|
|
1995
|
+
updated_at = excluded.updated_at`)
|
|
1996
|
+
.run(id, parsed.label, parsed.provider, parsed.baseUrl, parsed.model, secretId, parsed.systemPrompt, parsed.enabled ? 1 : 0, JSON.stringify(metadata), now, now);
|
|
1997
|
+
return listWikiLlmProfiles().find((entry) => entry.id === id);
|
|
1998
|
+
}
|
|
1999
|
+
export function upsertWikiEmbeddingProfile(input, secrets) {
|
|
2000
|
+
const parsed = upsertWikiEmbeddingProfileSchema.parse(input);
|
|
2001
|
+
const now = nowIso();
|
|
2002
|
+
const id = parsed.id?.trim() ||
|
|
2003
|
+
`wiki_embed_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
2004
|
+
let secretId = listWikiEmbeddingProfiles().find((entry) => entry.id === id)?.secretId ??
|
|
2005
|
+
null;
|
|
2006
|
+
if (parsed.apiKey?.trim()) {
|
|
2007
|
+
secretId =
|
|
2008
|
+
secretId ??
|
|
2009
|
+
`wiki_embed_secret_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
2010
|
+
storeEncryptedSecret(secretId, secrets.sealJson({ apiKey: parsed.apiKey.trim() }), `${parsed.label} wiki embedding profile`);
|
|
2011
|
+
}
|
|
2012
|
+
getDatabase()
|
|
2013
|
+
.prepare(`INSERT INTO wiki_embedding_profiles (
|
|
2014
|
+
id, label, provider, base_url, model, secret_id, dimensions, chunk_size, chunk_overlap, enabled, metadata_json, created_at, updated_at
|
|
2015
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2016
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
2017
|
+
label = excluded.label,
|
|
2018
|
+
provider = excluded.provider,
|
|
2019
|
+
base_url = excluded.base_url,
|
|
2020
|
+
model = excluded.model,
|
|
2021
|
+
secret_id = excluded.secret_id,
|
|
2022
|
+
dimensions = excluded.dimensions,
|
|
2023
|
+
chunk_size = excluded.chunk_size,
|
|
2024
|
+
chunk_overlap = excluded.chunk_overlap,
|
|
2025
|
+
enabled = excluded.enabled,
|
|
2026
|
+
metadata_json = excluded.metadata_json,
|
|
2027
|
+
updated_at = excluded.updated_at`)
|
|
2028
|
+
.run(id, parsed.label, parsed.provider, parsed.baseUrl, parsed.model, secretId, parsed.dimensions ?? null, parsed.chunkSize, parsed.chunkOverlap, parsed.enabled ? 1 : 0, JSON.stringify(parsed.metadata), now, now);
|
|
2029
|
+
return listWikiEmbeddingProfiles().find((entry) => entry.id === id);
|
|
2030
|
+
}
|
|
2031
|
+
export function deleteWikiProfile(kind, profileId) {
|
|
2032
|
+
if (kind === "llm") {
|
|
2033
|
+
const profile = listWikiLlmProfiles().find((entry) => entry.id === profileId);
|
|
2034
|
+
if (profile?.secretId) {
|
|
2035
|
+
deleteEncryptedSecret(profile.secretId);
|
|
2036
|
+
}
|
|
2037
|
+
getDatabase()
|
|
2038
|
+
.prepare(`DELETE FROM wiki_llm_profiles WHERE id = ?`)
|
|
2039
|
+
.run(profileId);
|
|
2040
|
+
return;
|
|
2041
|
+
}
|
|
2042
|
+
const profile = listWikiEmbeddingProfiles().find((entry) => entry.id === profileId);
|
|
2043
|
+
if (profile?.secretId) {
|
|
2044
|
+
deleteEncryptedSecret(profile.secretId);
|
|
2045
|
+
}
|
|
2046
|
+
getDatabase()
|
|
2047
|
+
.prepare(`DELETE FROM wiki_embedding_profiles WHERE id = ?`)
|
|
2048
|
+
.run(profileId);
|
|
2049
|
+
getDatabase()
|
|
2050
|
+
.prepare(`DELETE FROM wiki_embedding_chunks WHERE profile_id = ?`)
|
|
2051
|
+
.run(profileId);
|
|
2052
|
+
}
|
|
2053
|
+
export async function reindexWikiEmbeddings(input, secrets) {
|
|
2054
|
+
const parsed = reindexWikiEmbeddingsSchema.parse(input);
|
|
2055
|
+
const profiles = listWikiEmbeddingProfiles().filter((entry) => entry.enabled && (!parsed.profileId || entry.id === parsed.profileId));
|
|
2056
|
+
const pages = listWikiPages({ spaceId: parsed.spaceId, limit: 10_000 });
|
|
2057
|
+
let chunkCount = 0;
|
|
2058
|
+
for (const profile of profiles) {
|
|
2059
|
+
for (const page of pages) {
|
|
2060
|
+
getDatabase()
|
|
2061
|
+
.prepare(`DELETE FROM wiki_embedding_chunks WHERE note_id = ? AND profile_id = ?`)
|
|
2062
|
+
.run(page.id, profile.id);
|
|
2063
|
+
const chunks = chunkHeadingAware(page.contentMarkdown, profile.chunkSize, profile.chunkOverlap);
|
|
2064
|
+
if (chunks.length === 0) {
|
|
2065
|
+
continue;
|
|
2066
|
+
}
|
|
2067
|
+
const vectors = await embedTexts(profile, secrets, chunks.map((chunk) => chunk.contentText));
|
|
2068
|
+
const insert = getDatabase().prepare(`INSERT INTO wiki_embedding_chunks (
|
|
2069
|
+
id, note_id, space_id, profile_id, chunk_key, heading_path, content_text, vector_json, created_at, updated_at
|
|
2070
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
2071
|
+
const now = nowIso();
|
|
2072
|
+
chunks.forEach((chunk, index) => {
|
|
2073
|
+
const vector = vectors[index];
|
|
2074
|
+
if (!vector || vector.length === 0) {
|
|
2075
|
+
return;
|
|
2076
|
+
}
|
|
2077
|
+
insert.run(`wiki_chunk_${randomUUID().replaceAll("-", "").slice(0, 10)}`, page.id, page.spaceId, profile.id, chunk.key, chunk.headingPath, chunk.contentText, JSON.stringify(vector), now, now);
|
|
2078
|
+
chunkCount += 1;
|
|
2079
|
+
});
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
return {
|
|
2083
|
+
profilesIndexed: profiles.length,
|
|
2084
|
+
pagesIndexed: pages.length,
|
|
2085
|
+
chunkCount
|
|
2086
|
+
};
|
|
2087
|
+
}
|
|
2088
|
+
function getWikiIngestJobDir(jobId) {
|
|
2089
|
+
return path.join(resolveDataDir(), "wiki-ingest", jobId);
|
|
2090
|
+
}
|
|
2091
|
+
function getWikiIngestUploadsDir(jobId) {
|
|
2092
|
+
return path.join(getWikiIngestJobDir(jobId), "uploads");
|
|
2093
|
+
}
|
|
2094
|
+
function readWikiIngestJobRow(jobId) {
|
|
2095
|
+
return getDatabase()
|
|
2096
|
+
.prepare(`SELECT id, space_id, llm_profile_id, status, phase, progress_percent, total_files, processed_files,
|
|
2097
|
+
created_page_count, created_entity_count, accepted_count, rejected_count, latest_message,
|
|
2098
|
+
source_kind, source_locator, mime_type, title_hint, summary, page_note_id, created_by_actor,
|
|
2099
|
+
error_message, input_json, created_at, updated_at, completed_at
|
|
2100
|
+
FROM wiki_ingest_jobs
|
|
2101
|
+
WHERE id = ?`)
|
|
2102
|
+
.get(jobId);
|
|
2103
|
+
}
|
|
2104
|
+
function mapWikiIngestJobRow(job) {
|
|
2105
|
+
return {
|
|
2106
|
+
id: job.id,
|
|
2107
|
+
spaceId: job.space_id,
|
|
2108
|
+
llmProfileId: job.llm_profile_id,
|
|
2109
|
+
status: job.status,
|
|
2110
|
+
phase: job.phase,
|
|
2111
|
+
progressPercent: job.progress_percent,
|
|
2112
|
+
totalFiles: job.total_files,
|
|
2113
|
+
processedFiles: job.processed_files,
|
|
2114
|
+
createdPageCount: job.created_page_count,
|
|
2115
|
+
createdEntityCount: job.created_entity_count,
|
|
2116
|
+
acceptedCount: job.accepted_count,
|
|
2117
|
+
rejectedCount: job.rejected_count,
|
|
2118
|
+
latestMessage: job.latest_message,
|
|
2119
|
+
sourceKind: job.source_kind,
|
|
2120
|
+
sourceLocator: job.source_locator,
|
|
2121
|
+
mimeType: job.mime_type,
|
|
2122
|
+
titleHint: job.title_hint,
|
|
2123
|
+
summary: job.summary,
|
|
2124
|
+
pageNoteId: job.page_note_id,
|
|
2125
|
+
createdByActor: job.created_by_actor,
|
|
2126
|
+
errorMessage: job.error_message,
|
|
2127
|
+
createdAt: job.created_at,
|
|
2128
|
+
updatedAt: job.updated_at,
|
|
2129
|
+
completedAt: job.completed_at
|
|
2130
|
+
};
|
|
2131
|
+
}
|
|
2132
|
+
function listWikiIngestJobLogsInternal(jobId) {
|
|
2133
|
+
return getDatabase()
|
|
2134
|
+
.prepare(`SELECT id, level, message, metadata_json, created_at
|
|
2135
|
+
FROM wiki_ingest_job_logs
|
|
2136
|
+
WHERE job_id = ?
|
|
2137
|
+
ORDER BY created_at ASC`)
|
|
2138
|
+
.all(jobId);
|
|
2139
|
+
}
|
|
2140
|
+
function listWikiIngestJobAssetsInternal(jobId) {
|
|
2141
|
+
return getDatabase()
|
|
2142
|
+
.prepare(`SELECT id, status, source_kind, source_locator, file_name, mime_type, file_path,
|
|
2143
|
+
size_bytes, checksum, metadata_json, created_at, updated_at
|
|
2144
|
+
FROM wiki_ingest_job_assets
|
|
2145
|
+
WHERE job_id = ?
|
|
2146
|
+
ORDER BY created_at ASC`)
|
|
2147
|
+
.all(jobId);
|
|
2148
|
+
}
|
|
2149
|
+
function listWikiIngestCandidatesInternal(jobId) {
|
|
2150
|
+
return getDatabase()
|
|
2151
|
+
.prepare(`SELECT id, source_asset_id, candidate_type, status, title, summary, target_key,
|
|
2152
|
+
payload_json, published_note_id, published_entity_type, published_entity_id,
|
|
2153
|
+
created_at, updated_at
|
|
2154
|
+
FROM wiki_ingest_job_candidates
|
|
2155
|
+
WHERE job_id = ?
|
|
2156
|
+
ORDER BY created_at ASC`)
|
|
2157
|
+
.all(jobId);
|
|
2158
|
+
}
|
|
2159
|
+
function updateWikiIngestJob(jobId, patch) {
|
|
2160
|
+
const current = readWikiIngestJobRow(jobId);
|
|
2161
|
+
if (!current) {
|
|
2162
|
+
return null;
|
|
2163
|
+
}
|
|
2164
|
+
const next = {
|
|
2165
|
+
status: patch.status ?? current.status,
|
|
2166
|
+
phase: patch.phase ?? current.phase,
|
|
2167
|
+
progressPercent: patch.progressPercent ?? current.progress_percent,
|
|
2168
|
+
totalFiles: patch.totalFiles ?? current.total_files,
|
|
2169
|
+
processedFiles: patch.processedFiles ?? current.processed_files,
|
|
2170
|
+
createdPageCount: patch.createdPageCount ?? current.created_page_count,
|
|
2171
|
+
createdEntityCount: patch.createdEntityCount ?? current.created_entity_count,
|
|
2172
|
+
acceptedCount: patch.acceptedCount ?? current.accepted_count,
|
|
2173
|
+
rejectedCount: patch.rejectedCount ?? current.rejected_count,
|
|
2174
|
+
latestMessage: patch.latestMessage ?? current.latest_message,
|
|
2175
|
+
sourceLocator: patch.sourceLocator ?? current.source_locator,
|
|
2176
|
+
mimeType: patch.mimeType ?? current.mime_type,
|
|
2177
|
+
summary: patch.summary ?? current.summary,
|
|
2178
|
+
pageNoteId: patch.pageNoteId === undefined ? current.page_note_id : patch.pageNoteId,
|
|
2179
|
+
errorMessage: patch.errorMessage ?? current.error_message,
|
|
2180
|
+
completedAt: patch.completedAt === undefined
|
|
2181
|
+
? current.completed_at
|
|
2182
|
+
: patch.completedAt,
|
|
2183
|
+
updatedAt: nowIso()
|
|
2184
|
+
};
|
|
2185
|
+
getDatabase()
|
|
2186
|
+
.prepare(`UPDATE wiki_ingest_jobs
|
|
2187
|
+
SET status = ?, phase = ?, progress_percent = ?, total_files = ?, processed_files = ?,
|
|
2188
|
+
created_page_count = ?, created_entity_count = ?, accepted_count = ?, rejected_count = ?,
|
|
2189
|
+
latest_message = ?, source_locator = ?, mime_type = ?, summary = ?, page_note_id = ?,
|
|
2190
|
+
error_message = ?, completed_at = ?, updated_at = ?
|
|
2191
|
+
WHERE id = ?`)
|
|
2192
|
+
.run(next.status, next.phase, next.progressPercent, next.totalFiles, next.processedFiles, next.createdPageCount, next.createdEntityCount, next.acceptedCount, next.rejectedCount, next.latestMessage, next.sourceLocator, next.mimeType, next.summary, next.pageNoteId, next.errorMessage, next.completedAt, next.updatedAt, jobId);
|
|
2193
|
+
return getWikiIngestJob(jobId);
|
|
2194
|
+
}
|
|
2195
|
+
function createWikiIngestLog(jobId, message, level = "info", metadata = {}, options = {}) {
|
|
2196
|
+
const createdAt = nowIso();
|
|
2197
|
+
const logMetadata = options.aggregateKey
|
|
2198
|
+
? { ...metadata, aggregateKey: options.aggregateKey }
|
|
2199
|
+
: metadata;
|
|
2200
|
+
const aggregateKey = options.aggregateKey?.trim() || null;
|
|
2201
|
+
if (aggregateKey) {
|
|
2202
|
+
const current = [...listWikiIngestJobLogsInternal(jobId)]
|
|
2203
|
+
.reverse()
|
|
2204
|
+
.find((entry) => {
|
|
2205
|
+
const parsed = parseJsonRecord(entry.metadata_json);
|
|
2206
|
+
return parsed.aggregateKey === aggregateKey;
|
|
2207
|
+
});
|
|
2208
|
+
if (current) {
|
|
2209
|
+
getDatabase()
|
|
2210
|
+
.prepare(`UPDATE wiki_ingest_job_logs
|
|
2211
|
+
SET level = ?, message = ?, metadata_json = ?, created_at = ?
|
|
2212
|
+
WHERE id = ?`)
|
|
2213
|
+
.run(level, message, JSON.stringify(logMetadata), createdAt, current.id);
|
|
2214
|
+
}
|
|
2215
|
+
else {
|
|
2216
|
+
getDatabase()
|
|
2217
|
+
.prepare(`INSERT INTO wiki_ingest_job_logs (id, job_id, level, message, metadata_json, created_at)
|
|
2218
|
+
VALUES (?, ?, ?, ?, ?, ?)`)
|
|
2219
|
+
.run(`wiki_ingest_log_${randomUUID().replaceAll("-", "").slice(0, 10)}`, jobId, level, message, JSON.stringify(logMetadata), createdAt);
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
else {
|
|
2223
|
+
getDatabase()
|
|
2224
|
+
.prepare(`INSERT INTO wiki_ingest_job_logs (id, job_id, level, message, metadata_json, created_at)
|
|
2225
|
+
VALUES (?, ?, ?, ?, ?, ?)`)
|
|
2226
|
+
.run(`wiki_ingest_log_${randomUUID().replaceAll("-", "").slice(0, 10)}`, jobId, level, message, JSON.stringify(logMetadata), createdAt);
|
|
2227
|
+
}
|
|
2228
|
+
if (options.recordDiagnostic !== false) {
|
|
2229
|
+
recordDiagnosticLog({
|
|
2230
|
+
level,
|
|
2231
|
+
source: "server",
|
|
2232
|
+
scope: typeof metadata.scope === "string" && metadata.scope.trim()
|
|
2233
|
+
? metadata.scope
|
|
2234
|
+
: "wiki_ingest",
|
|
2235
|
+
eventKey: typeof metadata.eventKey === "string" && metadata.eventKey.trim()
|
|
2236
|
+
? metadata.eventKey
|
|
2237
|
+
: "wiki_ingest_log",
|
|
2238
|
+
message,
|
|
2239
|
+
functionName: "createWikiIngestLog",
|
|
2240
|
+
entityType: "wiki_ingest_job",
|
|
2241
|
+
entityId: jobId,
|
|
2242
|
+
jobId,
|
|
2243
|
+
details: logMetadata
|
|
2244
|
+
});
|
|
2245
|
+
}
|
|
2246
|
+
}
|
|
2247
|
+
function findOpenAiResponseIdForJobAsset(input) {
|
|
2248
|
+
const normalizedFileName = input.fileName.trim().toLowerCase();
|
|
2249
|
+
const normalizedSourceLocator = input.sourceLocator.trim().toLowerCase();
|
|
2250
|
+
const normalizedChecksum = input.checksum.trim().toLowerCase();
|
|
2251
|
+
const sameNamedAssets = listWikiIngestJobAssetsInternal(input.jobId).filter((asset) => asset.file_name.trim().toLowerCase() === normalizedFileName).length;
|
|
2252
|
+
const logs = [...listWikiIngestJobLogsInternal(input.jobId)].reverse();
|
|
2253
|
+
for (const entry of logs) {
|
|
2254
|
+
const metadata = parseJsonRecord(entry.metadata_json);
|
|
2255
|
+
const responseId = readStringRecordValue(metadata, "responseId");
|
|
2256
|
+
if (!responseId) {
|
|
2257
|
+
continue;
|
|
2258
|
+
}
|
|
2259
|
+
const loggedAssetId = readStringRecordValue(metadata, "sourceAssetId") ??
|
|
2260
|
+
readStringRecordValue(metadata, "assetId");
|
|
2261
|
+
if (loggedAssetId) {
|
|
2262
|
+
if (loggedAssetId === input.assetId) {
|
|
2263
|
+
return responseId;
|
|
2264
|
+
}
|
|
2265
|
+
continue;
|
|
2266
|
+
}
|
|
2267
|
+
const loggedSourceLocator = readStringRecordValue(metadata, "sourceLocator");
|
|
2268
|
+
if (loggedSourceLocator) {
|
|
2269
|
+
if (loggedSourceLocator.trim().toLowerCase() === normalizedSourceLocator) {
|
|
2270
|
+
return responseId;
|
|
2271
|
+
}
|
|
2272
|
+
continue;
|
|
2273
|
+
}
|
|
2274
|
+
const loggedChecksum = readStringRecordValue(metadata, "checksum");
|
|
2275
|
+
if (loggedChecksum) {
|
|
2276
|
+
if (loggedChecksum.trim().toLowerCase() === normalizedChecksum) {
|
|
2277
|
+
return responseId;
|
|
2278
|
+
}
|
|
2279
|
+
continue;
|
|
2280
|
+
}
|
|
2281
|
+
const loggedFileName = readStringRecordValue(metadata, "currentFileName") ??
|
|
2282
|
+
readStringRecordValue(metadata, "fileName");
|
|
2283
|
+
if (!loggedFileName) {
|
|
2284
|
+
return responseId;
|
|
2285
|
+
}
|
|
2286
|
+
if (sameNamedAssets === 1 &&
|
|
2287
|
+
loggedFileName.trim().toLowerCase() === normalizedFileName) {
|
|
2288
|
+
return responseId;
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
return null;
|
|
2292
|
+
}
|
|
2293
|
+
function createWikiIngestAssetRecord(input) {
|
|
2294
|
+
const id = `wiki_ingest_asset_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
2295
|
+
const now = nowIso();
|
|
2296
|
+
getDatabase()
|
|
2297
|
+
.prepare(`INSERT INTO wiki_ingest_job_assets (
|
|
2298
|
+
id, job_id, status, source_kind, source_locator, file_name, mime_type, file_path,
|
|
2299
|
+
size_bytes, checksum, metadata_json, created_at, updated_at
|
|
2300
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
2301
|
+
.run(id, input.jobId, input.status ?? "queued", input.sourceKind, input.sourceLocator, input.fileName, input.mimeType, input.filePath, input.sizeBytes, input.checksum, JSON.stringify(input.metadata ?? {}), now, now);
|
|
2302
|
+
return id;
|
|
2303
|
+
}
|
|
2304
|
+
function updateWikiIngestAsset(assetId, patch) {
|
|
2305
|
+
const current = getDatabase()
|
|
2306
|
+
.prepare(`SELECT file_path, mime_type, size_bytes, checksum, metadata_json, status
|
|
2307
|
+
FROM wiki_ingest_job_assets
|
|
2308
|
+
WHERE id = ?`)
|
|
2309
|
+
.get(assetId);
|
|
2310
|
+
if (!current) {
|
|
2311
|
+
return;
|
|
2312
|
+
}
|
|
2313
|
+
getDatabase()
|
|
2314
|
+
.prepare(`UPDATE wiki_ingest_job_assets
|
|
2315
|
+
SET status = ?, file_path = ?, mime_type = ?, size_bytes = ?, checksum = ?, metadata_json = ?, updated_at = ?
|
|
2316
|
+
WHERE id = ?`)
|
|
2317
|
+
.run(patch.status ?? current.status, patch.filePath ?? current.file_path, patch.mimeType ?? current.mime_type, patch.sizeBytes ?? current.size_bytes, patch.checksum ?? current.checksum, JSON.stringify(patch.metadata ?? parseJsonRecord(current.metadata_json)), nowIso(), assetId);
|
|
2318
|
+
}
|
|
2319
|
+
function createWikiIngestCandidate(input) {
|
|
2320
|
+
const existing = listWikiIngestCandidatesInternal(input.jobId).find((candidate) => candidate.source_asset_id === (input.sourceAssetId ?? null) &&
|
|
2321
|
+
candidate.candidate_type === input.candidateType &&
|
|
2322
|
+
candidate.title === (input.title ?? "") &&
|
|
2323
|
+
candidate.summary === (input.summary ?? "") &&
|
|
2324
|
+
candidate.target_key === (input.targetKey ?? "") &&
|
|
2325
|
+
candidate.payload_json === JSON.stringify(input.payload));
|
|
2326
|
+
if (existing) {
|
|
2327
|
+
return existing.id;
|
|
2328
|
+
}
|
|
2329
|
+
const id = `wiki_ingest_candidate_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
2330
|
+
const now = nowIso();
|
|
2331
|
+
getDatabase()
|
|
2332
|
+
.prepare(`INSERT INTO wiki_ingest_job_candidates (
|
|
2333
|
+
id, job_id, source_asset_id, candidate_type, status, title, summary, target_key,
|
|
2334
|
+
payload_json, published_note_id, published_entity_type, published_entity_id, created_at, updated_at
|
|
2335
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
2336
|
+
.run(id, input.jobId, input.sourceAssetId ?? null, input.candidateType, "suggested", input.title ?? "", input.summary ?? "", input.targetKey ?? "", JSON.stringify(input.payload), null, null, null, now, now);
|
|
2337
|
+
return id;
|
|
2338
|
+
}
|
|
2339
|
+
function updateWikiIngestCandidate(candidateId, patch) {
|
|
2340
|
+
const current = getDatabase()
|
|
2341
|
+
.prepare(`SELECT payload_json, status, published_note_id, published_entity_type, published_entity_id
|
|
2342
|
+
FROM wiki_ingest_job_candidates
|
|
2343
|
+
WHERE id = ?`)
|
|
2344
|
+
.get(candidateId);
|
|
2345
|
+
if (!current) {
|
|
2346
|
+
return;
|
|
2347
|
+
}
|
|
2348
|
+
getDatabase()
|
|
2349
|
+
.prepare(`UPDATE wiki_ingest_job_candidates
|
|
2350
|
+
SET status = ?, payload_json = ?, published_note_id = ?, published_entity_type = ?, published_entity_id = ?, updated_at = ?
|
|
2351
|
+
WHERE id = ?`)
|
|
2352
|
+
.run(patch.status ?? current.status, JSON.stringify(patch.payload ?? parseJsonRecord(current.payload_json)), patch.publishedNoteId === undefined
|
|
2353
|
+
? current.published_note_id
|
|
2354
|
+
: patch.publishedNoteId, patch.publishedEntityType === undefined
|
|
2355
|
+
? current.published_entity_type
|
|
2356
|
+
: patch.publishedEntityType, patch.publishedEntityId === undefined
|
|
2357
|
+
? current.published_entity_id
|
|
2358
|
+
: patch.publishedEntityId, nowIso(), candidateId);
|
|
2359
|
+
}
|
|
2360
|
+
function calculateProgress(totalFiles, processedFiles) {
|
|
2361
|
+
if (totalFiles <= 0) {
|
|
2362
|
+
return 0;
|
|
2363
|
+
}
|
|
2364
|
+
return Math.max(0, Math.min(100, Math.round((processedFiles / totalFiles) * 100)));
|
|
2365
|
+
}
|
|
2366
|
+
async function persistIngestUpload(options) {
|
|
2367
|
+
const uploadDir = getWikiIngestUploadsDir(options.jobId);
|
|
2368
|
+
await mkdir(uploadDir, { recursive: true });
|
|
2369
|
+
const safeName = sanitizeFileName(options.fileName || "upload.bin");
|
|
2370
|
+
const nextPath = path.join(uploadDir, `${Date.now()}-${randomUUID().slice(0, 8)}-${safeName}`);
|
|
2371
|
+
await writeFile(nextPath, options.payload);
|
|
2372
|
+
return {
|
|
2373
|
+
filePath: nextPath,
|
|
2374
|
+
sizeBytes: options.payload.byteLength,
|
|
2375
|
+
checksum: hashBuffer(options.payload)
|
|
2376
|
+
};
|
|
2377
|
+
}
|
|
2378
|
+
function readWikiIngestInput(jobId) {
|
|
2379
|
+
const row = readWikiIngestJobRow(jobId);
|
|
2380
|
+
if (!row) {
|
|
2381
|
+
return null;
|
|
2382
|
+
}
|
|
2383
|
+
const payload = parseJsonRecord(row.input_json);
|
|
2384
|
+
return createWikiIngestJobSchema.parse({
|
|
2385
|
+
spaceId: row.space_id,
|
|
2386
|
+
titleHint: typeof payload.titleHint === "string"
|
|
2387
|
+
? payload.titleHint
|
|
2388
|
+
: row.title_hint,
|
|
2389
|
+
sourceKind: payload.sourceKind === "local_path" || payload.sourceKind === "url"
|
|
2390
|
+
? payload.sourceKind
|
|
2391
|
+
: "raw_text",
|
|
2392
|
+
sourceText: typeof payload.sourceText === "string" ? payload.sourceText : "",
|
|
2393
|
+
sourcePath: typeof payload.sourcePath === "string" ? payload.sourcePath : undefined,
|
|
2394
|
+
sourceUrl: typeof payload.sourceUrl === "string" ? payload.sourceUrl : undefined,
|
|
2395
|
+
mimeType: typeof payload.mimeType === "string" ? payload.mimeType : row.mime_type,
|
|
2396
|
+
llmProfileId: typeof payload.llmProfileId === "string"
|
|
2397
|
+
? payload.llmProfileId
|
|
2398
|
+
: (row.llm_profile_id ?? undefined),
|
|
2399
|
+
parseStrategy: payload.parseStrategy === "text_only" ||
|
|
2400
|
+
payload.parseStrategy === "multimodal"
|
|
2401
|
+
? payload.parseStrategy
|
|
2402
|
+
: "auto",
|
|
2403
|
+
entityProposalMode: payload.entityProposalMode === "none" ? "none" : "suggest",
|
|
2404
|
+
userId: typeof payload.userId === "string"
|
|
2405
|
+
? payload.userId
|
|
2406
|
+
: payload.userId === null
|
|
2407
|
+
? null
|
|
2408
|
+
: row.created_by_actor,
|
|
2409
|
+
createAsKind: payload.createAsKind === "evidence" ? "evidence" : "wiki",
|
|
2410
|
+
linkedEntityHints: Array.isArray(payload.linkedEntityHints)
|
|
2411
|
+
? payload.linkedEntityHints
|
|
2412
|
+
: []
|
|
2413
|
+
});
|
|
2414
|
+
}
|
|
2415
|
+
export async function createUploadedWikiIngestJob(input, files, options = {}) {
|
|
2416
|
+
const parsed = createWikiIngestJobSchema.parse({
|
|
2417
|
+
...input,
|
|
2418
|
+
sourceKind: "raw_text",
|
|
2419
|
+
sourceText: "",
|
|
2420
|
+
mimeType: ""
|
|
2421
|
+
});
|
|
2422
|
+
const spaceId = resolveSpaceId(parsed.spaceId, parsed.userId);
|
|
2423
|
+
const now = nowIso();
|
|
2424
|
+
const jobId = `wiki_ingest_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
2425
|
+
getDatabase()
|
|
2426
|
+
.prepare(`INSERT INTO wiki_ingest_jobs (
|
|
2427
|
+
id, space_id, llm_profile_id, status, phase, progress_percent, total_files, processed_files,
|
|
2428
|
+
created_page_count, created_entity_count, accepted_count, rejected_count, latest_message,
|
|
2429
|
+
source_kind, source_locator, mime_type, title_hint, summary, page_note_id, created_by_actor,
|
|
2430
|
+
error_message, input_json, created_at, updated_at, completed_at
|
|
2431
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
2432
|
+
.run(jobId, spaceId, parsed.llmProfileId ?? null, "queued", "queued", 0, files.length, 0, 0, 0, 0, 0, files.length === 1
|
|
2433
|
+
? "Waiting to ingest 1 uploaded file"
|
|
2434
|
+
: `Waiting to ingest ${files.length} uploaded files`, "upload", "", "", parsed.titleHint, "", null, options.actor ?? parsed.userId ?? null, "", JSON.stringify({
|
|
2435
|
+
...parsed,
|
|
2436
|
+
sourceKind: "upload"
|
|
2437
|
+
}), now, now, null);
|
|
2438
|
+
for (const file of files) {
|
|
2439
|
+
const persisted = await persistIngestUpload({
|
|
2440
|
+
jobId,
|
|
2441
|
+
fileName: file.fileName,
|
|
2442
|
+
mimeType: file.mimeType || inferMimeTypeFromPath(file.fileName),
|
|
2443
|
+
payload: file.payload
|
|
2444
|
+
});
|
|
2445
|
+
createWikiIngestAssetRecord({
|
|
2446
|
+
jobId,
|
|
2447
|
+
sourceKind: "upload",
|
|
2448
|
+
sourceLocator: file.fileName,
|
|
2449
|
+
fileName: file.fileName,
|
|
2450
|
+
mimeType: file.mimeType || inferMimeTypeFromPath(file.fileName),
|
|
2451
|
+
filePath: persisted.filePath,
|
|
2452
|
+
sizeBytes: persisted.sizeBytes,
|
|
2453
|
+
checksum: persisted.checksum
|
|
2454
|
+
});
|
|
2455
|
+
}
|
|
2456
|
+
createWikiIngestLog(jobId, files.length === 1
|
|
2457
|
+
? "Queued 1 uploaded file for Forge wiki ingestion."
|
|
2458
|
+
: `Queued ${files.length} uploaded files for Forge wiki ingestion.`);
|
|
2459
|
+
return {
|
|
2460
|
+
job: getWikiIngestJob(jobId),
|
|
2461
|
+
page: null
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
export async function ingestWikiSource(input, options = {}) {
|
|
2465
|
+
const parsed = createWikiIngestJobSchema.parse(input);
|
|
2466
|
+
const spaceId = resolveSpaceId(parsed.spaceId, parsed.userId);
|
|
2467
|
+
const now = nowIso();
|
|
2468
|
+
const jobId = `wiki_ingest_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
|
|
2469
|
+
let filePath = "";
|
|
2470
|
+
let sizeBytes = 0;
|
|
2471
|
+
let checksum = "";
|
|
2472
|
+
let sourceLocator = "";
|
|
2473
|
+
let fileName = "source.bin";
|
|
2474
|
+
let mimeType = parsed.mimeType;
|
|
2475
|
+
if (parsed.sourceKind === "raw_text") {
|
|
2476
|
+
const payload = Buffer.from(parsed.sourceText, "utf8");
|
|
2477
|
+
const persisted = await persistIngestUpload({
|
|
2478
|
+
jobId,
|
|
2479
|
+
fileName: "inline.txt",
|
|
2480
|
+
mimeType: parsed.mimeType || "text/plain",
|
|
2481
|
+
payload
|
|
2482
|
+
});
|
|
2483
|
+
filePath = persisted.filePath;
|
|
2484
|
+
sizeBytes = persisted.sizeBytes;
|
|
2485
|
+
checksum = persisted.checksum;
|
|
2486
|
+
sourceLocator = "raw_text";
|
|
2487
|
+
fileName = "inline.txt";
|
|
2488
|
+
mimeType = parsed.mimeType || "text/plain";
|
|
2489
|
+
}
|
|
2490
|
+
else if (parsed.sourceKind === "local_path") {
|
|
2491
|
+
const sourcePath = parsed.sourcePath?.trim();
|
|
2492
|
+
if (!sourcePath) {
|
|
2493
|
+
throw new Error("sourcePath is required for local_path ingest.");
|
|
2494
|
+
}
|
|
2495
|
+
const payload = await readFile(sourcePath);
|
|
2496
|
+
const persisted = await persistIngestUpload({
|
|
2497
|
+
jobId,
|
|
2498
|
+
fileName: path.basename(sourcePath),
|
|
2499
|
+
mimeType: parsed.mimeType || inferMimeTypeFromPath(sourcePath),
|
|
2500
|
+
payload
|
|
2501
|
+
});
|
|
2502
|
+
filePath = persisted.filePath;
|
|
2503
|
+
sizeBytes = persisted.sizeBytes;
|
|
2504
|
+
checksum = persisted.checksum;
|
|
2505
|
+
sourceLocator = sourcePath;
|
|
2506
|
+
fileName = path.basename(sourcePath);
|
|
2507
|
+
mimeType = parsed.mimeType || inferMimeTypeFromPath(sourcePath);
|
|
2508
|
+
}
|
|
2509
|
+
else {
|
|
2510
|
+
sourceLocator = parsed.sourceUrl?.trim() || "";
|
|
2511
|
+
fileName =
|
|
2512
|
+
sourceLocator.split("/").pop()?.split("?")[0]?.trim() ||
|
|
2513
|
+
"remote-source.bin";
|
|
2514
|
+
mimeType = parsed.mimeType || "application/octet-stream";
|
|
2515
|
+
}
|
|
2516
|
+
getDatabase()
|
|
2517
|
+
.prepare(`INSERT INTO wiki_ingest_jobs (
|
|
2518
|
+
id, space_id, llm_profile_id, status, phase, progress_percent, total_files, processed_files,
|
|
2519
|
+
created_page_count, created_entity_count, accepted_count, rejected_count, latest_message,
|
|
2520
|
+
source_kind, source_locator, mime_type, title_hint, summary, page_note_id, created_by_actor,
|
|
2521
|
+
error_message, input_json, created_at, updated_at, completed_at
|
|
2522
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
2523
|
+
.run(jobId, spaceId, parsed.llmProfileId ?? null, "queued", "queued", 0, 1, 0, 0, 0, 0, 0, "Waiting to ingest source", parsed.sourceKind, sourceLocator, mimeType, parsed.titleHint, "", null, options.actor ?? parsed.userId ?? null, "", JSON.stringify(parsed), now, now, null);
|
|
2524
|
+
createWikiIngestAssetRecord({
|
|
2525
|
+
jobId,
|
|
2526
|
+
sourceKind: parsed.sourceKind,
|
|
2527
|
+
sourceLocator,
|
|
2528
|
+
fileName,
|
|
2529
|
+
mimeType,
|
|
2530
|
+
filePath,
|
|
2531
|
+
sizeBytes,
|
|
2532
|
+
checksum
|
|
2533
|
+
});
|
|
2534
|
+
createWikiIngestLog(jobId, "Queued source for Forge wiki ingestion.");
|
|
2535
|
+
return {
|
|
2536
|
+
job: getWikiIngestJob(jobId),
|
|
2537
|
+
page: null
|
|
2538
|
+
};
|
|
2539
|
+
}
|
|
2540
|
+
function createPageCandidatePayload(input) {
|
|
2541
|
+
return {
|
|
2542
|
+
title: input.title,
|
|
2543
|
+
summary: input.summary,
|
|
2544
|
+
contentMarkdown: input.markdown,
|
|
2545
|
+
kind: input.createAsKind,
|
|
2546
|
+
aliases: normalizeTags(input.aliases),
|
|
2547
|
+
parentSlug: typeof input.parentSlug === "string" && input.parentSlug.trim().length > 0
|
|
2548
|
+
? input.parentSlug.trim()
|
|
2549
|
+
: null,
|
|
2550
|
+
sourceLocator: input.sourceLocator,
|
|
2551
|
+
links: input.linkedEntityHints,
|
|
2552
|
+
tags: normalizeTags(input.tags)
|
|
2553
|
+
};
|
|
2554
|
+
}
|
|
2555
|
+
export async function processWikiIngestJob(jobId, options) {
|
|
2556
|
+
const job = readWikiIngestJobRow(jobId);
|
|
2557
|
+
if (!job) {
|
|
2558
|
+
return null;
|
|
2559
|
+
}
|
|
2560
|
+
const parsed = readWikiIngestInput(jobId);
|
|
2561
|
+
if (!parsed) {
|
|
2562
|
+
return null;
|
|
2563
|
+
}
|
|
2564
|
+
const llmProfile = parsed.llmProfileId
|
|
2565
|
+
? (listWikiLlmProfiles().find((entry) => entry.id === parsed.llmProfileId) ?? null)
|
|
2566
|
+
: null;
|
|
2567
|
+
const space = getWikiSpaceById(job.space_id) ?? ensureSharedWikiSpace();
|
|
2568
|
+
if (parsed.llmProfileId && !llmProfile) {
|
|
2569
|
+
updateWikiIngestJob(jobId, {
|
|
2570
|
+
status: "failed",
|
|
2571
|
+
phase: "failed",
|
|
2572
|
+
latestMessage: "The selected LLM profile could not be found.",
|
|
2573
|
+
errorMessage: "The selected LLM profile could not be found.",
|
|
2574
|
+
completedAt: nowIso()
|
|
2575
|
+
});
|
|
2576
|
+
createWikiIngestLog(jobId, "The selected LLM profile could not be found.", "error");
|
|
2577
|
+
return getWikiIngestJob(jobId);
|
|
2578
|
+
}
|
|
2579
|
+
updateWikiIngestJob(jobId, {
|
|
2580
|
+
status: "processing",
|
|
2581
|
+
phase: "processing",
|
|
2582
|
+
latestMessage: "Processing queued wiki ingest sources.",
|
|
2583
|
+
errorMessage: "",
|
|
2584
|
+
completedAt: null
|
|
2585
|
+
});
|
|
2586
|
+
createWikiIngestLog(jobId, "Started background wiki ingestion.");
|
|
2587
|
+
let currentAssetContext = null;
|
|
2588
|
+
let currentPollCount = 0;
|
|
2589
|
+
const updateCurrentAssetMetadata = (patch) => {
|
|
2590
|
+
if (!currentAssetContext) {
|
|
2591
|
+
return;
|
|
2592
|
+
}
|
|
2593
|
+
const asset = listWikiIngestJobAssetsInternal(jobId).find((entry) => entry.id === currentAssetContext?.assetId);
|
|
2594
|
+
if (!asset) {
|
|
2595
|
+
return;
|
|
2596
|
+
}
|
|
2597
|
+
updateWikiIngestAsset(asset.id, {
|
|
2598
|
+
metadata: {
|
|
2599
|
+
...parseJsonRecord(asset.metadata_json),
|
|
2600
|
+
...patch
|
|
2601
|
+
}
|
|
2602
|
+
});
|
|
2603
|
+
};
|
|
2604
|
+
const llmDiagnosticLogger = ({ level, message, details = {} }) => {
|
|
2605
|
+
const enrichedDetails = {
|
|
2606
|
+
...details,
|
|
2607
|
+
sourceAssetId: currentAssetContext?.assetId ?? null,
|
|
2608
|
+
currentFileName: currentAssetContext?.fileName ?? null,
|
|
2609
|
+
sourceLocator: currentAssetContext?.sourceLocator ?? null,
|
|
2610
|
+
checksum: currentAssetContext?.checksum ?? null,
|
|
2611
|
+
currentFileIndex: currentAssetContext?.fileIndex ?? null,
|
|
2612
|
+
currentFileTotal: currentAssetContext?.totalFiles ?? null,
|
|
2613
|
+
chunkIndex: typeof details.chunkIndex === "number"
|
|
2614
|
+
? details.chunkIndex
|
|
2615
|
+
: currentAssetContext?.chunkIndex ?? null,
|
|
2616
|
+
chunkCount: typeof details.chunkCount === "number"
|
|
2617
|
+
? details.chunkCount
|
|
2618
|
+
: currentAssetContext?.chunkCount ?? null
|
|
2619
|
+
};
|
|
2620
|
+
const eventKey = typeof details.eventKey === "string" ? details.eventKey : "";
|
|
2621
|
+
if (eventKey === "llm_compile_background_started") {
|
|
2622
|
+
currentPollCount = 0;
|
|
2623
|
+
updateCurrentAssetMetadata({
|
|
2624
|
+
openAiResponseId: typeof details.responseId === "string" ? details.responseId : null,
|
|
2625
|
+
openAiResponseStatus: typeof details.status === "string" ? details.status : "queued",
|
|
2626
|
+
openAiLastPolledAt: nowIso()
|
|
2627
|
+
});
|
|
2628
|
+
}
|
|
2629
|
+
if (eventKey === "llm_compile_background_polled") {
|
|
2630
|
+
currentPollCount += 1;
|
|
2631
|
+
const pollStatus = typeof details.status === "string" ? details.status : "in_progress";
|
|
2632
|
+
const chunkSuffix = (currentAssetContext?.chunkCount ?? 1) > 1
|
|
2633
|
+
? ` · chunk ${currentAssetContext?.chunkIndex ?? 1}/${currentAssetContext?.chunkCount ?? 1}`
|
|
2634
|
+
: "";
|
|
2635
|
+
const fileLabel = currentAssetContext?.fileName ?? "current source";
|
|
2636
|
+
const progressMessage = `Waiting for OpenAI on ${fileLabel}${chunkSuffix}. Poll ${currentPollCount} · ${pollStatus}.`;
|
|
2637
|
+
updateWikiIngestJob(jobId, {
|
|
2638
|
+
latestMessage: progressMessage
|
|
2639
|
+
});
|
|
2640
|
+
updateCurrentAssetMetadata({
|
|
2641
|
+
openAiResponseId: typeof details.responseId === "string" ? details.responseId : null,
|
|
2642
|
+
openAiResponseStatus: pollStatus,
|
|
2643
|
+
openAiLastPolledAt: nowIso(),
|
|
2644
|
+
openAiPollCount: currentPollCount
|
|
2645
|
+
});
|
|
2646
|
+
createWikiIngestLog(jobId, progressMessage, "info", {
|
|
2647
|
+
...enrichedDetails,
|
|
2648
|
+
pollCount: currentPollCount,
|
|
2649
|
+
status: pollStatus
|
|
2650
|
+
}, {
|
|
2651
|
+
aggregateKey: `llm_compile_background_polled:${currentAssetContext?.assetId ?? "job"}`,
|
|
2652
|
+
recordDiagnostic: false
|
|
2653
|
+
});
|
|
2654
|
+
return;
|
|
2655
|
+
}
|
|
2656
|
+
if (eventKey === "llm_compile_success" ||
|
|
2657
|
+
eventKey === "llm_compile_unparseable" ||
|
|
2658
|
+
eventKey === "llm_compile_background_terminal_error") {
|
|
2659
|
+
updateCurrentAssetMetadata({
|
|
2660
|
+
openAiResponseId: typeof details.responseId === "string" ? details.responseId : null,
|
|
2661
|
+
openAiResponseStatus: eventKey === "llm_compile_background_terminal_error"
|
|
2662
|
+
? typeof details.status === "string"
|
|
2663
|
+
? details.status
|
|
2664
|
+
: "failed"
|
|
2665
|
+
: "completed",
|
|
2666
|
+
openAiLastPolledAt: nowIso(),
|
|
2667
|
+
openAiPollCount: currentPollCount
|
|
2668
|
+
});
|
|
2669
|
+
}
|
|
2670
|
+
createWikiIngestLog(jobId, message, level === "debug" ? "info" : level, {
|
|
2671
|
+
scope: "wiki_llm",
|
|
2672
|
+
eventKey: "wiki_llm_event",
|
|
2673
|
+
...enrichedDetails
|
|
2674
|
+
});
|
|
2675
|
+
};
|
|
2676
|
+
const refreshCounts = () => {
|
|
2677
|
+
const candidates = listWikiIngestCandidatesInternal(jobId);
|
|
2678
|
+
const pageCount = candidates.filter((candidate) => candidate.candidate_type === "page").length;
|
|
2679
|
+
const entityCount = candidates.filter((candidate) => candidate.candidate_type === "entity").length;
|
|
2680
|
+
return { pageCount, entityCount };
|
|
2681
|
+
};
|
|
2682
|
+
const assetQueue = () => listWikiIngestJobAssetsInternal(jobId).filter((asset) => ["queued", "processing"].includes(asset.status));
|
|
2683
|
+
const initialAssets = listWikiIngestJobAssetsInternal(jobId);
|
|
2684
|
+
let processedFiles = initialAssets.filter((asset) => asset.status === "completed").length;
|
|
2685
|
+
let totalFiles = Math.max(job.total_files, initialAssets.length);
|
|
2686
|
+
let hadSuccess = false;
|
|
2687
|
+
while (assetQueue().length > 0) {
|
|
2688
|
+
const nextAsset = assetQueue().find((asset) => ["processing", "queued"].includes(asset.status));
|
|
2689
|
+
if (!nextAsset) {
|
|
2690
|
+
break;
|
|
2691
|
+
}
|
|
2692
|
+
if (nextAsset.file_name.toLowerCase().endsWith(".zip") ||
|
|
2693
|
+
nextAsset.mime_type === "application/zip") {
|
|
2694
|
+
updateWikiIngestAsset(nextAsset.id, { status: "processing" });
|
|
2695
|
+
try {
|
|
2696
|
+
const archive = new AdmZip(nextAsset.file_path);
|
|
2697
|
+
const entries = archive
|
|
2698
|
+
.getEntries()
|
|
2699
|
+
.filter((entry) => !entry.isDirectory);
|
|
2700
|
+
for (const entry of entries) {
|
|
2701
|
+
const payload = entry.getData();
|
|
2702
|
+
const fileName = path.basename(entry.entryName);
|
|
2703
|
+
const mimeType = inferMimeTypeFromPath(fileName);
|
|
2704
|
+
const persisted = await persistIngestUpload({
|
|
2705
|
+
jobId,
|
|
2706
|
+
fileName,
|
|
2707
|
+
mimeType,
|
|
2708
|
+
payload
|
|
2709
|
+
});
|
|
2710
|
+
createWikiIngestAssetRecord({
|
|
2711
|
+
jobId,
|
|
2712
|
+
sourceKind: "upload",
|
|
2713
|
+
sourceLocator: entry.entryName,
|
|
2714
|
+
fileName,
|
|
2715
|
+
mimeType,
|
|
2716
|
+
filePath: persisted.filePath,
|
|
2717
|
+
sizeBytes: persisted.sizeBytes,
|
|
2718
|
+
checksum: persisted.checksum,
|
|
2719
|
+
metadata: {
|
|
2720
|
+
parentAssetId: nextAsset.id
|
|
2721
|
+
}
|
|
2722
|
+
});
|
|
2723
|
+
}
|
|
2724
|
+
totalFiles = Math.max(0, totalFiles - 1 + entries.length);
|
|
2725
|
+
updateWikiIngestAsset(nextAsset.id, {
|
|
2726
|
+
status: "completed",
|
|
2727
|
+
metadata: {
|
|
2728
|
+
...(parseJsonRecord(nextAsset.metadata_json) ?? {}),
|
|
2729
|
+
extractedCount: entries.length
|
|
2730
|
+
}
|
|
2731
|
+
});
|
|
2732
|
+
updateWikiIngestJob(jobId, {
|
|
2733
|
+
totalFiles,
|
|
2734
|
+
latestMessage: entries.length === 1
|
|
2735
|
+
? "Expanded 1 file from ZIP archive."
|
|
2736
|
+
: `Expanded ${entries.length} files from ZIP archive.`
|
|
2737
|
+
});
|
|
2738
|
+
createWikiIngestLog(jobId, entries.length === 1
|
|
2739
|
+
? "Expanded 1 file from ZIP archive."
|
|
2740
|
+
: `Expanded ${entries.length} files from ZIP archive.`);
|
|
2741
|
+
}
|
|
2742
|
+
catch (error) {
|
|
2743
|
+
updateWikiIngestAsset(nextAsset.id, { status: "failed" });
|
|
2744
|
+
createWikiIngestLog(jobId, error instanceof Error ? error.message : "ZIP extraction failed.", "error");
|
|
2745
|
+
}
|
|
2746
|
+
continue;
|
|
2747
|
+
}
|
|
2748
|
+
updateWikiIngestAsset(nextAsset.id, { status: "processing" });
|
|
2749
|
+
currentAssetContext = {
|
|
2750
|
+
assetId: nextAsset.id,
|
|
2751
|
+
fileName: nextAsset.file_name || nextAsset.source_locator || "Source",
|
|
2752
|
+
sourceLocator: nextAsset.source_locator,
|
|
2753
|
+
checksum: nextAsset.checksum,
|
|
2754
|
+
fileIndex: Math.min(totalFiles, processedFiles + 1),
|
|
2755
|
+
totalFiles,
|
|
2756
|
+
chunkIndex: 1,
|
|
2757
|
+
chunkCount: 1
|
|
2758
|
+
};
|
|
2759
|
+
currentPollCount = 0;
|
|
2760
|
+
createWikiIngestLog(jobId, `Processing ${currentAssetContext.fileName} (${currentAssetContext.fileIndex}/${currentAssetContext.totalFiles}).`, "info", {
|
|
2761
|
+
sourceAssetId: currentAssetContext.assetId,
|
|
2762
|
+
fileName: currentAssetContext.fileName,
|
|
2763
|
+
sourceLocator: currentAssetContext.sourceLocator,
|
|
2764
|
+
checksum: currentAssetContext.checksum,
|
|
2765
|
+
fileIndex: currentAssetContext.fileIndex,
|
|
2766
|
+
totalFiles: currentAssetContext.totalFiles,
|
|
2767
|
+
chunkIndex: currentAssetContext.chunkIndex,
|
|
2768
|
+
chunkCount: currentAssetContext.chunkCount
|
|
2769
|
+
});
|
|
2770
|
+
try {
|
|
2771
|
+
const fetched = nextAsset.source_kind === "url"
|
|
2772
|
+
? await getFetchedContent("url", {
|
|
2773
|
+
sourceText: "",
|
|
2774
|
+
sourceUrl: nextAsset.source_locator,
|
|
2775
|
+
mimeType: nextAsset.mime_type
|
|
2776
|
+
})
|
|
2777
|
+
: await getFetchedContent("local_path", {
|
|
2778
|
+
sourceText: "",
|
|
2779
|
+
sourcePath: nextAsset.file_path,
|
|
2780
|
+
mimeType: nextAsset.mime_type
|
|
2781
|
+
});
|
|
2782
|
+
const rawSource = await persistWikiRawSource({
|
|
2783
|
+
space,
|
|
2784
|
+
jobId,
|
|
2785
|
+
fetched
|
|
2786
|
+
});
|
|
2787
|
+
const existingAssetMetadata = parseJsonRecord(nextAsset.metadata_json);
|
|
2788
|
+
const resumeResponseId = readStringRecordValue(existingAssetMetadata, "openAiResponseId") ??
|
|
2789
|
+
findOpenAiResponseIdForJobAsset({
|
|
2790
|
+
jobId,
|
|
2791
|
+
assetId: nextAsset.id,
|
|
2792
|
+
fileName: nextAsset.file_name,
|
|
2793
|
+
sourceLocator: nextAsset.source_locator,
|
|
2794
|
+
checksum: nextAsset.checksum
|
|
2795
|
+
});
|
|
2796
|
+
const compiled = llmProfile && parsed.parseStrategy !== "text_only"
|
|
2797
|
+
? await options.llm.compileWikiIngest(llmProfile, {
|
|
2798
|
+
titleHint: parsed.titleHint,
|
|
2799
|
+
rawText: fetched.contentText,
|
|
2800
|
+
binary: fetched.binary,
|
|
2801
|
+
mimeType: fetched.mimeType,
|
|
2802
|
+
parseStrategy: parsed.parseStrategy
|
|
2803
|
+
}, {
|
|
2804
|
+
resumeResponseId
|
|
2805
|
+
}, llmDiagnosticLogger)
|
|
2806
|
+
: llmProfile && fetched.contentText
|
|
2807
|
+
? await options.llm.compileWikiIngest(llmProfile, {
|
|
2808
|
+
titleHint: parsed.titleHint,
|
|
2809
|
+
rawText: fetched.contentText,
|
|
2810
|
+
binary: fetched.binary,
|
|
2811
|
+
mimeType: fetched.mimeType,
|
|
2812
|
+
parseStrategy: "text_only"
|
|
2813
|
+
}, {
|
|
2814
|
+
resumeResponseId
|
|
2815
|
+
}, llmDiagnosticLogger)
|
|
2816
|
+
: null;
|
|
2817
|
+
if (llmProfile && !compiled) {
|
|
2818
|
+
throw new Error("The LLM did not produce structured draft candidates. Check the OpenAI settings and try again.");
|
|
2819
|
+
}
|
|
2820
|
+
const title = compiled?.title ||
|
|
2821
|
+
parsed.titleHint ||
|
|
2822
|
+
inferTitle(fetched.contentText || fetched.fileName, "Imported source");
|
|
2823
|
+
const summary = compiled?.summary ||
|
|
2824
|
+
(fetched.contentText ? inferSummary(fetched.contentText) : "");
|
|
2825
|
+
const markdown = compiled?.markdown
|
|
2826
|
+
? compiled.markdown
|
|
2827
|
+
: fetched.contentText
|
|
2828
|
+
? `# ${title}\n\n${fetched.contentText.trim()}\n`
|
|
2829
|
+
: `# ${title}\n\nImported media asset \`${fetched.fileName}\` (${fetched.mimeType}).\n`;
|
|
2830
|
+
createWikiIngestCandidate({
|
|
2831
|
+
jobId,
|
|
2832
|
+
sourceAssetId: nextAsset.id,
|
|
2833
|
+
candidateType: "page",
|
|
2834
|
+
title,
|
|
2835
|
+
summary,
|
|
2836
|
+
targetKey: "",
|
|
2837
|
+
payload: createPageCandidatePayload({
|
|
2838
|
+
title,
|
|
2839
|
+
summary,
|
|
2840
|
+
markdown,
|
|
2841
|
+
sourceLocator: fetched.locator,
|
|
2842
|
+
createAsKind: parsed.createAsKind,
|
|
2843
|
+
tags: compiled?.tags,
|
|
2844
|
+
linkedEntityHints: parsed.linkedEntityHints
|
|
2845
|
+
})
|
|
2846
|
+
});
|
|
2847
|
+
if (parsed.entityProposalMode === "suggest") {
|
|
2848
|
+
for (const proposal of compiled?.entityProposals ?? []) {
|
|
2849
|
+
createWikiIngestCandidate({
|
|
2850
|
+
jobId,
|
|
2851
|
+
sourceAssetId: nextAsset.id,
|
|
2852
|
+
candidateType: "entity",
|
|
2853
|
+
title: typeof proposal.title === "string"
|
|
2854
|
+
? proposal.title
|
|
2855
|
+
: "Entity proposal",
|
|
2856
|
+
summary: typeof proposal.summary === "string" ? proposal.summary : "",
|
|
2857
|
+
targetKey: typeof proposal.entityType === "string"
|
|
2858
|
+
? proposal.entityType
|
|
2859
|
+
: "",
|
|
2860
|
+
payload: proposal
|
|
2861
|
+
});
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
for (const suggestion of compiled?.pageUpdateSuggestions ?? []) {
|
|
2865
|
+
const patchSummary = typeof suggestion.patchSummary === "string"
|
|
2866
|
+
? suggestion.patchSummary
|
|
2867
|
+
: "";
|
|
2868
|
+
createWikiIngestCandidate({
|
|
2869
|
+
jobId,
|
|
2870
|
+
sourceAssetId: nextAsset.id,
|
|
2871
|
+
candidateType: "page_update",
|
|
2872
|
+
title: typeof suggestion.targetSlug === "string"
|
|
2873
|
+
? suggestion.targetSlug
|
|
2874
|
+
: "Page update",
|
|
2875
|
+
summary: typeof suggestion.rationale === "string"
|
|
2876
|
+
? suggestion.rationale
|
|
2877
|
+
: patchSummary,
|
|
2878
|
+
targetKey: typeof suggestion.targetSlug === "string"
|
|
2879
|
+
? suggestion.targetSlug
|
|
2880
|
+
: "",
|
|
2881
|
+
payload: {
|
|
2882
|
+
...suggestion,
|
|
2883
|
+
patchSummary
|
|
2884
|
+
}
|
|
2885
|
+
});
|
|
2886
|
+
}
|
|
2887
|
+
for (const candidate of compiled?.articleCandidates ?? []) {
|
|
2888
|
+
const articleTitle = typeof candidate.title === "string" &&
|
|
2889
|
+
candidate.title.trim().length > 0
|
|
2890
|
+
? candidate.title.trim()
|
|
2891
|
+
: "Suggested article";
|
|
2892
|
+
const articleSummary = typeof candidate.summary === "string" ? candidate.summary : "";
|
|
2893
|
+
const articleRationale = typeof candidate.rationale === "string" ? candidate.rationale : "";
|
|
2894
|
+
createWikiIngestCandidate({
|
|
2895
|
+
jobId,
|
|
2896
|
+
sourceAssetId: nextAsset.id,
|
|
2897
|
+
candidateType: "page",
|
|
2898
|
+
title: articleTitle,
|
|
2899
|
+
summary: articleSummary,
|
|
2900
|
+
targetKey: typeof candidate.slug === "string" ? candidate.slug : "",
|
|
2901
|
+
payload: createPageCandidatePayload({
|
|
2902
|
+
title: articleTitle,
|
|
2903
|
+
summary: articleSummary,
|
|
2904
|
+
markdown: typeof candidate.markdown === "string" &&
|
|
2905
|
+
candidate.markdown.trim().length > 0
|
|
2906
|
+
? candidate.markdown
|
|
2907
|
+
: `# ${articleTitle}\n\n${articleSummary}\n\n${articleRationale
|
|
2908
|
+
? `## Why this page\n\n${articleRationale}\n`
|
|
2909
|
+
: ""}`,
|
|
2910
|
+
sourceLocator: fetched.locator,
|
|
2911
|
+
createAsKind: "wiki",
|
|
2912
|
+
aliases: Array.isArray(candidate.aliases)
|
|
2913
|
+
? candidate.aliases.filter((alias) => typeof alias === "string")
|
|
2914
|
+
: [],
|
|
2915
|
+
parentSlug: typeof candidate.parentSlug === "string"
|
|
2916
|
+
? candidate.parentSlug
|
|
2917
|
+
: null,
|
|
2918
|
+
tags: Array.isArray(candidate.tags)
|
|
2919
|
+
? candidate.tags.filter((tag) => typeof tag === "string")
|
|
2920
|
+
: [],
|
|
2921
|
+
linkedEntityHints: parsed.linkedEntityHints
|
|
2922
|
+
})
|
|
2923
|
+
});
|
|
2924
|
+
}
|
|
2925
|
+
updateWikiIngestAsset(nextAsset.id, {
|
|
2926
|
+
status: "completed",
|
|
2927
|
+
filePath: rawSource.filePath,
|
|
2928
|
+
mimeType: fetched.mimeType,
|
|
2929
|
+
sizeBytes: rawSource.sizeBytes,
|
|
2930
|
+
checksum: rawSource.checksum,
|
|
2931
|
+
metadata: {
|
|
2932
|
+
...(parseJsonRecord(nextAsset.metadata_json) ?? {}),
|
|
2933
|
+
rawSourcePath: rawSource.filePath,
|
|
2934
|
+
openAiResponseStatus: llmProfile ? "completed" : null,
|
|
2935
|
+
openAiLastPolledAt: llmProfile ? nowIso() : null
|
|
2936
|
+
}
|
|
2937
|
+
});
|
|
2938
|
+
hadSuccess = true;
|
|
2939
|
+
processedFiles += 1;
|
|
2940
|
+
const counts = refreshCounts();
|
|
2941
|
+
updateWikiIngestJob(jobId, {
|
|
2942
|
+
processedFiles,
|
|
2943
|
+
totalFiles,
|
|
2944
|
+
progressPercent: calculateProgress(totalFiles, processedFiles),
|
|
2945
|
+
createdPageCount: counts.pageCount,
|
|
2946
|
+
createdEntityCount: counts.entityCount,
|
|
2947
|
+
sourceLocator: fetched.locator,
|
|
2948
|
+
mimeType: fetched.mimeType,
|
|
2949
|
+
summary,
|
|
2950
|
+
latestMessage: `Prepared candidates from ${fetched.fileName}.`
|
|
2951
|
+
});
|
|
2952
|
+
createWikiIngestLog(jobId, `Prepared candidates from ${fetched.fileName}.`, "info", {
|
|
2953
|
+
fileName: currentAssetContext?.fileName ?? fetched.fileName,
|
|
2954
|
+
sourceAssetId: currentAssetContext?.assetId ?? null,
|
|
2955
|
+
fileIndex: currentAssetContext?.fileIndex ?? processedFiles,
|
|
2956
|
+
totalFiles: currentAssetContext?.totalFiles ?? totalFiles,
|
|
2957
|
+
pageCandidates: counts.pageCount,
|
|
2958
|
+
entityCandidates: counts.entityCount
|
|
2959
|
+
});
|
|
2960
|
+
currentAssetContext = null;
|
|
2961
|
+
currentPollCount = 0;
|
|
2962
|
+
}
|
|
2963
|
+
catch (error) {
|
|
2964
|
+
processedFiles += 1;
|
|
2965
|
+
updateWikiIngestAsset(nextAsset.id, { status: "failed" });
|
|
2966
|
+
updateCurrentAssetMetadata({
|
|
2967
|
+
openAiResponseStatus: "failed",
|
|
2968
|
+
openAiLastPolledAt: nowIso(),
|
|
2969
|
+
openAiPollCount: currentPollCount
|
|
2970
|
+
});
|
|
2971
|
+
updateWikiIngestJob(jobId, {
|
|
2972
|
+
processedFiles,
|
|
2973
|
+
totalFiles,
|
|
2974
|
+
progressPercent: calculateProgress(totalFiles, processedFiles),
|
|
2975
|
+
latestMessage: error instanceof Error ? error.message : "Source ingest failed."
|
|
2976
|
+
});
|
|
2977
|
+
createWikiIngestLog(jobId, error instanceof Error ? error.message : "Source ingest failed.", "error", {
|
|
2978
|
+
fileName: currentAssetContext?.fileName ?? null,
|
|
2979
|
+
fileIndex: currentAssetContext?.fileIndex ?? null,
|
|
2980
|
+
totalFiles: currentAssetContext?.totalFiles ?? totalFiles,
|
|
2981
|
+
pollCount: currentPollCount
|
|
2982
|
+
});
|
|
2983
|
+
currentAssetContext = null;
|
|
2984
|
+
currentPollCount = 0;
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
const counts = refreshCounts();
|
|
2988
|
+
updateWikiIngestJob(jobId, {
|
|
2989
|
+
status: hadSuccess ? "completed" : "failed",
|
|
2990
|
+
phase: hadSuccess ? "review" : "failed",
|
|
2991
|
+
progressPercent: 100,
|
|
2992
|
+
totalFiles,
|
|
2993
|
+
processedFiles: Math.max(processedFiles, totalFiles),
|
|
2994
|
+
createdPageCount: counts.pageCount,
|
|
2995
|
+
createdEntityCount: counts.entityCount,
|
|
2996
|
+
latestMessage: hadSuccess
|
|
2997
|
+
? "Ingest finished. Review the proposed pages and entities."
|
|
2998
|
+
: "Ingest failed before any candidates could be prepared.",
|
|
2999
|
+
errorMessage: hadSuccess ? "" : "No candidates were produced.",
|
|
3000
|
+
completedAt: nowIso()
|
|
3001
|
+
});
|
|
3002
|
+
createWikiIngestLog(jobId, hadSuccess
|
|
3003
|
+
? "Background wiki ingestion completed and is ready for review."
|
|
3004
|
+
: "Background wiki ingestion failed.", hadSuccess ? "info" : "error");
|
|
3005
|
+
return getWikiIngestJob(jobId);
|
|
3006
|
+
}
|
|
3007
|
+
export function listWikiIngestJobs(input = {}) {
|
|
3008
|
+
const parsed = listWikiIngestJobsQuerySchema.parse(input);
|
|
3009
|
+
const rows = getDatabase()
|
|
3010
|
+
.prepare(`SELECT id
|
|
3011
|
+
FROM wiki_ingest_jobs
|
|
3012
|
+
WHERE (? IS NULL OR space_id = ?)
|
|
3013
|
+
ORDER BY created_at DESC
|
|
3014
|
+
LIMIT ?`)
|
|
3015
|
+
.all(parsed.spaceId ?? null, parsed.spaceId ?? null, parsed.limit);
|
|
3016
|
+
return rows
|
|
3017
|
+
.map((row) => getWikiIngestJob(row.id))
|
|
3018
|
+
.filter((job) => job !== null);
|
|
3019
|
+
}
|
|
3020
|
+
export async function rerunWikiIngestJob(jobId, options = {}) {
|
|
3021
|
+
const job = readWikiIngestJobRow(jobId);
|
|
3022
|
+
if (!job) {
|
|
3023
|
+
return null;
|
|
3024
|
+
}
|
|
3025
|
+
if (["queued", "processing"].includes(job.status)) {
|
|
3026
|
+
throw new Error("Wiki ingest jobs can only be rerun after processing ends.");
|
|
3027
|
+
}
|
|
3028
|
+
const ingestInput = readWikiIngestInput(jobId);
|
|
3029
|
+
if (!ingestInput) {
|
|
3030
|
+
throw new Error("Wiki ingest input could not be restored.");
|
|
3031
|
+
}
|
|
3032
|
+
const rootAssets = listWikiIngestJobAssetsInternal(jobId).filter((asset) => {
|
|
3033
|
+
const metadata = parseJsonRecord(asset.metadata_json);
|
|
3034
|
+
return !metadata?.parentAssetId && asset.source_kind !== "url";
|
|
3035
|
+
});
|
|
3036
|
+
const replayResult = rootAssets.length > 0
|
|
3037
|
+
? await createUploadedWikiIngestJob({
|
|
3038
|
+
spaceId: ingestInput.spaceId,
|
|
3039
|
+
titleHint: ingestInput.titleHint,
|
|
3040
|
+
llmProfileId: ingestInput.llmProfileId,
|
|
3041
|
+
parseStrategy: ingestInput.parseStrategy,
|
|
3042
|
+
entityProposalMode: ingestInput.entityProposalMode,
|
|
3043
|
+
userId: ingestInput.userId ?? null,
|
|
3044
|
+
createAsKind: ingestInput.createAsKind,
|
|
3045
|
+
linkedEntityHints: ingestInput.linkedEntityHints
|
|
3046
|
+
}, await Promise.all(rootAssets.map(async (asset) => ({
|
|
3047
|
+
fileName: asset.file_name,
|
|
3048
|
+
mimeType: asset.mime_type,
|
|
3049
|
+
payload: await readFile(asset.file_path)
|
|
3050
|
+
}))), {
|
|
3051
|
+
actor: options.actor ?? job.created_by_actor ?? null
|
|
3052
|
+
})
|
|
3053
|
+
: await ingestWikiSource(ingestInput, {
|
|
3054
|
+
actor: options.actor ?? job.created_by_actor ?? null
|
|
3055
|
+
});
|
|
3056
|
+
const nextJobId = replayResult.job?.job.id ?? null;
|
|
3057
|
+
if (nextJobId) {
|
|
3058
|
+
createWikiIngestLog(nextJobId, `Reran ingest from ${jobId}.`, "info", {
|
|
3059
|
+
scope: "wiki_ingest",
|
|
3060
|
+
eventKey: "wiki_ingest_rerun",
|
|
3061
|
+
sourceJobId: jobId
|
|
3062
|
+
});
|
|
3063
|
+
createWikiIngestLog(jobId, `Created rerun job ${nextJobId}.`, "info", {
|
|
3064
|
+
scope: "wiki_ingest",
|
|
3065
|
+
eventKey: "wiki_ingest_rerun_requested",
|
|
3066
|
+
rerunJobId: nextJobId
|
|
3067
|
+
});
|
|
3068
|
+
}
|
|
3069
|
+
return replayResult;
|
|
3070
|
+
}
|
|
3071
|
+
export function deleteWikiIngestJob(jobId) {
|
|
3072
|
+
const job = readWikiIngestJobRow(jobId);
|
|
3073
|
+
if (!job) {
|
|
3074
|
+
return null;
|
|
3075
|
+
}
|
|
3076
|
+
if (["queued", "processing"].includes(job.status)) {
|
|
3077
|
+
throw new Error("Wiki ingest jobs can only be deleted after processing ends.");
|
|
3078
|
+
}
|
|
3079
|
+
const candidates = listWikiIngestCandidatesInternal(jobId);
|
|
3080
|
+
const assets = listWikiIngestJobAssetsInternal(jobId);
|
|
3081
|
+
const preservedAssetIds = new Set(candidates
|
|
3082
|
+
.filter((candidate) => candidate.status === "applied")
|
|
3083
|
+
.map((candidate) => candidate.source_asset_id)
|
|
3084
|
+
.filter((assetId) => typeof assetId === "string"));
|
|
3085
|
+
for (const asset of assets) {
|
|
3086
|
+
if (preservedAssetIds.has(asset.id)) {
|
|
3087
|
+
continue;
|
|
3088
|
+
}
|
|
3089
|
+
const metadata = parseJsonRecord(asset.metadata_json);
|
|
3090
|
+
const rawSourcePath = typeof metadata?.rawSourcePath === "string"
|
|
3091
|
+
? metadata.rawSourcePath
|
|
3092
|
+
: null;
|
|
3093
|
+
if (rawSourcePath && existsSync(rawSourcePath)) {
|
|
3094
|
+
unlinkSync(rawSourcePath);
|
|
3095
|
+
}
|
|
3096
|
+
}
|
|
3097
|
+
const jobDir = getWikiIngestJobDir(jobId);
|
|
3098
|
+
if (existsSync(jobDir)) {
|
|
3099
|
+
rmSync(jobDir, { recursive: true, force: true });
|
|
3100
|
+
}
|
|
3101
|
+
getDatabase()
|
|
3102
|
+
.prepare(`DELETE FROM wiki_ingest_jobs WHERE id = ?`)
|
|
3103
|
+
.run(jobId);
|
|
3104
|
+
return {
|
|
3105
|
+
id: jobId
|
|
3106
|
+
};
|
|
3107
|
+
}
|
|
3108
|
+
export async function reviewWikiIngestJob(jobId, input, options) {
|
|
3109
|
+
const parsed = reviewWikiIngestJobSchema.parse(input);
|
|
3110
|
+
const job = readWikiIngestJobRow(jobId);
|
|
3111
|
+
if (!job) {
|
|
3112
|
+
return null;
|
|
3113
|
+
}
|
|
3114
|
+
const ingestInput = readWikiIngestInput(jobId);
|
|
3115
|
+
if (!ingestInput) {
|
|
3116
|
+
return null;
|
|
3117
|
+
}
|
|
3118
|
+
const candidates = listWikiIngestCandidatesInternal(jobId);
|
|
3119
|
+
let acceptedCount = 0;
|
|
3120
|
+
let mappedCount = 0;
|
|
3121
|
+
let rejectedCount = 0;
|
|
3122
|
+
let firstPublishedPageId = null;
|
|
3123
|
+
for (const decision of parsed.decisions) {
|
|
3124
|
+
const candidate = candidates.find((entry) => entry.id === decision.candidateId);
|
|
3125
|
+
if (!candidate) {
|
|
3126
|
+
continue;
|
|
3127
|
+
}
|
|
3128
|
+
const action = decision.action ??
|
|
3129
|
+
(decision.keep === false ? "discard" : "keep");
|
|
3130
|
+
if (action === "discard") {
|
|
3131
|
+
rejectedCount += 1;
|
|
3132
|
+
updateWikiIngestCandidate(candidate.id, { status: "rejected" });
|
|
3133
|
+
continue;
|
|
3134
|
+
}
|
|
3135
|
+
try {
|
|
3136
|
+
const payload = parseJsonRecord(candidate.payload_json);
|
|
3137
|
+
if (candidate.candidate_type === "page") {
|
|
3138
|
+
if (action === "merge_existing") {
|
|
3139
|
+
const target = decision.targetNoteId
|
|
3140
|
+
? (getWikiPageDetail(decision.targetNoteId)?.page ?? null)
|
|
3141
|
+
: null;
|
|
3142
|
+
if (!target || target.spaceId !== job.space_id) {
|
|
3143
|
+
throw new Error("Merge target page was not found.");
|
|
3144
|
+
}
|
|
3145
|
+
const incomingTitle = typeof payload.title === "string" ? payload.title : candidate.title;
|
|
3146
|
+
const incomingMarkdown = typeof payload.contentMarkdown === "string"
|
|
3147
|
+
? payload.contentMarkdown
|
|
3148
|
+
: `# ${candidate.title}\n`;
|
|
3149
|
+
const mergedContentMarkdown = mergeWikiPageContent(target.contentMarkdown, {
|
|
3150
|
+
title: incomingTitle,
|
|
3151
|
+
markdown: incomingMarkdown
|
|
3152
|
+
});
|
|
3153
|
+
const mergedSummary = inferSummary(mergedContentMarkdown);
|
|
3154
|
+
const updated = options.updateNote(target.id, {
|
|
3155
|
+
contentMarkdown: mergedContentMarkdown,
|
|
3156
|
+
summary: mergedSummary
|
|
3157
|
+
});
|
|
3158
|
+
if (!updated) {
|
|
3159
|
+
throw new Error("Merge target page could not be updated.");
|
|
3160
|
+
}
|
|
3161
|
+
acceptedCount += 1;
|
|
3162
|
+
firstPublishedPageId = firstPublishedPageId ?? updated.id;
|
|
3163
|
+
updateWikiIngestCandidate(candidate.id, {
|
|
3164
|
+
status: "applied",
|
|
3165
|
+
publishedNoteId: updated.id
|
|
3166
|
+
});
|
|
3167
|
+
}
|
|
3168
|
+
else {
|
|
3169
|
+
const note = options.createNote({
|
|
3170
|
+
kind: payload.kind === "evidence" ? "evidence" : "wiki",
|
|
3171
|
+
title: typeof payload.title === "string"
|
|
3172
|
+
? payload.title
|
|
3173
|
+
: candidate.title,
|
|
3174
|
+
slug: "",
|
|
3175
|
+
indexOrder: 0,
|
|
3176
|
+
aliases: Array.isArray(payload.aliases)
|
|
3177
|
+
? payload.aliases.filter((alias) => typeof alias === "string")
|
|
3178
|
+
: [],
|
|
3179
|
+
summary: typeof payload.summary === "string"
|
|
3180
|
+
? payload.summary
|
|
3181
|
+
: candidate.summary,
|
|
3182
|
+
sourcePath: "",
|
|
3183
|
+
frontmatter: {},
|
|
3184
|
+
revisionHash: "",
|
|
3185
|
+
spaceId: job.space_id,
|
|
3186
|
+
parentSlug: typeof payload.parentSlug === "string"
|
|
3187
|
+
? payload.parentSlug
|
|
3188
|
+
: null,
|
|
3189
|
+
contentMarkdown: typeof payload.contentMarkdown === "string"
|
|
3190
|
+
? payload.contentMarkdown
|
|
3191
|
+
: `# ${candidate.title}\n`,
|
|
3192
|
+
author: ingestInput.userId ?? null,
|
|
3193
|
+
links: Array.isArray(payload.links)
|
|
3194
|
+
? payload.links
|
|
3195
|
+
: ingestInput.linkedEntityHints,
|
|
3196
|
+
tags: Array.isArray(payload.tags)
|
|
3197
|
+
? payload.tags.filter((tag) => typeof tag === "string")
|
|
3198
|
+
: [],
|
|
3199
|
+
destroyAt: null,
|
|
3200
|
+
userId: ingestInput.userId ?? null
|
|
3201
|
+
});
|
|
3202
|
+
acceptedCount += 1;
|
|
3203
|
+
firstPublishedPageId = firstPublishedPageId ?? note.id;
|
|
3204
|
+
updateWikiIngestCandidate(candidate.id, {
|
|
3205
|
+
status: "applied",
|
|
3206
|
+
publishedNoteId: note.id
|
|
3207
|
+
});
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
else if (candidate.candidate_type === "page_update") {
|
|
3211
|
+
const targetSlug = typeof payload.targetSlug === "string"
|
|
3212
|
+
? payload.targetSlug
|
|
3213
|
+
: candidate.target_key;
|
|
3214
|
+
const target = targetSlug.trim().length > 0
|
|
3215
|
+
? (getWikiPageDetailBySlug({
|
|
3216
|
+
slug: targetSlug,
|
|
3217
|
+
spaceId: job.space_id
|
|
3218
|
+
})?.page ?? null)
|
|
3219
|
+
: null;
|
|
3220
|
+
if (target) {
|
|
3221
|
+
const patchSummary = typeof payload.patchSummary === "string"
|
|
3222
|
+
? payload.patchSummary
|
|
3223
|
+
: "";
|
|
3224
|
+
const rationale = typeof payload.rationale === "string" ? payload.rationale : "";
|
|
3225
|
+
options.updateNote(target.id, {
|
|
3226
|
+
contentMarkdown: `${target.contentMarkdown.trim()}\n\n## Imported update\n\n${patchSummary}\n${rationale ? `\n${rationale}\n` : ""}`,
|
|
3227
|
+
summary: target.summary || (typeof rationale === "string" ? rationale : "")
|
|
3228
|
+
});
|
|
3229
|
+
acceptedCount += 1;
|
|
3230
|
+
updateWikiIngestCandidate(candidate.id, {
|
|
3231
|
+
status: "applied",
|
|
3232
|
+
publishedNoteId: target.id
|
|
3233
|
+
});
|
|
3234
|
+
}
|
|
3235
|
+
else {
|
|
3236
|
+
updateWikiIngestCandidate(candidate.id, { status: "failed" });
|
|
3237
|
+
}
|
|
3238
|
+
}
|
|
3239
|
+
else if (candidate.candidate_type === "entity") {
|
|
3240
|
+
if (action === "map_existing") {
|
|
3241
|
+
const proposedType = typeof payload.entityType === "string" ? payload.entityType : "";
|
|
3242
|
+
if (decision.mappedEntityType !== proposedType) {
|
|
3243
|
+
throw new Error("Mapped entity type must match the proposed entity type.");
|
|
3244
|
+
}
|
|
3245
|
+
const mapped = options.resolveMappedEntity(decision.mappedEntityType, decision.mappedEntityId);
|
|
3246
|
+
if (!mapped) {
|
|
3247
|
+
throw new Error("Mapped entity was not found.");
|
|
3248
|
+
}
|
|
3249
|
+
acceptedCount += 1;
|
|
3250
|
+
mappedCount += 1;
|
|
3251
|
+
updateWikiIngestCandidate(candidate.id, {
|
|
3252
|
+
status: "applied",
|
|
3253
|
+
publishedEntityType: mapped.entityType,
|
|
3254
|
+
publishedEntityId: mapped.entityId
|
|
3255
|
+
});
|
|
3256
|
+
}
|
|
3257
|
+
else {
|
|
3258
|
+
const result = options.publishEntity(payload);
|
|
3259
|
+
acceptedCount += 1;
|
|
3260
|
+
updateWikiIngestCandidate(candidate.id, {
|
|
3261
|
+
status: "applied",
|
|
3262
|
+
publishedEntityType: result.entityType,
|
|
3263
|
+
publishedEntityId: result.entityId
|
|
3264
|
+
});
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
catch {
|
|
3269
|
+
updateWikiIngestCandidate(candidate.id, { status: "failed" });
|
|
3270
|
+
}
|
|
3271
|
+
}
|
|
3272
|
+
updateWikiIngestJob(jobId, {
|
|
3273
|
+
phase: "reviewed",
|
|
3274
|
+
acceptedCount,
|
|
3275
|
+
rejectedCount,
|
|
3276
|
+
pageNoteId: firstPublishedPageId,
|
|
3277
|
+
latestMessage: acceptedCount > 0 || rejectedCount > 0
|
|
3278
|
+
? `Review saved with ${acceptedCount} kept${mappedCount > 0 ? ` (${mappedCount} mapped)` : ""} and ${rejectedCount} discarded.`
|
|
3279
|
+
: "Review saved."
|
|
3280
|
+
});
|
|
3281
|
+
createWikiIngestLog(jobId, `Review saved with ${acceptedCount} kept${mappedCount > 0 ? ` (${mappedCount} mapped)` : ""} and ${rejectedCount} discarded.`);
|
|
3282
|
+
return getWikiIngestJob(jobId);
|
|
3283
|
+
}
|
|
3284
|
+
export function getWikiIngestJob(jobId) {
|
|
3285
|
+
const job = readWikiIngestJobRow(jobId);
|
|
3286
|
+
if (!job) {
|
|
3287
|
+
return null;
|
|
3288
|
+
}
|
|
3289
|
+
const items = getDatabase()
|
|
3290
|
+
.prepare(`SELECT id, job_id, item_type, status, note_id, media_asset_id, payload_json, created_at, updated_at
|
|
3291
|
+
FROM wiki_ingest_job_items
|
|
3292
|
+
WHERE job_id = ?
|
|
3293
|
+
ORDER BY created_at ASC`)
|
|
3294
|
+
.all(jobId);
|
|
3295
|
+
const assets = listWikiIngestJobAssetsInternal(jobId);
|
|
3296
|
+
const normalizedItems = items.length > 0
|
|
3297
|
+
? items.map((item) => ({
|
|
3298
|
+
id: item.id,
|
|
3299
|
+
itemType: item.item_type,
|
|
3300
|
+
status: item.status,
|
|
3301
|
+
noteId: item.note_id,
|
|
3302
|
+
mediaAssetId: item.media_asset_id,
|
|
3303
|
+
payload: parseJsonRecord(item.payload_json),
|
|
3304
|
+
createdAt: item.created_at,
|
|
3305
|
+
updatedAt: item.updated_at
|
|
3306
|
+
}))
|
|
3307
|
+
: assets.map((asset) => ({
|
|
3308
|
+
id: asset.id,
|
|
3309
|
+
itemType: "raw_source",
|
|
3310
|
+
status: asset.status,
|
|
3311
|
+
noteId: null,
|
|
3312
|
+
mediaAssetId: null,
|
|
3313
|
+
payload: {
|
|
3314
|
+
sourceKind: asset.source_kind,
|
|
3315
|
+
sourceLocator: asset.source_locator,
|
|
3316
|
+
fileName: asset.file_name,
|
|
3317
|
+
mimeType: asset.mime_type,
|
|
3318
|
+
filePath: asset.file_path,
|
|
3319
|
+
sizeBytes: asset.size_bytes,
|
|
3320
|
+
checksum: asset.checksum,
|
|
3321
|
+
...(parseJsonRecord(asset.metadata_json) ?? {})
|
|
3322
|
+
},
|
|
3323
|
+
createdAt: asset.created_at,
|
|
3324
|
+
updatedAt: asset.updated_at
|
|
3325
|
+
}));
|
|
3326
|
+
return wikiIngestJobPayloadSchema.parse({
|
|
3327
|
+
job: mapWikiIngestJobRow(job),
|
|
3328
|
+
items: normalizedItems,
|
|
3329
|
+
logs: listWikiIngestJobLogsInternal(jobId).map((log) => ({
|
|
3330
|
+
id: log.id,
|
|
3331
|
+
level: log.level,
|
|
3332
|
+
message: log.message,
|
|
3333
|
+
metadata: parseJsonRecord(log.metadata_json),
|
|
3334
|
+
createdAt: log.created_at
|
|
3335
|
+
})),
|
|
3336
|
+
assets: assets.map((asset) => ({
|
|
3337
|
+
id: asset.id,
|
|
3338
|
+
status: asset.status,
|
|
3339
|
+
sourceKind: asset.source_kind,
|
|
3340
|
+
sourceLocator: asset.source_locator,
|
|
3341
|
+
fileName: asset.file_name,
|
|
3342
|
+
mimeType: asset.mime_type,
|
|
3343
|
+
filePath: asset.file_path,
|
|
3344
|
+
sizeBytes: asset.size_bytes,
|
|
3345
|
+
checksum: asset.checksum,
|
|
3346
|
+
metadata: parseJsonRecord(asset.metadata_json),
|
|
3347
|
+
createdAt: asset.created_at,
|
|
3348
|
+
updatedAt: asset.updated_at
|
|
3349
|
+
})),
|
|
3350
|
+
candidates: listWikiIngestCandidatesInternal(jobId).map((candidate) => ({
|
|
3351
|
+
id: candidate.id,
|
|
3352
|
+
sourceAssetId: candidate.source_asset_id,
|
|
3353
|
+
candidateType: candidate.candidate_type,
|
|
3354
|
+
status: candidate.status,
|
|
3355
|
+
title: candidate.title,
|
|
3356
|
+
summary: candidate.summary,
|
|
3357
|
+
targetKey: candidate.target_key,
|
|
3358
|
+
payload: parseJsonRecord(candidate.payload_json),
|
|
3359
|
+
publishedNoteId: candidate.published_note_id,
|
|
3360
|
+
publishedEntityType: candidate.published_entity_type,
|
|
3361
|
+
publishedEntityId: candidate.published_entity_id,
|
|
3362
|
+
createdAt: candidate.created_at,
|
|
3363
|
+
updatedAt: candidate.updated_at
|
|
3364
|
+
}))
|
|
3365
|
+
});
|
|
3366
|
+
}
|