forge-openclaw-plugin 0.2.19 → 0.2.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +133 -2
  2. package/dist/assets/board-_C6oMy5w.js +6 -0
  3. package/dist/assets/{board-8L3uX7_O.js.map → board-_C6oMy5w.js.map} +1 -1
  4. package/dist/assets/index-B4A6TooJ.js +63 -0
  5. package/dist/assets/index-B4A6TooJ.js.map +1 -0
  6. package/dist/assets/index-D6Xs_2mo.css +1 -0
  7. package/dist/assets/{motion-1GAqqi8M.js → motion-D4sZgCHd.js} +2 -2
  8. package/dist/assets/{motion-1GAqqi8M.js.map → motion-D4sZgCHd.js.map} +1 -1
  9. package/dist/assets/{table-DBGlgRjk.js → table-BWzTaky1.js} +2 -2
  10. package/dist/assets/{table-DBGlgRjk.js.map → table-BWzTaky1.js.map} +1 -1
  11. package/dist/assets/{ui-iTluWjC4.js → ui-BzK4azQb.js} +7 -7
  12. package/dist/assets/{ui-iTluWjC4.js.map → ui-BzK4azQb.js.map} +1 -1
  13. package/dist/assets/vendor-DT3pnAKJ.css +1 -0
  14. package/dist/assets/vendor-De38P6YR.js +729 -0
  15. package/dist/assets/vendor-De38P6YR.js.map +1 -0
  16. package/dist/assets/viz-C6hfyqzu.js +34 -0
  17. package/dist/assets/viz-C6hfyqzu.js.map +1 -0
  18. package/dist/index.html +9 -9
  19. package/dist/openclaw/parity.d.ts +1 -1
  20. package/dist/openclaw/parity.js +29 -2
  21. package/dist/openclaw/routes.js +207 -24
  22. package/dist/openclaw/tools.js +324 -35
  23. package/dist/server/app.js +2080 -92
  24. package/dist/server/db.js +3 -0
  25. package/dist/server/health.js +1284 -0
  26. package/dist/server/managers/platform/background-job-manager.js +138 -2
  27. package/dist/server/managers/platform/llm-manager.js +126 -0
  28. package/dist/server/managers/platform/openai-responses-provider.js +773 -0
  29. package/dist/server/managers/runtime.js +6 -1
  30. package/dist/server/openapi.js +718 -0
  31. package/dist/server/preferences-seeds.js +409 -0
  32. package/dist/server/preferences-types.js +368 -0
  33. package/dist/server/psyche-types.js +42 -18
  34. package/dist/server/repositories/activity-events.js +53 -4
  35. package/dist/server/repositories/calendar.js +89 -15
  36. package/dist/server/repositories/collaboration.js +8 -3
  37. package/dist/server/repositories/diagnostic-logs.js +243 -0
  38. package/dist/server/repositories/entity-ownership.js +92 -0
  39. package/dist/server/repositories/goals.js +7 -2
  40. package/dist/server/repositories/habits.js +122 -16
  41. package/dist/server/repositories/notes.js +119 -41
  42. package/dist/server/repositories/preferences.js +1765 -0
  43. package/dist/server/repositories/projects.js +18 -7
  44. package/dist/server/repositories/psyche.js +84 -27
  45. package/dist/server/repositories/rewards.js +112 -4
  46. package/dist/server/repositories/strategies.js +450 -0
  47. package/dist/server/repositories/tags.js +11 -6
  48. package/dist/server/repositories/task-runs.js +10 -2
  49. package/dist/server/repositories/tasks.js +99 -17
  50. package/dist/server/repositories/users.js +417 -0
  51. package/dist/server/repositories/wiki-memory.js +3366 -0
  52. package/dist/server/services/context.js +20 -18
  53. package/dist/server/services/dashboard.js +29 -6
  54. package/dist/server/services/entity-crud.js +21 -3
  55. package/dist/server/services/insights.js +9 -7
  56. package/dist/server/services/projects.js +2 -1
  57. package/dist/server/services/psyche.js +10 -9
  58. package/dist/server/types.js +594 -30
  59. package/openclaw.plugin.json +1 -1
  60. package/package.json +1 -1
  61. package/server/migrations/015_multi_user_and_strategies.sql +244 -0
  62. package/server/migrations/016_health_companion.sql +158 -0
  63. package/server/migrations/016_strategy_contracts_and_user_graph.sql +22 -0
  64. package/server/migrations/017_preferences.sql +131 -0
  65. package/server/migrations/018_preference_catalogs.sql +31 -0
  66. package/server/migrations/019_wiki_memory.sql +255 -0
  67. package/server/migrations/020_wiki_page_hierarchy.sql +11 -0
  68. package/server/migrations/021_hide_evidence_from_wiki_index.sql +3 -0
  69. package/server/migrations/022_wiki_ingest_background.sql +85 -0
  70. package/server/migrations/023_diagnostic_logs.sql +28 -0
  71. package/skills/forge-openclaw/SKILL.md +126 -34
  72. package/skills/forge-openclaw/entity_conversation_playbooks.md +337 -0
  73. package/skills/forge-openclaw/psyche_entity_playbooks.md +404 -0
  74. package/dist/assets/board-8L3uX7_O.js +0 -6
  75. package/dist/assets/index-Cj1IBH_w.js +0 -36
  76. package/dist/assets/index-Cj1IBH_w.js.map +0 -1
  77. package/dist/assets/index-DQT6EbuS.css +0 -1
  78. package/dist/assets/vendor-BvM2F9Dp.js +0 -503
  79. package/dist/assets/vendor-BvM2F9Dp.js.map +0 -1
  80. package/dist/assets/vendor-CRS-psbw.css +0 -1
  81. package/dist/assets/viz-CNeunkfu.js +0 -34
  82. package/dist/assets/viz-CNeunkfu.js.map +0 -1
@@ -0,0 +1,3366 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import AdmZip from "adm-zip";
3
+ import { existsSync, mkdirSync, readdirSync, unlinkSync, rmSync, writeFileSync } from "node:fs";
4
+ import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
5
+ import path from "node:path";
6
+ import { z } from "zod";
7
+ import { resolveDataDir, getDatabase } from "../db.js";
8
+ import { decorateOwnedEntity } from "./entity-ownership.js";
9
+ import { createNoteLinkSchema, crudEntityTypeSchema, noteKindSchema, noteSchema as persistedNoteSchema, wikiSearchModeSchema, wikiSpaceVisibilitySchema } from "../types.js";
10
+ import { deleteEncryptedSecret, readEncryptedSecret, storeEncryptedSecret } from "./calendar.js";
11
+ import { recordDiagnosticLog } from "./diagnostic-logs.js";
12
+ const wikiSpaceSchema = z.object({
13
+ id: z.string(),
14
+ slug: z.string(),
15
+ label: z.string(),
16
+ description: z.string(),
17
+ ownerUserId: z.string().nullable(),
18
+ visibility: wikiSpaceVisibilitySchema,
19
+ createdAt: z.string(),
20
+ updatedAt: z.string()
21
+ });
22
+ const wikiLinkEdgeSchema = z.object({
23
+ sourceNoteId: z.string(),
24
+ targetType: z.enum(["page", "entity", "unresolved"]),
25
+ targetNoteId: z.string().nullable(),
26
+ targetEntityType: crudEntityTypeSchema.nullable(),
27
+ targetEntityId: z.string().nullable(),
28
+ label: z.string(),
29
+ rawTarget: z.string(),
30
+ isEmbed: z.boolean(),
31
+ createdAt: z.string(),
32
+ updatedAt: z.string()
33
+ });
34
+ const wikiMediaAssetSchema = z.object({
35
+ id: z.string(),
36
+ spaceId: z.string(),
37
+ noteId: z.string().nullable(),
38
+ label: z.string(),
39
+ mimeType: z.string(),
40
+ fileName: z.string(),
41
+ filePath: z.string(),
42
+ sizeBytes: z.number().int().nonnegative(),
43
+ checksum: z.string(),
44
+ transcriptNoteId: z.string().nullable(),
45
+ metadata: z.record(z.string(), z.unknown()),
46
+ createdAt: z.string(),
47
+ updatedAt: z.string()
48
+ });
49
+ const wikiLlmProfileSchema = z.object({
50
+ id: z.string(),
51
+ label: z.string(),
52
+ provider: z.string(),
53
+ baseUrl: z.string(),
54
+ model: z.string(),
55
+ secretId: z.string().nullable(),
56
+ systemPrompt: z.string(),
57
+ enabled: z.boolean(),
58
+ metadata: z.record(z.string(), z.unknown()),
59
+ createdAt: z.string(),
60
+ updatedAt: z.string()
61
+ });
62
+ const wikiEmbeddingProfileSchema = z.object({
63
+ id: z.string(),
64
+ label: z.string(),
65
+ provider: z.string(),
66
+ baseUrl: z.string(),
67
+ model: z.string(),
68
+ secretId: z.string().nullable(),
69
+ dimensions: z.number().int().positive().nullable(),
70
+ chunkSize: z.number().int().positive(),
71
+ chunkOverlap: z.number().int().nonnegative(),
72
+ enabled: z.boolean(),
73
+ metadata: z.record(z.string(), z.unknown()),
74
+ createdAt: z.string(),
75
+ updatedAt: z.string()
76
+ });
77
+ const wikiSettingsPayloadSchema = z.object({
78
+ spaces: z.array(wikiSpaceSchema),
79
+ llmProfiles: z.array(wikiLlmProfileSchema),
80
+ embeddingProfiles: z.array(wikiEmbeddingProfileSchema)
81
+ });
82
+ const wikiPageTreeNodeSchema = z.lazy(() => z.object({
83
+ page: persistedNoteSchema,
84
+ children: z.array(wikiPageTreeNodeSchema)
85
+ }));
86
+ const wikiHealthPayloadSchema = z.object({
87
+ space: wikiSpaceSchema,
88
+ indexPath: z.string(),
89
+ rawDirectoryPath: z.string(),
90
+ pageCount: z.number().int().nonnegative(),
91
+ wikiPageCount: z.number().int().nonnegative(),
92
+ evidencePageCount: z.number().int().nonnegative(),
93
+ assetCount: z.number().int().nonnegative(),
94
+ rawSourceCount: z.number().int().nonnegative(),
95
+ unresolvedLinks: z.array(z.object({
96
+ sourceNoteId: z.string(),
97
+ sourceSlug: z.string(),
98
+ sourceTitle: z.string(),
99
+ rawTarget: z.string(),
100
+ updatedAt: z.string()
101
+ })),
102
+ orphanPages: z.array(z.object({
103
+ id: z.string(),
104
+ slug: z.string(),
105
+ title: z.string(),
106
+ kind: noteKindSchema,
107
+ updatedAt: z.string()
108
+ })),
109
+ missingSummaries: z.array(z.object({
110
+ id: z.string(),
111
+ slug: z.string(),
112
+ title: z.string(),
113
+ updatedAt: z.string()
114
+ })),
115
+ enabledEmbeddingProfiles: z.array(z.object({
116
+ id: z.string(),
117
+ label: z.string(),
118
+ model: z.string()
119
+ })),
120
+ enabledLlmProfiles: z.array(z.object({
121
+ id: z.string(),
122
+ label: z.string(),
123
+ model: z.string()
124
+ }))
125
+ });
126
+ const wikiIngestJobLogSchema = z.object({
127
+ id: z.string(),
128
+ level: z.string(),
129
+ message: z.string(),
130
+ metadata: z.record(z.string(), z.unknown()),
131
+ createdAt: z.string()
132
+ });
133
+ const wikiIngestJobAssetSchema = z.object({
134
+ id: z.string(),
135
+ status: z.string(),
136
+ sourceKind: z.string(),
137
+ sourceLocator: z.string(),
138
+ fileName: z.string(),
139
+ mimeType: z.string(),
140
+ filePath: z.string(),
141
+ sizeBytes: z.number().int().nonnegative(),
142
+ checksum: z.string(),
143
+ metadata: z.record(z.string(), z.unknown()),
144
+ createdAt: z.string(),
145
+ updatedAt: z.string()
146
+ });
147
+ const wikiIngestJobCandidateSchema = z.object({
148
+ id: z.string(),
149
+ sourceAssetId: z.string().nullable(),
150
+ candidateType: z.string(),
151
+ status: z.string(),
152
+ title: z.string(),
153
+ summary: z.string(),
154
+ targetKey: z.string(),
155
+ payload: z.record(z.string(), z.unknown()),
156
+ publishedNoteId: z.string().nullable(),
157
+ publishedEntityType: z.string().nullable(),
158
+ publishedEntityId: z.string().nullable(),
159
+ createdAt: z.string(),
160
+ updatedAt: z.string()
161
+ });
162
+ const wikiIngestJobPayloadSchema = z.object({
163
+ job: z.object({
164
+ id: z.string(),
165
+ spaceId: z.string(),
166
+ llmProfileId: z.string().nullable(),
167
+ status: z.string(),
168
+ phase: z.string(),
169
+ progressPercent: z.number().int().nonnegative(),
170
+ totalFiles: z.number().int().nonnegative(),
171
+ processedFiles: z.number().int().nonnegative(),
172
+ createdPageCount: z.number().int().nonnegative(),
173
+ createdEntityCount: z.number().int().nonnegative(),
174
+ acceptedCount: z.number().int().nonnegative(),
175
+ rejectedCount: z.number().int().nonnegative(),
176
+ latestMessage: z.string(),
177
+ sourceKind: z.string(),
178
+ sourceLocator: z.string(),
179
+ mimeType: z.string(),
180
+ titleHint: z.string(),
181
+ summary: z.string(),
182
+ pageNoteId: z.string().nullable(),
183
+ createdByActor: z.string().nullable(),
184
+ errorMessage: z.string(),
185
+ createdAt: z.string(),
186
+ updatedAt: z.string(),
187
+ completedAt: z.string().nullable()
188
+ }),
189
+ items: z.array(z.object({
190
+ id: z.string(),
191
+ itemType: z.string(),
192
+ status: z.string(),
193
+ noteId: z.string().nullable(),
194
+ mediaAssetId: z.string().nullable(),
195
+ payload: z.record(z.string(), z.unknown()),
196
+ createdAt: z.string(),
197
+ updatedAt: z.string()
198
+ })),
199
+ logs: z.array(wikiIngestJobLogSchema),
200
+ assets: z.array(wikiIngestJobAssetSchema),
201
+ candidates: z.array(wikiIngestJobCandidateSchema)
202
+ });
203
+ const listWikiIngestJobsQuerySchema = z.object({
204
+ spaceId: z.string().trim().optional(),
205
+ limit: z.coerce.number().int().positive().max(200).default(20)
206
+ });
207
+ export const reviewWikiIngestJobSchema = z.object({
208
+ decisions: z
209
+ .array(z
210
+ .object({
211
+ candidateId: z.string().trim().min(1),
212
+ keep: z.boolean().optional(),
213
+ action: z
214
+ .enum(["keep", "discard", "map_existing", "merge_existing"])
215
+ .optional(),
216
+ mappedEntityType: crudEntityTypeSchema.optional(),
217
+ mappedEntityId: z.string().trim().min(1).optional(),
218
+ targetNoteId: z.string().trim().min(1).optional()
219
+ })
220
+ .superRefine((value, context) => {
221
+ if (value.action === "map_existing") {
222
+ if (!value.mappedEntityType) {
223
+ context.addIssue({
224
+ code: z.ZodIssueCode.custom,
225
+ path: ["mappedEntityType"],
226
+ message: "mappedEntityType is required when action is map_existing"
227
+ });
228
+ }
229
+ if (!value.mappedEntityId) {
230
+ context.addIssue({
231
+ code: z.ZodIssueCode.custom,
232
+ path: ["mappedEntityId"],
233
+ message: "mappedEntityId is required when action is map_existing"
234
+ });
235
+ }
236
+ }
237
+ if (value.action === "merge_existing" && !value.targetNoteId) {
238
+ context.addIssue({
239
+ code: z.ZodIssueCode.custom,
240
+ path: ["targetNoteId"],
241
+ message: "targetNoteId is required when action is merge_existing"
242
+ });
243
+ }
244
+ if (value.action === undefined && value.keep === undefined) {
245
+ context.addIssue({
246
+ code: z.ZodIssueCode.custom,
247
+ path: ["action"],
248
+ message: "Either keep or action is required"
249
+ });
250
+ }
251
+ }))
252
+ .min(1)
253
+ });
254
+ export const createWikiSpaceSchema = z.object({
255
+ label: z.string().trim().min(1),
256
+ slug: z.string().trim().optional(),
257
+ description: z.string().trim().default(""),
258
+ ownerUserId: z.string().trim().nullable().optional(),
259
+ visibility: wikiSpaceVisibilitySchema.default("personal")
260
+ });
261
+ const wikiLlmReasoningEffortSchema = z.enum([
262
+ "none",
263
+ "low",
264
+ "medium",
265
+ "high",
266
+ "xhigh"
267
+ ]);
268
+ const wikiLlmVerbositySchema = z.enum(["low", "medium", "high"]);
269
+ export const upsertWikiLlmProfileSchema = z.object({
270
+ id: z.string().trim().optional(),
271
+ label: z.string().trim().min(1),
272
+ provider: z.string().trim().min(1).default("openai-responses"),
273
+ baseUrl: z.string().trim().default("https://api.openai.com/v1"),
274
+ model: z.string().trim().min(1),
275
+ apiKey: z.string().trim().optional(),
276
+ systemPrompt: z.string().trim().default(""),
277
+ reasoningEffort: wikiLlmReasoningEffortSchema.optional(),
278
+ verbosity: wikiLlmVerbositySchema.optional(),
279
+ enabled: z.boolean().default(true),
280
+ metadata: z.record(z.string(), z.unknown()).default({})
281
+ });
282
+ export const testWikiLlmProfileSchema = z.object({
283
+ profileId: z.string().trim().optional(),
284
+ provider: z.string().trim().min(1).default("openai-responses"),
285
+ baseUrl: z.string().trim().default("https://api.openai.com/v1"),
286
+ model: z.string().trim().min(1),
287
+ apiKey: z.string().trim().optional(),
288
+ reasoningEffort: wikiLlmReasoningEffortSchema.optional(),
289
+ verbosity: wikiLlmVerbositySchema.optional()
290
+ });
291
+ export const upsertWikiEmbeddingProfileSchema = z.object({
292
+ id: z.string().trim().optional(),
293
+ label: z.string().trim().min(1),
294
+ provider: z.string().trim().min(1).default("openai-compatible"),
295
+ baseUrl: z.string().trim().default("https://api.openai.com/v1"),
296
+ model: z.string().trim().min(1).default("text-embedding-3-small"),
297
+ dimensions: z.number().int().positive().nullable().optional(),
298
+ chunkSize: z.number().int().positive().default(1200),
299
+ chunkOverlap: z.number().int().nonnegative().default(200),
300
+ apiKey: z.string().trim().optional(),
301
+ enabled: z.boolean().default(true),
302
+ metadata: z.record(z.string(), z.unknown()).default({})
303
+ });
304
+ export const wikiSearchQuerySchema = z.object({
305
+ spaceId: z.string().trim().optional(),
306
+ kind: noteKindSchema.optional(),
307
+ mode: wikiSearchModeSchema.default("hybrid"),
308
+ query: z.string().trim().default(""),
309
+ profileId: z.string().trim().optional(),
310
+ linkedEntity: z
311
+ .object({
312
+ entityType: crudEntityTypeSchema,
313
+ entityId: z.string().trim().min(1)
314
+ })
315
+ .optional(),
316
+ limit: z.coerce.number().int().positive().max(50).default(20)
317
+ });
318
+ export const syncWikiVaultSchema = z.object({
319
+ spaceId: z.string().trim().optional()
320
+ });
321
+ export const reindexWikiEmbeddingsSchema = z.object({
322
+ spaceId: z.string().trim().optional(),
323
+ profileId: z.string().trim().optional()
324
+ });
325
+ export const createWikiIngestJobSchema = z.object({
326
+ spaceId: z.string().trim().optional(),
327
+ titleHint: z.string().trim().default(""),
328
+ sourceKind: z.enum(["raw_text", "local_path", "url"]),
329
+ sourceText: z.string().default(""),
330
+ sourcePath: z.string().trim().optional(),
331
+ sourceUrl: z.string().trim().optional(),
332
+ mimeType: z.string().trim().default(""),
333
+ llmProfileId: z.string().trim().optional(),
334
+ parseStrategy: z.enum(["auto", "text_only", "multimodal"]).default("auto"),
335
+ entityProposalMode: z.enum(["none", "suggest"]).default("suggest"),
336
+ userId: z.string().trim().nullable().optional(),
337
+ createAsKind: noteKindSchema.default("wiki"),
338
+ linkedEntityHints: z.array(createNoteLinkSchema).default([])
339
+ });
340
+ function nowIso() {
341
+ return new Date().toISOString();
342
+ }
343
+ const WIKI_STARTER_PAGES = [
344
+ {
345
+ slug: "index",
346
+ title: "Home",
347
+ parentSlug: null,
348
+ indexOrder: 0,
349
+ summary: "Top-level home page for this wiki space."
350
+ },
351
+ {
352
+ slug: "people",
353
+ title: "People",
354
+ parentSlug: "index",
355
+ indexOrder: 10,
356
+ summary: "People, collaborators, and relationship context."
357
+ },
358
+ {
359
+ slug: "projects",
360
+ title: "Projects",
361
+ parentSlug: "index",
362
+ indexOrder: 20,
363
+ summary: "Active projects, initiatives, and workstreams."
364
+ },
365
+ {
366
+ slug: "concepts",
367
+ title: "Concepts",
368
+ parentSlug: "index",
369
+ indexOrder: 30,
370
+ summary: "Ideas, themes, philosophies, and operating concepts."
371
+ },
372
+ {
373
+ slug: "sources",
374
+ title: "Sources",
375
+ parentSlug: "index",
376
+ indexOrder: 40,
377
+ summary: "Raw materials, references, imports, and citations."
378
+ },
379
+ {
380
+ slug: "chronicle",
381
+ title: "Chronicle",
382
+ parentSlug: "index",
383
+ indexOrder: 50,
384
+ summary: "Timeline-style notes, field logs, and ongoing narrative."
385
+ }
386
+ ];
387
+ function normalizeAnchorKey(anchorKey) {
388
+ return anchorKey.trim().length > 0 ? anchorKey.trim() : null;
389
+ }
390
+ function normalizeTags(tags) {
391
+ if (!tags) {
392
+ return [];
393
+ }
394
+ const seen = new Set();
395
+ return tags
396
+ .map((tag) => tag.trim())
397
+ .filter(Boolean)
398
+ .filter((tag) => {
399
+ const normalized = tag.toLowerCase();
400
+ if (seen.has(normalized)) {
401
+ return false;
402
+ }
403
+ seen.add(normalized);
404
+ return true;
405
+ });
406
+ }
407
+ function normalizeAliases(aliases) {
408
+ if (!aliases) {
409
+ return [];
410
+ }
411
+ const seen = new Set();
412
+ return aliases
413
+ .map((alias) => alias.trim())
414
+ .filter(Boolean)
415
+ .filter((alias) => {
416
+ const normalized = alias.toLowerCase();
417
+ if (seen.has(normalized)) {
418
+ return false;
419
+ }
420
+ seen.add(normalized);
421
+ return true;
422
+ });
423
+ }
424
+ function parseJsonRecord(raw) {
425
+ if (!raw) {
426
+ return {};
427
+ }
428
+ try {
429
+ const parsed = JSON.parse(raw);
430
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed)
431
+ ? parsed
432
+ : {};
433
+ }
434
+ catch {
435
+ return {};
436
+ }
437
+ }
438
+ function parseJsonStringArray(raw) {
439
+ if (!raw) {
440
+ return [];
441
+ }
442
+ try {
443
+ const parsed = JSON.parse(raw);
444
+ return Array.isArray(parsed)
445
+ ? parsed.filter((entry) => typeof entry === "string")
446
+ : [];
447
+ }
448
+ catch {
449
+ return [];
450
+ }
451
+ }
452
+ function readStringRecordValue(record, key) {
453
+ const value = record[key];
454
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
455
+ }
456
+ function listLinkRowsForNotes(noteIds) {
457
+ if (noteIds.length === 0) {
458
+ return [];
459
+ }
460
+ const placeholders = noteIds.map(() => "?").join(", ");
461
+ return getDatabase()
462
+ .prepare(`SELECT note_id, entity_type, entity_id, anchor_key, created_at
463
+ FROM note_links
464
+ WHERE note_id IN (${placeholders})
465
+ ORDER BY created_at ASC`)
466
+ .all(...noteIds);
467
+ }
468
+ function mapLinks(rows) {
469
+ return rows.map((row) => ({
470
+ entityType: row.entity_type,
471
+ entityId: row.entity_id,
472
+ anchorKey: normalizeAnchorKey(row.anchor_key)
473
+ }));
474
+ }
475
+ function mapNoteRow(row, linkRows) {
476
+ return persistedNoteSchema.parse(decorateOwnedEntity("note", {
477
+ id: row.id,
478
+ kind: row.kind,
479
+ title: row.title,
480
+ slug: row.slug,
481
+ spaceId: row.space_id,
482
+ parentSlug: row.parent_slug,
483
+ indexOrder: row.index_order,
484
+ showInIndex: row.show_in_index === 1,
485
+ aliases: normalizeAliases(parseJsonStringArray(row.aliases_json)),
486
+ summary: row.summary,
487
+ contentMarkdown: row.content_markdown,
488
+ contentPlain: row.content_plain,
489
+ author: row.author,
490
+ source: row.source,
491
+ sourcePath: row.source_path,
492
+ frontmatter: parseJsonRecord(row.frontmatter_json),
493
+ revisionHash: row.revision_hash,
494
+ lastSyncedAt: row.last_synced_at,
495
+ createdAt: row.created_at,
496
+ updatedAt: row.updated_at,
497
+ links: mapLinks(linkRows),
498
+ tags: normalizeTags(parseJsonStringArray(row.tags_json)),
499
+ destroyAt: row.destroy_at
500
+ }));
501
+ }
502
+ function getNoteRows(whereClause = "", params = []) {
503
+ return getDatabase()
504
+ .prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
505
+ tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
506
+ FROM notes
507
+ ${whereClause}
508
+ ORDER BY updated_at DESC`)
509
+ .all(...params);
510
+ }
511
+ function getNoteByIdRaw(noteId) {
512
+ return getDatabase()
513
+ .prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
514
+ tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
515
+ FROM notes
516
+ WHERE id = ?`)
517
+ .get(noteId);
518
+ }
519
+ function getNoteBySlugRaw(spaceId, slug, exceptNoteId) {
520
+ const row = getDatabase()
521
+ .prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
522
+ tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
523
+ FROM notes
524
+ WHERE space_id = ?
525
+ AND lower(slug) = lower(?)
526
+ ${exceptNoteId ? "AND id != ?" : ""}
527
+ LIMIT 1`)
528
+ .get(...(exceptNoteId ? [spaceId, slug, exceptNoteId] : [spaceId, slug]));
529
+ return row;
530
+ }
531
+ function buildContentPlain(markdown) {
532
+ return markdown
533
+ .replace(/^---[\s\S]*?---\s*/m, "")
534
+ .replace(/```[\s\S]*?```/g, (block) => block.replace(/```/g, "").trim())
535
+ .replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1")
536
+ .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
537
+ .replace(/\[\[([^\]|]+)\|?([^\]]*)\]\]/g, (_match, left, right) => (right || left).trim())
538
+ .replace(/`([^`]+)`/g, "$1")
539
+ .replace(/\*\*([^*]+)\*\*/g, "$1")
540
+ .replace(/\*([^*]+)\*/g, "$1")
541
+ .replace(/^>\s?/gm, "")
542
+ .replace(/^#{1,6}\s+/gm, "")
543
+ .replace(/^[-*+]\s+/gm, "")
544
+ .replace(/^\d+\.\s+/gm, "")
545
+ .replace(/\r/g, "")
546
+ .trim();
547
+ }
548
+ function inferTitle(markdown, fallback) {
549
+ const headingMatch = markdown.match(/^#{1,6}\s+(.+)$/m);
550
+ if (headingMatch?.[1]?.trim()) {
551
+ return headingMatch[1].trim().slice(0, 160);
552
+ }
553
+ const plain = buildContentPlain(markdown);
554
+ if (plain.trim()) {
555
+ return plain.trim().split("\n")[0].slice(0, 160);
556
+ }
557
+ return fallback;
558
+ }
559
+ function inferSummary(markdown) {
560
+ const plain = buildContentPlain(markdown).replace(/\s+/g, " ").trim();
561
+ return plain.slice(0, 240);
562
+ }
563
+ function stripLeadingHeading(markdown, title) {
564
+ const normalizedTitle = title.trim().toLowerCase();
565
+ const trimmed = markdown.trim();
566
+ const match = trimmed.match(/^#\s+(.+?)\n+/);
567
+ if (!match) {
568
+ return trimmed;
569
+ }
570
+ const heading = match[1]?.trim().toLowerCase() ?? "";
571
+ if (heading !== normalizedTitle) {
572
+ return trimmed;
573
+ }
574
+ return trimmed.slice(match[0].length).trim();
575
+ }
576
+ function mergeWikiPageContent(targetMarkdown, incoming) {
577
+ const mergedBody = stripLeadingHeading(incoming.markdown, incoming.title) ||
578
+ incoming.markdown.trim();
579
+ return [
580
+ targetMarkdown.trim(),
581
+ "",
582
+ `## ${incoming.title}`,
583
+ "",
584
+ mergedBody
585
+ ]
586
+ .filter((part) => part.trim().length > 0)
587
+ .join("\n");
588
+ }
589
+ function slugify(value) {
590
+ const normalized = value
591
+ .toLowerCase()
592
+ .normalize("NFKD")
593
+ .replace(/[^\w\s-]/g, "")
594
+ .trim()
595
+ .replace(/[\s_]+/g, "-")
596
+ .replace(/-+/g, "-");
597
+ return normalized || `page-${randomUUID().replaceAll("-", "").slice(0, 8)}`;
598
+ }
599
+ function buildUniqueSlug(spaceId, requestedSlug, noteId) {
600
+ const base = slugify(requestedSlug);
601
+ let candidate = base;
602
+ let suffix = 2;
603
+ while (getNoteBySlugRaw(spaceId, candidate, noteId)) {
604
+ candidate = `${base}-${suffix}`;
605
+ suffix += 1;
606
+ }
607
+ return candidate;
608
+ }
609
+ function mapWikiSpace(row) {
610
+ return wikiSpaceSchema.parse({
611
+ id: row.id,
612
+ slug: row.slug,
613
+ label: row.label,
614
+ description: row.description,
615
+ ownerUserId: row.owner_user_id,
616
+ visibility: row.visibility,
617
+ createdAt: row.created_at,
618
+ updatedAt: row.updated_at
619
+ });
620
+ }
621
+ function getWikiRootDir() {
622
+ return path.join(resolveDataDir(), "wiki");
623
+ }
624
+ function getSpaceStorageDir(space) {
625
+ if (space.visibility === "shared") {
626
+ return path.join(getWikiRootDir(), "shared", space.slug);
627
+ }
628
+ return path.join(getWikiRootDir(), "users", space.ownerUserId ?? space.slug);
629
+ }
630
+ function getSpaceIndexPath(space) {
631
+ return path.join(getSpaceStorageDir(space), "index.md");
632
+ }
633
+ function getSpaceRawDir(space) {
634
+ return path.join(getSpaceStorageDir(space), "raw");
635
+ }
636
+ function getNoteStoragePath(note, space) {
637
+ const directory = note.kind === "wiki" ? "pages" : "evidence";
638
+ return path.join(getSpaceStorageDir(space), directory, `${note.slug}.md`);
639
+ }
640
+ function buildNoteFrontmatter(note) {
641
+ return {
642
+ id: note.id,
643
+ kind: note.kind,
644
+ title: note.title,
645
+ slug: note.slug,
646
+ spaceId: note.spaceId,
647
+ parentSlug: note.parentSlug,
648
+ indexOrder: note.indexOrder,
649
+ showInIndex: note.showInIndex,
650
+ aliases: note.aliases,
651
+ summary: note.summary,
652
+ tags: note.tags ?? [],
653
+ linkedEntities: note.links.map((link) => ({
654
+ entityType: link.entityType,
655
+ entityId: link.entityId,
656
+ anchorKey: link.anchorKey
657
+ })),
658
+ createdAt: note.createdAt,
659
+ updatedAt: note.updatedAt,
660
+ lastSyncedAt: note.lastSyncedAt,
661
+ author: note.author
662
+ };
663
+ }
664
+ function stringifyFrontmatterValue(value) {
665
+ if (typeof value === "string") {
666
+ return JSON.stringify(value);
667
+ }
668
+ return JSON.stringify(value);
669
+ }
670
+ function renderFrontmatter(frontmatter) {
671
+ const lines = ["---"];
672
+ for (const [key, value] of Object.entries(frontmatter)) {
673
+ lines.push(`${key}: ${stringifyFrontmatterValue(value)}`);
674
+ }
675
+ lines.push("---", "");
676
+ return lines.join("\n");
677
+ }
678
+ function hashContent(value) {
679
+ return createHash("sha256").update(value).digest("hex");
680
+ }
681
+ function hashBuffer(value) {
682
+ return createHash("sha256").update(value).digest("hex");
683
+ }
684
+ function buildLinkedEntityTokens(note) {
685
+ return note.links
686
+ .map((link) => `${link.entityType}:${link.entityId}`)
687
+ .join(" ");
688
+ }
689
+ function buildWikiFtsQuery(query) {
690
+ const tokens = query
691
+ .trim()
692
+ .split(/\s+/)
693
+ .map((token) => token.replace(/["*']/g, "").trim())
694
+ .filter(Boolean);
695
+ if (tokens.length === 0) {
696
+ return null;
697
+ }
698
+ return tokens.map((token) => `${token}*`).join(" AND ");
699
+ }
700
+ function deleteWikiSearchRow(noteId) {
701
+ getDatabase()
702
+ .prepare(`DELETE FROM wiki_pages_fts WHERE note_id = ?`)
703
+ .run(noteId);
704
+ }
705
+ function chunkHeadingAware(markdown, chunkSize, chunkOverlap) {
706
+ const stripped = markdown
707
+ .replace(/^---[\s\S]*?---\s*/m, "")
708
+ .replace(/\r/g, "");
709
+ const sections = [];
710
+ const lines = stripped.split("\n");
711
+ let currentHeading = "Document";
712
+ let currentLines = [];
713
+ const flush = () => {
714
+ const text = currentLines.join("\n").trim();
715
+ if (text) {
716
+ sections.push({ headingPath: currentHeading, text });
717
+ }
718
+ currentLines = [];
719
+ };
720
+ for (const line of lines) {
721
+ const match = line.match(/^(#{1,6})\s+(.+)$/);
722
+ if (match) {
723
+ flush();
724
+ currentHeading = match[2].trim();
725
+ continue;
726
+ }
727
+ currentLines.push(line);
728
+ }
729
+ flush();
730
+ const chunks = [];
731
+ sections.forEach((section, sectionIndex) => {
732
+ const content = section.text.replace(/\s+/g, " ").trim();
733
+ if (!content) {
734
+ return;
735
+ }
736
+ if (content.length <= chunkSize) {
737
+ chunks.push({
738
+ key: `${sectionIndex}-0`,
739
+ headingPath: section.headingPath,
740
+ contentText: content
741
+ });
742
+ return;
743
+ }
744
+ let offset = 0;
745
+ let partIndex = 0;
746
+ while (offset < content.length) {
747
+ const slice = content.slice(offset, offset + chunkSize).trim();
748
+ if (slice) {
749
+ chunks.push({
750
+ key: `${sectionIndex}-${partIndex}`,
751
+ headingPath: section.headingPath,
752
+ contentText: slice
753
+ });
754
+ }
755
+ if (offset + chunkSize >= content.length) {
756
+ break;
757
+ }
758
+ offset += Math.max(1, chunkSize - chunkOverlap);
759
+ partIndex += 1;
760
+ }
761
+ });
762
+ return chunks;
763
+ }
764
+ function cosineSimilarity(left, right) {
765
+ if (left.length !== right.length || left.length === 0) {
766
+ return 0;
767
+ }
768
+ let dot = 0;
769
+ let leftNorm = 0;
770
+ let rightNorm = 0;
771
+ for (let index = 0; index < left.length; index += 1) {
772
+ dot += left[index] * right[index];
773
+ leftNorm += left[index] * left[index];
774
+ rightNorm += right[index] * right[index];
775
+ }
776
+ if (leftNorm === 0 || rightNorm === 0) {
777
+ return 0;
778
+ }
779
+ return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
780
+ }
781
+ async function getFetchedContent(sourceKind, options) {
782
+ if (sourceKind === "raw_text") {
783
+ return {
784
+ locator: "raw_text",
785
+ contentText: options.sourceText,
786
+ mimeType: options.mimeType || "text/plain",
787
+ fileName: "inline.txt",
788
+ binary: null
789
+ };
790
+ }
791
+ if (sourceKind === "local_path") {
792
+ const filePath = options.sourcePath?.trim();
793
+ if (!filePath) {
794
+ throw new Error("sourcePath is required for local_path ingest.");
795
+ }
796
+ const payload = await readFile(filePath);
797
+ const fileName = path.basename(filePath);
798
+ const mimeType = options.mimeType?.trim() || inferMimeTypeFromPath(fileName);
799
+ return {
800
+ locator: filePath,
801
+ contentText: mimeType.startsWith("text/") ||
802
+ mimeType === "application/json" ||
803
+ mimeType === "text/markdown"
804
+ ? payload.toString("utf8")
805
+ : "",
806
+ mimeType,
807
+ fileName,
808
+ binary: payload
809
+ };
810
+ }
811
+ const sourceUrl = options.sourceUrl?.trim();
812
+ if (!sourceUrl) {
813
+ throw new Error("sourceUrl is required for url ingest.");
814
+ }
815
+ const response = await fetch(sourceUrl);
816
+ if (!response.ok) {
817
+ throw new Error(`Could not fetch ${sourceUrl}: ${response.status}`);
818
+ }
819
+ const mimeType = options.mimeType?.trim() ||
820
+ response.headers.get("content-type")?.split(";")[0]?.trim() ||
821
+ "application/octet-stream";
822
+ const arrayBuffer = await response.arrayBuffer();
823
+ const binary = Buffer.from(arrayBuffer);
824
+ const fileName = sourceUrl.split("/").pop()?.split("?")[0]?.trim() || "remote-source.bin";
825
+ return {
826
+ locator: sourceUrl,
827
+ contentText: mimeType.startsWith("text/") ||
828
+ mimeType === "application/json" ||
829
+ mimeType === "text/markdown"
830
+ ? binary.toString("utf8")
831
+ : "",
832
+ mimeType,
833
+ fileName,
834
+ binary
835
+ };
836
+ }
837
+ function inferExtensionFromMimeType(mimeType) {
838
+ switch (mimeType) {
839
+ case "text/markdown":
840
+ return ".md";
841
+ case "text/plain":
842
+ return ".txt";
843
+ case "text/html":
844
+ return ".html";
845
+ case "application/json":
846
+ return ".json";
847
+ case "application/pdf":
848
+ return ".pdf";
849
+ case "image/png":
850
+ return ".png";
851
+ case "image/jpeg":
852
+ return ".jpg";
853
+ case "image/gif":
854
+ return ".gif";
855
+ case "image/webp":
856
+ return ".webp";
857
+ case "audio/mpeg":
858
+ return ".mp3";
859
+ case "audio/wav":
860
+ return ".wav";
861
+ case "audio/mp4":
862
+ return ".m4a";
863
+ case "video/mp4":
864
+ return ".mp4";
865
+ case "video/quicktime":
866
+ return ".mov";
867
+ default:
868
+ return "";
869
+ }
870
+ }
871
+ function inferMimeTypeFromPath(fileName) {
872
+ const extension = path.extname(fileName).toLowerCase();
873
+ switch (extension) {
874
+ case ".md":
875
+ case ".markdown":
876
+ return "text/markdown";
877
+ case ".txt":
878
+ return "text/plain";
879
+ case ".html":
880
+ case ".htm":
881
+ return "text/html";
882
+ case ".json":
883
+ return "application/json";
884
+ case ".png":
885
+ return "image/png";
886
+ case ".jpg":
887
+ case ".jpeg":
888
+ return "image/jpeg";
889
+ case ".gif":
890
+ return "image/gif";
891
+ case ".webp":
892
+ return "image/webp";
893
+ case ".mp3":
894
+ return "audio/mpeg";
895
+ case ".wav":
896
+ return "audio/wav";
897
+ case ".m4a":
898
+ return "audio/mp4";
899
+ case ".mp4":
900
+ return "video/mp4";
901
+ case ".mov":
902
+ return "video/quicktime";
903
+ case ".pdf":
904
+ return "application/pdf";
905
+ default:
906
+ return "application/octet-stream";
907
+ }
908
+ }
909
+ function sanitizeFileName(value) {
910
+ return value.replace(/[^a-zA-Z0-9._-]+/g, "-");
911
+ }
912
+ async function readSecretApiKey(secretId, secrets) {
913
+ if (!secretId) {
914
+ return null;
915
+ }
916
+ const cipherText = readEncryptedSecret(secretId);
917
+ if (!cipherText) {
918
+ return null;
919
+ }
920
+ const payload = secrets.openJson(cipherText);
921
+ return payload.apiKey || null;
922
+ }
923
+ async function compileTextWithLlm(profile, secrets, input) {
924
+ const apiKey = await readSecretApiKey(profile.secretId, secrets);
925
+ if (!apiKey) {
926
+ return null;
927
+ }
928
+ const prompt = [
929
+ "You compile user-provided source material into a local wiki page.",
930
+ "Return JSON with keys title, summary, markdown, tags, entityProposals, pageUpdateSuggestions, articleCandidates.",
931
+ "The markdown should be concise, structured, and agent-readable.",
932
+ "entityProposals should be an array of objects with entityType, title, summary, rationale, confidence, and suggestedFields.",
933
+ "pageUpdateSuggestions should be an array of objects with targetSlug, rationale, and patchSummary.",
934
+ "articleCandidates should be an array of objects with title, slug, rationale, and summary.",
935
+ profile.systemPrompt.trim()
936
+ ]
937
+ .filter(Boolean)
938
+ .join("\n");
939
+ const response = await fetch(`${profile.baseUrl.replace(/\/$/, "")}/chat/completions`, {
940
+ method: "POST",
941
+ headers: {
942
+ "content-type": "application/json",
943
+ authorization: `Bearer ${apiKey}`
944
+ },
945
+ body: JSON.stringify({
946
+ model: profile.model,
947
+ temperature: 0.2,
948
+ response_format: { type: "json_object" },
949
+ messages: [
950
+ {
951
+ role: "system",
952
+ content: prompt
953
+ },
954
+ {
955
+ role: "user",
956
+ content: `Title hint: ${input.titleHint || "none"}\nMime type: ${input.mimeType}\n\nSource:\n${input.rawText.slice(0, 24_000)}`
957
+ }
958
+ ]
959
+ })
960
+ });
961
+ if (!response.ok) {
962
+ throw new Error(`LLM compilation failed: ${response.status}`);
963
+ }
964
+ const payload = (await response.json());
965
+ const content = payload.choices?.[0]?.message?.content;
966
+ if (!content) {
967
+ return null;
968
+ }
969
+ try {
970
+ const parsed = JSON.parse(content);
971
+ return {
972
+ title: parsed.title?.trim() || input.titleHint || "Imported source",
973
+ summary: parsed.summary?.trim() || "",
974
+ markdown: parsed.markdown?.trim() || input.rawText.trim(),
975
+ tags: normalizeTags(parsed.tags),
976
+ entityProposals: Array.isArray(parsed.entityProposals)
977
+ ? parsed.entityProposals.filter((entry) => entry !== null && typeof entry === "object")
978
+ : [],
979
+ pageUpdateSuggestions: Array.isArray(parsed.pageUpdateSuggestions)
980
+ ? parsed.pageUpdateSuggestions.filter((entry) => entry !== null && typeof entry === "object")
981
+ : [],
982
+ articleCandidates: Array.isArray(parsed.articleCandidates)
983
+ ? parsed.articleCandidates.filter((entry) => entry !== null && typeof entry === "object")
984
+ : []
985
+ };
986
+ }
987
+ catch {
988
+ return null;
989
+ }
990
+ }
991
+ async function compileImageWithLlm(profile, secrets, input) {
992
+ const apiKey = await readSecretApiKey(profile.secretId, secrets);
993
+ if (!apiKey) {
994
+ return null;
995
+ }
996
+ const prompt = [
997
+ "You compile a user-provided image into a local wiki page.",
998
+ "Return JSON with keys title, summary, markdown, tags, entityProposals, pageUpdateSuggestions, articleCandidates.",
999
+ "Describe the image, capture useful extracted text when visible, and keep the markdown structured for an agent memory wiki.",
1000
+ "entityProposals should be an array of objects with entityType, title, summary, rationale, confidence, and suggestedFields.",
1001
+ "pageUpdateSuggestions should be an array of objects with targetSlug, rationale, and patchSummary.",
1002
+ "articleCandidates should be an array of objects with title, slug, rationale, and summary.",
1003
+ profile.systemPrompt.trim()
1004
+ ]
1005
+ .filter(Boolean)
1006
+ .join("\n");
1007
+ const response = await fetch(`${profile.baseUrl.replace(/\/$/, "")}/chat/completions`, {
1008
+ method: "POST",
1009
+ headers: {
1010
+ "content-type": "application/json",
1011
+ authorization: `Bearer ${apiKey}`
1012
+ },
1013
+ body: JSON.stringify({
1014
+ model: profile.model,
1015
+ temperature: 0.2,
1016
+ response_format: { type: "json_object" },
1017
+ messages: [
1018
+ {
1019
+ role: "system",
1020
+ content: prompt
1021
+ },
1022
+ {
1023
+ role: "user",
1024
+ content: [
1025
+ {
1026
+ type: "text",
1027
+ text: `Title hint: ${input.titleHint || "none"}\nMime type: ${input.mimeType}`
1028
+ },
1029
+ {
1030
+ type: "image_url",
1031
+ image_url: {
1032
+ url: `data:${input.mimeType};base64,${input.binary.toString("base64")}`,
1033
+ detail: "low"
1034
+ }
1035
+ }
1036
+ ]
1037
+ }
1038
+ ]
1039
+ })
1040
+ });
1041
+ if (!response.ok) {
1042
+ throw new Error(`LLM image compilation failed: ${response.status}`);
1043
+ }
1044
+ const payload = (await response.json());
1045
+ const content = payload.choices?.[0]?.message?.content;
1046
+ if (!content) {
1047
+ return null;
1048
+ }
1049
+ try {
1050
+ const parsed = JSON.parse(content);
1051
+ return {
1052
+ title: parsed.title?.trim() || input.titleHint || "Imported image",
1053
+ summary: parsed.summary?.trim() || "",
1054
+ markdown: parsed.markdown?.trim() ||
1055
+ `# ${parsed.title?.trim() || input.titleHint || "Imported image"}\n\nImage imported into the wiki vault.\n`,
1056
+ tags: normalizeTags(parsed.tags),
1057
+ entityProposals: Array.isArray(parsed.entityProposals)
1058
+ ? parsed.entityProposals.filter((entry) => entry !== null && typeof entry === "object")
1059
+ : [],
1060
+ pageUpdateSuggestions: Array.isArray(parsed.pageUpdateSuggestions)
1061
+ ? parsed.pageUpdateSuggestions.filter((entry) => entry !== null && typeof entry === "object")
1062
+ : [],
1063
+ articleCandidates: Array.isArray(parsed.articleCandidates)
1064
+ ? parsed.articleCandidates.filter((entry) => entry !== null && typeof entry === "object")
1065
+ : []
1066
+ };
1067
+ }
1068
+ catch {
1069
+ return null;
1070
+ }
1071
+ }
1072
+ async function compileSourceWithLlm(profile, secrets, input) {
1073
+ if (input.rawText.trim()) {
1074
+ return compileTextWithLlm(profile, secrets, {
1075
+ titleHint: input.titleHint,
1076
+ rawText: input.rawText,
1077
+ mimeType: input.mimeType
1078
+ });
1079
+ }
1080
+ if (input.binary &&
1081
+ input.parseStrategy !== "text_only" &&
1082
+ input.mimeType.startsWith("image/")) {
1083
+ return compileImageWithLlm(profile, secrets, {
1084
+ titleHint: input.titleHint,
1085
+ binary: input.binary,
1086
+ mimeType: input.mimeType
1087
+ });
1088
+ }
1089
+ return null;
1090
+ }
1091
+ async function embedTexts(profile, secrets, inputs) {
1092
+ const apiKey = await readSecretApiKey(profile.secretId, secrets);
1093
+ if (!apiKey || inputs.length === 0) {
1094
+ return [];
1095
+ }
1096
+ const response = await fetch(`${profile.baseUrl.replace(/\/$/, "")}/embeddings`, {
1097
+ method: "POST",
1098
+ headers: {
1099
+ "content-type": "application/json",
1100
+ authorization: `Bearer ${apiKey}`
1101
+ },
1102
+ body: JSON.stringify({
1103
+ model: profile.model,
1104
+ input: inputs,
1105
+ ...(profile.dimensions ? { dimensions: profile.dimensions } : {})
1106
+ })
1107
+ });
1108
+ if (!response.ok) {
1109
+ throw new Error(`Embedding request failed: ${response.status}`);
1110
+ }
1111
+ const payload = (await response.json());
1112
+ return (payload.data?.map((entry) => Array.isArray(entry.embedding) ? entry.embedding : []) ?? []);
1113
+ }
1114
+ function findExistingSpaceByOwner(ownerUserId) {
1115
+ const row = getDatabase()
1116
+ .prepare(`SELECT id, slug, label, description, owner_user_id, visibility, created_at, updated_at
1117
+ FROM wiki_spaces
1118
+ WHERE owner_user_id = ?
1119
+ ORDER BY created_at ASC
1120
+ LIMIT 1`)
1121
+ .get(ownerUserId);
1122
+ return row ? mapWikiSpace(row) : null;
1123
+ }
1124
+ function getWikiSpaceById(spaceId) {
1125
+ const row = getDatabase()
1126
+ .prepare(`SELECT id, slug, label, description, owner_user_id, visibility, created_at, updated_at
1127
+ FROM wiki_spaces
1128
+ WHERE id = ?`)
1129
+ .get(spaceId);
1130
+ return row ? mapWikiSpace(row) : null;
1131
+ }
1132
+ function ensureSharedWikiSpace() {
1133
+ const existing = getWikiSpaceById("wiki_space_shared");
1134
+ if (existing) {
1135
+ ensureWikiSpaceSeedPages(existing.id);
1136
+ return existing;
1137
+ }
1138
+ const now = nowIso();
1139
+ getDatabase()
1140
+ .prepare(`INSERT INTO wiki_spaces (id, slug, label, description, owner_user_id, visibility, created_at, updated_at)
1141
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
1142
+ .run("wiki_space_shared", "shared", "Shared Forge Memory", "Shared wiki space for file-backed Forge knowledge.", null, "shared", now, now);
1143
+ const space = getWikiSpaceById("wiki_space_shared");
1144
+ ensureWikiSpaceSeedPages(space.id);
1145
+ return space;
1146
+ }
1147
+ function ensurePersonalWikiSpace(userId) {
1148
+ const existing = findExistingSpaceByOwner(userId);
1149
+ if (existing) {
1150
+ ensureWikiSpaceSeedPages(existing.id);
1151
+ return existing;
1152
+ }
1153
+ const now = nowIso();
1154
+ const id = `wiki_space_user_${slugify(userId)}`;
1155
+ const slug = `user-${slugify(userId)}`;
1156
+ getDatabase()
1157
+ .prepare(`INSERT INTO wiki_spaces (id, slug, label, description, owner_user_id, visibility, created_at, updated_at)
1158
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
1159
+ .run(id, slug, `${userId} Wiki`, "Personal Forge wiki space.", userId, "personal", now, now);
1160
+ const space = getWikiSpaceById(id);
1161
+ ensureWikiSpaceSeedPages(space.id);
1162
+ return space;
1163
+ }
1164
+ function buildStarterPageMarkdown(page, space) {
1165
+ if (page.slug === "index") {
1166
+ return [
1167
+ `# ${space.label}`,
1168
+ "",
1169
+ "This wiki is the explicit memory surface for Forge.",
1170
+ "",
1171
+ "Use it to maintain durable context, connect pages to Forge entities, and keep knowledge readable for both humans and agents.",
1172
+ "",
1173
+ "## Starting Points",
1174
+ "",
1175
+ "- [[people]]",
1176
+ "- [[projects]]",
1177
+ "- [[concepts]]",
1178
+ "- [[sources]]",
1179
+ "- [[chronicle]]",
1180
+ ""
1181
+ ].join("\n");
1182
+ }
1183
+ return [
1184
+ `# ${page.title}`,
1185
+ "",
1186
+ page.summary,
1187
+ "",
1188
+ `Return to [[index|Home]].`,
1189
+ ""
1190
+ ].join("\n");
1191
+ }
1192
+ function insertSeedNote(space, seed) {
1193
+ const now = nowIso();
1194
+ const noteId = `note_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
1195
+ const markdown = buildStarterPageMarkdown(seed, space);
1196
+ const contentPlain = buildContentPlain(markdown);
1197
+ const note = persistedNoteSchema.parse({
1198
+ id: noteId,
1199
+ kind: "wiki",
1200
+ title: seed.title,
1201
+ slug: seed.slug,
1202
+ spaceId: space.id,
1203
+ parentSlug: seed.parentSlug,
1204
+ indexOrder: seed.indexOrder,
1205
+ showInIndex: true,
1206
+ aliases: [],
1207
+ summary: seed.summary,
1208
+ contentMarkdown: markdown,
1209
+ contentPlain,
1210
+ author: null,
1211
+ source: "system",
1212
+ sourcePath: "",
1213
+ frontmatter: {},
1214
+ revisionHash: "",
1215
+ lastSyncedAt: null,
1216
+ createdAt: now,
1217
+ updatedAt: now,
1218
+ links: [],
1219
+ tags: []
1220
+ });
1221
+ getDatabase()
1222
+ .prepare(`INSERT INTO notes (
1223
+ id, kind, title, slug, space_id, parent_slug, index_order, show_in_index, aliases_json, summary, content_markdown, content_plain, author, source, tags_json, destroy_at,
1224
+ source_path, frontmatter_json, revision_hash, last_synced_at, created_at, updated_at
1225
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
1226
+ .run(note.id, note.kind, note.title, note.slug, note.spaceId, note.parentSlug, note.indexOrder, note.showInIndex ? 1 : 0, JSON.stringify(note.aliases), note.summary, note.contentMarkdown, note.contentPlain, note.author, note.source, JSON.stringify(note.tags), null, "", "{}", "", null, now, now);
1227
+ return note;
1228
+ }
1229
+ function ensureWikiSpaceSeedPages(spaceId) {
1230
+ const space = getWikiSpaceById(spaceId);
1231
+ if (!space) {
1232
+ return;
1233
+ }
1234
+ const existingSlugs = new Set(getNoteRows("WHERE space_id = ?", [spaceId]).map((row) => row.slug.toLowerCase()));
1235
+ let inserted = false;
1236
+ const insertedNotes = [];
1237
+ for (const seed of WIKI_STARTER_PAGES) {
1238
+ if (existingSlugs.has(seed.slug)) {
1239
+ continue;
1240
+ }
1241
+ insertedNotes.push(insertSeedNote(space, seed));
1242
+ existingSlugs.add(seed.slug);
1243
+ inserted = true;
1244
+ }
1245
+ if (inserted) {
1246
+ for (const note of insertedNotes) {
1247
+ syncNoteWikiArtifacts(note);
1248
+ }
1249
+ syncWikiSpaceIndex(spaceId);
1250
+ }
1251
+ }
1252
+ function resolveSpaceId(spaceId, userId) {
1253
+ if (spaceId?.trim()) {
1254
+ const existing = getWikiSpaceById(spaceId.trim());
1255
+ if (existing) {
1256
+ return existing.id;
1257
+ }
1258
+ }
1259
+ if (userId?.trim()) {
1260
+ return ensurePersonalWikiSpace(userId.trim()).id;
1261
+ }
1262
+ return ensureSharedWikiSpace().id;
1263
+ }
1264
+ export function prepareNoteWikiFields(input) {
1265
+ const kind = input.kind ?? input.existing?.kind ?? "evidence";
1266
+ const spaceId = resolveSpaceId(input.spaceId ?? input.existing?.spaceId, input.userId);
1267
+ const title = input.title?.trim() ||
1268
+ input.existing?.title?.trim() ||
1269
+ inferTitle(input.contentMarkdown, kind === "wiki" ? "Untitled wiki page" : "Untitled note");
1270
+ const slug = buildUniqueSlug(spaceId, input.slug?.trim() || input.existing?.slug || title, input.id);
1271
+ return {
1272
+ kind,
1273
+ title,
1274
+ slug,
1275
+ spaceId,
1276
+ parentSlug: input.parentSlug === undefined
1277
+ ? (input.existing?.parentSlug ?? null)
1278
+ : input.parentSlug?.trim() || null,
1279
+ indexOrder: input.indexOrder ?? input.existing?.indexOrder ?? 0,
1280
+ showInIndex: input.showInIndex ?? input.existing?.showInIndex ?? kind === "wiki",
1281
+ aliases: normalizeAliases(input.aliases ?? input.existing?.aliases),
1282
+ summary: input.summary?.trim() ||
1283
+ input.existing?.summary?.trim() ||
1284
+ inferSummary(input.contentMarkdown)
1285
+ };
1286
+ }
1287
+ export function syncNoteWikiArtifacts(note) {
1288
+ const space = getWikiSpaceById(note.spaceId) ?? ensureSharedWikiSpace();
1289
+ const filePath = getNoteStoragePath(note, space);
1290
+ const frontmatter = buildNoteFrontmatter(note);
1291
+ const payload = `${renderFrontmatter(frontmatter)}${note.contentMarkdown.trim()}\n`;
1292
+ const revisionHash = hashContent(payload);
1293
+ mkdirSync(path.dirname(filePath), { recursive: true });
1294
+ if (note.sourcePath && note.sourcePath !== filePath) {
1295
+ if (existsSync(note.sourcePath)) {
1296
+ rmSync(note.sourcePath, { force: true });
1297
+ }
1298
+ }
1299
+ writeFileSync(filePath, payload, "utf8");
1300
+ const now = nowIso();
1301
+ getDatabase()
1302
+ .prepare(`UPDATE notes
1303
+ SET source_path = ?, frontmatter_json = ?, revision_hash = ?, last_synced_at = ?
1304
+ WHERE id = ?`)
1305
+ .run(filePath, JSON.stringify(frontmatter), revisionHash, now, note.id);
1306
+ upsertWikiSearchRow({
1307
+ ...note,
1308
+ sourcePath: filePath,
1309
+ frontmatter,
1310
+ revisionHash,
1311
+ lastSyncedAt: now
1312
+ });
1313
+ rebuildWikiLinkEdges({
1314
+ ...note,
1315
+ sourcePath: filePath,
1316
+ frontmatter,
1317
+ revisionHash,
1318
+ lastSyncedAt: now
1319
+ });
1320
+ syncWikiSpaceIndex(space.id);
1321
+ }
1322
+ export function deleteNoteWikiArtifacts(note) {
1323
+ if (note.sourcePath && existsSync(note.sourcePath)) {
1324
+ rmSync(note.sourcePath, { force: true });
1325
+ }
1326
+ deleteWikiSearchRow(note.id);
1327
+ getDatabase()
1328
+ .prepare(`DELETE FROM wiki_link_edges WHERE source_note_id = ?`)
1329
+ .run(note.id);
1330
+ getDatabase()
1331
+ .prepare(`DELETE FROM wiki_embedding_chunks WHERE note_id = ?`)
1332
+ .run(note.id);
1333
+ getDatabase()
1334
+ .prepare(`DELETE FROM wiki_media_assets WHERE note_id = ? OR transcript_note_id = ?`)
1335
+ .run(note.id, note.id);
1336
+ syncWikiSpaceIndex(note.spaceId);
1337
+ }
1338
+ function buildWikiIndexMarkdown(space, pages) {
1339
+ const wikiPages = [...pages]
1340
+ .filter((page) => page.kind === "wiki")
1341
+ .sort((left, right) => left.parentSlug === right.parentSlug
1342
+ ? left.indexOrder - right.indexOrder ||
1343
+ left.title.localeCompare(right.title)
1344
+ : (left.parentSlug ?? "").localeCompare(right.parentSlug ?? ""));
1345
+ const evidencePages = [...pages]
1346
+ .filter((page) => page.kind === "evidence")
1347
+ .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
1348
+ const lines = [
1349
+ `# ${space.label}`,
1350
+ "",
1351
+ "Explicit Forge wiki index generated from the local vault.",
1352
+ "",
1353
+ "## How To Use",
1354
+ "",
1355
+ "- Start here when an agent needs a crawlable catalog of the space.",
1356
+ "- `pages/` contains durable wiki articles.",
1357
+ "- `evidence/` contains shorter notes and work traces.",
1358
+ "- `raw/` contains imported source material for future recompilation.",
1359
+ "",
1360
+ `Generated at ${nowIso()}.`,
1361
+ "",
1362
+ "## Wiki Index",
1363
+ ""
1364
+ ];
1365
+ if (wikiPages.length === 0) {
1366
+ lines.push("_No wiki pages yet._", "");
1367
+ }
1368
+ else {
1369
+ for (const page of wikiPages) {
1370
+ const depth = page.parentSlug ? 1 : 0;
1371
+ const prefix = `${" ".repeat(depth)}- `;
1372
+ lines.push(`${prefix}[[${page.slug}]]${page.summary ? ` - ${page.summary}` : ""}`);
1373
+ }
1374
+ lines.push("");
1375
+ }
1376
+ lines.push("## Evidence Pages", "");
1377
+ if (evidencePages.length === 0) {
1378
+ lines.push("_No evidence pages yet._", "");
1379
+ }
1380
+ else {
1381
+ for (const page of evidencePages.slice(0, 200)) {
1382
+ lines.push(`- [[${page.slug}]]${page.summary ? ` - ${page.summary}` : ""}`);
1383
+ }
1384
+ lines.push("");
1385
+ }
1386
+ return `${lines.join("\n")}\n`;
1387
+ }
1388
+ function syncWikiSpaceIndex(spaceId) {
1389
+ const space = getWikiSpaceById(spaceId) ?? ensureSharedWikiSpace();
1390
+ const rootDir = getSpaceStorageDir(space);
1391
+ mkdirSync(path.join(rootDir, "pages"), { recursive: true });
1392
+ mkdirSync(path.join(rootDir, "evidence"), { recursive: true });
1393
+ mkdirSync(path.join(rootDir, "assets"), { recursive: true });
1394
+ mkdirSync(path.join(rootDir, "raw"), { recursive: true });
1395
+ writeFileSync(getSpaceIndexPath(space), buildWikiIndexMarkdown(space, listWikiPages({ spaceId, limit: 10_000 })), "utf8");
1396
+ }
1397
+ function upsertWikiSearchRow(note) {
1398
+ deleteWikiSearchRow(note.id);
1399
+ getDatabase()
1400
+ .prepare(`INSERT INTO wiki_pages_fts (note_id, title, slug, aliases, summary, content_plain, linked_entities)
1401
+ VALUES (?, ?, ?, ?, ?, ?, ?)`)
1402
+ .run(note.id, note.title, note.slug, JSON.stringify(note.aliases ?? []), note.summary ?? "", note.contentPlain, buildLinkedEntityTokens(note));
1403
+ }
1404
+ function rebuildWikiLinkEdges(note) {
1405
+ const now = nowIso();
1406
+ getDatabase()
1407
+ .prepare(`DELETE FROM wiki_link_edges WHERE source_note_id = ?`)
1408
+ .run(note.id);
1409
+ const matches = [...note.contentMarkdown.matchAll(/(!)?\[\[([^[\]]+)\]\]/g)];
1410
+ const insert = getDatabase().prepare(`INSERT INTO wiki_link_edges (
1411
+ source_note_id, target_type, target_note_id, target_entity_type, target_entity_id, label, raw_target, is_embed, created_at, updated_at
1412
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
1413
+ for (const match of matches) {
1414
+ const isEmbed = Boolean(match[1]);
1415
+ const token = (match[2] ?? "").trim();
1416
+ if (!token) {
1417
+ continue;
1418
+ }
1419
+ const [left, right] = token.split("|");
1420
+ const label = right?.trim() || left.trim();
1421
+ if (left.startsWith("forge:")) {
1422
+ const parts = left.split(":");
1423
+ const entityType = parts[1];
1424
+ const entityId = parts.slice(2).join(":");
1425
+ const parsedEntityType = crudEntityTypeSchema.safeParse(entityType);
1426
+ if (parsedEntityType.success && entityId.trim()) {
1427
+ insert.run(note.id, "entity", null, parsedEntityType.data, entityId.trim(), label, left, isEmbed ? 1 : 0, now, now);
1428
+ continue;
1429
+ }
1430
+ }
1431
+ const targetNote = getNoteBySlugRaw(note.spaceId, left.trim(), note.id);
1432
+ if (targetNote) {
1433
+ insert.run(note.id, "page", targetNote.id, null, null, label, left.trim(), isEmbed ? 1 : 0, now, now);
1434
+ continue;
1435
+ }
1436
+ insert.run(note.id, "unresolved", null, null, null, label, left.trim(), isEmbed ? 1 : 0, now, now);
1437
+ }
1438
+ }
1439
+ function loadNotesByIds(noteIds) {
1440
+ if (noteIds.length === 0) {
1441
+ return [];
1442
+ }
1443
+ const placeholders = noteIds.map(() => "?").join(", ");
1444
+ const rows = getDatabase()
1445
+ .prepare(`SELECT id, kind, title, slug, space_id, aliases_json, summary, content_markdown, content_plain, author, source,
1446
+ tags_json, destroy_at, source_path, frontmatter_json, revision_hash, last_synced_at, parent_slug, index_order, show_in_index, created_at, updated_at
1447
+ FROM notes
1448
+ WHERE id IN (${placeholders})`)
1449
+ .all(...noteIds);
1450
+ const links = listLinkRowsForNotes(noteIds);
1451
+ const linksByNoteId = new Map();
1452
+ for (const link of links) {
1453
+ const current = linksByNoteId.get(link.note_id) ?? [];
1454
+ current.push(link);
1455
+ linksByNoteId.set(link.note_id, current);
1456
+ }
1457
+ return rows.map((row) => mapNoteRow(row, linksByNoteId.get(row.id) ?? []));
1458
+ }
1459
+ function listAllNotes() {
1460
+ const rows = getNoteRows();
1461
+ const links = listLinkRowsForNotes(rows.map((row) => row.id));
1462
+ const linksByNoteId = new Map();
1463
+ for (const link of links) {
1464
+ const current = linksByNoteId.get(link.note_id) ?? [];
1465
+ current.push(link);
1466
+ linksByNoteId.set(link.note_id, current);
1467
+ }
1468
+ return rows.map((row) => mapNoteRow(row, linksByNoteId.get(row.id) ?? []));
1469
+ }
1470
+ export function listWikiSpaces() {
1471
+ ensureSharedWikiSpace();
1472
+ const rows = getDatabase()
1473
+ .prepare(`SELECT id, slug, label, description, owner_user_id, visibility, created_at, updated_at
1474
+ FROM wiki_spaces
1475
+ ORDER BY visibility ASC, updated_at DESC`)
1476
+ .all();
1477
+ const spaces = rows.map(mapWikiSpace);
1478
+ for (const space of spaces) {
1479
+ ensureWikiSpaceSeedPages(space.id);
1480
+ }
1481
+ return spaces;
1482
+ }
1483
+ export function createWikiSpace(input) {
1484
+ const parsed = createWikiSpaceSchema.parse(input);
1485
+ const now = nowIso();
1486
+ const id = `wiki_space_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
1487
+ const slug = slugify(parsed.slug || parsed.label);
1488
+ getDatabase()
1489
+ .prepare(`INSERT INTO wiki_spaces (id, slug, label, description, owner_user_id, visibility, created_at, updated_at)
1490
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
1491
+ .run(id, slug, parsed.label, parsed.description, parsed.ownerUserId ?? null, parsed.visibility, now, now);
1492
+ ensureWikiSpaceSeedPages(id);
1493
+ return getWikiSpaceById(id);
1494
+ }
1495
+ function compareWikiPageOrder(left, right) {
1496
+ if ((left.parentSlug ?? "") !== (right.parentSlug ?? "")) {
1497
+ return (left.parentSlug ?? "").localeCompare(right.parentSlug ?? "");
1498
+ }
1499
+ if (left.indexOrder !== right.indexOrder) {
1500
+ return left.indexOrder - right.indexOrder;
1501
+ }
1502
+ return left.title.localeCompare(right.title);
1503
+ }
1504
+ export function listWikiPages(query) {
1505
+ const spaceId = resolveSpaceId(query.spaceId, null);
1506
+ ensureWikiSpaceSeedPages(spaceId);
1507
+ return listAllNotes()
1508
+ .filter((note) => note.spaceId === spaceId)
1509
+ .filter((note) => (query.kind ? note.kind === query.kind : true))
1510
+ .sort(compareWikiPageOrder)
1511
+ .slice(0, query.limit ?? 100);
1512
+ }
1513
+ export function listWikiPageTree(query) {
1514
+ const pages = listWikiPages({ ...query, limit: 10_000 }).filter((page) => page.kind === "wiki" && page.showInIndex);
1515
+ const childrenByParent = new Map();
1516
+ for (const page of pages) {
1517
+ const key = page.parentSlug ?? null;
1518
+ const current = childrenByParent.get(key) ?? [];
1519
+ current.push(page);
1520
+ childrenByParent.set(key, current);
1521
+ }
1522
+ const build = (parentSlug) => (childrenByParent.get(parentSlug) ?? [])
1523
+ .sort(compareWikiPageOrder)
1524
+ .map((page) => ({
1525
+ page,
1526
+ children: build(page.slug)
1527
+ }));
1528
+ return z.array(wikiPageTreeNodeSchema).parse(build(null));
1529
+ }
1530
+ export function getWikiHomePageDetail(input = {}) {
1531
+ const spaceId = resolveSpaceId(input.spaceId, null);
1532
+ ensureWikiSpaceSeedPages(spaceId);
1533
+ const home = getNoteBySlugRaw(spaceId, "index");
1534
+ if (!home) {
1535
+ return null;
1536
+ }
1537
+ return getWikiPageDetail(home.id);
1538
+ }
1539
+ export function getWikiPageDetailBySlug(input) {
1540
+ const spaceId = resolveSpaceId(input.spaceId, null);
1541
+ ensureWikiSpaceSeedPages(spaceId);
1542
+ const row = getNoteBySlugRaw(spaceId, input.slug.trim());
1543
+ if (!row) {
1544
+ return null;
1545
+ }
1546
+ return getWikiPageDetail(row.id);
1547
+ }
1548
+ export function getWikiPageDetail(noteId) {
1549
+ const row = getNoteByIdRaw(noteId);
1550
+ if (!row) {
1551
+ return null;
1552
+ }
1553
+ const note = mapNoteRow(row, listLinkRowsForNotes([row.id]));
1554
+ const backlinkRows = getDatabase()
1555
+ .prepare(`SELECT source_note_id, target_type, target_note_id, target_entity_type, target_entity_id, label, raw_target, is_embed, created_at, updated_at
1556
+ FROM wiki_link_edges
1557
+ WHERE target_note_id = ?
1558
+ ORDER BY updated_at DESC`)
1559
+ .all(noteId);
1560
+ const assets = getDatabase()
1561
+ .prepare(`SELECT id, space_id, note_id, label, mime_type, file_name, file_path, size_bytes, checksum, transcript_note_id, metadata_json, created_at, updated_at
1562
+ FROM wiki_media_assets
1563
+ WHERE note_id = ? OR transcript_note_id = ?
1564
+ ORDER BY updated_at DESC`)
1565
+ .all(noteId, noteId);
1566
+ const backlinkSourceNotes = loadNotesByIds(Array.from(new Set(backlinkRows.map((row) => row.source_note_id))));
1567
+ const backlinkSourceById = new Map(backlinkSourceNotes.map((entry) => [entry.id, entry]));
1568
+ return {
1569
+ page: note,
1570
+ backlinks: backlinkRows.map((row) => wikiLinkEdgeSchema.parse({
1571
+ sourceNoteId: row.source_note_id,
1572
+ targetType: row.target_type,
1573
+ targetNoteId: row.target_note_id,
1574
+ targetEntityType: row.target_entity_type,
1575
+ targetEntityId: row.target_entity_id,
1576
+ label: row.label,
1577
+ rawTarget: row.raw_target,
1578
+ isEmbed: row.is_embed === 1,
1579
+ createdAt: row.created_at,
1580
+ updatedAt: row.updated_at
1581
+ })),
1582
+ backlinkSourceNotes,
1583
+ assets: assets.map((row) => wikiMediaAssetSchema.parse({
1584
+ id: row.id,
1585
+ spaceId: row.space_id,
1586
+ noteId: row.note_id,
1587
+ label: row.label,
1588
+ mimeType: row.mime_type,
1589
+ fileName: row.file_name,
1590
+ filePath: row.file_path,
1591
+ sizeBytes: row.size_bytes,
1592
+ checksum: row.checksum,
1593
+ transcriptNoteId: row.transcript_note_id,
1594
+ metadata: parseJsonRecord(row.metadata_json),
1595
+ createdAt: row.created_at,
1596
+ updatedAt: row.updated_at
1597
+ })),
1598
+ backlinksBySourceId: Object.fromEntries(backlinkRows.map((row) => [
1599
+ row.source_note_id,
1600
+ backlinkSourceById.get(row.source_note_id) ?? null
1601
+ ]))
1602
+ };
1603
+ }
1604
+ export async function syncWikiVaultFromDisk(input) {
1605
+ const parsed = syncWikiVaultSchema.parse(input);
1606
+ const spaces = parsed.spaceId
1607
+ ? [getWikiSpaceById(parsed.spaceId)].filter((entry) => entry !== null)
1608
+ : listWikiSpaces();
1609
+ let updated = 0;
1610
+ for (const space of spaces) {
1611
+ for (const directoryName of ["pages", "evidence"]) {
1612
+ const directory = path.join(getSpaceStorageDir(space), directoryName);
1613
+ try {
1614
+ const entries = await readdir(directory);
1615
+ for (const entry of entries) {
1616
+ if (!entry.endsWith(".md")) {
1617
+ continue;
1618
+ }
1619
+ const filePath = path.join(directory, entry);
1620
+ const content = await readFile(filePath, "utf8");
1621
+ const parsedFile = parseFrontmatter(content);
1622
+ const noteId = typeof parsedFile.frontmatter.id === "string"
1623
+ ? parsedFile.frontmatter.id
1624
+ : null;
1625
+ if (!noteId) {
1626
+ continue;
1627
+ }
1628
+ const existing = getNoteByIdRaw(noteId);
1629
+ if (!existing) {
1630
+ continue;
1631
+ }
1632
+ const markdown = parsedFile.body.trim();
1633
+ const contentPlain = buildContentPlain(markdown);
1634
+ const title = typeof parsedFile.frontmatter.title === "string"
1635
+ ? parsedFile.frontmatter.title
1636
+ : inferTitle(markdown, existing.title);
1637
+ const aliases = normalizeAliases(Array.isArray(parsedFile.frontmatter.aliases)
1638
+ ? parsedFile.frontmatter.aliases.filter((entry) => typeof entry === "string")
1639
+ : []);
1640
+ const summary = typeof parsedFile.frontmatter.summary === "string"
1641
+ ? parsedFile.frontmatter.summary
1642
+ : inferSummary(markdown);
1643
+ const payload = `${renderFrontmatter(parsedFile.frontmatter)}${markdown}\n`;
1644
+ const revisionHash = hashContent(payload);
1645
+ const now = nowIso();
1646
+ getDatabase()
1647
+ .prepare(`UPDATE notes
1648
+ SET title = ?, slug = ?, kind = ?, space_id = ?, parent_slug = ?, index_order = ?, show_in_index = ?, aliases_json = ?, summary = ?, content_markdown = ?, content_plain = ?,
1649
+ source_path = ?, frontmatter_json = ?, revision_hash = ?, last_synced_at = ?, updated_at = ?
1650
+ WHERE id = ?`)
1651
+ .run(title, typeof parsedFile.frontmatter.slug === "string"
1652
+ ? parsedFile.frontmatter.slug
1653
+ : existing.slug, directoryName === "pages" ? "wiki" : "evidence", space.id, typeof parsedFile.frontmatter.parentSlug === "string"
1654
+ ? parsedFile.frontmatter.parentSlug
1655
+ : existing.parent_slug, typeof parsedFile.frontmatter.indexOrder === "number"
1656
+ ? Math.trunc(parsedFile.frontmatter.indexOrder)
1657
+ : existing.index_order, parsedFile.frontmatter.showInIndex === false ? 0 : 1, JSON.stringify(aliases), summary, markdown, contentPlain, filePath, JSON.stringify(parsedFile.frontmatter), revisionHash, now, now, noteId);
1658
+ const note = mapNoteRow(getNoteByIdRaw(noteId), listLinkRowsForNotes([noteId]));
1659
+ upsertWikiSearchRow(note);
1660
+ rebuildWikiLinkEdges(note);
1661
+ updated += 1;
1662
+ }
1663
+ }
1664
+ catch {
1665
+ continue;
1666
+ }
1667
+ }
1668
+ syncWikiSpaceIndex(space.id);
1669
+ }
1670
+ return { updated };
1671
+ }
1672
+ function parseFrontmatter(markdown) {
1673
+ const match = markdown.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
1674
+ if (!match) {
1675
+ return { frontmatter: {}, body: markdown };
1676
+ }
1677
+ const frontmatter = {};
1678
+ for (const line of match[1].split("\n")) {
1679
+ const separatorIndex = line.indexOf(":");
1680
+ if (separatorIndex <= 0) {
1681
+ continue;
1682
+ }
1683
+ const key = line.slice(0, separatorIndex).trim();
1684
+ const rawValue = line.slice(separatorIndex + 1).trim();
1685
+ if (!key) {
1686
+ continue;
1687
+ }
1688
+ try {
1689
+ frontmatter[key] = JSON.parse(rawValue);
1690
+ }
1691
+ catch {
1692
+ frontmatter[key] = rawValue.replace(/^"(.*)"$/, "$1");
1693
+ }
1694
+ }
1695
+ return { frontmatter, body: match[2] };
1696
+ }
1697
+ function findMatchingWikiNoteIds(query) {
1698
+ const ftsQuery = buildWikiFtsQuery(query);
1699
+ if (!ftsQuery) {
1700
+ return new Set();
1701
+ }
1702
+ const rows = getDatabase()
1703
+ .prepare(`SELECT note_id FROM wiki_pages_fts WHERE wiki_pages_fts MATCH ?`)
1704
+ .all(ftsQuery);
1705
+ return new Set(rows.map((row) => row.note_id));
1706
+ }
1707
+ export async function searchWikiPages(input, secrets) {
1708
+ const parsed = wikiSearchQuerySchema.parse(input);
1709
+ const pages = listAllNotes()
1710
+ .filter((page) => (parsed.spaceId ? page.spaceId === parsed.spaceId : true))
1711
+ .filter((page) => (parsed.kind ? page.kind === parsed.kind : true));
1712
+ const scores = new Map();
1713
+ const addScore = (noteId, value) => {
1714
+ scores.set(noteId, (scores.get(noteId) ?? 0) + value);
1715
+ };
1716
+ if (parsed.mode === "text" ||
1717
+ parsed.mode === "hybrid" ||
1718
+ parsed.mode === "entity") {
1719
+ if (parsed.query) {
1720
+ for (const noteId of findMatchingWikiNoteIds(parsed.query)) {
1721
+ addScore(noteId, 4);
1722
+ }
1723
+ }
1724
+ }
1725
+ if (parsed.linkedEntity) {
1726
+ for (const page of pages) {
1727
+ if (page.links.some((link) => link.entityType === parsed.linkedEntity?.entityType &&
1728
+ link.entityId === parsed.linkedEntity?.entityId)) {
1729
+ addScore(page.id, 6);
1730
+ }
1731
+ }
1732
+ }
1733
+ if (secrets &&
1734
+ parsed.query &&
1735
+ (parsed.mode === "semantic" || parsed.mode === "hybrid")) {
1736
+ const profile = listWikiEmbeddingProfiles().find((entry) => entry.enabled && (!parsed.profileId || entry.id === parsed.profileId)) ?? null;
1737
+ if (profile) {
1738
+ const [queryVector] = await embedTexts(profile, secrets, [parsed.query]);
1739
+ if (queryVector && queryVector.length > 0) {
1740
+ const chunkRows = getDatabase()
1741
+ .prepare(`SELECT note_id, vector_json
1742
+ FROM wiki_embedding_chunks
1743
+ WHERE profile_id = ?
1744
+ ${parsed.spaceId ? "AND space_id = ?" : ""}
1745
+ ORDER BY updated_at DESC`)
1746
+ .all(...(parsed.spaceId ? [profile.id, parsed.spaceId] : [profile.id]));
1747
+ for (const row of chunkRows) {
1748
+ try {
1749
+ const score = cosineSimilarity(JSON.parse(row.vector_json), queryVector);
1750
+ if (score > 0) {
1751
+ addScore(row.note_id, score * 5);
1752
+ }
1753
+ }
1754
+ catch {
1755
+ continue;
1756
+ }
1757
+ }
1758
+ }
1759
+ }
1760
+ }
1761
+ if (parsed.query) {
1762
+ const normalizedQuery = parsed.query.toLowerCase();
1763
+ for (const page of pages) {
1764
+ if (page.slug.toLowerCase() === normalizedQuery) {
1765
+ addScore(page.id, 12);
1766
+ }
1767
+ else if (page.title.toLowerCase() === normalizedQuery) {
1768
+ addScore(page.id, 10);
1769
+ }
1770
+ else if (page.title.toLowerCase().includes(normalizedQuery)) {
1771
+ addScore(page.id, 2);
1772
+ }
1773
+ }
1774
+ }
1775
+ const ranked = [...pages]
1776
+ .filter((page) => {
1777
+ if (!parsed.query && !parsed.linkedEntity) {
1778
+ return true;
1779
+ }
1780
+ return scores.has(page.id);
1781
+ })
1782
+ .sort((left, right) => {
1783
+ const scoreDelta = (scores.get(right.id) ?? 0) - (scores.get(left.id) ?? 0);
1784
+ if (scoreDelta !== 0) {
1785
+ return scoreDelta;
1786
+ }
1787
+ return right.updatedAt.localeCompare(left.updatedAt);
1788
+ })
1789
+ .slice(0, parsed.limit);
1790
+ return {
1791
+ mode: parsed.mode,
1792
+ profileId: parsed.profileId ?? null,
1793
+ results: ranked.map((page) => ({
1794
+ page,
1795
+ score: Number((scores.get(page.id) ?? 0).toFixed(4))
1796
+ }))
1797
+ };
1798
+ }
1799
+ export function listWikiLlmProfiles() {
1800
+ const rows = getDatabase()
1801
+ .prepare(`SELECT id, label, provider, base_url, model, secret_id, system_prompt, enabled, metadata_json, created_at, updated_at
1802
+ FROM wiki_llm_profiles
1803
+ ORDER BY updated_at DESC`)
1804
+ .all();
1805
+ return rows.map((row) => wikiLlmProfileSchema.parse({
1806
+ id: row.id,
1807
+ label: row.label,
1808
+ provider: row.provider,
1809
+ baseUrl: row.base_url,
1810
+ model: row.model,
1811
+ secretId: row.secret_id,
1812
+ systemPrompt: row.system_prompt,
1813
+ enabled: row.enabled === 1,
1814
+ metadata: parseJsonRecord(row.metadata_json),
1815
+ createdAt: row.created_at,
1816
+ updatedAt: row.updated_at
1817
+ }));
1818
+ }
1819
+ export function listWikiEmbeddingProfiles() {
1820
+ const rows = getDatabase()
1821
+ .prepare(`SELECT id, label, provider, base_url, model, secret_id, dimensions, chunk_size, chunk_overlap, enabled, metadata_json, created_at, updated_at
1822
+ FROM wiki_embedding_profiles
1823
+ ORDER BY updated_at DESC`)
1824
+ .all();
1825
+ return rows.map((row) => wikiEmbeddingProfileSchema.parse({
1826
+ id: row.id,
1827
+ label: row.label,
1828
+ provider: row.provider,
1829
+ baseUrl: row.base_url,
1830
+ model: row.model,
1831
+ secretId: row.secret_id,
1832
+ dimensions: row.dimensions,
1833
+ chunkSize: row.chunk_size,
1834
+ chunkOverlap: row.chunk_overlap,
1835
+ enabled: row.enabled === 1,
1836
+ metadata: parseJsonRecord(row.metadata_json),
1837
+ createdAt: row.created_at,
1838
+ updatedAt: row.updated_at
1839
+ }));
1840
+ }
1841
+ export function getWikiSettingsPayload() {
1842
+ return wikiSettingsPayloadSchema.parse({
1843
+ spaces: listWikiSpaces(),
1844
+ llmProfiles: listWikiLlmProfiles(),
1845
+ embeddingProfiles: listWikiEmbeddingProfiles()
1846
+ });
1847
+ }
1848
+ export function getWikiHealth(input = {}) {
1849
+ const spaceId = resolveSpaceId(input.spaceId, null);
1850
+ const space = getWikiSpaceById(spaceId) ?? ensureSharedWikiSpace();
1851
+ const pages = listWikiPages({ spaceId, limit: 10_000 });
1852
+ const noteIds = pages.map((page) => page.id);
1853
+ const noteIdSet = new Set(noteIds);
1854
+ const rootDir = getSpaceStorageDir(space);
1855
+ const indexPath = getSpaceIndexPath(space);
1856
+ const rawDirectoryPath = getSpaceRawDir(space);
1857
+ const edgeRows = getDatabase()
1858
+ .prepare(`SELECT e.source_note_id, e.target_type, e.target_note_id, e.raw_target, e.updated_at, n.slug AS source_slug, n.title AS source_title
1859
+ FROM wiki_link_edges e
1860
+ JOIN notes n ON n.id = e.source_note_id
1861
+ WHERE n.space_id = ?
1862
+ ORDER BY e.updated_at DESC`)
1863
+ .all(spaceId);
1864
+ const backlinkCounts = new Map();
1865
+ const outboundCounts = new Map();
1866
+ const unresolvedLinks = edgeRows
1867
+ .filter((row) => row.target_type === "unresolved")
1868
+ .map((row) => ({
1869
+ sourceNoteId: row.source_note_id,
1870
+ sourceSlug: row.source_slug,
1871
+ sourceTitle: row.source_title,
1872
+ rawTarget: row.raw_target,
1873
+ updatedAt: row.updated_at
1874
+ }));
1875
+ for (const row of edgeRows) {
1876
+ outboundCounts.set(row.source_note_id, (outboundCounts.get(row.source_note_id) ?? 0) + 1);
1877
+ if (row.target_note_id && noteIdSet.has(row.target_note_id)) {
1878
+ backlinkCounts.set(row.target_note_id, (backlinkCounts.get(row.target_note_id) ?? 0) + 1);
1879
+ }
1880
+ }
1881
+ let rawSourceCount = 0;
1882
+ try {
1883
+ rawSourceCount = readdirSync(rawDirectoryPath).length;
1884
+ }
1885
+ catch {
1886
+ rawSourceCount = 0;
1887
+ }
1888
+ const assetCount = getDatabase()
1889
+ .prepare(`SELECT COUNT(*) AS count
1890
+ FROM wiki_media_assets
1891
+ WHERE space_id = ?`)
1892
+ .get(spaceId).count;
1893
+ return wikiHealthPayloadSchema.parse({
1894
+ space,
1895
+ indexPath,
1896
+ rawDirectoryPath,
1897
+ pageCount: pages.length,
1898
+ wikiPageCount: pages.filter((page) => page.kind === "wiki").length,
1899
+ evidencePageCount: pages.filter((page) => page.kind === "evidence").length,
1900
+ assetCount,
1901
+ rawSourceCount,
1902
+ unresolvedLinks,
1903
+ orphanPages: pages
1904
+ .filter((page) => page.kind === "wiki")
1905
+ .filter((page) => (backlinkCounts.get(page.id) ?? 0) === 0 &&
1906
+ (outboundCounts.get(page.id) ?? 0) === 0)
1907
+ .map((page) => ({
1908
+ id: page.id,
1909
+ slug: page.slug,
1910
+ title: page.title,
1911
+ kind: page.kind,
1912
+ updatedAt: page.updatedAt
1913
+ }))
1914
+ .slice(0, 50),
1915
+ missingSummaries: pages
1916
+ .filter((page) => !page.summary.trim())
1917
+ .map((page) => ({
1918
+ id: page.id,
1919
+ slug: page.slug,
1920
+ title: page.title,
1921
+ updatedAt: page.updatedAt
1922
+ }))
1923
+ .slice(0, 50),
1924
+ enabledEmbeddingProfiles: listWikiEmbeddingProfiles()
1925
+ .filter((profile) => profile.enabled)
1926
+ .map((profile) => ({
1927
+ id: profile.id,
1928
+ label: profile.label,
1929
+ model: profile.model
1930
+ })),
1931
+ enabledLlmProfiles: listWikiLlmProfiles()
1932
+ .filter((profile) => profile.enabled)
1933
+ .map((profile) => ({
1934
+ id: profile.id,
1935
+ label: profile.label,
1936
+ model: profile.model
1937
+ }))
1938
+ });
1939
+ }
1940
+ async function persistWikiRawSource(options) {
1941
+ const rawDir = getSpaceRawDir(options.space);
1942
+ await mkdir(rawDir, { recursive: true });
1943
+ const extension = path.extname(options.fetched.fileName) ||
1944
+ inferExtensionFromMimeType(options.fetched.mimeType) ||
1945
+ (options.fetched.contentText ? ".txt" : ".bin");
1946
+ const baseName = sanitizeFileName(path.basename(options.fetched.fileName, path.extname(options.fetched.fileName)) || options.jobId);
1947
+ const rawPath = path.join(rawDir, `${options.jobId}-${baseName}${extension}`);
1948
+ const payload = options.fetched.binary ?? Buffer.from(options.fetched.contentText, "utf8");
1949
+ await writeFile(rawPath, payload);
1950
+ return {
1951
+ filePath: rawPath,
1952
+ sizeBytes: payload.byteLength,
1953
+ checksum: hashBuffer(payload)
1954
+ };
1955
+ }
1956
+ export function upsertWikiLlmProfile(input, secrets) {
1957
+ const parsed = upsertWikiLlmProfileSchema.parse(input);
1958
+ const now = nowIso();
1959
+ const id = parsed.id?.trim() ||
1960
+ `wiki_llm_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
1961
+ let secretId = listWikiLlmProfiles().find((entry) => entry.id === id)?.secretId ?? null;
1962
+ if (parsed.apiKey?.trim()) {
1963
+ secretId =
1964
+ secretId ??
1965
+ `wiki_llm_secret_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
1966
+ storeEncryptedSecret(secretId, secrets.sealJson({ apiKey: parsed.apiKey.trim() }), `${parsed.label} wiki LLM profile`);
1967
+ }
1968
+ const metadata = {
1969
+ ...parsed.metadata
1970
+ };
1971
+ if (parsed.reasoningEffort) {
1972
+ metadata.reasoningEffort = parsed.reasoningEffort;
1973
+ }
1974
+ else {
1975
+ delete metadata.reasoningEffort;
1976
+ }
1977
+ if (parsed.verbosity) {
1978
+ metadata.verbosity = parsed.verbosity;
1979
+ }
1980
+ else {
1981
+ delete metadata.verbosity;
1982
+ }
1983
+ getDatabase()
1984
+ .prepare(`INSERT INTO wiki_llm_profiles (id, label, provider, base_url, model, secret_id, system_prompt, enabled, metadata_json, created_at, updated_at)
1985
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1986
+ ON CONFLICT(id) DO UPDATE SET
1987
+ label = excluded.label,
1988
+ provider = excluded.provider,
1989
+ base_url = excluded.base_url,
1990
+ model = excluded.model,
1991
+ secret_id = excluded.secret_id,
1992
+ system_prompt = excluded.system_prompt,
1993
+ enabled = excluded.enabled,
1994
+ metadata_json = excluded.metadata_json,
1995
+ updated_at = excluded.updated_at`)
1996
+ .run(id, parsed.label, parsed.provider, parsed.baseUrl, parsed.model, secretId, parsed.systemPrompt, parsed.enabled ? 1 : 0, JSON.stringify(metadata), now, now);
1997
+ return listWikiLlmProfiles().find((entry) => entry.id === id);
1998
+ }
1999
+ export function upsertWikiEmbeddingProfile(input, secrets) {
2000
+ const parsed = upsertWikiEmbeddingProfileSchema.parse(input);
2001
+ const now = nowIso();
2002
+ const id = parsed.id?.trim() ||
2003
+ `wiki_embed_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
2004
+ let secretId = listWikiEmbeddingProfiles().find((entry) => entry.id === id)?.secretId ??
2005
+ null;
2006
+ if (parsed.apiKey?.trim()) {
2007
+ secretId =
2008
+ secretId ??
2009
+ `wiki_embed_secret_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
2010
+ storeEncryptedSecret(secretId, secrets.sealJson({ apiKey: parsed.apiKey.trim() }), `${parsed.label} wiki embedding profile`);
2011
+ }
2012
+ getDatabase()
2013
+ .prepare(`INSERT INTO wiki_embedding_profiles (
2014
+ id, label, provider, base_url, model, secret_id, dimensions, chunk_size, chunk_overlap, enabled, metadata_json, created_at, updated_at
2015
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2016
+ ON CONFLICT(id) DO UPDATE SET
2017
+ label = excluded.label,
2018
+ provider = excluded.provider,
2019
+ base_url = excluded.base_url,
2020
+ model = excluded.model,
2021
+ secret_id = excluded.secret_id,
2022
+ dimensions = excluded.dimensions,
2023
+ chunk_size = excluded.chunk_size,
2024
+ chunk_overlap = excluded.chunk_overlap,
2025
+ enabled = excluded.enabled,
2026
+ metadata_json = excluded.metadata_json,
2027
+ updated_at = excluded.updated_at`)
2028
+ .run(id, parsed.label, parsed.provider, parsed.baseUrl, parsed.model, secretId, parsed.dimensions ?? null, parsed.chunkSize, parsed.chunkOverlap, parsed.enabled ? 1 : 0, JSON.stringify(parsed.metadata), now, now);
2029
+ return listWikiEmbeddingProfiles().find((entry) => entry.id === id);
2030
+ }
2031
+ export function deleteWikiProfile(kind, profileId) {
2032
+ if (kind === "llm") {
2033
+ const profile = listWikiLlmProfiles().find((entry) => entry.id === profileId);
2034
+ if (profile?.secretId) {
2035
+ deleteEncryptedSecret(profile.secretId);
2036
+ }
2037
+ getDatabase()
2038
+ .prepare(`DELETE FROM wiki_llm_profiles WHERE id = ?`)
2039
+ .run(profileId);
2040
+ return;
2041
+ }
2042
+ const profile = listWikiEmbeddingProfiles().find((entry) => entry.id === profileId);
2043
+ if (profile?.secretId) {
2044
+ deleteEncryptedSecret(profile.secretId);
2045
+ }
2046
+ getDatabase()
2047
+ .prepare(`DELETE FROM wiki_embedding_profiles WHERE id = ?`)
2048
+ .run(profileId);
2049
+ getDatabase()
2050
+ .prepare(`DELETE FROM wiki_embedding_chunks WHERE profile_id = ?`)
2051
+ .run(profileId);
2052
+ }
2053
+ export async function reindexWikiEmbeddings(input, secrets) {
2054
+ const parsed = reindexWikiEmbeddingsSchema.parse(input);
2055
+ const profiles = listWikiEmbeddingProfiles().filter((entry) => entry.enabled && (!parsed.profileId || entry.id === parsed.profileId));
2056
+ const pages = listWikiPages({ spaceId: parsed.spaceId, limit: 10_000 });
2057
+ let chunkCount = 0;
2058
+ for (const profile of profiles) {
2059
+ for (const page of pages) {
2060
+ getDatabase()
2061
+ .prepare(`DELETE FROM wiki_embedding_chunks WHERE note_id = ? AND profile_id = ?`)
2062
+ .run(page.id, profile.id);
2063
+ const chunks = chunkHeadingAware(page.contentMarkdown, profile.chunkSize, profile.chunkOverlap);
2064
+ if (chunks.length === 0) {
2065
+ continue;
2066
+ }
2067
+ const vectors = await embedTexts(profile, secrets, chunks.map((chunk) => chunk.contentText));
2068
+ const insert = getDatabase().prepare(`INSERT INTO wiki_embedding_chunks (
2069
+ id, note_id, space_id, profile_id, chunk_key, heading_path, content_text, vector_json, created_at, updated_at
2070
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
2071
+ const now = nowIso();
2072
+ chunks.forEach((chunk, index) => {
2073
+ const vector = vectors[index];
2074
+ if (!vector || vector.length === 0) {
2075
+ return;
2076
+ }
2077
+ insert.run(`wiki_chunk_${randomUUID().replaceAll("-", "").slice(0, 10)}`, page.id, page.spaceId, profile.id, chunk.key, chunk.headingPath, chunk.contentText, JSON.stringify(vector), now, now);
2078
+ chunkCount += 1;
2079
+ });
2080
+ }
2081
+ }
2082
+ return {
2083
+ profilesIndexed: profiles.length,
2084
+ pagesIndexed: pages.length,
2085
+ chunkCount
2086
+ };
2087
+ }
2088
+ function getWikiIngestJobDir(jobId) {
2089
+ return path.join(resolveDataDir(), "wiki-ingest", jobId);
2090
+ }
2091
+ function getWikiIngestUploadsDir(jobId) {
2092
+ return path.join(getWikiIngestJobDir(jobId), "uploads");
2093
+ }
2094
+ function readWikiIngestJobRow(jobId) {
2095
+ return getDatabase()
2096
+ .prepare(`SELECT id, space_id, llm_profile_id, status, phase, progress_percent, total_files, processed_files,
2097
+ created_page_count, created_entity_count, accepted_count, rejected_count, latest_message,
2098
+ source_kind, source_locator, mime_type, title_hint, summary, page_note_id, created_by_actor,
2099
+ error_message, input_json, created_at, updated_at, completed_at
2100
+ FROM wiki_ingest_jobs
2101
+ WHERE id = ?`)
2102
+ .get(jobId);
2103
+ }
2104
+ function mapWikiIngestJobRow(job) {
2105
+ return {
2106
+ id: job.id,
2107
+ spaceId: job.space_id,
2108
+ llmProfileId: job.llm_profile_id,
2109
+ status: job.status,
2110
+ phase: job.phase,
2111
+ progressPercent: job.progress_percent,
2112
+ totalFiles: job.total_files,
2113
+ processedFiles: job.processed_files,
2114
+ createdPageCount: job.created_page_count,
2115
+ createdEntityCount: job.created_entity_count,
2116
+ acceptedCount: job.accepted_count,
2117
+ rejectedCount: job.rejected_count,
2118
+ latestMessage: job.latest_message,
2119
+ sourceKind: job.source_kind,
2120
+ sourceLocator: job.source_locator,
2121
+ mimeType: job.mime_type,
2122
+ titleHint: job.title_hint,
2123
+ summary: job.summary,
2124
+ pageNoteId: job.page_note_id,
2125
+ createdByActor: job.created_by_actor,
2126
+ errorMessage: job.error_message,
2127
+ createdAt: job.created_at,
2128
+ updatedAt: job.updated_at,
2129
+ completedAt: job.completed_at
2130
+ };
2131
+ }
2132
+ function listWikiIngestJobLogsInternal(jobId) {
2133
+ return getDatabase()
2134
+ .prepare(`SELECT id, level, message, metadata_json, created_at
2135
+ FROM wiki_ingest_job_logs
2136
+ WHERE job_id = ?
2137
+ ORDER BY created_at ASC`)
2138
+ .all(jobId);
2139
+ }
2140
+ function listWikiIngestJobAssetsInternal(jobId) {
2141
+ return getDatabase()
2142
+ .prepare(`SELECT id, status, source_kind, source_locator, file_name, mime_type, file_path,
2143
+ size_bytes, checksum, metadata_json, created_at, updated_at
2144
+ FROM wiki_ingest_job_assets
2145
+ WHERE job_id = ?
2146
+ ORDER BY created_at ASC`)
2147
+ .all(jobId);
2148
+ }
2149
+ function listWikiIngestCandidatesInternal(jobId) {
2150
+ return getDatabase()
2151
+ .prepare(`SELECT id, source_asset_id, candidate_type, status, title, summary, target_key,
2152
+ payload_json, published_note_id, published_entity_type, published_entity_id,
2153
+ created_at, updated_at
2154
+ FROM wiki_ingest_job_candidates
2155
+ WHERE job_id = ?
2156
+ ORDER BY created_at ASC`)
2157
+ .all(jobId);
2158
+ }
2159
+ function updateWikiIngestJob(jobId, patch) {
2160
+ const current = readWikiIngestJobRow(jobId);
2161
+ if (!current) {
2162
+ return null;
2163
+ }
2164
+ const next = {
2165
+ status: patch.status ?? current.status,
2166
+ phase: patch.phase ?? current.phase,
2167
+ progressPercent: patch.progressPercent ?? current.progress_percent,
2168
+ totalFiles: patch.totalFiles ?? current.total_files,
2169
+ processedFiles: patch.processedFiles ?? current.processed_files,
2170
+ createdPageCount: patch.createdPageCount ?? current.created_page_count,
2171
+ createdEntityCount: patch.createdEntityCount ?? current.created_entity_count,
2172
+ acceptedCount: patch.acceptedCount ?? current.accepted_count,
2173
+ rejectedCount: patch.rejectedCount ?? current.rejected_count,
2174
+ latestMessage: patch.latestMessage ?? current.latest_message,
2175
+ sourceLocator: patch.sourceLocator ?? current.source_locator,
2176
+ mimeType: patch.mimeType ?? current.mime_type,
2177
+ summary: patch.summary ?? current.summary,
2178
+ pageNoteId: patch.pageNoteId === undefined ? current.page_note_id : patch.pageNoteId,
2179
+ errorMessage: patch.errorMessage ?? current.error_message,
2180
+ completedAt: patch.completedAt === undefined
2181
+ ? current.completed_at
2182
+ : patch.completedAt,
2183
+ updatedAt: nowIso()
2184
+ };
2185
+ getDatabase()
2186
+ .prepare(`UPDATE wiki_ingest_jobs
2187
+ SET status = ?, phase = ?, progress_percent = ?, total_files = ?, processed_files = ?,
2188
+ created_page_count = ?, created_entity_count = ?, accepted_count = ?, rejected_count = ?,
2189
+ latest_message = ?, source_locator = ?, mime_type = ?, summary = ?, page_note_id = ?,
2190
+ error_message = ?, completed_at = ?, updated_at = ?
2191
+ WHERE id = ?`)
2192
+ .run(next.status, next.phase, next.progressPercent, next.totalFiles, next.processedFiles, next.createdPageCount, next.createdEntityCount, next.acceptedCount, next.rejectedCount, next.latestMessage, next.sourceLocator, next.mimeType, next.summary, next.pageNoteId, next.errorMessage, next.completedAt, next.updatedAt, jobId);
2193
+ return getWikiIngestJob(jobId);
2194
+ }
2195
+ function createWikiIngestLog(jobId, message, level = "info", metadata = {}, options = {}) {
2196
+ const createdAt = nowIso();
2197
+ const logMetadata = options.aggregateKey
2198
+ ? { ...metadata, aggregateKey: options.aggregateKey }
2199
+ : metadata;
2200
+ const aggregateKey = options.aggregateKey?.trim() || null;
2201
+ if (aggregateKey) {
2202
+ const current = [...listWikiIngestJobLogsInternal(jobId)]
2203
+ .reverse()
2204
+ .find((entry) => {
2205
+ const parsed = parseJsonRecord(entry.metadata_json);
2206
+ return parsed.aggregateKey === aggregateKey;
2207
+ });
2208
+ if (current) {
2209
+ getDatabase()
2210
+ .prepare(`UPDATE wiki_ingest_job_logs
2211
+ SET level = ?, message = ?, metadata_json = ?, created_at = ?
2212
+ WHERE id = ?`)
2213
+ .run(level, message, JSON.stringify(logMetadata), createdAt, current.id);
2214
+ }
2215
+ else {
2216
+ getDatabase()
2217
+ .prepare(`INSERT INTO wiki_ingest_job_logs (id, job_id, level, message, metadata_json, created_at)
2218
+ VALUES (?, ?, ?, ?, ?, ?)`)
2219
+ .run(`wiki_ingest_log_${randomUUID().replaceAll("-", "").slice(0, 10)}`, jobId, level, message, JSON.stringify(logMetadata), createdAt);
2220
+ }
2221
+ }
2222
+ else {
2223
+ getDatabase()
2224
+ .prepare(`INSERT INTO wiki_ingest_job_logs (id, job_id, level, message, metadata_json, created_at)
2225
+ VALUES (?, ?, ?, ?, ?, ?)`)
2226
+ .run(`wiki_ingest_log_${randomUUID().replaceAll("-", "").slice(0, 10)}`, jobId, level, message, JSON.stringify(logMetadata), createdAt);
2227
+ }
2228
+ if (options.recordDiagnostic !== false) {
2229
+ recordDiagnosticLog({
2230
+ level,
2231
+ source: "server",
2232
+ scope: typeof metadata.scope === "string" && metadata.scope.trim()
2233
+ ? metadata.scope
2234
+ : "wiki_ingest",
2235
+ eventKey: typeof metadata.eventKey === "string" && metadata.eventKey.trim()
2236
+ ? metadata.eventKey
2237
+ : "wiki_ingest_log",
2238
+ message,
2239
+ functionName: "createWikiIngestLog",
2240
+ entityType: "wiki_ingest_job",
2241
+ entityId: jobId,
2242
+ jobId,
2243
+ details: logMetadata
2244
+ });
2245
+ }
2246
+ }
2247
+ function findOpenAiResponseIdForJobAsset(input) {
2248
+ const normalizedFileName = input.fileName.trim().toLowerCase();
2249
+ const normalizedSourceLocator = input.sourceLocator.trim().toLowerCase();
2250
+ const normalizedChecksum = input.checksum.trim().toLowerCase();
2251
+ const sameNamedAssets = listWikiIngestJobAssetsInternal(input.jobId).filter((asset) => asset.file_name.trim().toLowerCase() === normalizedFileName).length;
2252
+ const logs = [...listWikiIngestJobLogsInternal(input.jobId)].reverse();
2253
+ for (const entry of logs) {
2254
+ const metadata = parseJsonRecord(entry.metadata_json);
2255
+ const responseId = readStringRecordValue(metadata, "responseId");
2256
+ if (!responseId) {
2257
+ continue;
2258
+ }
2259
+ const loggedAssetId = readStringRecordValue(metadata, "sourceAssetId") ??
2260
+ readStringRecordValue(metadata, "assetId");
2261
+ if (loggedAssetId) {
2262
+ if (loggedAssetId === input.assetId) {
2263
+ return responseId;
2264
+ }
2265
+ continue;
2266
+ }
2267
+ const loggedSourceLocator = readStringRecordValue(metadata, "sourceLocator");
2268
+ if (loggedSourceLocator) {
2269
+ if (loggedSourceLocator.trim().toLowerCase() === normalizedSourceLocator) {
2270
+ return responseId;
2271
+ }
2272
+ continue;
2273
+ }
2274
+ const loggedChecksum = readStringRecordValue(metadata, "checksum");
2275
+ if (loggedChecksum) {
2276
+ if (loggedChecksum.trim().toLowerCase() === normalizedChecksum) {
2277
+ return responseId;
2278
+ }
2279
+ continue;
2280
+ }
2281
+ const loggedFileName = readStringRecordValue(metadata, "currentFileName") ??
2282
+ readStringRecordValue(metadata, "fileName");
2283
+ if (!loggedFileName) {
2284
+ return responseId;
2285
+ }
2286
+ if (sameNamedAssets === 1 &&
2287
+ loggedFileName.trim().toLowerCase() === normalizedFileName) {
2288
+ return responseId;
2289
+ }
2290
+ }
2291
+ return null;
2292
+ }
2293
+ function createWikiIngestAssetRecord(input) {
2294
+ const id = `wiki_ingest_asset_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
2295
+ const now = nowIso();
2296
+ getDatabase()
2297
+ .prepare(`INSERT INTO wiki_ingest_job_assets (
2298
+ id, job_id, status, source_kind, source_locator, file_name, mime_type, file_path,
2299
+ size_bytes, checksum, metadata_json, created_at, updated_at
2300
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
2301
+ .run(id, input.jobId, input.status ?? "queued", input.sourceKind, input.sourceLocator, input.fileName, input.mimeType, input.filePath, input.sizeBytes, input.checksum, JSON.stringify(input.metadata ?? {}), now, now);
2302
+ return id;
2303
+ }
2304
+ function updateWikiIngestAsset(assetId, patch) {
2305
+ const current = getDatabase()
2306
+ .prepare(`SELECT file_path, mime_type, size_bytes, checksum, metadata_json, status
2307
+ FROM wiki_ingest_job_assets
2308
+ WHERE id = ?`)
2309
+ .get(assetId);
2310
+ if (!current) {
2311
+ return;
2312
+ }
2313
+ getDatabase()
2314
+ .prepare(`UPDATE wiki_ingest_job_assets
2315
+ SET status = ?, file_path = ?, mime_type = ?, size_bytes = ?, checksum = ?, metadata_json = ?, updated_at = ?
2316
+ WHERE id = ?`)
2317
+ .run(patch.status ?? current.status, patch.filePath ?? current.file_path, patch.mimeType ?? current.mime_type, patch.sizeBytes ?? current.size_bytes, patch.checksum ?? current.checksum, JSON.stringify(patch.metadata ?? parseJsonRecord(current.metadata_json)), nowIso(), assetId);
2318
+ }
2319
+ function createWikiIngestCandidate(input) {
2320
+ const existing = listWikiIngestCandidatesInternal(input.jobId).find((candidate) => candidate.source_asset_id === (input.sourceAssetId ?? null) &&
2321
+ candidate.candidate_type === input.candidateType &&
2322
+ candidate.title === (input.title ?? "") &&
2323
+ candidate.summary === (input.summary ?? "") &&
2324
+ candidate.target_key === (input.targetKey ?? "") &&
2325
+ candidate.payload_json === JSON.stringify(input.payload));
2326
+ if (existing) {
2327
+ return existing.id;
2328
+ }
2329
+ const id = `wiki_ingest_candidate_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
2330
+ const now = nowIso();
2331
+ getDatabase()
2332
+ .prepare(`INSERT INTO wiki_ingest_job_candidates (
2333
+ id, job_id, source_asset_id, candidate_type, status, title, summary, target_key,
2334
+ payload_json, published_note_id, published_entity_type, published_entity_id, created_at, updated_at
2335
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
2336
+ .run(id, input.jobId, input.sourceAssetId ?? null, input.candidateType, "suggested", input.title ?? "", input.summary ?? "", input.targetKey ?? "", JSON.stringify(input.payload), null, null, null, now, now);
2337
+ return id;
2338
+ }
2339
+ function updateWikiIngestCandidate(candidateId, patch) {
2340
+ const current = getDatabase()
2341
+ .prepare(`SELECT payload_json, status, published_note_id, published_entity_type, published_entity_id
2342
+ FROM wiki_ingest_job_candidates
2343
+ WHERE id = ?`)
2344
+ .get(candidateId);
2345
+ if (!current) {
2346
+ return;
2347
+ }
2348
+ getDatabase()
2349
+ .prepare(`UPDATE wiki_ingest_job_candidates
2350
+ SET status = ?, payload_json = ?, published_note_id = ?, published_entity_type = ?, published_entity_id = ?, updated_at = ?
2351
+ WHERE id = ?`)
2352
+ .run(patch.status ?? current.status, JSON.stringify(patch.payload ?? parseJsonRecord(current.payload_json)), patch.publishedNoteId === undefined
2353
+ ? current.published_note_id
2354
+ : patch.publishedNoteId, patch.publishedEntityType === undefined
2355
+ ? current.published_entity_type
2356
+ : patch.publishedEntityType, patch.publishedEntityId === undefined
2357
+ ? current.published_entity_id
2358
+ : patch.publishedEntityId, nowIso(), candidateId);
2359
+ }
2360
+ function calculateProgress(totalFiles, processedFiles) {
2361
+ if (totalFiles <= 0) {
2362
+ return 0;
2363
+ }
2364
+ return Math.max(0, Math.min(100, Math.round((processedFiles / totalFiles) * 100)));
2365
+ }
2366
+ async function persistIngestUpload(options) {
2367
+ const uploadDir = getWikiIngestUploadsDir(options.jobId);
2368
+ await mkdir(uploadDir, { recursive: true });
2369
+ const safeName = sanitizeFileName(options.fileName || "upload.bin");
2370
+ const nextPath = path.join(uploadDir, `${Date.now()}-${randomUUID().slice(0, 8)}-${safeName}`);
2371
+ await writeFile(nextPath, options.payload);
2372
+ return {
2373
+ filePath: nextPath,
2374
+ sizeBytes: options.payload.byteLength,
2375
+ checksum: hashBuffer(options.payload)
2376
+ };
2377
+ }
2378
+ function readWikiIngestInput(jobId) {
2379
+ const row = readWikiIngestJobRow(jobId);
2380
+ if (!row) {
2381
+ return null;
2382
+ }
2383
+ const payload = parseJsonRecord(row.input_json);
2384
+ return createWikiIngestJobSchema.parse({
2385
+ spaceId: row.space_id,
2386
+ titleHint: typeof payload.titleHint === "string"
2387
+ ? payload.titleHint
2388
+ : row.title_hint,
2389
+ sourceKind: payload.sourceKind === "local_path" || payload.sourceKind === "url"
2390
+ ? payload.sourceKind
2391
+ : "raw_text",
2392
+ sourceText: typeof payload.sourceText === "string" ? payload.sourceText : "",
2393
+ sourcePath: typeof payload.sourcePath === "string" ? payload.sourcePath : undefined,
2394
+ sourceUrl: typeof payload.sourceUrl === "string" ? payload.sourceUrl : undefined,
2395
+ mimeType: typeof payload.mimeType === "string" ? payload.mimeType : row.mime_type,
2396
+ llmProfileId: typeof payload.llmProfileId === "string"
2397
+ ? payload.llmProfileId
2398
+ : (row.llm_profile_id ?? undefined),
2399
+ parseStrategy: payload.parseStrategy === "text_only" ||
2400
+ payload.parseStrategy === "multimodal"
2401
+ ? payload.parseStrategy
2402
+ : "auto",
2403
+ entityProposalMode: payload.entityProposalMode === "none" ? "none" : "suggest",
2404
+ userId: typeof payload.userId === "string"
2405
+ ? payload.userId
2406
+ : payload.userId === null
2407
+ ? null
2408
+ : row.created_by_actor,
2409
+ createAsKind: payload.createAsKind === "evidence" ? "evidence" : "wiki",
2410
+ linkedEntityHints: Array.isArray(payload.linkedEntityHints)
2411
+ ? payload.linkedEntityHints
2412
+ : []
2413
+ });
2414
+ }
2415
+ export async function createUploadedWikiIngestJob(input, files, options = {}) {
2416
+ const parsed = createWikiIngestJobSchema.parse({
2417
+ ...input,
2418
+ sourceKind: "raw_text",
2419
+ sourceText: "",
2420
+ mimeType: ""
2421
+ });
2422
+ const spaceId = resolveSpaceId(parsed.spaceId, parsed.userId);
2423
+ const now = nowIso();
2424
+ const jobId = `wiki_ingest_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
2425
+ getDatabase()
2426
+ .prepare(`INSERT INTO wiki_ingest_jobs (
2427
+ id, space_id, llm_profile_id, status, phase, progress_percent, total_files, processed_files,
2428
+ created_page_count, created_entity_count, accepted_count, rejected_count, latest_message,
2429
+ source_kind, source_locator, mime_type, title_hint, summary, page_note_id, created_by_actor,
2430
+ error_message, input_json, created_at, updated_at, completed_at
2431
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
2432
+ .run(jobId, spaceId, parsed.llmProfileId ?? null, "queued", "queued", 0, files.length, 0, 0, 0, 0, 0, files.length === 1
2433
+ ? "Waiting to ingest 1 uploaded file"
2434
+ : `Waiting to ingest ${files.length} uploaded files`, "upload", "", "", parsed.titleHint, "", null, options.actor ?? parsed.userId ?? null, "", JSON.stringify({
2435
+ ...parsed,
2436
+ sourceKind: "upload"
2437
+ }), now, now, null);
2438
+ for (const file of files) {
2439
+ const persisted = await persistIngestUpload({
2440
+ jobId,
2441
+ fileName: file.fileName,
2442
+ mimeType: file.mimeType || inferMimeTypeFromPath(file.fileName),
2443
+ payload: file.payload
2444
+ });
2445
+ createWikiIngestAssetRecord({
2446
+ jobId,
2447
+ sourceKind: "upload",
2448
+ sourceLocator: file.fileName,
2449
+ fileName: file.fileName,
2450
+ mimeType: file.mimeType || inferMimeTypeFromPath(file.fileName),
2451
+ filePath: persisted.filePath,
2452
+ sizeBytes: persisted.sizeBytes,
2453
+ checksum: persisted.checksum
2454
+ });
2455
+ }
2456
+ createWikiIngestLog(jobId, files.length === 1
2457
+ ? "Queued 1 uploaded file for Forge wiki ingestion."
2458
+ : `Queued ${files.length} uploaded files for Forge wiki ingestion.`);
2459
+ return {
2460
+ job: getWikiIngestJob(jobId),
2461
+ page: null
2462
+ };
2463
+ }
2464
+ export async function ingestWikiSource(input, options = {}) {
2465
+ const parsed = createWikiIngestJobSchema.parse(input);
2466
+ const spaceId = resolveSpaceId(parsed.spaceId, parsed.userId);
2467
+ const now = nowIso();
2468
+ const jobId = `wiki_ingest_${randomUUID().replaceAll("-", "").slice(0, 10)}`;
2469
+ let filePath = "";
2470
+ let sizeBytes = 0;
2471
+ let checksum = "";
2472
+ let sourceLocator = "";
2473
+ let fileName = "source.bin";
2474
+ let mimeType = parsed.mimeType;
2475
+ if (parsed.sourceKind === "raw_text") {
2476
+ const payload = Buffer.from(parsed.sourceText, "utf8");
2477
+ const persisted = await persistIngestUpload({
2478
+ jobId,
2479
+ fileName: "inline.txt",
2480
+ mimeType: parsed.mimeType || "text/plain",
2481
+ payload
2482
+ });
2483
+ filePath = persisted.filePath;
2484
+ sizeBytes = persisted.sizeBytes;
2485
+ checksum = persisted.checksum;
2486
+ sourceLocator = "raw_text";
2487
+ fileName = "inline.txt";
2488
+ mimeType = parsed.mimeType || "text/plain";
2489
+ }
2490
+ else if (parsed.sourceKind === "local_path") {
2491
+ const sourcePath = parsed.sourcePath?.trim();
2492
+ if (!sourcePath) {
2493
+ throw new Error("sourcePath is required for local_path ingest.");
2494
+ }
2495
+ const payload = await readFile(sourcePath);
2496
+ const persisted = await persistIngestUpload({
2497
+ jobId,
2498
+ fileName: path.basename(sourcePath),
2499
+ mimeType: parsed.mimeType || inferMimeTypeFromPath(sourcePath),
2500
+ payload
2501
+ });
2502
+ filePath = persisted.filePath;
2503
+ sizeBytes = persisted.sizeBytes;
2504
+ checksum = persisted.checksum;
2505
+ sourceLocator = sourcePath;
2506
+ fileName = path.basename(sourcePath);
2507
+ mimeType = parsed.mimeType || inferMimeTypeFromPath(sourcePath);
2508
+ }
2509
+ else {
2510
+ sourceLocator = parsed.sourceUrl?.trim() || "";
2511
+ fileName =
2512
+ sourceLocator.split("/").pop()?.split("?")[0]?.trim() ||
2513
+ "remote-source.bin";
2514
+ mimeType = parsed.mimeType || "application/octet-stream";
2515
+ }
2516
+ getDatabase()
2517
+ .prepare(`INSERT INTO wiki_ingest_jobs (
2518
+ id, space_id, llm_profile_id, status, phase, progress_percent, total_files, processed_files,
2519
+ created_page_count, created_entity_count, accepted_count, rejected_count, latest_message,
2520
+ source_kind, source_locator, mime_type, title_hint, summary, page_note_id, created_by_actor,
2521
+ error_message, input_json, created_at, updated_at, completed_at
2522
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
2523
+ .run(jobId, spaceId, parsed.llmProfileId ?? null, "queued", "queued", 0, 1, 0, 0, 0, 0, 0, "Waiting to ingest source", parsed.sourceKind, sourceLocator, mimeType, parsed.titleHint, "", null, options.actor ?? parsed.userId ?? null, "", JSON.stringify(parsed), now, now, null);
2524
+ createWikiIngestAssetRecord({
2525
+ jobId,
2526
+ sourceKind: parsed.sourceKind,
2527
+ sourceLocator,
2528
+ fileName,
2529
+ mimeType,
2530
+ filePath,
2531
+ sizeBytes,
2532
+ checksum
2533
+ });
2534
+ createWikiIngestLog(jobId, "Queued source for Forge wiki ingestion.");
2535
+ return {
2536
+ job: getWikiIngestJob(jobId),
2537
+ page: null
2538
+ };
2539
+ }
2540
+ function createPageCandidatePayload(input) {
2541
+ return {
2542
+ title: input.title,
2543
+ summary: input.summary,
2544
+ contentMarkdown: input.markdown,
2545
+ kind: input.createAsKind,
2546
+ aliases: normalizeTags(input.aliases),
2547
+ parentSlug: typeof input.parentSlug === "string" && input.parentSlug.trim().length > 0
2548
+ ? input.parentSlug.trim()
2549
+ : null,
2550
+ sourceLocator: input.sourceLocator,
2551
+ links: input.linkedEntityHints,
2552
+ tags: normalizeTags(input.tags)
2553
+ };
2554
+ }
2555
+ export async function processWikiIngestJob(jobId, options) {
2556
+ const job = readWikiIngestJobRow(jobId);
2557
+ if (!job) {
2558
+ return null;
2559
+ }
2560
+ const parsed = readWikiIngestInput(jobId);
2561
+ if (!parsed) {
2562
+ return null;
2563
+ }
2564
+ const llmProfile = parsed.llmProfileId
2565
+ ? (listWikiLlmProfiles().find((entry) => entry.id === parsed.llmProfileId) ?? null)
2566
+ : null;
2567
+ const space = getWikiSpaceById(job.space_id) ?? ensureSharedWikiSpace();
2568
+ if (parsed.llmProfileId && !llmProfile) {
2569
+ updateWikiIngestJob(jobId, {
2570
+ status: "failed",
2571
+ phase: "failed",
2572
+ latestMessage: "The selected LLM profile could not be found.",
2573
+ errorMessage: "The selected LLM profile could not be found.",
2574
+ completedAt: nowIso()
2575
+ });
2576
+ createWikiIngestLog(jobId, "The selected LLM profile could not be found.", "error");
2577
+ return getWikiIngestJob(jobId);
2578
+ }
2579
+ updateWikiIngestJob(jobId, {
2580
+ status: "processing",
2581
+ phase: "processing",
2582
+ latestMessage: "Processing queued wiki ingest sources.",
2583
+ errorMessage: "",
2584
+ completedAt: null
2585
+ });
2586
+ createWikiIngestLog(jobId, "Started background wiki ingestion.");
2587
+ let currentAssetContext = null;
2588
+ let currentPollCount = 0;
2589
+ const updateCurrentAssetMetadata = (patch) => {
2590
+ if (!currentAssetContext) {
2591
+ return;
2592
+ }
2593
+ const asset = listWikiIngestJobAssetsInternal(jobId).find((entry) => entry.id === currentAssetContext?.assetId);
2594
+ if (!asset) {
2595
+ return;
2596
+ }
2597
+ updateWikiIngestAsset(asset.id, {
2598
+ metadata: {
2599
+ ...parseJsonRecord(asset.metadata_json),
2600
+ ...patch
2601
+ }
2602
+ });
2603
+ };
2604
+ const llmDiagnosticLogger = ({ level, message, details = {} }) => {
2605
+ const enrichedDetails = {
2606
+ ...details,
2607
+ sourceAssetId: currentAssetContext?.assetId ?? null,
2608
+ currentFileName: currentAssetContext?.fileName ?? null,
2609
+ sourceLocator: currentAssetContext?.sourceLocator ?? null,
2610
+ checksum: currentAssetContext?.checksum ?? null,
2611
+ currentFileIndex: currentAssetContext?.fileIndex ?? null,
2612
+ currentFileTotal: currentAssetContext?.totalFiles ?? null,
2613
+ chunkIndex: typeof details.chunkIndex === "number"
2614
+ ? details.chunkIndex
2615
+ : currentAssetContext?.chunkIndex ?? null,
2616
+ chunkCount: typeof details.chunkCount === "number"
2617
+ ? details.chunkCount
2618
+ : currentAssetContext?.chunkCount ?? null
2619
+ };
2620
+ const eventKey = typeof details.eventKey === "string" ? details.eventKey : "";
2621
+ if (eventKey === "llm_compile_background_started") {
2622
+ currentPollCount = 0;
2623
+ updateCurrentAssetMetadata({
2624
+ openAiResponseId: typeof details.responseId === "string" ? details.responseId : null,
2625
+ openAiResponseStatus: typeof details.status === "string" ? details.status : "queued",
2626
+ openAiLastPolledAt: nowIso()
2627
+ });
2628
+ }
2629
+ if (eventKey === "llm_compile_background_polled") {
2630
+ currentPollCount += 1;
2631
+ const pollStatus = typeof details.status === "string" ? details.status : "in_progress";
2632
+ const chunkSuffix = (currentAssetContext?.chunkCount ?? 1) > 1
2633
+ ? ` · chunk ${currentAssetContext?.chunkIndex ?? 1}/${currentAssetContext?.chunkCount ?? 1}`
2634
+ : "";
2635
+ const fileLabel = currentAssetContext?.fileName ?? "current source";
2636
+ const progressMessage = `Waiting for OpenAI on ${fileLabel}${chunkSuffix}. Poll ${currentPollCount} · ${pollStatus}.`;
2637
+ updateWikiIngestJob(jobId, {
2638
+ latestMessage: progressMessage
2639
+ });
2640
+ updateCurrentAssetMetadata({
2641
+ openAiResponseId: typeof details.responseId === "string" ? details.responseId : null,
2642
+ openAiResponseStatus: pollStatus,
2643
+ openAiLastPolledAt: nowIso(),
2644
+ openAiPollCount: currentPollCount
2645
+ });
2646
+ createWikiIngestLog(jobId, progressMessage, "info", {
2647
+ ...enrichedDetails,
2648
+ pollCount: currentPollCount,
2649
+ status: pollStatus
2650
+ }, {
2651
+ aggregateKey: `llm_compile_background_polled:${currentAssetContext?.assetId ?? "job"}`,
2652
+ recordDiagnostic: false
2653
+ });
2654
+ return;
2655
+ }
2656
+ if (eventKey === "llm_compile_success" ||
2657
+ eventKey === "llm_compile_unparseable" ||
2658
+ eventKey === "llm_compile_background_terminal_error") {
2659
+ updateCurrentAssetMetadata({
2660
+ openAiResponseId: typeof details.responseId === "string" ? details.responseId : null,
2661
+ openAiResponseStatus: eventKey === "llm_compile_background_terminal_error"
2662
+ ? typeof details.status === "string"
2663
+ ? details.status
2664
+ : "failed"
2665
+ : "completed",
2666
+ openAiLastPolledAt: nowIso(),
2667
+ openAiPollCount: currentPollCount
2668
+ });
2669
+ }
2670
+ createWikiIngestLog(jobId, message, level === "debug" ? "info" : level, {
2671
+ scope: "wiki_llm",
2672
+ eventKey: "wiki_llm_event",
2673
+ ...enrichedDetails
2674
+ });
2675
+ };
2676
+ const refreshCounts = () => {
2677
+ const candidates = listWikiIngestCandidatesInternal(jobId);
2678
+ const pageCount = candidates.filter((candidate) => candidate.candidate_type === "page").length;
2679
+ const entityCount = candidates.filter((candidate) => candidate.candidate_type === "entity").length;
2680
+ return { pageCount, entityCount };
2681
+ };
2682
+ const assetQueue = () => listWikiIngestJobAssetsInternal(jobId).filter((asset) => ["queued", "processing"].includes(asset.status));
2683
+ const initialAssets = listWikiIngestJobAssetsInternal(jobId);
2684
+ let processedFiles = initialAssets.filter((asset) => asset.status === "completed").length;
2685
+ let totalFiles = Math.max(job.total_files, initialAssets.length);
2686
+ let hadSuccess = false;
2687
+ while (assetQueue().length > 0) {
2688
+ const nextAsset = assetQueue().find((asset) => ["processing", "queued"].includes(asset.status));
2689
+ if (!nextAsset) {
2690
+ break;
2691
+ }
2692
+ if (nextAsset.file_name.toLowerCase().endsWith(".zip") ||
2693
+ nextAsset.mime_type === "application/zip") {
2694
+ updateWikiIngestAsset(nextAsset.id, { status: "processing" });
2695
+ try {
2696
+ const archive = new AdmZip(nextAsset.file_path);
2697
+ const entries = archive
2698
+ .getEntries()
2699
+ .filter((entry) => !entry.isDirectory);
2700
+ for (const entry of entries) {
2701
+ const payload = entry.getData();
2702
+ const fileName = path.basename(entry.entryName);
2703
+ const mimeType = inferMimeTypeFromPath(fileName);
2704
+ const persisted = await persistIngestUpload({
2705
+ jobId,
2706
+ fileName,
2707
+ mimeType,
2708
+ payload
2709
+ });
2710
+ createWikiIngestAssetRecord({
2711
+ jobId,
2712
+ sourceKind: "upload",
2713
+ sourceLocator: entry.entryName,
2714
+ fileName,
2715
+ mimeType,
2716
+ filePath: persisted.filePath,
2717
+ sizeBytes: persisted.sizeBytes,
2718
+ checksum: persisted.checksum,
2719
+ metadata: {
2720
+ parentAssetId: nextAsset.id
2721
+ }
2722
+ });
2723
+ }
2724
+ totalFiles = Math.max(0, totalFiles - 1 + entries.length);
2725
+ updateWikiIngestAsset(nextAsset.id, {
2726
+ status: "completed",
2727
+ metadata: {
2728
+ ...(parseJsonRecord(nextAsset.metadata_json) ?? {}),
2729
+ extractedCount: entries.length
2730
+ }
2731
+ });
2732
+ updateWikiIngestJob(jobId, {
2733
+ totalFiles,
2734
+ latestMessage: entries.length === 1
2735
+ ? "Expanded 1 file from ZIP archive."
2736
+ : `Expanded ${entries.length} files from ZIP archive.`
2737
+ });
2738
+ createWikiIngestLog(jobId, entries.length === 1
2739
+ ? "Expanded 1 file from ZIP archive."
2740
+ : `Expanded ${entries.length} files from ZIP archive.`);
2741
+ }
2742
+ catch (error) {
2743
+ updateWikiIngestAsset(nextAsset.id, { status: "failed" });
2744
+ createWikiIngestLog(jobId, error instanceof Error ? error.message : "ZIP extraction failed.", "error");
2745
+ }
2746
+ continue;
2747
+ }
2748
+ updateWikiIngestAsset(nextAsset.id, { status: "processing" });
2749
+ currentAssetContext = {
2750
+ assetId: nextAsset.id,
2751
+ fileName: nextAsset.file_name || nextAsset.source_locator || "Source",
2752
+ sourceLocator: nextAsset.source_locator,
2753
+ checksum: nextAsset.checksum,
2754
+ fileIndex: Math.min(totalFiles, processedFiles + 1),
2755
+ totalFiles,
2756
+ chunkIndex: 1,
2757
+ chunkCount: 1
2758
+ };
2759
+ currentPollCount = 0;
2760
+ createWikiIngestLog(jobId, `Processing ${currentAssetContext.fileName} (${currentAssetContext.fileIndex}/${currentAssetContext.totalFiles}).`, "info", {
2761
+ sourceAssetId: currentAssetContext.assetId,
2762
+ fileName: currentAssetContext.fileName,
2763
+ sourceLocator: currentAssetContext.sourceLocator,
2764
+ checksum: currentAssetContext.checksum,
2765
+ fileIndex: currentAssetContext.fileIndex,
2766
+ totalFiles: currentAssetContext.totalFiles,
2767
+ chunkIndex: currentAssetContext.chunkIndex,
2768
+ chunkCount: currentAssetContext.chunkCount
2769
+ });
2770
+ try {
2771
+ const fetched = nextAsset.source_kind === "url"
2772
+ ? await getFetchedContent("url", {
2773
+ sourceText: "",
2774
+ sourceUrl: nextAsset.source_locator,
2775
+ mimeType: nextAsset.mime_type
2776
+ })
2777
+ : await getFetchedContent("local_path", {
2778
+ sourceText: "",
2779
+ sourcePath: nextAsset.file_path,
2780
+ mimeType: nextAsset.mime_type
2781
+ });
2782
+ const rawSource = await persistWikiRawSource({
2783
+ space,
2784
+ jobId,
2785
+ fetched
2786
+ });
2787
+ const existingAssetMetadata = parseJsonRecord(nextAsset.metadata_json);
2788
+ const resumeResponseId = readStringRecordValue(existingAssetMetadata, "openAiResponseId") ??
2789
+ findOpenAiResponseIdForJobAsset({
2790
+ jobId,
2791
+ assetId: nextAsset.id,
2792
+ fileName: nextAsset.file_name,
2793
+ sourceLocator: nextAsset.source_locator,
2794
+ checksum: nextAsset.checksum
2795
+ });
2796
+ const compiled = llmProfile && parsed.parseStrategy !== "text_only"
2797
+ ? await options.llm.compileWikiIngest(llmProfile, {
2798
+ titleHint: parsed.titleHint,
2799
+ rawText: fetched.contentText,
2800
+ binary: fetched.binary,
2801
+ mimeType: fetched.mimeType,
2802
+ parseStrategy: parsed.parseStrategy
2803
+ }, {
2804
+ resumeResponseId
2805
+ }, llmDiagnosticLogger)
2806
+ : llmProfile && fetched.contentText
2807
+ ? await options.llm.compileWikiIngest(llmProfile, {
2808
+ titleHint: parsed.titleHint,
2809
+ rawText: fetched.contentText,
2810
+ binary: fetched.binary,
2811
+ mimeType: fetched.mimeType,
2812
+ parseStrategy: "text_only"
2813
+ }, {
2814
+ resumeResponseId
2815
+ }, llmDiagnosticLogger)
2816
+ : null;
2817
+ if (llmProfile && !compiled) {
2818
+ throw new Error("The LLM did not produce structured draft candidates. Check the OpenAI settings and try again.");
2819
+ }
2820
+ const title = compiled?.title ||
2821
+ parsed.titleHint ||
2822
+ inferTitle(fetched.contentText || fetched.fileName, "Imported source");
2823
+ const summary = compiled?.summary ||
2824
+ (fetched.contentText ? inferSummary(fetched.contentText) : "");
2825
+ const markdown = compiled?.markdown
2826
+ ? compiled.markdown
2827
+ : fetched.contentText
2828
+ ? `# ${title}\n\n${fetched.contentText.trim()}\n`
2829
+ : `# ${title}\n\nImported media asset \`${fetched.fileName}\` (${fetched.mimeType}).\n`;
2830
+ createWikiIngestCandidate({
2831
+ jobId,
2832
+ sourceAssetId: nextAsset.id,
2833
+ candidateType: "page",
2834
+ title,
2835
+ summary,
2836
+ targetKey: "",
2837
+ payload: createPageCandidatePayload({
2838
+ title,
2839
+ summary,
2840
+ markdown,
2841
+ sourceLocator: fetched.locator,
2842
+ createAsKind: parsed.createAsKind,
2843
+ tags: compiled?.tags,
2844
+ linkedEntityHints: parsed.linkedEntityHints
2845
+ })
2846
+ });
2847
+ if (parsed.entityProposalMode === "suggest") {
2848
+ for (const proposal of compiled?.entityProposals ?? []) {
2849
+ createWikiIngestCandidate({
2850
+ jobId,
2851
+ sourceAssetId: nextAsset.id,
2852
+ candidateType: "entity",
2853
+ title: typeof proposal.title === "string"
2854
+ ? proposal.title
2855
+ : "Entity proposal",
2856
+ summary: typeof proposal.summary === "string" ? proposal.summary : "",
2857
+ targetKey: typeof proposal.entityType === "string"
2858
+ ? proposal.entityType
2859
+ : "",
2860
+ payload: proposal
2861
+ });
2862
+ }
2863
+ }
2864
+ for (const suggestion of compiled?.pageUpdateSuggestions ?? []) {
2865
+ const patchSummary = typeof suggestion.patchSummary === "string"
2866
+ ? suggestion.patchSummary
2867
+ : "";
2868
+ createWikiIngestCandidate({
2869
+ jobId,
2870
+ sourceAssetId: nextAsset.id,
2871
+ candidateType: "page_update",
2872
+ title: typeof suggestion.targetSlug === "string"
2873
+ ? suggestion.targetSlug
2874
+ : "Page update",
2875
+ summary: typeof suggestion.rationale === "string"
2876
+ ? suggestion.rationale
2877
+ : patchSummary,
2878
+ targetKey: typeof suggestion.targetSlug === "string"
2879
+ ? suggestion.targetSlug
2880
+ : "",
2881
+ payload: {
2882
+ ...suggestion,
2883
+ patchSummary
2884
+ }
2885
+ });
2886
+ }
2887
+ for (const candidate of compiled?.articleCandidates ?? []) {
2888
+ const articleTitle = typeof candidate.title === "string" &&
2889
+ candidate.title.trim().length > 0
2890
+ ? candidate.title.trim()
2891
+ : "Suggested article";
2892
+ const articleSummary = typeof candidate.summary === "string" ? candidate.summary : "";
2893
+ const articleRationale = typeof candidate.rationale === "string" ? candidate.rationale : "";
2894
+ createWikiIngestCandidate({
2895
+ jobId,
2896
+ sourceAssetId: nextAsset.id,
2897
+ candidateType: "page",
2898
+ title: articleTitle,
2899
+ summary: articleSummary,
2900
+ targetKey: typeof candidate.slug === "string" ? candidate.slug : "",
2901
+ payload: createPageCandidatePayload({
2902
+ title: articleTitle,
2903
+ summary: articleSummary,
2904
+ markdown: typeof candidate.markdown === "string" &&
2905
+ candidate.markdown.trim().length > 0
2906
+ ? candidate.markdown
2907
+ : `# ${articleTitle}\n\n${articleSummary}\n\n${articleRationale
2908
+ ? `## Why this page\n\n${articleRationale}\n`
2909
+ : ""}`,
2910
+ sourceLocator: fetched.locator,
2911
+ createAsKind: "wiki",
2912
+ aliases: Array.isArray(candidate.aliases)
2913
+ ? candidate.aliases.filter((alias) => typeof alias === "string")
2914
+ : [],
2915
+ parentSlug: typeof candidate.parentSlug === "string"
2916
+ ? candidate.parentSlug
2917
+ : null,
2918
+ tags: Array.isArray(candidate.tags)
2919
+ ? candidate.tags.filter((tag) => typeof tag === "string")
2920
+ : [],
2921
+ linkedEntityHints: parsed.linkedEntityHints
2922
+ })
2923
+ });
2924
+ }
2925
+ updateWikiIngestAsset(nextAsset.id, {
2926
+ status: "completed",
2927
+ filePath: rawSource.filePath,
2928
+ mimeType: fetched.mimeType,
2929
+ sizeBytes: rawSource.sizeBytes,
2930
+ checksum: rawSource.checksum,
2931
+ metadata: {
2932
+ ...(parseJsonRecord(nextAsset.metadata_json) ?? {}),
2933
+ rawSourcePath: rawSource.filePath,
2934
+ openAiResponseStatus: llmProfile ? "completed" : null,
2935
+ openAiLastPolledAt: llmProfile ? nowIso() : null
2936
+ }
2937
+ });
2938
+ hadSuccess = true;
2939
+ processedFiles += 1;
2940
+ const counts = refreshCounts();
2941
+ updateWikiIngestJob(jobId, {
2942
+ processedFiles,
2943
+ totalFiles,
2944
+ progressPercent: calculateProgress(totalFiles, processedFiles),
2945
+ createdPageCount: counts.pageCount,
2946
+ createdEntityCount: counts.entityCount,
2947
+ sourceLocator: fetched.locator,
2948
+ mimeType: fetched.mimeType,
2949
+ summary,
2950
+ latestMessage: `Prepared candidates from ${fetched.fileName}.`
2951
+ });
2952
+ createWikiIngestLog(jobId, `Prepared candidates from ${fetched.fileName}.`, "info", {
2953
+ fileName: currentAssetContext?.fileName ?? fetched.fileName,
2954
+ sourceAssetId: currentAssetContext?.assetId ?? null,
2955
+ fileIndex: currentAssetContext?.fileIndex ?? processedFiles,
2956
+ totalFiles: currentAssetContext?.totalFiles ?? totalFiles,
2957
+ pageCandidates: counts.pageCount,
2958
+ entityCandidates: counts.entityCount
2959
+ });
2960
+ currentAssetContext = null;
2961
+ currentPollCount = 0;
2962
+ }
2963
+ catch (error) {
2964
+ processedFiles += 1;
2965
+ updateWikiIngestAsset(nextAsset.id, { status: "failed" });
2966
+ updateCurrentAssetMetadata({
2967
+ openAiResponseStatus: "failed",
2968
+ openAiLastPolledAt: nowIso(),
2969
+ openAiPollCount: currentPollCount
2970
+ });
2971
+ updateWikiIngestJob(jobId, {
2972
+ processedFiles,
2973
+ totalFiles,
2974
+ progressPercent: calculateProgress(totalFiles, processedFiles),
2975
+ latestMessage: error instanceof Error ? error.message : "Source ingest failed."
2976
+ });
2977
+ createWikiIngestLog(jobId, error instanceof Error ? error.message : "Source ingest failed.", "error", {
2978
+ fileName: currentAssetContext?.fileName ?? null,
2979
+ fileIndex: currentAssetContext?.fileIndex ?? null,
2980
+ totalFiles: currentAssetContext?.totalFiles ?? totalFiles,
2981
+ pollCount: currentPollCount
2982
+ });
2983
+ currentAssetContext = null;
2984
+ currentPollCount = 0;
2985
+ }
2986
+ }
2987
+ const counts = refreshCounts();
2988
+ updateWikiIngestJob(jobId, {
2989
+ status: hadSuccess ? "completed" : "failed",
2990
+ phase: hadSuccess ? "review" : "failed",
2991
+ progressPercent: 100,
2992
+ totalFiles,
2993
+ processedFiles: Math.max(processedFiles, totalFiles),
2994
+ createdPageCount: counts.pageCount,
2995
+ createdEntityCount: counts.entityCount,
2996
+ latestMessage: hadSuccess
2997
+ ? "Ingest finished. Review the proposed pages and entities."
2998
+ : "Ingest failed before any candidates could be prepared.",
2999
+ errorMessage: hadSuccess ? "" : "No candidates were produced.",
3000
+ completedAt: nowIso()
3001
+ });
3002
+ createWikiIngestLog(jobId, hadSuccess
3003
+ ? "Background wiki ingestion completed and is ready for review."
3004
+ : "Background wiki ingestion failed.", hadSuccess ? "info" : "error");
3005
+ return getWikiIngestJob(jobId);
3006
+ }
3007
+ export function listWikiIngestJobs(input = {}) {
3008
+ const parsed = listWikiIngestJobsQuerySchema.parse(input);
3009
+ const rows = getDatabase()
3010
+ .prepare(`SELECT id
3011
+ FROM wiki_ingest_jobs
3012
+ WHERE (? IS NULL OR space_id = ?)
3013
+ ORDER BY created_at DESC
3014
+ LIMIT ?`)
3015
+ .all(parsed.spaceId ?? null, parsed.spaceId ?? null, parsed.limit);
3016
+ return rows
3017
+ .map((row) => getWikiIngestJob(row.id))
3018
+ .filter((job) => job !== null);
3019
+ }
3020
+ export async function rerunWikiIngestJob(jobId, options = {}) {
3021
+ const job = readWikiIngestJobRow(jobId);
3022
+ if (!job) {
3023
+ return null;
3024
+ }
3025
+ if (["queued", "processing"].includes(job.status)) {
3026
+ throw new Error("Wiki ingest jobs can only be rerun after processing ends.");
3027
+ }
3028
+ const ingestInput = readWikiIngestInput(jobId);
3029
+ if (!ingestInput) {
3030
+ throw new Error("Wiki ingest input could not be restored.");
3031
+ }
3032
+ const rootAssets = listWikiIngestJobAssetsInternal(jobId).filter((asset) => {
3033
+ const metadata = parseJsonRecord(asset.metadata_json);
3034
+ return !metadata?.parentAssetId && asset.source_kind !== "url";
3035
+ });
3036
+ const replayResult = rootAssets.length > 0
3037
+ ? await createUploadedWikiIngestJob({
3038
+ spaceId: ingestInput.spaceId,
3039
+ titleHint: ingestInput.titleHint,
3040
+ llmProfileId: ingestInput.llmProfileId,
3041
+ parseStrategy: ingestInput.parseStrategy,
3042
+ entityProposalMode: ingestInput.entityProposalMode,
3043
+ userId: ingestInput.userId ?? null,
3044
+ createAsKind: ingestInput.createAsKind,
3045
+ linkedEntityHints: ingestInput.linkedEntityHints
3046
+ }, await Promise.all(rootAssets.map(async (asset) => ({
3047
+ fileName: asset.file_name,
3048
+ mimeType: asset.mime_type,
3049
+ payload: await readFile(asset.file_path)
3050
+ }))), {
3051
+ actor: options.actor ?? job.created_by_actor ?? null
3052
+ })
3053
+ : await ingestWikiSource(ingestInput, {
3054
+ actor: options.actor ?? job.created_by_actor ?? null
3055
+ });
3056
+ const nextJobId = replayResult.job?.job.id ?? null;
3057
+ if (nextJobId) {
3058
+ createWikiIngestLog(nextJobId, `Reran ingest from ${jobId}.`, "info", {
3059
+ scope: "wiki_ingest",
3060
+ eventKey: "wiki_ingest_rerun",
3061
+ sourceJobId: jobId
3062
+ });
3063
+ createWikiIngestLog(jobId, `Created rerun job ${nextJobId}.`, "info", {
3064
+ scope: "wiki_ingest",
3065
+ eventKey: "wiki_ingest_rerun_requested",
3066
+ rerunJobId: nextJobId
3067
+ });
3068
+ }
3069
+ return replayResult;
3070
+ }
3071
+ export function deleteWikiIngestJob(jobId) {
3072
+ const job = readWikiIngestJobRow(jobId);
3073
+ if (!job) {
3074
+ return null;
3075
+ }
3076
+ if (["queued", "processing"].includes(job.status)) {
3077
+ throw new Error("Wiki ingest jobs can only be deleted after processing ends.");
3078
+ }
3079
+ const candidates = listWikiIngestCandidatesInternal(jobId);
3080
+ const assets = listWikiIngestJobAssetsInternal(jobId);
3081
+ const preservedAssetIds = new Set(candidates
3082
+ .filter((candidate) => candidate.status === "applied")
3083
+ .map((candidate) => candidate.source_asset_id)
3084
+ .filter((assetId) => typeof assetId === "string"));
3085
+ for (const asset of assets) {
3086
+ if (preservedAssetIds.has(asset.id)) {
3087
+ continue;
3088
+ }
3089
+ const metadata = parseJsonRecord(asset.metadata_json);
3090
+ const rawSourcePath = typeof metadata?.rawSourcePath === "string"
3091
+ ? metadata.rawSourcePath
3092
+ : null;
3093
+ if (rawSourcePath && existsSync(rawSourcePath)) {
3094
+ unlinkSync(rawSourcePath);
3095
+ }
3096
+ }
3097
+ const jobDir = getWikiIngestJobDir(jobId);
3098
+ if (existsSync(jobDir)) {
3099
+ rmSync(jobDir, { recursive: true, force: true });
3100
+ }
3101
+ getDatabase()
3102
+ .prepare(`DELETE FROM wiki_ingest_jobs WHERE id = ?`)
3103
+ .run(jobId);
3104
+ return {
3105
+ id: jobId
3106
+ };
3107
+ }
3108
+ export async function reviewWikiIngestJob(jobId, input, options) {
3109
+ const parsed = reviewWikiIngestJobSchema.parse(input);
3110
+ const job = readWikiIngestJobRow(jobId);
3111
+ if (!job) {
3112
+ return null;
3113
+ }
3114
+ const ingestInput = readWikiIngestInput(jobId);
3115
+ if (!ingestInput) {
3116
+ return null;
3117
+ }
3118
+ const candidates = listWikiIngestCandidatesInternal(jobId);
3119
+ let acceptedCount = 0;
3120
+ let mappedCount = 0;
3121
+ let rejectedCount = 0;
3122
+ let firstPublishedPageId = null;
3123
+ for (const decision of parsed.decisions) {
3124
+ const candidate = candidates.find((entry) => entry.id === decision.candidateId);
3125
+ if (!candidate) {
3126
+ continue;
3127
+ }
3128
+ const action = decision.action ??
3129
+ (decision.keep === false ? "discard" : "keep");
3130
+ if (action === "discard") {
3131
+ rejectedCount += 1;
3132
+ updateWikiIngestCandidate(candidate.id, { status: "rejected" });
3133
+ continue;
3134
+ }
3135
+ try {
3136
+ const payload = parseJsonRecord(candidate.payload_json);
3137
+ if (candidate.candidate_type === "page") {
3138
+ if (action === "merge_existing") {
3139
+ const target = decision.targetNoteId
3140
+ ? (getWikiPageDetail(decision.targetNoteId)?.page ?? null)
3141
+ : null;
3142
+ if (!target || target.spaceId !== job.space_id) {
3143
+ throw new Error("Merge target page was not found.");
3144
+ }
3145
+ const incomingTitle = typeof payload.title === "string" ? payload.title : candidate.title;
3146
+ const incomingMarkdown = typeof payload.contentMarkdown === "string"
3147
+ ? payload.contentMarkdown
3148
+ : `# ${candidate.title}\n`;
3149
+ const mergedContentMarkdown = mergeWikiPageContent(target.contentMarkdown, {
3150
+ title: incomingTitle,
3151
+ markdown: incomingMarkdown
3152
+ });
3153
+ const mergedSummary = inferSummary(mergedContentMarkdown);
3154
+ const updated = options.updateNote(target.id, {
3155
+ contentMarkdown: mergedContentMarkdown,
3156
+ summary: mergedSummary
3157
+ });
3158
+ if (!updated) {
3159
+ throw new Error("Merge target page could not be updated.");
3160
+ }
3161
+ acceptedCount += 1;
3162
+ firstPublishedPageId = firstPublishedPageId ?? updated.id;
3163
+ updateWikiIngestCandidate(candidate.id, {
3164
+ status: "applied",
3165
+ publishedNoteId: updated.id
3166
+ });
3167
+ }
3168
+ else {
3169
+ const note = options.createNote({
3170
+ kind: payload.kind === "evidence" ? "evidence" : "wiki",
3171
+ title: typeof payload.title === "string"
3172
+ ? payload.title
3173
+ : candidate.title,
3174
+ slug: "",
3175
+ indexOrder: 0,
3176
+ aliases: Array.isArray(payload.aliases)
3177
+ ? payload.aliases.filter((alias) => typeof alias === "string")
3178
+ : [],
3179
+ summary: typeof payload.summary === "string"
3180
+ ? payload.summary
3181
+ : candidate.summary,
3182
+ sourcePath: "",
3183
+ frontmatter: {},
3184
+ revisionHash: "",
3185
+ spaceId: job.space_id,
3186
+ parentSlug: typeof payload.parentSlug === "string"
3187
+ ? payload.parentSlug
3188
+ : null,
3189
+ contentMarkdown: typeof payload.contentMarkdown === "string"
3190
+ ? payload.contentMarkdown
3191
+ : `# ${candidate.title}\n`,
3192
+ author: ingestInput.userId ?? null,
3193
+ links: Array.isArray(payload.links)
3194
+ ? payload.links
3195
+ : ingestInput.linkedEntityHints,
3196
+ tags: Array.isArray(payload.tags)
3197
+ ? payload.tags.filter((tag) => typeof tag === "string")
3198
+ : [],
3199
+ destroyAt: null,
3200
+ userId: ingestInput.userId ?? null
3201
+ });
3202
+ acceptedCount += 1;
3203
+ firstPublishedPageId = firstPublishedPageId ?? note.id;
3204
+ updateWikiIngestCandidate(candidate.id, {
3205
+ status: "applied",
3206
+ publishedNoteId: note.id
3207
+ });
3208
+ }
3209
+ }
3210
+ else if (candidate.candidate_type === "page_update") {
3211
+ const targetSlug = typeof payload.targetSlug === "string"
3212
+ ? payload.targetSlug
3213
+ : candidate.target_key;
3214
+ const target = targetSlug.trim().length > 0
3215
+ ? (getWikiPageDetailBySlug({
3216
+ slug: targetSlug,
3217
+ spaceId: job.space_id
3218
+ })?.page ?? null)
3219
+ : null;
3220
+ if (target) {
3221
+ const patchSummary = typeof payload.patchSummary === "string"
3222
+ ? payload.patchSummary
3223
+ : "";
3224
+ const rationale = typeof payload.rationale === "string" ? payload.rationale : "";
3225
+ options.updateNote(target.id, {
3226
+ contentMarkdown: `${target.contentMarkdown.trim()}\n\n## Imported update\n\n${patchSummary}\n${rationale ? `\n${rationale}\n` : ""}`,
3227
+ summary: target.summary || (typeof rationale === "string" ? rationale : "")
3228
+ });
3229
+ acceptedCount += 1;
3230
+ updateWikiIngestCandidate(candidate.id, {
3231
+ status: "applied",
3232
+ publishedNoteId: target.id
3233
+ });
3234
+ }
3235
+ else {
3236
+ updateWikiIngestCandidate(candidate.id, { status: "failed" });
3237
+ }
3238
+ }
3239
+ else if (candidate.candidate_type === "entity") {
3240
+ if (action === "map_existing") {
3241
+ const proposedType = typeof payload.entityType === "string" ? payload.entityType : "";
3242
+ if (decision.mappedEntityType !== proposedType) {
3243
+ throw new Error("Mapped entity type must match the proposed entity type.");
3244
+ }
3245
+ const mapped = options.resolveMappedEntity(decision.mappedEntityType, decision.mappedEntityId);
3246
+ if (!mapped) {
3247
+ throw new Error("Mapped entity was not found.");
3248
+ }
3249
+ acceptedCount += 1;
3250
+ mappedCount += 1;
3251
+ updateWikiIngestCandidate(candidate.id, {
3252
+ status: "applied",
3253
+ publishedEntityType: mapped.entityType,
3254
+ publishedEntityId: mapped.entityId
3255
+ });
3256
+ }
3257
+ else {
3258
+ const result = options.publishEntity(payload);
3259
+ acceptedCount += 1;
3260
+ updateWikiIngestCandidate(candidate.id, {
3261
+ status: "applied",
3262
+ publishedEntityType: result.entityType,
3263
+ publishedEntityId: result.entityId
3264
+ });
3265
+ }
3266
+ }
3267
+ }
3268
+ catch {
3269
+ updateWikiIngestCandidate(candidate.id, { status: "failed" });
3270
+ }
3271
+ }
3272
+ updateWikiIngestJob(jobId, {
3273
+ phase: "reviewed",
3274
+ acceptedCount,
3275
+ rejectedCount,
3276
+ pageNoteId: firstPublishedPageId,
3277
+ latestMessage: acceptedCount > 0 || rejectedCount > 0
3278
+ ? `Review saved with ${acceptedCount} kept${mappedCount > 0 ? ` (${mappedCount} mapped)` : ""} and ${rejectedCount} discarded.`
3279
+ : "Review saved."
3280
+ });
3281
+ createWikiIngestLog(jobId, `Review saved with ${acceptedCount} kept${mappedCount > 0 ? ` (${mappedCount} mapped)` : ""} and ${rejectedCount} discarded.`);
3282
+ return getWikiIngestJob(jobId);
3283
+ }
3284
+ export function getWikiIngestJob(jobId) {
3285
+ const job = readWikiIngestJobRow(jobId);
3286
+ if (!job) {
3287
+ return null;
3288
+ }
3289
+ const items = getDatabase()
3290
+ .prepare(`SELECT id, job_id, item_type, status, note_id, media_asset_id, payload_json, created_at, updated_at
3291
+ FROM wiki_ingest_job_items
3292
+ WHERE job_id = ?
3293
+ ORDER BY created_at ASC`)
3294
+ .all(jobId);
3295
+ const assets = listWikiIngestJobAssetsInternal(jobId);
3296
+ const normalizedItems = items.length > 0
3297
+ ? items.map((item) => ({
3298
+ id: item.id,
3299
+ itemType: item.item_type,
3300
+ status: item.status,
3301
+ noteId: item.note_id,
3302
+ mediaAssetId: item.media_asset_id,
3303
+ payload: parseJsonRecord(item.payload_json),
3304
+ createdAt: item.created_at,
3305
+ updatedAt: item.updated_at
3306
+ }))
3307
+ : assets.map((asset) => ({
3308
+ id: asset.id,
3309
+ itemType: "raw_source",
3310
+ status: asset.status,
3311
+ noteId: null,
3312
+ mediaAssetId: null,
3313
+ payload: {
3314
+ sourceKind: asset.source_kind,
3315
+ sourceLocator: asset.source_locator,
3316
+ fileName: asset.file_name,
3317
+ mimeType: asset.mime_type,
3318
+ filePath: asset.file_path,
3319
+ sizeBytes: asset.size_bytes,
3320
+ checksum: asset.checksum,
3321
+ ...(parseJsonRecord(asset.metadata_json) ?? {})
3322
+ },
3323
+ createdAt: asset.created_at,
3324
+ updatedAt: asset.updated_at
3325
+ }));
3326
+ return wikiIngestJobPayloadSchema.parse({
3327
+ job: mapWikiIngestJobRow(job),
3328
+ items: normalizedItems,
3329
+ logs: listWikiIngestJobLogsInternal(jobId).map((log) => ({
3330
+ id: log.id,
3331
+ level: log.level,
3332
+ message: log.message,
3333
+ metadata: parseJsonRecord(log.metadata_json),
3334
+ createdAt: log.created_at
3335
+ })),
3336
+ assets: assets.map((asset) => ({
3337
+ id: asset.id,
3338
+ status: asset.status,
3339
+ sourceKind: asset.source_kind,
3340
+ sourceLocator: asset.source_locator,
3341
+ fileName: asset.file_name,
3342
+ mimeType: asset.mime_type,
3343
+ filePath: asset.file_path,
3344
+ sizeBytes: asset.size_bytes,
3345
+ checksum: asset.checksum,
3346
+ metadata: parseJsonRecord(asset.metadata_json),
3347
+ createdAt: asset.created_at,
3348
+ updatedAt: asset.updated_at
3349
+ })),
3350
+ candidates: listWikiIngestCandidatesInternal(jobId).map((candidate) => ({
3351
+ id: candidate.id,
3352
+ sourceAssetId: candidate.source_asset_id,
3353
+ candidateType: candidate.candidate_type,
3354
+ status: candidate.status,
3355
+ title: candidate.title,
3356
+ summary: candidate.summary,
3357
+ targetKey: candidate.target_key,
3358
+ payload: parseJsonRecord(candidate.payload_json),
3359
+ publishedNoteId: candidate.published_note_id,
3360
+ publishedEntityType: candidate.published_entity_type,
3361
+ publishedEntityId: candidate.published_entity_id,
3362
+ createdAt: candidate.created_at,
3363
+ updatedAt: candidate.updated_at
3364
+ }))
3365
+ });
3366
+ }