@swarmvaultai/engine 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -6
- package/dist/index.d.ts +104 -21
- package/dist/index.js +2003 -973
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,9 +1,19 @@
|
|
|
1
|
+
// src/agents.ts
|
|
2
|
+
import fs3 from "fs/promises";
|
|
3
|
+
import path3 from "path";
|
|
4
|
+
|
|
1
5
|
// src/config.ts
|
|
2
|
-
import path2 from "path";
|
|
3
6
|
import fs2 from "fs/promises";
|
|
7
|
+
import path2 from "path";
|
|
4
8
|
import { fileURLToPath } from "url";
|
|
5
9
|
import { z as z2 } from "zod";
|
|
6
10
|
|
|
11
|
+
// src/types.ts
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
var providerCapabilitySchema = z.enum(["responses", "chat", "structured", "tools", "vision", "embeddings", "streaming", "local"]);
|
|
14
|
+
var providerTypeSchema = z.enum(["heuristic", "openai", "ollama", "anthropic", "gemini", "openai-compatible", "custom"]);
|
|
15
|
+
var webSearchProviderTypeSchema = z.enum(["http-json", "custom"]);
|
|
16
|
+
|
|
7
17
|
// src/utils.ts
|
|
8
18
|
import crypto from "crypto";
|
|
9
19
|
import fs from "fs/promises";
|
|
@@ -110,28 +120,6 @@ async function listFilesRecursive(rootDir) {
|
|
|
110
120
|
return files;
|
|
111
121
|
}
|
|
112
122
|
|
|
113
|
-
// src/types.ts
|
|
114
|
-
import { z } from "zod";
|
|
115
|
-
var providerCapabilitySchema = z.enum([
|
|
116
|
-
"responses",
|
|
117
|
-
"chat",
|
|
118
|
-
"structured",
|
|
119
|
-
"tools",
|
|
120
|
-
"vision",
|
|
121
|
-
"embeddings",
|
|
122
|
-
"streaming",
|
|
123
|
-
"local"
|
|
124
|
-
]);
|
|
125
|
-
var providerTypeSchema = z.enum([
|
|
126
|
-
"heuristic",
|
|
127
|
-
"openai",
|
|
128
|
-
"ollama",
|
|
129
|
-
"anthropic",
|
|
130
|
-
"gemini",
|
|
131
|
-
"openai-compatible",
|
|
132
|
-
"custom"
|
|
133
|
-
]);
|
|
134
|
-
|
|
135
123
|
// src/config.ts
|
|
136
124
|
var PRIMARY_CONFIG_FILENAME = "swarmvault.config.json";
|
|
137
125
|
var LEGACY_CONFIG_FILENAME = "vault.config.json";
|
|
@@ -148,6 +136,22 @@ var providerConfigSchema = z2.object({
|
|
|
148
136
|
capabilities: z2.array(providerCapabilitySchema).optional(),
|
|
149
137
|
apiStyle: z2.enum(["responses", "chat"]).optional()
|
|
150
138
|
});
|
|
139
|
+
var webSearchProviderConfigSchema = z2.object({
|
|
140
|
+
type: webSearchProviderTypeSchema,
|
|
141
|
+
endpoint: z2.string().url().optional(),
|
|
142
|
+
method: z2.enum(["GET", "POST"]).optional(),
|
|
143
|
+
apiKeyEnv: z2.string().min(1).optional(),
|
|
144
|
+
apiKeyHeader: z2.string().min(1).optional(),
|
|
145
|
+
apiKeyPrefix: z2.string().optional(),
|
|
146
|
+
headers: z2.record(z2.string(), z2.string()).optional(),
|
|
147
|
+
queryParam: z2.string().min(1).optional(),
|
|
148
|
+
limitParam: z2.string().min(1).optional(),
|
|
149
|
+
resultsPath: z2.string().min(1).optional(),
|
|
150
|
+
titleField: z2.string().min(1).optional(),
|
|
151
|
+
urlField: z2.string().min(1).optional(),
|
|
152
|
+
snippetField: z2.string().min(1).optional(),
|
|
153
|
+
module: z2.string().min(1).optional()
|
|
154
|
+
});
|
|
151
155
|
var vaultConfigSchema = z2.object({
|
|
152
156
|
workspace: z2.object({
|
|
153
157
|
rawDir: z2.string().min(1),
|
|
@@ -166,7 +170,13 @@ var vaultConfigSchema = z2.object({
|
|
|
166
170
|
viewer: z2.object({
|
|
167
171
|
port: z2.number().int().positive()
|
|
168
172
|
}),
|
|
169
|
-
agents: z2.array(z2.enum(["codex", "claude", "cursor"])).default(["codex", "claude", "cursor"])
|
|
173
|
+
agents: z2.array(z2.enum(["codex", "claude", "cursor"])).default(["codex", "claude", "cursor"]),
|
|
174
|
+
webSearch: z2.object({
|
|
175
|
+
providers: z2.record(z2.string(), webSearchProviderConfigSchema),
|
|
176
|
+
tasks: z2.object({
|
|
177
|
+
deepLintProvider: z2.string().min(1)
|
|
178
|
+
})
|
|
179
|
+
}).optional()
|
|
170
180
|
});
|
|
171
181
|
function defaultVaultConfig() {
|
|
172
182
|
return {
|
|
@@ -333,25 +343,101 @@ async function initWorkspace(rootDir) {
|
|
|
333
343
|
return { config, paths };
|
|
334
344
|
}
|
|
335
345
|
|
|
346
|
+
// src/agents.ts
|
|
347
|
+
var managedStart = "<!-- swarmvault:managed:start -->";
|
|
348
|
+
var managedEnd = "<!-- swarmvault:managed:end -->";
|
|
349
|
+
var legacyManagedStart = "<!-- vault:managed:start -->";
|
|
350
|
+
var legacyManagedEnd = "<!-- vault:managed:end -->";
|
|
351
|
+
function buildManagedBlock(agent) {
|
|
352
|
+
const body = [
|
|
353
|
+
managedStart,
|
|
354
|
+
`# SwarmVault Rules (${agent})`,
|
|
355
|
+
"",
|
|
356
|
+
"- Read `swarmvault.schema.md` before compile or query style work. If only `schema.md` exists, treat it as the legacy schema path.",
|
|
357
|
+
"- Treat `raw/` as immutable source input.",
|
|
358
|
+
"- Treat `wiki/` as generated markdown owned by the agent and compiler workflow.",
|
|
359
|
+
"- Read `wiki/index.md` before broad file searching when answering SwarmVault questions.",
|
|
360
|
+
"- Preserve frontmatter fields including `page_id`, `source_ids`, `node_ids`, `freshness`, and `source_hashes`.",
|
|
361
|
+
"- Save high-value answers back into `wiki/outputs/` instead of leaving them only in chat.",
|
|
362
|
+
"- Prefer `swarmvault ingest`, `swarmvault compile`, `swarmvault query`, and `swarmvault lint` for SwarmVault maintenance tasks.",
|
|
363
|
+
managedEnd,
|
|
364
|
+
""
|
|
365
|
+
].join("\n");
|
|
366
|
+
if (agent === "cursor") {
|
|
367
|
+
return body;
|
|
368
|
+
}
|
|
369
|
+
return body;
|
|
370
|
+
}
|
|
371
|
+
async function upsertManagedBlock(filePath, block) {
|
|
372
|
+
const existing = await fileExists(filePath) ? await fs3.readFile(filePath, "utf8") : "";
|
|
373
|
+
if (!existing) {
|
|
374
|
+
await ensureDir(path3.dirname(filePath));
|
|
375
|
+
await fs3.writeFile(filePath, `${block}
|
|
376
|
+
`, "utf8");
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
const startIndex = existing.includes(managedStart) ? existing.indexOf(managedStart) : existing.indexOf(legacyManagedStart);
|
|
380
|
+
const endIndex = existing.includes(managedEnd) ? existing.indexOf(managedEnd) : existing.indexOf(legacyManagedEnd);
|
|
381
|
+
if (startIndex !== -1 && endIndex !== -1) {
|
|
382
|
+
const next = `${existing.slice(0, startIndex)}${block}${existing.slice(endIndex + managedEnd.length)}`;
|
|
383
|
+
await fs3.writeFile(filePath, next, "utf8");
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
await fs3.writeFile(filePath, `${existing.trimEnd()}
|
|
387
|
+
|
|
388
|
+
${block}
|
|
389
|
+
`, "utf8");
|
|
390
|
+
}
|
|
391
|
+
async function installAgent(rootDir, agent) {
|
|
392
|
+
await initWorkspace(rootDir);
|
|
393
|
+
const block = buildManagedBlock(agent);
|
|
394
|
+
switch (agent) {
|
|
395
|
+
case "codex": {
|
|
396
|
+
const target = path3.join(rootDir, "AGENTS.md");
|
|
397
|
+
await upsertManagedBlock(target, block);
|
|
398
|
+
return target;
|
|
399
|
+
}
|
|
400
|
+
case "claude": {
|
|
401
|
+
const target = path3.join(rootDir, "CLAUDE.md");
|
|
402
|
+
await upsertManagedBlock(target, block);
|
|
403
|
+
return target;
|
|
404
|
+
}
|
|
405
|
+
case "cursor": {
|
|
406
|
+
const rulesDir = path3.join(rootDir, ".cursor", "rules");
|
|
407
|
+
await ensureDir(rulesDir);
|
|
408
|
+
const target = path3.join(rulesDir, "swarmvault.mdc");
|
|
409
|
+
await fs3.writeFile(target, `${block}
|
|
410
|
+
`, "utf8");
|
|
411
|
+
return target;
|
|
412
|
+
}
|
|
413
|
+
default:
|
|
414
|
+
throw new Error(`Unsupported agent ${String(agent)}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
async function installConfiguredAgents(rootDir) {
|
|
418
|
+
const { config } = await initWorkspace(rootDir);
|
|
419
|
+
return Promise.all(config.agents.map((agent) => installAgent(rootDir, agent)));
|
|
420
|
+
}
|
|
421
|
+
|
|
336
422
|
// src/ingest.ts
|
|
337
|
-
import
|
|
338
|
-
import
|
|
339
|
-
import { JSDOM } from "jsdom";
|
|
340
|
-
import TurndownService from "turndown";
|
|
423
|
+
import fs5 from "fs/promises";
|
|
424
|
+
import path5 from "path";
|
|
341
425
|
import { Readability } from "@mozilla/readability";
|
|
426
|
+
import { JSDOM } from "jsdom";
|
|
342
427
|
import mime from "mime-types";
|
|
428
|
+
import TurndownService from "turndown";
|
|
343
429
|
|
|
344
430
|
// src/logs.ts
|
|
345
|
-
import
|
|
346
|
-
import
|
|
431
|
+
import fs4 from "fs/promises";
|
|
432
|
+
import path4 from "path";
|
|
347
433
|
async function appendLogEntry(rootDir, action, title, lines = []) {
|
|
348
434
|
const { paths } = await initWorkspace(rootDir);
|
|
349
435
|
await ensureDir(paths.wikiDir);
|
|
350
|
-
const logPath =
|
|
436
|
+
const logPath = path4.join(paths.wikiDir, "log.md");
|
|
351
437
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().slice(0, 19).replace("T", " ");
|
|
352
438
|
const entry = [`## [${timestamp}] ${action} | ${title}`, ...lines.map((line) => `- ${line}`), ""].join("\n");
|
|
353
|
-
const existing = await fileExists(logPath) ? await
|
|
354
|
-
await
|
|
439
|
+
const existing = await fileExists(logPath) ? await fs4.readFile(logPath, "utf8") : "# Log\n\n";
|
|
440
|
+
await fs4.writeFile(logPath, `${existing}${entry}
|
|
355
441
|
`, "utf8");
|
|
356
442
|
}
|
|
357
443
|
async function appendWatchRun(rootDir, run) {
|
|
@@ -393,7 +479,7 @@ function buildCompositeHash(payloadBytes, attachments = []) {
|
|
|
393
479
|
return sha256(`${sha256(payloadBytes)}|${attachmentSignature}`);
|
|
394
480
|
}
|
|
395
481
|
function sanitizeAssetRelativePath(value) {
|
|
396
|
-
const normalized =
|
|
482
|
+
const normalized = path5.posix.normalize(value.replace(/\\/g, "/"));
|
|
397
483
|
const segments = normalized.split("/").filter(Boolean).map((segment) => {
|
|
398
484
|
if (segment === ".") {
|
|
399
485
|
return "";
|
|
@@ -413,7 +499,7 @@ function normalizeLocalReference(value) {
|
|
|
413
499
|
return null;
|
|
414
500
|
}
|
|
415
501
|
const lowered = candidate.toLowerCase();
|
|
416
|
-
if (lowered.startsWith("http://") || lowered.startsWith("https://") || lowered.startsWith("data:") || lowered.startsWith("mailto:") || lowered.startsWith("#") ||
|
|
502
|
+
if (lowered.startsWith("http://") || lowered.startsWith("https://") || lowered.startsWith("data:") || lowered.startsWith("mailto:") || lowered.startsWith("#") || path5.isAbsolute(candidate)) {
|
|
417
503
|
return null;
|
|
418
504
|
}
|
|
419
505
|
return candidate.replace(/\\/g, "/");
|
|
@@ -441,12 +527,12 @@ async function convertHtmlToMarkdown(html, url) {
|
|
|
441
527
|
};
|
|
442
528
|
}
|
|
443
529
|
async function readManifestByHash(manifestsDir, contentHash) {
|
|
444
|
-
const entries = await
|
|
530
|
+
const entries = await fs5.readdir(manifestsDir, { withFileTypes: true }).catch(() => []);
|
|
445
531
|
for (const entry of entries) {
|
|
446
532
|
if (!entry.isFile() || !entry.name.endsWith(".json")) {
|
|
447
533
|
continue;
|
|
448
534
|
}
|
|
449
|
-
const manifest = await readJsonFile(
|
|
535
|
+
const manifest = await readJsonFile(path5.join(manifestsDir, entry.name));
|
|
450
536
|
if (manifest?.contentHash === contentHash) {
|
|
451
537
|
return manifest;
|
|
452
538
|
}
|
|
@@ -466,20 +552,20 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
466
552
|
}
|
|
467
553
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
468
554
|
const sourceId = `${slugify(prepared.title)}-${contentHash.slice(0, 8)}`;
|
|
469
|
-
const storedPath =
|
|
470
|
-
await
|
|
555
|
+
const storedPath = path5.join(paths.rawSourcesDir, `${sourceId}${prepared.storedExtension}`);
|
|
556
|
+
await fs5.writeFile(storedPath, prepared.payloadBytes);
|
|
471
557
|
let extractedTextPath;
|
|
472
558
|
if (prepared.extractedText) {
|
|
473
|
-
extractedTextPath =
|
|
474
|
-
await
|
|
559
|
+
extractedTextPath = path5.join(paths.extractsDir, `${sourceId}.md`);
|
|
560
|
+
await fs5.writeFile(extractedTextPath, prepared.extractedText, "utf8");
|
|
475
561
|
}
|
|
476
562
|
const manifestAttachments = [];
|
|
477
563
|
for (const attachment of attachments) {
|
|
478
|
-
const absoluteAttachmentPath =
|
|
479
|
-
await ensureDir(
|
|
480
|
-
await
|
|
564
|
+
const absoluteAttachmentPath = path5.join(paths.rawAssetsDir, sourceId, attachment.relativePath);
|
|
565
|
+
await ensureDir(path5.dirname(absoluteAttachmentPath));
|
|
566
|
+
await fs5.writeFile(absoluteAttachmentPath, attachment.bytes);
|
|
481
567
|
manifestAttachments.push({
|
|
482
|
-
path: toPosix(
|
|
568
|
+
path: toPosix(path5.relative(rootDir, absoluteAttachmentPath)),
|
|
483
569
|
mimeType: attachment.mimeType,
|
|
484
570
|
originalPath: attachment.originalPath
|
|
485
571
|
});
|
|
@@ -491,15 +577,15 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
491
577
|
sourceKind: prepared.sourceKind,
|
|
492
578
|
originalPath: prepared.originalPath,
|
|
493
579
|
url: prepared.url,
|
|
494
|
-
storedPath: toPosix(
|
|
495
|
-
extractedTextPath: extractedTextPath ? toPosix(
|
|
580
|
+
storedPath: toPosix(path5.relative(rootDir, storedPath)),
|
|
581
|
+
extractedTextPath: extractedTextPath ? toPosix(path5.relative(rootDir, extractedTextPath)) : void 0,
|
|
496
582
|
mimeType: prepared.mimeType,
|
|
497
583
|
contentHash,
|
|
498
584
|
createdAt: now,
|
|
499
585
|
updatedAt: now,
|
|
500
586
|
attachments: manifestAttachments.length ? manifestAttachments : void 0
|
|
501
587
|
};
|
|
502
|
-
await writeJsonFile(
|
|
588
|
+
await writeJsonFile(path5.join(paths.manifestsDir, `${sourceId}.json`), manifest);
|
|
503
589
|
await appendLogEntry(rootDir, "ingest", prepared.title, [
|
|
504
590
|
`source_id=${sourceId}`,
|
|
505
591
|
`kind=${prepared.sourceKind}`,
|
|
@@ -507,18 +593,18 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
507
593
|
]);
|
|
508
594
|
return { manifest, isNew: true };
|
|
509
595
|
}
|
|
510
|
-
async function prepareFileInput(
|
|
511
|
-
const payloadBytes = await
|
|
596
|
+
async function prepareFileInput(_rootDir, absoluteInput) {
|
|
597
|
+
const payloadBytes = await fs5.readFile(absoluteInput);
|
|
512
598
|
const mimeType = guessMimeType(absoluteInput);
|
|
513
599
|
const sourceKind = inferKind(mimeType, absoluteInput);
|
|
514
|
-
const storedExtension =
|
|
600
|
+
const storedExtension = path5.extname(absoluteInput) || `.${mime.extension(mimeType) || "bin"}`;
|
|
515
601
|
let title;
|
|
516
602
|
let extractedText;
|
|
517
603
|
if (sourceKind === "markdown" || sourceKind === "text") {
|
|
518
604
|
extractedText = payloadBytes.toString("utf8");
|
|
519
|
-
title = titleFromText(
|
|
605
|
+
title = titleFromText(path5.basename(absoluteInput, path5.extname(absoluteInput)), extractedText);
|
|
520
606
|
} else {
|
|
521
|
-
title =
|
|
607
|
+
title = path5.basename(absoluteInput, path5.extname(absoluteInput));
|
|
522
608
|
}
|
|
523
609
|
return {
|
|
524
610
|
title,
|
|
@@ -552,7 +638,7 @@ async function prepareUrlInput(input) {
|
|
|
552
638
|
sourceKind = "markdown";
|
|
553
639
|
storedExtension = ".md";
|
|
554
640
|
} else {
|
|
555
|
-
const extension =
|
|
641
|
+
const extension = path5.extname(new URL(input).pathname);
|
|
556
642
|
storedExtension = extension || `.${mime.extension(mimeType) || "bin"}`;
|
|
557
643
|
if (sourceKind === "markdown" || sourceKind === "text") {
|
|
558
644
|
extractedText = payloadBytes.toString("utf8");
|
|
@@ -578,14 +664,14 @@ async function collectInboxAttachmentRefs(inputDir, files) {
|
|
|
578
664
|
if (sourceKind !== "markdown") {
|
|
579
665
|
continue;
|
|
580
666
|
}
|
|
581
|
-
const content = await
|
|
667
|
+
const content = await fs5.readFile(absolutePath, "utf8");
|
|
582
668
|
const refs = extractMarkdownReferences(content);
|
|
583
669
|
if (!refs.length) {
|
|
584
670
|
continue;
|
|
585
671
|
}
|
|
586
672
|
const sourceRefs = [];
|
|
587
673
|
for (const ref of refs) {
|
|
588
|
-
const resolved =
|
|
674
|
+
const resolved = path5.resolve(path5.dirname(absolutePath), ref);
|
|
589
675
|
if (!resolved.startsWith(inputDir) || !await fileExists(resolved)) {
|
|
590
676
|
continue;
|
|
591
677
|
}
|
|
@@ -619,12 +705,12 @@ function rewriteMarkdownReferences(content, replacements) {
|
|
|
619
705
|
});
|
|
620
706
|
}
|
|
621
707
|
async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
|
|
622
|
-
const originalBytes = await
|
|
708
|
+
const originalBytes = await fs5.readFile(absolutePath);
|
|
623
709
|
const originalText = originalBytes.toString("utf8");
|
|
624
|
-
const title = titleFromText(
|
|
710
|
+
const title = titleFromText(path5.basename(absolutePath, path5.extname(absolutePath)), originalText);
|
|
625
711
|
const attachments = [];
|
|
626
712
|
for (const attachmentRef of attachmentRefs) {
|
|
627
|
-
const bytes = await
|
|
713
|
+
const bytes = await fs5.readFile(attachmentRef.absolutePath);
|
|
628
714
|
attachments.push({
|
|
629
715
|
relativePath: sanitizeAssetRelativePath(attachmentRef.relativeRef),
|
|
630
716
|
mimeType: guessMimeType(attachmentRef.absolutePath),
|
|
@@ -647,7 +733,7 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
|
|
|
647
733
|
sourceKind: "markdown",
|
|
648
734
|
originalPath: toPosix(absolutePath),
|
|
649
735
|
mimeType: "text/markdown",
|
|
650
|
-
storedExtension:
|
|
736
|
+
storedExtension: path5.extname(absolutePath) || ".md",
|
|
651
737
|
payloadBytes: Buffer.from(rewrittenText, "utf8"),
|
|
652
738
|
extractedText: rewrittenText,
|
|
653
739
|
attachments,
|
|
@@ -659,50 +745,48 @@ function isSupportedInboxKind(sourceKind) {
|
|
|
659
745
|
}
|
|
660
746
|
async function ingestInput(rootDir, input) {
|
|
661
747
|
const { paths } = await initWorkspace(rootDir);
|
|
662
|
-
const prepared = /^https?:\/\//i.test(input) ? await prepareUrlInput(input) : await prepareFileInput(rootDir,
|
|
748
|
+
const prepared = /^https?:\/\//i.test(input) ? await prepareUrlInput(input) : await prepareFileInput(rootDir, path5.resolve(rootDir, input));
|
|
663
749
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
664
750
|
return result.manifest;
|
|
665
751
|
}
|
|
666
752
|
async function importInbox(rootDir, inputDir) {
|
|
667
753
|
const { paths } = await initWorkspace(rootDir);
|
|
668
|
-
const effectiveInputDir =
|
|
754
|
+
const effectiveInputDir = path5.resolve(rootDir, inputDir ?? paths.inboxDir);
|
|
669
755
|
if (!await fileExists(effectiveInputDir)) {
|
|
670
756
|
throw new Error(`Inbox directory not found: ${effectiveInputDir}`);
|
|
671
757
|
}
|
|
672
758
|
const files = (await listFilesRecursive(effectiveInputDir)).sort();
|
|
673
759
|
const refsBySource = await collectInboxAttachmentRefs(effectiveInputDir, files);
|
|
674
|
-
const claimedAttachments = new Set(
|
|
675
|
-
[...refsBySource.values()].flatMap((refs) => refs.map((ref) => ref.absolutePath))
|
|
676
|
-
);
|
|
760
|
+
const claimedAttachments = new Set([...refsBySource.values()].flatMap((refs) => refs.map((ref) => ref.absolutePath)));
|
|
677
761
|
const imported = [];
|
|
678
762
|
const skipped = [];
|
|
679
763
|
let attachmentCount = 0;
|
|
680
764
|
for (const absolutePath of files) {
|
|
681
|
-
const basename =
|
|
765
|
+
const basename = path5.basename(absolutePath);
|
|
682
766
|
if (basename.startsWith(".")) {
|
|
683
|
-
skipped.push({ path: toPosix(
|
|
767
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: "hidden_file" });
|
|
684
768
|
continue;
|
|
685
769
|
}
|
|
686
770
|
if (claimedAttachments.has(absolutePath)) {
|
|
687
|
-
skipped.push({ path: toPosix(
|
|
771
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: "referenced_attachment" });
|
|
688
772
|
continue;
|
|
689
773
|
}
|
|
690
774
|
const mimeType = guessMimeType(absolutePath);
|
|
691
775
|
const sourceKind = inferKind(mimeType, absolutePath);
|
|
692
776
|
if (!isSupportedInboxKind(sourceKind)) {
|
|
693
|
-
skipped.push({ path: toPosix(
|
|
777
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: `unsupported_kind:${sourceKind}` });
|
|
694
778
|
continue;
|
|
695
779
|
}
|
|
696
780
|
const prepared = sourceKind === "markdown" && refsBySource.has(absolutePath) ? await prepareInboxMarkdownInput(absolutePath, refsBySource.get(absolutePath) ?? []) : await prepareFileInput(rootDir, absolutePath);
|
|
697
781
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
698
782
|
if (!result.isNew) {
|
|
699
|
-
skipped.push({ path: toPosix(
|
|
783
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: "duplicate_content" });
|
|
700
784
|
continue;
|
|
701
785
|
}
|
|
702
786
|
attachmentCount += result.manifest.attachments?.length ?? 0;
|
|
703
787
|
imported.push(result.manifest);
|
|
704
788
|
}
|
|
705
|
-
await appendLogEntry(rootDir, "inbox_import", toPosix(
|
|
789
|
+
await appendLogEntry(rootDir, "inbox_import", toPosix(path5.relative(rootDir, effectiveInputDir)) || ".", [
|
|
706
790
|
`scanned=${files.length}`,
|
|
707
791
|
`imported=${imported.length}`,
|
|
708
792
|
`attachments=${attachmentCount}`,
|
|
@@ -721,9 +805,9 @@ async function listManifests(rootDir) {
|
|
|
721
805
|
if (!await fileExists(paths.manifestsDir)) {
|
|
722
806
|
return [];
|
|
723
807
|
}
|
|
724
|
-
const entries = await
|
|
808
|
+
const entries = await fs5.readdir(paths.manifestsDir);
|
|
725
809
|
const manifests = await Promise.all(
|
|
726
|
-
entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(
|
|
810
|
+
entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(path5.join(paths.manifestsDir, entry)))
|
|
727
811
|
);
|
|
728
812
|
return manifests.filter((manifest) => Boolean(manifest));
|
|
729
813
|
}
|
|
@@ -731,33 +815,62 @@ async function readExtractedText(rootDir, manifest) {
|
|
|
731
815
|
if (!manifest.extractedTextPath) {
|
|
732
816
|
return void 0;
|
|
733
817
|
}
|
|
734
|
-
const absolutePath =
|
|
818
|
+
const absolutePath = path5.resolve(rootDir, manifest.extractedTextPath);
|
|
735
819
|
if (!await fileExists(absolutePath)) {
|
|
736
820
|
return void 0;
|
|
737
821
|
}
|
|
738
|
-
return
|
|
822
|
+
return fs5.readFile(absolutePath, "utf8");
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
// src/mcp.ts
|
|
826
|
+
import fs12 from "fs/promises";
|
|
827
|
+
import path14 from "path";
|
|
828
|
+
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
829
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
830
|
+
import { z as z9 } from "zod";
|
|
831
|
+
|
|
832
|
+
// src/schema.ts
|
|
833
|
+
import fs6 from "fs/promises";
|
|
834
|
+
import path6 from "path";
|
|
835
|
+
async function loadVaultSchema(rootDir) {
|
|
836
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
837
|
+
const schemaPath = paths.schemaPath;
|
|
838
|
+
const content = await fileExists(schemaPath) ? await fs6.readFile(schemaPath, "utf8") : defaultVaultSchema();
|
|
839
|
+
const normalized = content.trim() ? content.trim() : defaultVaultSchema().trim();
|
|
840
|
+
return {
|
|
841
|
+
path: schemaPath,
|
|
842
|
+
content: normalized,
|
|
843
|
+
hash: sha256(normalized),
|
|
844
|
+
isLegacyPath: path6.basename(schemaPath) === LEGACY_SCHEMA_FILENAME && path6.basename(schemaPath) !== PRIMARY_SCHEMA_FILENAME
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
function buildSchemaPrompt(schema, instruction) {
|
|
848
|
+
return [instruction, "", `Vault schema path: ${schema.path}`, "", "Vault schema instructions:", schema.content].join("\n");
|
|
739
849
|
}
|
|
740
850
|
|
|
741
851
|
// src/vault.ts
|
|
742
|
-
import
|
|
743
|
-
import
|
|
744
|
-
import
|
|
852
|
+
import fs11 from "fs/promises";
|
|
853
|
+
import path13 from "path";
|
|
854
|
+
import matter5 from "gray-matter";
|
|
855
|
+
import { z as z8 } from "zod";
|
|
745
856
|
|
|
746
857
|
// src/analysis.ts
|
|
747
|
-
import
|
|
858
|
+
import path7 from "path";
|
|
748
859
|
import { z as z3 } from "zod";
|
|
749
860
|
var sourceAnalysisSchema = z3.object({
|
|
750
861
|
title: z3.string().min(1),
|
|
751
862
|
summary: z3.string().min(1),
|
|
752
863
|
concepts: z3.array(z3.object({ name: z3.string().min(1), description: z3.string().default("") })).max(12).default([]),
|
|
753
864
|
entities: z3.array(z3.object({ name: z3.string().min(1), description: z3.string().default("") })).max(12).default([]),
|
|
754
|
-
claims: z3.array(
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
865
|
+
claims: z3.array(
|
|
866
|
+
z3.object({
|
|
867
|
+
text: z3.string().min(1),
|
|
868
|
+
confidence: z3.number().min(0).max(1).default(0.6),
|
|
869
|
+
status: z3.enum(["extracted", "inferred", "conflicted", "stale"]).default("extracted"),
|
|
870
|
+
polarity: z3.enum(["positive", "negative", "neutral"]).default("neutral"),
|
|
871
|
+
citation: z3.string().min(1)
|
|
872
|
+
})
|
|
873
|
+
).max(8).default([]),
|
|
761
874
|
questions: z3.array(z3.string()).max(6).default([])
|
|
762
875
|
});
|
|
763
876
|
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
@@ -812,7 +925,10 @@ function extractTopTerms(text, count) {
|
|
|
812
925
|
}
|
|
813
926
|
function extractEntities(text, count) {
|
|
814
927
|
const matches = text.match(/\b[A-Z][A-Za-z0-9-]+(?:\s+[A-Z][A-Za-z0-9-]+){0,2}\b/g) ?? [];
|
|
815
|
-
return uniqueBy(
|
|
928
|
+
return uniqueBy(
|
|
929
|
+
matches.map((value) => normalizeWhitespace(value)),
|
|
930
|
+
(value) => value.toLowerCase()
|
|
931
|
+
).slice(0, count);
|
|
816
932
|
}
|
|
817
933
|
function detectPolarity(text) {
|
|
818
934
|
if (/\b(no|not|never|cannot|can't|won't|without)\b/i.test(text)) {
|
|
@@ -913,7 +1029,7 @@ ${truncate(text, 18e3)}`
|
|
|
913
1029
|
};
|
|
914
1030
|
}
|
|
915
1031
|
async function analyzeSource(manifest, extractedText, provider, paths, schema) {
|
|
916
|
-
const cachePath =
|
|
1032
|
+
const cachePath = path7.join(paths.analysesDir, `${manifest.sourceId}.json`);
|
|
917
1033
|
const cached = await readJsonFile(cachePath);
|
|
918
1034
|
if (cached && cached.sourceHash === manifest.contentHash && cached.schemaHash === schema.hash) {
|
|
919
1035
|
return cached;
|
|
@@ -949,352 +1065,42 @@ function analysisSignature(analysis) {
|
|
|
949
1065
|
return sha256(JSON.stringify(analysis));
|
|
950
1066
|
}
|
|
951
1067
|
|
|
952
|
-
// src/
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
var managedStart = "<!-- swarmvault:managed:start -->";
|
|
956
|
-
var managedEnd = "<!-- swarmvault:managed:end -->";
|
|
957
|
-
var legacyManagedStart = "<!-- vault:managed:start -->";
|
|
958
|
-
var legacyManagedEnd = "<!-- vault:managed:end -->";
|
|
959
|
-
function buildManagedBlock(agent) {
|
|
960
|
-
const body = [
|
|
961
|
-
managedStart,
|
|
962
|
-
`# SwarmVault Rules (${agent})`,
|
|
963
|
-
"",
|
|
964
|
-
"- Read `swarmvault.schema.md` before compile or query style work. If only `schema.md` exists, treat it as the legacy schema path.",
|
|
965
|
-
"- Treat `raw/` as immutable source input.",
|
|
966
|
-
"- Treat `wiki/` as generated markdown owned by the agent and compiler workflow.",
|
|
967
|
-
"- Read `wiki/index.md` before broad file searching when answering SwarmVault questions.",
|
|
968
|
-
"- Preserve frontmatter fields including `page_id`, `source_ids`, `node_ids`, `freshness`, and `source_hashes`.",
|
|
969
|
-
"- Save high-value answers back into `wiki/outputs/` instead of leaving them only in chat.",
|
|
970
|
-
"- Prefer `swarmvault ingest`, `swarmvault compile`, `swarmvault query`, and `swarmvault lint` for SwarmVault maintenance tasks.",
|
|
971
|
-
managedEnd,
|
|
972
|
-
""
|
|
973
|
-
].join("\n");
|
|
974
|
-
if (agent === "cursor") {
|
|
975
|
-
return body;
|
|
976
|
-
}
|
|
977
|
-
return body;
|
|
978
|
-
}
|
|
979
|
-
async function upsertManagedBlock(filePath, block) {
|
|
980
|
-
const existing = await fileExists(filePath) ? await fs5.readFile(filePath, "utf8") : "";
|
|
981
|
-
if (!existing) {
|
|
982
|
-
await ensureDir(path6.dirname(filePath));
|
|
983
|
-
await fs5.writeFile(filePath, `${block}
|
|
984
|
-
`, "utf8");
|
|
985
|
-
return;
|
|
986
|
-
}
|
|
987
|
-
const startIndex = existing.includes(managedStart) ? existing.indexOf(managedStart) : existing.indexOf(legacyManagedStart);
|
|
988
|
-
const endIndex = existing.includes(managedEnd) ? existing.indexOf(managedEnd) : existing.indexOf(legacyManagedEnd);
|
|
989
|
-
if (startIndex !== -1 && endIndex !== -1) {
|
|
990
|
-
const next = `${existing.slice(0, startIndex)}${block}${existing.slice(endIndex + managedEnd.length)}`;
|
|
991
|
-
await fs5.writeFile(filePath, next, "utf8");
|
|
992
|
-
return;
|
|
993
|
-
}
|
|
994
|
-
await fs5.writeFile(filePath, `${existing.trimEnd()}
|
|
995
|
-
|
|
996
|
-
${block}
|
|
997
|
-
`, "utf8");
|
|
1068
|
+
// src/confidence.ts
|
|
1069
|
+
function nodeConfidence(sourceCount) {
|
|
1070
|
+
return Math.min(0.5 + sourceCount * 0.15, 0.95);
|
|
998
1071
|
}
|
|
999
|
-
|
|
1000
|
-
const
|
|
1001
|
-
const
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
const target = path6.join(rootDir, "AGENTS.md");
|
|
1005
|
-
await upsertManagedBlock(target, block);
|
|
1006
|
-
return target;
|
|
1007
|
-
}
|
|
1008
|
-
case "claude": {
|
|
1009
|
-
const target = path6.join(rootDir, "CLAUDE.md");
|
|
1010
|
-
await upsertManagedBlock(target, block);
|
|
1011
|
-
return target;
|
|
1012
|
-
}
|
|
1013
|
-
case "cursor": {
|
|
1014
|
-
const rulesDir = path6.join(rootDir, ".cursor", "rules");
|
|
1015
|
-
await ensureDir(rulesDir);
|
|
1016
|
-
const target = path6.join(rulesDir, "swarmvault.mdc");
|
|
1017
|
-
await fs5.writeFile(target, `${block}
|
|
1018
|
-
`, "utf8");
|
|
1019
|
-
return target;
|
|
1020
|
-
}
|
|
1021
|
-
default:
|
|
1022
|
-
throw new Error(`Unsupported agent ${String(agent)}`);
|
|
1072
|
+
function edgeConfidence(claims, conceptName) {
|
|
1073
|
+
const lower = conceptName.toLowerCase();
|
|
1074
|
+
const relevant = claims.filter((c) => c.text.toLowerCase().includes(lower));
|
|
1075
|
+
if (!relevant.length) {
|
|
1076
|
+
return 0.5;
|
|
1023
1077
|
}
|
|
1078
|
+
return relevant.reduce((sum, c) => sum + c.confidence, 0) / relevant.length;
|
|
1024
1079
|
}
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
return Promise.all(config.agents.map((agent) => installAgent(rootDir, agent)));
|
|
1080
|
+
function conflictConfidence(claimA, claimB) {
|
|
1081
|
+
return Math.min(claimA.confidence, claimB.confidence);
|
|
1028
1082
|
}
|
|
1029
1083
|
|
|
1030
|
-
// src/
|
|
1084
|
+
// src/deep-lint.ts
|
|
1085
|
+
import fs8 from "fs/promises";
|
|
1086
|
+
import path10 from "path";
|
|
1031
1087
|
import matter from "gray-matter";
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
const pageId = `source:${manifest.sourceId}`;
|
|
1049
|
-
const nodeIds = [
|
|
1050
|
-
`source:${manifest.sourceId}`,
|
|
1051
|
-
...analysis.concepts.map((item) => item.id),
|
|
1052
|
-
...analysis.entities.map((item) => item.id)
|
|
1053
|
-
];
|
|
1054
|
-
const backlinks = [
|
|
1055
|
-
...analysis.concepts.map((item) => `concept:${slugify(item.name)}`),
|
|
1056
|
-
...analysis.entities.map((item) => `entity:${slugify(item.name)}`)
|
|
1057
|
-
];
|
|
1058
|
-
const frontmatter = {
|
|
1059
|
-
page_id: pageId,
|
|
1060
|
-
kind: "source",
|
|
1061
|
-
title: analysis.title,
|
|
1062
|
-
tags: ["source"],
|
|
1063
|
-
source_ids: [manifest.sourceId],
|
|
1064
|
-
node_ids: nodeIds,
|
|
1065
|
-
freshness: "fresh",
|
|
1066
|
-
confidence: 0.8,
|
|
1067
|
-
updated_at: analysis.producedAt,
|
|
1068
|
-
backlinks,
|
|
1069
|
-
schema_hash: schemaHash,
|
|
1070
|
-
source_hashes: {
|
|
1071
|
-
[manifest.sourceId]: manifest.contentHash
|
|
1072
|
-
}
|
|
1073
|
-
};
|
|
1074
|
-
const body = [
|
|
1075
|
-
`# ${analysis.title}`,
|
|
1076
|
-
"",
|
|
1077
|
-
`Source ID: \`${manifest.sourceId}\``,
|
|
1078
|
-
manifest.url ? `Source URL: ${manifest.url}` : `Source Path: \`${manifest.originalPath ?? manifest.storedPath}\``,
|
|
1079
|
-
"",
|
|
1080
|
-
"## Summary",
|
|
1081
|
-
"",
|
|
1082
|
-
analysis.summary,
|
|
1083
|
-
"",
|
|
1084
|
-
"## Concepts",
|
|
1085
|
-
"",
|
|
1086
|
-
...analysis.concepts.length ? analysis.concepts.map((item) => `- [[${pagePathFor("concept", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`) : ["- None detected."],
|
|
1087
|
-
"",
|
|
1088
|
-
"## Entities",
|
|
1089
|
-
"",
|
|
1090
|
-
...analysis.entities.length ? analysis.entities.map((item) => `- [[${pagePathFor("entity", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`) : ["- None detected."],
|
|
1091
|
-
"",
|
|
1092
|
-
"## Claims",
|
|
1093
|
-
"",
|
|
1094
|
-
...analysis.claims.length ? analysis.claims.map((claim) => `- ${claim.text} [source:${claim.citation}]`) : ["- No claims extracted."],
|
|
1095
|
-
"",
|
|
1096
|
-
"## Questions",
|
|
1097
|
-
"",
|
|
1098
|
-
...analysis.questions.length ? analysis.questions.map((question) => `- ${question}`) : ["- No follow-up questions yet."],
|
|
1099
|
-
""
|
|
1100
|
-
].join("\n");
|
|
1101
|
-
return {
|
|
1102
|
-
page: {
|
|
1103
|
-
id: pageId,
|
|
1104
|
-
path: relativePath,
|
|
1105
|
-
title: analysis.title,
|
|
1106
|
-
kind: "source",
|
|
1107
|
-
sourceIds: [manifest.sourceId],
|
|
1108
|
-
nodeIds,
|
|
1109
|
-
freshness: "fresh",
|
|
1110
|
-
confidence: 0.8,
|
|
1111
|
-
backlinks,
|
|
1112
|
-
schemaHash,
|
|
1113
|
-
sourceHashes: { [manifest.sourceId]: manifest.contentHash }
|
|
1114
|
-
},
|
|
1115
|
-
content: matter.stringify(body, frontmatter)
|
|
1116
|
-
};
|
|
1117
|
-
}
|
|
1118
|
-
function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes, schemaHash) {
|
|
1119
|
-
const slug = slugify(name);
|
|
1120
|
-
const relativePath = pagePathFor(kind, slug);
|
|
1121
|
-
const pageId = `${kind}:${slug}`;
|
|
1122
|
-
const sourceIds = sourceAnalyses.map((item) => item.sourceId);
|
|
1123
|
-
const otherPages = sourceAnalyses.map((item) => `source:${item.sourceId}`);
|
|
1124
|
-
const summary = descriptions.find(Boolean) ?? `${kind} aggregated from ${sourceIds.length} source(s).`;
|
|
1125
|
-
const frontmatter = {
|
|
1126
|
-
page_id: pageId,
|
|
1127
|
-
kind,
|
|
1128
|
-
title: name,
|
|
1129
|
-
tags: [kind],
|
|
1130
|
-
source_ids: sourceIds,
|
|
1131
|
-
node_ids: [pageId],
|
|
1132
|
-
freshness: "fresh",
|
|
1133
|
-
confidence: 0.72,
|
|
1134
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1135
|
-
backlinks: otherPages,
|
|
1136
|
-
schema_hash: schemaHash,
|
|
1137
|
-
source_hashes: sourceHashes
|
|
1138
|
-
};
|
|
1139
|
-
const body = [
|
|
1140
|
-
`# ${name}`,
|
|
1141
|
-
"",
|
|
1142
|
-
"## Summary",
|
|
1143
|
-
"",
|
|
1144
|
-
summary,
|
|
1145
|
-
"",
|
|
1146
|
-
"## Seen In",
|
|
1147
|
-
"",
|
|
1148
|
-
...sourceAnalyses.map((item) => `- [[${pagePathFor("source", item.sourceId).replace(/\.md$/, "")}|${item.title}]]`),
|
|
1149
|
-
"",
|
|
1150
|
-
"## Source Claims",
|
|
1151
|
-
"",
|
|
1152
|
-
...sourceAnalyses.flatMap(
|
|
1153
|
-
(item) => item.claims.filter((claim) => claim.text.toLowerCase().includes(name.toLowerCase())).map((claim) => `- ${claim.text} [source:${claim.citation}]`)
|
|
1154
|
-
),
|
|
1155
|
-
""
|
|
1156
|
-
].join("\n");
|
|
1157
|
-
return {
|
|
1158
|
-
page: {
|
|
1159
|
-
id: pageId,
|
|
1160
|
-
path: relativePath,
|
|
1161
|
-
title: name,
|
|
1162
|
-
kind,
|
|
1163
|
-
sourceIds,
|
|
1164
|
-
nodeIds: [pageId],
|
|
1165
|
-
freshness: "fresh",
|
|
1166
|
-
confidence: 0.72,
|
|
1167
|
-
backlinks: otherPages,
|
|
1168
|
-
schemaHash,
|
|
1169
|
-
sourceHashes
|
|
1170
|
-
},
|
|
1171
|
-
content: matter.stringify(body, frontmatter)
|
|
1172
|
-
};
|
|
1173
|
-
}
|
|
1174
|
-
function buildIndexPage(pages, schemaHash) {
|
|
1175
|
-
const sources = pages.filter((page) => page.kind === "source");
|
|
1176
|
-
const concepts = pages.filter((page) => page.kind === "concept");
|
|
1177
|
-
const entities = pages.filter((page) => page.kind === "entity");
|
|
1178
|
-
return [
|
|
1179
|
-
"---",
|
|
1180
|
-
"page_id: index",
|
|
1181
|
-
"kind: index",
|
|
1182
|
-
"title: SwarmVault Index",
|
|
1183
|
-
"tags:",
|
|
1184
|
-
" - index",
|
|
1185
|
-
"source_ids: []",
|
|
1186
|
-
"node_ids: []",
|
|
1187
|
-
"freshness: fresh",
|
|
1188
|
-
"confidence: 1",
|
|
1189
|
-
`updated_at: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
1190
|
-
"backlinks: []",
|
|
1191
|
-
`schema_hash: ${schemaHash}`,
|
|
1192
|
-
"source_hashes: {}",
|
|
1193
|
-
"---",
|
|
1194
|
-
"",
|
|
1195
|
-
"# SwarmVault Index",
|
|
1196
|
-
"",
|
|
1197
|
-
"## Sources",
|
|
1198
|
-
"",
|
|
1199
|
-
...sources.length ? sources.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No sources yet."],
|
|
1200
|
-
"",
|
|
1201
|
-
"## Concepts",
|
|
1202
|
-
"",
|
|
1203
|
-
...concepts.length ? concepts.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No concepts yet."],
|
|
1204
|
-
"",
|
|
1205
|
-
"## Entities",
|
|
1206
|
-
"",
|
|
1207
|
-
...entities.length ? entities.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No entities yet."],
|
|
1208
|
-
""
|
|
1209
|
-
].join("\n");
|
|
1210
|
-
}
|
|
1211
|
-
function buildSectionIndex(kind, pages, schemaHash) {
|
|
1212
|
-
const title = kind.charAt(0).toUpperCase() + kind.slice(1);
|
|
1213
|
-
return matter.stringify(
|
|
1214
|
-
[
|
|
1215
|
-
`# ${title}`,
|
|
1216
|
-
"",
|
|
1217
|
-
...pages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`),
|
|
1218
|
-
""
|
|
1219
|
-
].join("\n"),
|
|
1220
|
-
{
|
|
1221
|
-
page_id: `${kind}:index`,
|
|
1222
|
-
kind: "index",
|
|
1223
|
-
title,
|
|
1224
|
-
tags: ["index", kind],
|
|
1225
|
-
source_ids: [],
|
|
1226
|
-
node_ids: [],
|
|
1227
|
-
freshness: "fresh",
|
|
1228
|
-
confidence: 1,
|
|
1229
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1230
|
-
backlinks: [],
|
|
1231
|
-
schema_hash: schemaHash,
|
|
1232
|
-
source_hashes: {}
|
|
1233
|
-
}
|
|
1234
|
-
);
|
|
1235
|
-
}
|
|
1236
|
-
function buildOutputPage(question, answer, citations, schemaHash) {
|
|
1237
|
-
const slug = slugify(question);
|
|
1238
|
-
const pageId = `output:${slug}`;
|
|
1239
|
-
const pathValue = pagePathFor("output", slug);
|
|
1240
|
-
const frontmatter = {
|
|
1241
|
-
page_id: pageId,
|
|
1242
|
-
kind: "output",
|
|
1243
|
-
title: question,
|
|
1244
|
-
tags: ["output"],
|
|
1245
|
-
source_ids: citations,
|
|
1246
|
-
node_ids: [],
|
|
1247
|
-
freshness: "fresh",
|
|
1248
|
-
confidence: 0.74,
|
|
1249
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1250
|
-
backlinks: citations.map((sourceId) => `source:${sourceId}`),
|
|
1251
|
-
schema_hash: schemaHash,
|
|
1252
|
-
source_hashes: {}
|
|
1253
|
-
};
|
|
1254
|
-
return {
|
|
1255
|
-
page: {
|
|
1256
|
-
id: pageId,
|
|
1257
|
-
path: pathValue,
|
|
1258
|
-
title: question,
|
|
1259
|
-
kind: "output",
|
|
1260
|
-
sourceIds: citations,
|
|
1261
|
-
nodeIds: [],
|
|
1262
|
-
freshness: "fresh",
|
|
1263
|
-
confidence: 0.74,
|
|
1264
|
-
backlinks: citations.map((sourceId) => `source:${sourceId}`),
|
|
1265
|
-
schemaHash,
|
|
1266
|
-
sourceHashes: {}
|
|
1267
|
-
},
|
|
1268
|
-
content: matter.stringify(
|
|
1269
|
-
[
|
|
1270
|
-
`# ${question}`,
|
|
1271
|
-
"",
|
|
1272
|
-
answer,
|
|
1273
|
-
"",
|
|
1274
|
-
"## Citations",
|
|
1275
|
-
"",
|
|
1276
|
-
...citations.map((citation) => `- [source:${citation}]`),
|
|
1277
|
-
""
|
|
1278
|
-
].join("\n"),
|
|
1279
|
-
frontmatter
|
|
1280
|
-
)
|
|
1281
|
-
};
|
|
1282
|
-
}
|
|
1283
|
-
|
|
1284
|
-
// src/providers/registry.ts
|
|
1285
|
-
import path7 from "path";
|
|
1286
|
-
import { pathToFileURL } from "url";
|
|
1287
|
-
import { z as z5 } from "zod";
|
|
1288
|
-
|
|
1289
|
-
// src/providers/base.ts
|
|
1290
|
-
import fs6 from "fs/promises";
|
|
1291
|
-
import { z as z4 } from "zod";
|
|
1292
|
-
var BaseProviderAdapter = class {
|
|
1293
|
-
constructor(id, type, model, capabilities) {
|
|
1294
|
-
this.id = id;
|
|
1295
|
-
this.type = type;
|
|
1296
|
-
this.model = model;
|
|
1297
|
-
this.capabilities = new Set(capabilities);
|
|
1088
|
+
import { z as z7 } from "zod";
|
|
1089
|
+
|
|
1090
|
+
// src/providers/registry.ts
|
|
1091
|
+
import path8 from "path";
|
|
1092
|
+
import { pathToFileURL } from "url";
|
|
1093
|
+
import { z as z5 } from "zod";
|
|
1094
|
+
|
|
1095
|
+
// src/providers/base.ts
|
|
1096
|
+
import fs7 from "fs/promises";
|
|
1097
|
+
import { z as z4 } from "zod";
|
|
1098
|
+
var BaseProviderAdapter = class {
|
|
1099
|
+
constructor(id, type, model, capabilities) {
|
|
1100
|
+
this.id = id;
|
|
1101
|
+
this.type = type;
|
|
1102
|
+
this.model = model;
|
|
1103
|
+
this.capabilities = new Set(capabilities);
|
|
1298
1104
|
}
|
|
1299
1105
|
id;
|
|
1300
1106
|
type;
|
|
@@ -1316,162 +1122,42 @@ ${schemaDescription}`
|
|
|
1316
1122
|
return Promise.all(
|
|
1317
1123
|
attachments.map(async (attachment) => ({
|
|
1318
1124
|
mimeType: attachment.mimeType,
|
|
1319
|
-
base64: await
|
|
1125
|
+
base64: await fs7.readFile(attachment.filePath, "base64")
|
|
1320
1126
|
}))
|
|
1321
1127
|
);
|
|
1322
1128
|
}
|
|
1323
1129
|
};
|
|
1324
1130
|
|
|
1325
|
-
// src/providers/
|
|
1326
|
-
|
|
1327
|
-
const cleaned = normalizeWhitespace(prompt);
|
|
1328
|
-
if (!cleaned) {
|
|
1329
|
-
return "No prompt content provided.";
|
|
1330
|
-
}
|
|
1331
|
-
return firstSentences(cleaned, 2) || cleaned.slice(0, 280);
|
|
1332
|
-
}
|
|
1333
|
-
var HeuristicProviderAdapter = class extends BaseProviderAdapter {
|
|
1334
|
-
constructor(id, model) {
|
|
1335
|
-
super(id, "heuristic", model, ["chat", "structured", "vision", "local"]);
|
|
1336
|
-
}
|
|
1337
|
-
async generateText(request) {
|
|
1338
|
-
const attachmentHint = request.attachments?.length ? ` Attachments: ${request.attachments.length}.` : "";
|
|
1339
|
-
return {
|
|
1340
|
-
text: `Heuristic provider response.${attachmentHint} ${summarizePrompt(request.prompt)}`.trim()
|
|
1341
|
-
};
|
|
1342
|
-
}
|
|
1343
|
-
};
|
|
1344
|
-
|
|
1345
|
-
// src/providers/openai-compatible.ts
|
|
1346
|
-
function buildAuthHeaders(apiKey) {
|
|
1347
|
-
return apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
|
|
1348
|
-
}
|
|
1349
|
-
var OpenAiCompatibleProviderAdapter = class extends BaseProviderAdapter {
|
|
1350
|
-
baseUrl;
|
|
1131
|
+
// src/providers/anthropic.ts
|
|
1132
|
+
var AnthropicProviderAdapter = class extends BaseProviderAdapter {
|
|
1351
1133
|
apiKey;
|
|
1352
1134
|
headers;
|
|
1353
|
-
|
|
1354
|
-
constructor(id,
|
|
1355
|
-
super(id,
|
|
1356
|
-
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
|
1135
|
+
baseUrl;
|
|
1136
|
+
constructor(id, model, options) {
|
|
1137
|
+
super(id, "anthropic", model, ["chat", "structured", "tools", "vision", "streaming"]);
|
|
1357
1138
|
this.apiKey = options.apiKey;
|
|
1358
1139
|
this.headers = options.headers;
|
|
1359
|
-
this.
|
|
1140
|
+
this.baseUrl = (options.baseUrl ?? "https://api.anthropic.com").replace(/\/+$/, "");
|
|
1360
1141
|
}
|
|
1361
1142
|
async generateText(request) {
|
|
1362
|
-
if (this.apiStyle === "chat") {
|
|
1363
|
-
return this.generateViaChatCompletions(request);
|
|
1364
|
-
}
|
|
1365
|
-
return this.generateViaResponses(request);
|
|
1366
|
-
}
|
|
1367
|
-
async generateViaResponses(request) {
|
|
1368
1143
|
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1369
|
-
const
|
|
1370
|
-
{
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
const response = await fetch(`${this.baseUrl}/responses`, {
|
|
1144
|
+
const content = [
|
|
1145
|
+
{ type: "text", text: request.prompt },
|
|
1146
|
+
...encodedAttachments.map((item) => ({
|
|
1147
|
+
type: "image",
|
|
1148
|
+
source: {
|
|
1149
|
+
type: "base64",
|
|
1150
|
+
media_type: item.mimeType,
|
|
1151
|
+
data: item.base64
|
|
1152
|
+
}
|
|
1153
|
+
}))
|
|
1154
|
+
];
|
|
1155
|
+
const response = await fetch(`${this.baseUrl}/v1/messages`, {
|
|
1382
1156
|
method: "POST",
|
|
1383
1157
|
headers: {
|
|
1384
1158
|
"content-type": "application/json",
|
|
1385
|
-
|
|
1386
|
-
...this.
|
|
1387
|
-
},
|
|
1388
|
-
body: JSON.stringify({
|
|
1389
|
-
model: this.model,
|
|
1390
|
-
input,
|
|
1391
|
-
instructions: request.system,
|
|
1392
|
-
max_output_tokens: request.maxOutputTokens
|
|
1393
|
-
})
|
|
1394
|
-
});
|
|
1395
|
-
if (!response.ok) {
|
|
1396
|
-
throw new Error(`Provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1397
|
-
}
|
|
1398
|
-
const payload = await response.json();
|
|
1399
|
-
return {
|
|
1400
|
-
text: payload.output_text ?? "",
|
|
1401
|
-
usage: payload.usage ? { inputTokens: payload.usage.input_tokens, outputTokens: payload.usage.output_tokens } : void 0
|
|
1402
|
-
};
|
|
1403
|
-
}
|
|
1404
|
-
async generateViaChatCompletions(request) {
|
|
1405
|
-
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1406
|
-
const content = encodedAttachments.length ? [
|
|
1407
|
-
{ type: "text", text: request.prompt },
|
|
1408
|
-
...encodedAttachments.map((item) => ({
|
|
1409
|
-
type: "image_url",
|
|
1410
|
-
image_url: {
|
|
1411
|
-
url: `data:${item.mimeType};base64,${item.base64}`
|
|
1412
|
-
}
|
|
1413
|
-
}))
|
|
1414
|
-
] : request.prompt;
|
|
1415
|
-
const messages = [
|
|
1416
|
-
...request.system ? [{ role: "system", content: request.system }] : [],
|
|
1417
|
-
{ role: "user", content }
|
|
1418
|
-
];
|
|
1419
|
-
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
1420
|
-
method: "POST",
|
|
1421
|
-
headers: {
|
|
1422
|
-
"content-type": "application/json",
|
|
1423
|
-
...buildAuthHeaders(this.apiKey),
|
|
1424
|
-
...this.headers
|
|
1425
|
-
},
|
|
1426
|
-
body: JSON.stringify({
|
|
1427
|
-
model: this.model,
|
|
1428
|
-
messages,
|
|
1429
|
-
max_tokens: request.maxOutputTokens
|
|
1430
|
-
})
|
|
1431
|
-
});
|
|
1432
|
-
if (!response.ok) {
|
|
1433
|
-
throw new Error(`Provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1434
|
-
}
|
|
1435
|
-
const payload = await response.json();
|
|
1436
|
-
const contentValue = payload.choices?.[0]?.message?.content;
|
|
1437
|
-
const text = Array.isArray(contentValue) ? contentValue.map((item) => item.text ?? "").join("\n") : contentValue ?? "";
|
|
1438
|
-
return {
|
|
1439
|
-
text,
|
|
1440
|
-
usage: payload.usage ? { inputTokens: payload.usage.prompt_tokens, outputTokens: payload.usage.completion_tokens } : void 0
|
|
1441
|
-
};
|
|
1442
|
-
}
|
|
1443
|
-
};
|
|
1444
|
-
|
|
1445
|
-
// src/providers/anthropic.ts
|
|
1446
|
-
var AnthropicProviderAdapter = class extends BaseProviderAdapter {
|
|
1447
|
-
apiKey;
|
|
1448
|
-
headers;
|
|
1449
|
-
baseUrl;
|
|
1450
|
-
constructor(id, model, options) {
|
|
1451
|
-
super(id, "anthropic", model, ["chat", "structured", "tools", "vision", "streaming"]);
|
|
1452
|
-
this.apiKey = options.apiKey;
|
|
1453
|
-
this.headers = options.headers;
|
|
1454
|
-
this.baseUrl = (options.baseUrl ?? "https://api.anthropic.com").replace(/\/+$/, "");
|
|
1455
|
-
}
|
|
1456
|
-
async generateText(request) {
|
|
1457
|
-
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1458
|
-
const content = [
|
|
1459
|
-
{ type: "text", text: request.prompt },
|
|
1460
|
-
...encodedAttachments.map((item) => ({
|
|
1461
|
-
type: "image",
|
|
1462
|
-
source: {
|
|
1463
|
-
type: "base64",
|
|
1464
|
-
media_type: item.mimeType,
|
|
1465
|
-
data: item.base64
|
|
1466
|
-
}
|
|
1467
|
-
}))
|
|
1468
|
-
];
|
|
1469
|
-
const response = await fetch(`${this.baseUrl}/v1/messages`, {
|
|
1470
|
-
method: "POST",
|
|
1471
|
-
headers: {
|
|
1472
|
-
"content-type": "application/json",
|
|
1473
|
-
"anthropic-version": "2023-06-01",
|
|
1474
|
-
...this.apiKey ? { "x-api-key": this.apiKey } : {},
|
|
1159
|
+
"anthropic-version": "2023-06-01",
|
|
1160
|
+
...this.apiKey ? { "x-api-key": this.apiKey } : {},
|
|
1475
1161
|
...this.headers
|
|
1476
1162
|
},
|
|
1477
1163
|
body: JSON.stringify({
|
|
@@ -1549,90 +1235,922 @@ ${request.system}` }] : [],
|
|
|
1549
1235
|
}
|
|
1550
1236
|
};
|
|
1551
1237
|
|
|
1552
|
-
// src/providers/
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
}
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1238
|
+
// src/providers/heuristic.ts
|
|
1239
|
+
function summarizePrompt(prompt) {
|
|
1240
|
+
const cleaned = normalizeWhitespace(prompt);
|
|
1241
|
+
if (!cleaned) {
|
|
1242
|
+
return "No prompt content provided.";
|
|
1243
|
+
}
|
|
1244
|
+
return firstSentences(cleaned, 2) || cleaned.slice(0, 280);
|
|
1245
|
+
}
|
|
1246
|
+
var HeuristicProviderAdapter = class extends BaseProviderAdapter {
|
|
1247
|
+
constructor(id, model) {
|
|
1248
|
+
super(id, "heuristic", model, ["chat", "structured", "vision", "local"]);
|
|
1249
|
+
}
|
|
1250
|
+
async generateText(request) {
|
|
1251
|
+
const attachmentHint = request.attachments?.length ? ` Attachments: ${request.attachments.length}.` : "";
|
|
1252
|
+
return {
|
|
1253
|
+
text: `Heuristic provider response.${attachmentHint} ${summarizePrompt(request.prompt)}`.trim()
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
};
|
|
1257
|
+
|
|
1258
|
+
// src/providers/openai-compatible.ts
|
|
1259
|
+
function buildAuthHeaders(apiKey) {
|
|
1260
|
+
return apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
|
|
1261
|
+
}
|
|
1262
|
+
var OpenAiCompatibleProviderAdapter = class extends BaseProviderAdapter {
|
|
1263
|
+
baseUrl;
|
|
1264
|
+
apiKey;
|
|
1265
|
+
headers;
|
|
1266
|
+
apiStyle;
|
|
1267
|
+
constructor(id, type, model, options) {
|
|
1268
|
+
super(id, type, model, options.capabilities);
|
|
1269
|
+
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
|
1270
|
+
this.apiKey = options.apiKey;
|
|
1271
|
+
this.headers = options.headers;
|
|
1272
|
+
this.apiStyle = options.apiStyle ?? "responses";
|
|
1273
|
+
}
|
|
1274
|
+
async generateText(request) {
|
|
1275
|
+
if (this.apiStyle === "chat") {
|
|
1276
|
+
return this.generateViaChatCompletions(request);
|
|
1277
|
+
}
|
|
1278
|
+
return this.generateViaResponses(request);
|
|
1279
|
+
}
|
|
1280
|
+
async generateViaResponses(request) {
|
|
1281
|
+
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1282
|
+
const input = encodedAttachments.length ? [
|
|
1283
|
+
{
|
|
1284
|
+
role: "user",
|
|
1285
|
+
content: [
|
|
1286
|
+
{ type: "input_text", text: request.prompt },
|
|
1287
|
+
...encodedAttachments.map((item) => ({
|
|
1288
|
+
type: "input_image",
|
|
1289
|
+
image_url: `data:${item.mimeType};base64,${item.base64}`
|
|
1290
|
+
}))
|
|
1291
|
+
]
|
|
1292
|
+
}
|
|
1293
|
+
] : request.prompt;
|
|
1294
|
+
const response = await fetch(`${this.baseUrl}/responses`, {
|
|
1295
|
+
method: "POST",
|
|
1296
|
+
headers: {
|
|
1297
|
+
"content-type": "application/json",
|
|
1298
|
+
...buildAuthHeaders(this.apiKey),
|
|
1299
|
+
...this.headers
|
|
1300
|
+
},
|
|
1301
|
+
body: JSON.stringify({
|
|
1302
|
+
model: this.model,
|
|
1303
|
+
input,
|
|
1304
|
+
instructions: request.system,
|
|
1305
|
+
max_output_tokens: request.maxOutputTokens
|
|
1306
|
+
})
|
|
1307
|
+
});
|
|
1308
|
+
if (!response.ok) {
|
|
1309
|
+
throw new Error(`Provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1310
|
+
}
|
|
1311
|
+
const payload = await response.json();
|
|
1312
|
+
return {
|
|
1313
|
+
text: payload.output_text ?? "",
|
|
1314
|
+
usage: payload.usage ? { inputTokens: payload.usage.input_tokens, outputTokens: payload.usage.output_tokens } : void 0
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
async generateViaChatCompletions(request) {
|
|
1318
|
+
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1319
|
+
const content = encodedAttachments.length ? [
|
|
1320
|
+
{ type: "text", text: request.prompt },
|
|
1321
|
+
...encodedAttachments.map((item) => ({
|
|
1322
|
+
type: "image_url",
|
|
1323
|
+
image_url: {
|
|
1324
|
+
url: `data:${item.mimeType};base64,${item.base64}`
|
|
1325
|
+
}
|
|
1326
|
+
}))
|
|
1327
|
+
] : request.prompt;
|
|
1328
|
+
const messages = [...request.system ? [{ role: "system", content: request.system }] : [], { role: "user", content }];
|
|
1329
|
+
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
1330
|
+
method: "POST",
|
|
1331
|
+
headers: {
|
|
1332
|
+
"content-type": "application/json",
|
|
1333
|
+
...buildAuthHeaders(this.apiKey),
|
|
1334
|
+
...this.headers
|
|
1335
|
+
},
|
|
1336
|
+
body: JSON.stringify({
|
|
1337
|
+
model: this.model,
|
|
1338
|
+
messages,
|
|
1339
|
+
max_tokens: request.maxOutputTokens
|
|
1340
|
+
})
|
|
1341
|
+
});
|
|
1342
|
+
if (!response.ok) {
|
|
1343
|
+
throw new Error(`Provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1344
|
+
}
|
|
1345
|
+
const payload = await response.json();
|
|
1346
|
+
const contentValue = payload.choices?.[0]?.message?.content;
|
|
1347
|
+
const text = Array.isArray(contentValue) ? contentValue.map((item) => item.text ?? "").join("\n") : contentValue ?? "";
|
|
1348
|
+
return {
|
|
1349
|
+
text,
|
|
1350
|
+
usage: payload.usage ? { inputTokens: payload.usage.prompt_tokens, outputTokens: payload.usage.completion_tokens } : void 0
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
};
|
|
1354
|
+
|
|
1355
|
+
// src/providers/registry.ts
|
|
1356
|
+
var customModuleSchema = z5.object({
|
|
1357
|
+
createAdapter: z5.function({
|
|
1358
|
+
input: [z5.string(), z5.custom(), z5.string()],
|
|
1359
|
+
output: z5.promise(z5.custom())
|
|
1360
|
+
})
|
|
1361
|
+
});
|
|
1362
|
+
function resolveCapabilities(config, fallback) {
|
|
1363
|
+
return config.capabilities?.length ? config.capabilities : fallback;
|
|
1364
|
+
}
|
|
1365
|
+
function envOrUndefined(name) {
|
|
1366
|
+
return name ? process.env[name] : void 0;
|
|
1367
|
+
}
|
|
1368
|
+
async function createProvider(id, config, rootDir) {
|
|
1369
|
+
switch (config.type) {
|
|
1370
|
+
case "heuristic":
|
|
1371
|
+
return new HeuristicProviderAdapter(id, config.model);
|
|
1372
|
+
case "openai":
|
|
1373
|
+
return new OpenAiCompatibleProviderAdapter(id, "openai", config.model, {
|
|
1374
|
+
baseUrl: config.baseUrl ?? "https://api.openai.com/v1",
|
|
1375
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1376
|
+
headers: config.headers,
|
|
1377
|
+
apiStyle: config.apiStyle ?? "responses",
|
|
1378
|
+
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming"])
|
|
1379
|
+
});
|
|
1380
|
+
case "ollama":
|
|
1381
|
+
return new OpenAiCompatibleProviderAdapter(id, "ollama", config.model, {
|
|
1382
|
+
baseUrl: config.baseUrl ?? "http://localhost:11434/v1",
|
|
1383
|
+
apiKey: envOrUndefined(config.apiKeyEnv) ?? "ollama",
|
|
1384
|
+
headers: config.headers,
|
|
1385
|
+
apiStyle: config.apiStyle ?? "responses",
|
|
1386
|
+
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming", "local"])
|
|
1387
|
+
});
|
|
1388
|
+
case "openai-compatible":
|
|
1389
|
+
return new OpenAiCompatibleProviderAdapter(id, "openai-compatible", config.model, {
|
|
1390
|
+
baseUrl: config.baseUrl ?? "http://localhost:8000/v1",
|
|
1391
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1392
|
+
headers: config.headers,
|
|
1393
|
+
apiStyle: config.apiStyle ?? "responses",
|
|
1394
|
+
capabilities: resolveCapabilities(config, ["chat", "structured"])
|
|
1395
|
+
});
|
|
1396
|
+
case "anthropic":
|
|
1397
|
+
return new AnthropicProviderAdapter(id, config.model, {
|
|
1398
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1399
|
+
headers: config.headers,
|
|
1400
|
+
baseUrl: config.baseUrl
|
|
1401
|
+
});
|
|
1402
|
+
case "gemini":
|
|
1403
|
+
return new GeminiProviderAdapter(id, config.model, {
|
|
1404
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1405
|
+
baseUrl: config.baseUrl
|
|
1406
|
+
});
|
|
1407
|
+
case "custom": {
|
|
1408
|
+
if (!config.module) {
|
|
1409
|
+
throw new Error(`Provider ${id} is type "custom" but no module path was configured.`);
|
|
1410
|
+
}
|
|
1411
|
+
const resolvedModule = path8.isAbsolute(config.module) ? config.module : path8.resolve(rootDir, config.module);
|
|
1412
|
+
const loaded = await import(pathToFileURL(resolvedModule).href);
|
|
1413
|
+
const parsed = customModuleSchema.parse(loaded);
|
|
1414
|
+
return parsed.createAdapter(id, config, rootDir);
|
|
1415
|
+
}
|
|
1416
|
+
default:
|
|
1417
|
+
throw new Error(`Unsupported provider type ${String(config.type)}`);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
async function getProviderForTask(rootDir, task) {
|
|
1421
|
+
const { config } = await loadVaultConfig(rootDir);
|
|
1422
|
+
const providerId = config.tasks[task];
|
|
1423
|
+
const providerConfig = config.providers[providerId];
|
|
1424
|
+
if (!providerConfig) {
|
|
1425
|
+
throw new Error(`No provider configured with id "${providerId}" for task "${task}".`);
|
|
1426
|
+
}
|
|
1427
|
+
return createProvider(providerId, providerConfig, rootDir);
|
|
1428
|
+
}
|
|
1429
|
+
function assertProviderCapability(provider, capability) {
|
|
1430
|
+
if (!provider.capabilities.has(capability)) {
|
|
1431
|
+
throw new Error(`Provider ${provider.id} does not support required capability "${capability}".`);
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
// src/web-search/registry.ts
|
|
1436
|
+
import path9 from "path";
|
|
1437
|
+
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
1438
|
+
import { z as z6 } from "zod";
|
|
1439
|
+
|
|
1440
|
+
// src/web-search/http-json.ts
|
|
1441
|
+
function deepGet(value, pathValue) {
|
|
1442
|
+
if (!pathValue) {
|
|
1443
|
+
return value;
|
|
1444
|
+
}
|
|
1445
|
+
return pathValue.split(".").filter(Boolean).reduce((current, segment) => {
|
|
1446
|
+
if (current && typeof current === "object" && segment in current) {
|
|
1447
|
+
return current[segment];
|
|
1448
|
+
}
|
|
1449
|
+
return void 0;
|
|
1450
|
+
}, value);
|
|
1451
|
+
}
|
|
1452
|
+
function envOrUndefined2(name) {
|
|
1453
|
+
return name ? process.env[name] : void 0;
|
|
1454
|
+
}
|
|
1455
|
+
var HttpJsonWebSearchAdapter = class {
|
|
1456
|
+
constructor(id, config) {
|
|
1457
|
+
this.id = id;
|
|
1458
|
+
this.config = config;
|
|
1459
|
+
}
|
|
1460
|
+
id;
|
|
1461
|
+
config;
|
|
1462
|
+
type = "http-json";
|
|
1463
|
+
async search(query, limit = 5) {
|
|
1464
|
+
if (!this.config.endpoint) {
|
|
1465
|
+
throw new Error(`Web search provider ${this.id} is missing an endpoint.`);
|
|
1466
|
+
}
|
|
1467
|
+
const method = this.config.method ?? "GET";
|
|
1468
|
+
const queryParam = this.config.queryParam ?? "q";
|
|
1469
|
+
const limitParam = this.config.limitParam ?? "limit";
|
|
1470
|
+
const headers = {
|
|
1471
|
+
accept: "application/json",
|
|
1472
|
+
...this.config.headers
|
|
1473
|
+
};
|
|
1474
|
+
const apiKey = envOrUndefined2(this.config.apiKeyEnv);
|
|
1475
|
+
if (apiKey) {
|
|
1476
|
+
headers[this.config.apiKeyHeader ?? "Authorization"] = `${this.config.apiKeyPrefix ?? "Bearer "}${apiKey}`;
|
|
1477
|
+
}
|
|
1478
|
+
const endpoint = new URL(this.config.endpoint);
|
|
1479
|
+
let body;
|
|
1480
|
+
if (method === "GET") {
|
|
1481
|
+
endpoint.searchParams.set(queryParam, query);
|
|
1482
|
+
endpoint.searchParams.set(limitParam, String(limit));
|
|
1483
|
+
} else {
|
|
1484
|
+
headers["content-type"] = "application/json";
|
|
1485
|
+
body = JSON.stringify({
|
|
1486
|
+
[queryParam]: query,
|
|
1487
|
+
[limitParam]: limit
|
|
1488
|
+
});
|
|
1489
|
+
}
|
|
1490
|
+
const response = await fetch(endpoint, {
|
|
1491
|
+
method,
|
|
1492
|
+
headers,
|
|
1493
|
+
body
|
|
1494
|
+
});
|
|
1495
|
+
if (!response.ok) {
|
|
1496
|
+
throw new Error(`Web search provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1497
|
+
}
|
|
1498
|
+
const payload = await response.json();
|
|
1499
|
+
const rawResults = deepGet(payload, this.config.resultsPath ?? "results");
|
|
1500
|
+
if (!Array.isArray(rawResults)) {
|
|
1501
|
+
return [];
|
|
1502
|
+
}
|
|
1503
|
+
return rawResults.map((item) => {
|
|
1504
|
+
const title = deepGet(item, this.config.titleField ?? "title");
|
|
1505
|
+
const url = deepGet(item, this.config.urlField ?? "url");
|
|
1506
|
+
const snippet = deepGet(item, this.config.snippetField ?? "snippet");
|
|
1507
|
+
if (typeof title !== "string" || typeof url !== "string") {
|
|
1508
|
+
return null;
|
|
1509
|
+
}
|
|
1510
|
+
return {
|
|
1511
|
+
title,
|
|
1512
|
+
url,
|
|
1513
|
+
snippet: typeof snippet === "string" ? snippet : ""
|
|
1514
|
+
};
|
|
1515
|
+
}).filter((item) => item !== null);
|
|
1516
|
+
}
|
|
1517
|
+
};
|
|
1518
|
+
|
|
1519
|
+
// src/web-search/registry.ts
|
|
1520
|
+
var customWebSearchModuleSchema = z6.object({
|
|
1521
|
+
createAdapter: z6.function({
|
|
1522
|
+
input: [z6.string(), z6.custom(), z6.string()],
|
|
1523
|
+
output: z6.promise(z6.custom())
|
|
1524
|
+
})
|
|
1525
|
+
});
|
|
1526
|
+
async function createWebSearchAdapter(id, config, rootDir) {
|
|
1527
|
+
switch (config.type) {
|
|
1528
|
+
case "http-json":
|
|
1529
|
+
return new HttpJsonWebSearchAdapter(id, config);
|
|
1530
|
+
case "custom": {
|
|
1531
|
+
if (!config.module) {
|
|
1532
|
+
throw new Error(`Web search provider ${id} is type "custom" but no module path was configured.`);
|
|
1533
|
+
}
|
|
1534
|
+
const resolvedModule = path9.isAbsolute(config.module) ? config.module : path9.resolve(rootDir, config.module);
|
|
1535
|
+
const loaded = await import(pathToFileURL2(resolvedModule).href);
|
|
1536
|
+
const parsed = customWebSearchModuleSchema.parse(loaded);
|
|
1537
|
+
return parsed.createAdapter(id, config, rootDir);
|
|
1538
|
+
}
|
|
1539
|
+
default:
|
|
1540
|
+
throw new Error(`Unsupported web search provider type ${String(config.type)}`);
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
async function getWebSearchAdapterForTask(rootDir, task) {
|
|
1544
|
+
const { config } = await loadVaultConfig(rootDir);
|
|
1545
|
+
const webSearchConfig = config.webSearch;
|
|
1546
|
+
if (!webSearchConfig) {
|
|
1547
|
+
throw new Error("No web search providers are configured. Add a webSearch block to swarmvault.config.json.");
|
|
1548
|
+
}
|
|
1549
|
+
const providerId = webSearchConfig.tasks[task];
|
|
1550
|
+
const providerConfig = webSearchConfig.providers[providerId];
|
|
1551
|
+
if (!providerConfig) {
|
|
1552
|
+
throw new Error(`No web search provider configured with id "${providerId}" for task "${task}".`);
|
|
1553
|
+
}
|
|
1554
|
+
return createWebSearchAdapter(providerId, providerConfig, rootDir);
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
// src/deep-lint.ts
|
|
1558
|
+
var deepLintResponseSchema = z7.object({
|
|
1559
|
+
findings: z7.array(
|
|
1560
|
+
z7.object({
|
|
1561
|
+
severity: z7.enum(["error", "warning", "info"]).default("info"),
|
|
1562
|
+
code: z7.enum(["coverage_gap", "contradiction_candidate", "missing_citation", "candidate_page", "follow_up_question"]),
|
|
1563
|
+
message: z7.string().min(1),
|
|
1564
|
+
relatedSourceIds: z7.array(z7.string()).default([]),
|
|
1565
|
+
relatedPageIds: z7.array(z7.string()).default([]),
|
|
1566
|
+
suggestedQuery: z7.string().optional()
|
|
1567
|
+
})
|
|
1568
|
+
).max(20)
|
|
1569
|
+
});
|
|
1570
|
+
async function loadContextPages(rootDir, graph) {
|
|
1571
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
1572
|
+
const contextPages = graph.pages.filter(
|
|
1573
|
+
(page) => page.kind === "source" || page.kind === "concept" || page.kind === "entity"
|
|
1574
|
+
);
|
|
1575
|
+
return Promise.all(
|
|
1576
|
+
contextPages.slice(0, 18).map(async (page) => {
|
|
1577
|
+
const absolutePath = path10.join(paths.wikiDir, page.path);
|
|
1578
|
+
const raw = await fs8.readFile(absolutePath, "utf8").catch(() => "");
|
|
1579
|
+
const parsed = matter(raw);
|
|
1580
|
+
return {
|
|
1581
|
+
id: page.id,
|
|
1582
|
+
title: page.title,
|
|
1583
|
+
path: page.path,
|
|
1584
|
+
kind: page.kind,
|
|
1585
|
+
sourceIds: page.sourceIds,
|
|
1586
|
+
excerpt: truncate(normalizeWhitespace(parsed.content), 1400)
|
|
1587
|
+
};
|
|
1588
|
+
})
|
|
1589
|
+
);
|
|
1590
|
+
}
|
|
1591
|
+
function heuristicDeepFindings(contextPages, structuralFindings) {
|
|
1592
|
+
const findings = [];
|
|
1593
|
+
for (const page of contextPages) {
|
|
1594
|
+
if (page.excerpt.includes("No claims extracted.")) {
|
|
1595
|
+
findings.push({
|
|
1596
|
+
severity: "warning",
|
|
1597
|
+
code: "coverage_gap",
|
|
1598
|
+
message: `Page ${page.title} has no extracted claims yet.`,
|
|
1599
|
+
pagePath: page.path,
|
|
1600
|
+
relatedSourceIds: page.sourceIds,
|
|
1601
|
+
relatedPageIds: [page.id],
|
|
1602
|
+
suggestedQuery: `What evidence or claims should ${page.title} contain?`
|
|
1603
|
+
});
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
for (const finding of structuralFindings.filter((item) => item.code === "uncited_claims").slice(0, 5)) {
|
|
1607
|
+
findings.push({
|
|
1608
|
+
severity: "warning",
|
|
1609
|
+
code: "missing_citation",
|
|
1610
|
+
message: finding.message,
|
|
1611
|
+
pagePath: finding.pagePath,
|
|
1612
|
+
suggestedQuery: finding.pagePath ? `Which sources support the claims in ${path10.basename(finding.pagePath, ".md")}?` : void 0
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1615
|
+
for (const page of contextPages.filter((item) => item.kind === "source").slice(0, 3)) {
|
|
1616
|
+
findings.push({
|
|
1617
|
+
severity: "info",
|
|
1618
|
+
code: "follow_up_question",
|
|
1619
|
+
message: `Investigate what broader implications ${page.title} has for the rest of the vault.`,
|
|
1620
|
+
pagePath: page.path,
|
|
1621
|
+
relatedSourceIds: page.sourceIds,
|
|
1622
|
+
relatedPageIds: [page.id],
|
|
1623
|
+
suggestedQuery: `What broader implications does ${page.title} have?`
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
return uniqueBy(findings, (item) => `${item.code}:${item.message}`);
|
|
1627
|
+
}
|
|
1628
|
+
async function runDeepLint(rootDir, structuralFindings, options = {}) {
|
|
1629
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
1630
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
1631
|
+
if (!graph) {
|
|
1632
|
+
return [];
|
|
1633
|
+
}
|
|
1634
|
+
const schema = await loadVaultSchema(rootDir);
|
|
1635
|
+
const provider = await getProviderForTask(rootDir, "lintProvider");
|
|
1636
|
+
const manifests = await listManifests(rootDir);
|
|
1637
|
+
const contextPages = await loadContextPages(rootDir, graph);
|
|
1638
|
+
let findings;
|
|
1639
|
+
if (provider.type === "heuristic") {
|
|
1640
|
+
findings = heuristicDeepFindings(contextPages, structuralFindings);
|
|
1641
|
+
} else {
|
|
1642
|
+
const response = await provider.generateStructured(
|
|
1643
|
+
{
|
|
1644
|
+
system: "You are an auditor for a local-first LLM knowledge vault. Return advisory findings only. Do not propose direct file edits.",
|
|
1645
|
+
prompt: [
|
|
1646
|
+
"Review this SwarmVault state and return high-signal advisory findings.",
|
|
1647
|
+
"",
|
|
1648
|
+
"Schema:",
|
|
1649
|
+
schema.content,
|
|
1650
|
+
"",
|
|
1651
|
+
"Vault summary:",
|
|
1652
|
+
`- sources: ${manifests.length}`,
|
|
1653
|
+
`- pages: ${graph.pages.length}`,
|
|
1654
|
+
`- structural_findings: ${structuralFindings.length}`,
|
|
1655
|
+
"",
|
|
1656
|
+
"Structural findings:",
|
|
1657
|
+
structuralFindings.map((item) => `- [${item.severity}] ${item.code}: ${item.message}`).join("\n") || "- none",
|
|
1658
|
+
"",
|
|
1659
|
+
"Page context:",
|
|
1660
|
+
contextPages.map(
|
|
1661
|
+
(page) => [
|
|
1662
|
+
`## ${page.title}`,
|
|
1663
|
+
`page_id: ${page.id}`,
|
|
1664
|
+
`path: ${page.path}`,
|
|
1665
|
+
`kind: ${page.kind}`,
|
|
1666
|
+
`source_ids: ${page.sourceIds.join(",") || "none"}`,
|
|
1667
|
+
page.excerpt
|
|
1668
|
+
].join("\n")
|
|
1669
|
+
).join("\n\n---\n\n")
|
|
1670
|
+
].join("\n")
|
|
1671
|
+
},
|
|
1672
|
+
deepLintResponseSchema
|
|
1673
|
+
);
|
|
1674
|
+
findings = response.findings.map((item) => ({
|
|
1675
|
+
severity: item.severity,
|
|
1676
|
+
code: item.code,
|
|
1677
|
+
message: item.message,
|
|
1678
|
+
relatedSourceIds: item.relatedSourceIds,
|
|
1679
|
+
relatedPageIds: item.relatedPageIds,
|
|
1680
|
+
suggestedQuery: item.suggestedQuery
|
|
1681
|
+
}));
|
|
1682
|
+
}
|
|
1683
|
+
if (!options.web) {
|
|
1684
|
+
return findings;
|
|
1685
|
+
}
|
|
1686
|
+
const webSearch = await getWebSearchAdapterForTask(rootDir, "deepLintProvider");
|
|
1687
|
+
const queryCache = /* @__PURE__ */ new Map();
|
|
1688
|
+
for (const finding of findings) {
|
|
1689
|
+
const query = finding.suggestedQuery ?? finding.message;
|
|
1690
|
+
if (!queryCache.has(query)) {
|
|
1691
|
+
queryCache.set(query, await webSearch.search(query, 3));
|
|
1692
|
+
}
|
|
1693
|
+
finding.evidence = queryCache.get(query);
|
|
1694
|
+
}
|
|
1695
|
+
return findings;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
// src/markdown.ts
|
|
1699
|
+
import matter2 from "gray-matter";
|
|
1700
|
+
function pagePathFor(kind, slug) {
|
|
1701
|
+
switch (kind) {
|
|
1702
|
+
case "source":
|
|
1703
|
+
return `sources/${slug}.md`;
|
|
1704
|
+
case "concept":
|
|
1705
|
+
return `concepts/${slug}.md`;
|
|
1706
|
+
case "entity":
|
|
1707
|
+
return `entities/${slug}.md`;
|
|
1708
|
+
case "output":
|
|
1709
|
+
return `outputs/${slug}.md`;
|
|
1710
|
+
default:
|
|
1711
|
+
return `${slug}.md`;
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
function pageLink(page) {
|
|
1715
|
+
return `[[${page.path.replace(/\.md$/, "")}|${page.title}]]`;
|
|
1716
|
+
}
|
|
1717
|
+
function relatedOutputsSection(relatedOutputs) {
|
|
1718
|
+
if (!relatedOutputs.length) {
|
|
1719
|
+
return [];
|
|
1720
|
+
}
|
|
1721
|
+
return ["## Related Outputs", "", ...relatedOutputs.map((page) => `- ${pageLink(page)}`), ""];
|
|
1722
|
+
}
|
|
1723
|
+
function buildSourcePage(manifest, analysis, schemaHash, confidence = 1, relatedOutputs = []) {
|
|
1724
|
+
const relativePath = pagePathFor("source", manifest.sourceId);
|
|
1725
|
+
const pageId = `source:${manifest.sourceId}`;
|
|
1726
|
+
const nodeIds = [`source:${manifest.sourceId}`, ...analysis.concepts.map((item) => item.id), ...analysis.entities.map((item) => item.id)];
|
|
1727
|
+
const backlinks = [
|
|
1728
|
+
...analysis.concepts.map((item) => `concept:${slugify(item.name)}`),
|
|
1729
|
+
...analysis.entities.map((item) => `entity:${slugify(item.name)}`),
|
|
1730
|
+
...relatedOutputs.map((page) => page.id)
|
|
1731
|
+
];
|
|
1732
|
+
const frontmatter = {
|
|
1733
|
+
page_id: pageId,
|
|
1734
|
+
kind: "source",
|
|
1735
|
+
title: analysis.title,
|
|
1736
|
+
tags: ["source"],
|
|
1737
|
+
source_ids: [manifest.sourceId],
|
|
1738
|
+
node_ids: nodeIds,
|
|
1739
|
+
freshness: "fresh",
|
|
1740
|
+
confidence,
|
|
1741
|
+
updated_at: analysis.producedAt,
|
|
1742
|
+
backlinks,
|
|
1743
|
+
schema_hash: schemaHash,
|
|
1744
|
+
source_hashes: {
|
|
1745
|
+
[manifest.sourceId]: manifest.contentHash
|
|
1746
|
+
}
|
|
1747
|
+
};
|
|
1748
|
+
const body = [
|
|
1749
|
+
`# ${analysis.title}`,
|
|
1750
|
+
"",
|
|
1751
|
+
`Source ID: \`${manifest.sourceId}\``,
|
|
1752
|
+
manifest.url ? `Source URL: ${manifest.url}` : `Source Path: \`${manifest.originalPath ?? manifest.storedPath}\``,
|
|
1753
|
+
"",
|
|
1754
|
+
"## Summary",
|
|
1755
|
+
"",
|
|
1756
|
+
analysis.summary,
|
|
1757
|
+
"",
|
|
1758
|
+
"## Concepts",
|
|
1759
|
+
"",
|
|
1760
|
+
...analysis.concepts.length ? analysis.concepts.map(
|
|
1761
|
+
(item) => `- [[${pagePathFor("concept", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`
|
|
1762
|
+
) : ["- None detected."],
|
|
1763
|
+
"",
|
|
1764
|
+
"## Entities",
|
|
1765
|
+
"",
|
|
1766
|
+
...analysis.entities.length ? analysis.entities.map(
|
|
1767
|
+
(item) => `- [[${pagePathFor("entity", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`
|
|
1768
|
+
) : ["- None detected."],
|
|
1769
|
+
"",
|
|
1770
|
+
"## Claims",
|
|
1771
|
+
"",
|
|
1772
|
+
...analysis.claims.length ? analysis.claims.map((claim) => `- ${claim.text} [source:${claim.citation}]`) : ["- No claims extracted."],
|
|
1773
|
+
"",
|
|
1774
|
+
"## Questions",
|
|
1775
|
+
"",
|
|
1776
|
+
...analysis.questions.length ? analysis.questions.map((question) => `- ${question}`) : ["- No follow-up questions yet."],
|
|
1777
|
+
"",
|
|
1778
|
+
...relatedOutputsSection(relatedOutputs),
|
|
1779
|
+
""
|
|
1780
|
+
].join("\n");
|
|
1781
|
+
return {
|
|
1782
|
+
page: {
|
|
1783
|
+
id: pageId,
|
|
1784
|
+
path: relativePath,
|
|
1785
|
+
title: analysis.title,
|
|
1786
|
+
kind: "source",
|
|
1787
|
+
sourceIds: [manifest.sourceId],
|
|
1788
|
+
nodeIds,
|
|
1789
|
+
freshness: "fresh",
|
|
1790
|
+
confidence,
|
|
1791
|
+
backlinks,
|
|
1792
|
+
schemaHash,
|
|
1793
|
+
sourceHashes: { [manifest.sourceId]: manifest.contentHash },
|
|
1794
|
+
relatedPageIds: relatedOutputs.map((page) => page.id),
|
|
1795
|
+
relatedNodeIds: [],
|
|
1796
|
+
relatedSourceIds: []
|
|
1797
|
+
},
|
|
1798
|
+
content: matter2.stringify(body, frontmatter)
|
|
1799
|
+
};
|
|
1800
|
+
}
|
|
1801
|
+
function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes, schemaHash, confidence = 0.72, relatedOutputs = []) {
|
|
1802
|
+
const slug = slugify(name);
|
|
1803
|
+
const relativePath = pagePathFor(kind, slug);
|
|
1804
|
+
const pageId = `${kind}:${slug}`;
|
|
1805
|
+
const sourceIds = sourceAnalyses.map((item) => item.sourceId);
|
|
1806
|
+
const otherPages = [...sourceAnalyses.map((item) => `source:${item.sourceId}`), ...relatedOutputs.map((page) => page.id)];
|
|
1807
|
+
const summary = descriptions.find(Boolean) ?? `${kind} aggregated from ${sourceIds.length} source(s).`;
|
|
1808
|
+
const frontmatter = {
|
|
1809
|
+
page_id: pageId,
|
|
1810
|
+
kind,
|
|
1811
|
+
title: name,
|
|
1812
|
+
tags: [kind],
|
|
1813
|
+
source_ids: sourceIds,
|
|
1814
|
+
node_ids: [pageId],
|
|
1815
|
+
freshness: "fresh",
|
|
1816
|
+
confidence,
|
|
1817
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1818
|
+
backlinks: otherPages,
|
|
1819
|
+
schema_hash: schemaHash,
|
|
1820
|
+
source_hashes: sourceHashes
|
|
1821
|
+
};
|
|
1822
|
+
const body = [
|
|
1823
|
+
`# ${name}`,
|
|
1824
|
+
"",
|
|
1825
|
+
"## Summary",
|
|
1826
|
+
"",
|
|
1827
|
+
summary,
|
|
1828
|
+
"",
|
|
1829
|
+
"## Seen In",
|
|
1830
|
+
"",
|
|
1831
|
+
...sourceAnalyses.map((item) => `- [[${pagePathFor("source", item.sourceId).replace(/\.md$/, "")}|${item.title}]]`),
|
|
1832
|
+
"",
|
|
1833
|
+
"## Source Claims",
|
|
1834
|
+
"",
|
|
1835
|
+
...sourceAnalyses.flatMap(
|
|
1836
|
+
(item) => item.claims.filter((claim) => claim.text.toLowerCase().includes(name.toLowerCase())).map((claim) => `- ${claim.text} [source:${claim.citation}]`)
|
|
1837
|
+
),
|
|
1838
|
+
"",
|
|
1839
|
+
...relatedOutputsSection(relatedOutputs),
|
|
1840
|
+
""
|
|
1841
|
+
].join("\n");
|
|
1842
|
+
return {
|
|
1843
|
+
page: {
|
|
1844
|
+
id: pageId,
|
|
1845
|
+
path: relativePath,
|
|
1846
|
+
title: name,
|
|
1847
|
+
kind,
|
|
1848
|
+
sourceIds,
|
|
1849
|
+
nodeIds: [pageId],
|
|
1850
|
+
freshness: "fresh",
|
|
1851
|
+
confidence,
|
|
1852
|
+
backlinks: otherPages,
|
|
1853
|
+
schemaHash,
|
|
1854
|
+
sourceHashes,
|
|
1855
|
+
relatedPageIds: relatedOutputs.map((page) => page.id),
|
|
1856
|
+
relatedNodeIds: [],
|
|
1857
|
+
relatedSourceIds: []
|
|
1858
|
+
},
|
|
1859
|
+
content: matter2.stringify(body, frontmatter)
|
|
1860
|
+
};
|
|
1861
|
+
}
|
|
1862
|
+
function buildIndexPage(pages, schemaHash) {
|
|
1863
|
+
const sources = pages.filter((page) => page.kind === "source");
|
|
1864
|
+
const concepts = pages.filter((page) => page.kind === "concept");
|
|
1865
|
+
const entities = pages.filter((page) => page.kind === "entity");
|
|
1866
|
+
const outputs = pages.filter((page) => page.kind === "output");
|
|
1867
|
+
return [
|
|
1868
|
+
"---",
|
|
1869
|
+
"page_id: index",
|
|
1870
|
+
"kind: index",
|
|
1871
|
+
"title: SwarmVault Index",
|
|
1872
|
+
"tags:",
|
|
1873
|
+
" - index",
|
|
1874
|
+
"source_ids: []",
|
|
1875
|
+
"node_ids: []",
|
|
1876
|
+
"freshness: fresh",
|
|
1877
|
+
"confidence: 1",
|
|
1878
|
+
`updated_at: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
1879
|
+
"backlinks: []",
|
|
1880
|
+
`schema_hash: ${schemaHash}`,
|
|
1881
|
+
"source_hashes: {}",
|
|
1882
|
+
"---",
|
|
1883
|
+
"",
|
|
1884
|
+
"# SwarmVault Index",
|
|
1885
|
+
"",
|
|
1886
|
+
"## Sources",
|
|
1887
|
+
"",
|
|
1888
|
+
...sources.length ? sources.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No sources yet."],
|
|
1889
|
+
"",
|
|
1890
|
+
"## Concepts",
|
|
1891
|
+
"",
|
|
1892
|
+
...concepts.length ? concepts.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No concepts yet."],
|
|
1893
|
+
"",
|
|
1894
|
+
"## Entities",
|
|
1895
|
+
"",
|
|
1896
|
+
...entities.length ? entities.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No entities yet."],
|
|
1897
|
+
"",
|
|
1898
|
+
"## Outputs",
|
|
1899
|
+
"",
|
|
1900
|
+
...outputs.length ? outputs.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No saved outputs yet."],
|
|
1901
|
+
""
|
|
1902
|
+
].join("\n");
|
|
1903
|
+
}
|
|
1904
|
+
function buildSectionIndex(kind, pages, schemaHash) {
|
|
1905
|
+
const title = kind.charAt(0).toUpperCase() + kind.slice(1);
|
|
1906
|
+
return matter2.stringify(
|
|
1907
|
+
[`# ${title}`, "", ...pages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`), ""].join("\n"),
|
|
1908
|
+
{
|
|
1909
|
+
page_id: `${kind}:index`,
|
|
1910
|
+
kind: "index",
|
|
1911
|
+
title,
|
|
1912
|
+
tags: ["index", kind],
|
|
1913
|
+
source_ids: [],
|
|
1914
|
+
node_ids: [],
|
|
1915
|
+
freshness: "fresh",
|
|
1916
|
+
confidence: 1,
|
|
1917
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1918
|
+
backlinks: [],
|
|
1919
|
+
schema_hash: schemaHash,
|
|
1920
|
+
source_hashes: {}
|
|
1921
|
+
}
|
|
1922
|
+
);
|
|
1923
|
+
}
|
|
1924
|
+
function buildOutputPage(input) {
|
|
1925
|
+
const slug = input.slug ?? slugify(input.question);
|
|
1926
|
+
const pageId = `output:${slug}`;
|
|
1927
|
+
const pathValue = pagePathFor("output", slug);
|
|
1928
|
+
const relatedPageIds = input.relatedPageIds ?? [];
|
|
1929
|
+
const relatedNodeIds = input.relatedNodeIds ?? [];
|
|
1930
|
+
const relatedSourceIds = input.relatedSourceIds ?? input.citations;
|
|
1931
|
+
const backlinks = [.../* @__PURE__ */ new Set([...relatedPageIds, ...relatedSourceIds.map((sourceId) => `source:${sourceId}`)])];
|
|
1932
|
+
const frontmatter = {
|
|
1933
|
+
page_id: pageId,
|
|
1934
|
+
kind: "output",
|
|
1935
|
+
title: input.title ?? input.question,
|
|
1936
|
+
tags: ["output"],
|
|
1937
|
+
source_ids: input.citations,
|
|
1938
|
+
node_ids: relatedNodeIds,
|
|
1939
|
+
freshness: "fresh",
|
|
1940
|
+
confidence: 0.74,
|
|
1941
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1942
|
+
backlinks,
|
|
1943
|
+
schema_hash: input.schemaHash,
|
|
1944
|
+
source_hashes: {},
|
|
1945
|
+
related_page_ids: relatedPageIds,
|
|
1946
|
+
related_node_ids: relatedNodeIds,
|
|
1947
|
+
related_source_ids: relatedSourceIds,
|
|
1948
|
+
origin: input.origin,
|
|
1949
|
+
question: input.question
|
|
1950
|
+
};
|
|
1951
|
+
return {
|
|
1952
|
+
page: {
|
|
1953
|
+
id: pageId,
|
|
1954
|
+
path: pathValue,
|
|
1955
|
+
title: input.title ?? input.question,
|
|
1956
|
+
kind: "output",
|
|
1957
|
+
sourceIds: input.citations,
|
|
1958
|
+
nodeIds: relatedNodeIds,
|
|
1959
|
+
freshness: "fresh",
|
|
1960
|
+
confidence: 0.74,
|
|
1961
|
+
backlinks,
|
|
1962
|
+
schemaHash: input.schemaHash,
|
|
1963
|
+
sourceHashes: {},
|
|
1964
|
+
relatedPageIds,
|
|
1965
|
+
relatedNodeIds,
|
|
1966
|
+
relatedSourceIds,
|
|
1967
|
+
origin: input.origin,
|
|
1968
|
+
question: input.question
|
|
1969
|
+
},
|
|
1970
|
+
content: matter2.stringify(
|
|
1971
|
+
[
|
|
1972
|
+
`# ${input.title ?? input.question}`,
|
|
1973
|
+
"",
|
|
1974
|
+
input.answer,
|
|
1975
|
+
"",
|
|
1976
|
+
"## Related Pages",
|
|
1977
|
+
"",
|
|
1978
|
+
...relatedPageIds.length ? relatedPageIds.map((pageId2) => `- \`${pageId2}\``) : ["- None recorded."],
|
|
1979
|
+
"",
|
|
1980
|
+
"## Citations",
|
|
1981
|
+
"",
|
|
1982
|
+
...input.citations.map((citation) => `- [source:${citation}]`),
|
|
1983
|
+
""
|
|
1984
|
+
].join("\n"),
|
|
1985
|
+
frontmatter
|
|
1986
|
+
)
|
|
1987
|
+
};
|
|
1988
|
+
}
|
|
1989
|
+
function buildExploreHubPage(input) {
|
|
1990
|
+
const slug = input.slug ?? `explore-${slugify(input.question)}`;
|
|
1991
|
+
const pageId = `output:${slug}`;
|
|
1992
|
+
const pathValue = pagePathFor("output", slug);
|
|
1993
|
+
const relatedPageIds = input.stepPages.map((page) => page.id);
|
|
1994
|
+
const relatedSourceIds = [...new Set(input.citations)];
|
|
1995
|
+
const relatedNodeIds = [...new Set(input.stepPages.flatMap((page) => page.nodeIds))];
|
|
1996
|
+
const backlinks = [.../* @__PURE__ */ new Set([...relatedPageIds, ...relatedSourceIds.map((sourceId) => `source:${sourceId}`)])];
|
|
1997
|
+
const title = `Explore: ${input.question}`;
|
|
1998
|
+
const frontmatter = {
|
|
1999
|
+
page_id: pageId,
|
|
2000
|
+
kind: "output",
|
|
2001
|
+
title,
|
|
2002
|
+
tags: ["output", "explore"],
|
|
2003
|
+
source_ids: relatedSourceIds,
|
|
2004
|
+
node_ids: relatedNodeIds,
|
|
2005
|
+
freshness: "fresh",
|
|
2006
|
+
confidence: 0.76,
|
|
2007
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2008
|
+
backlinks,
|
|
2009
|
+
schema_hash: input.schemaHash,
|
|
2010
|
+
source_hashes: {},
|
|
2011
|
+
related_page_ids: relatedPageIds,
|
|
2012
|
+
related_node_ids: relatedNodeIds,
|
|
2013
|
+
related_source_ids: relatedSourceIds,
|
|
2014
|
+
origin: "explore",
|
|
2015
|
+
question: input.question
|
|
2016
|
+
};
|
|
2017
|
+
return {
|
|
2018
|
+
page: {
|
|
2019
|
+
id: pageId,
|
|
2020
|
+
path: pathValue,
|
|
2021
|
+
title,
|
|
2022
|
+
kind: "output",
|
|
2023
|
+
sourceIds: relatedSourceIds,
|
|
2024
|
+
nodeIds: relatedNodeIds,
|
|
2025
|
+
freshness: "fresh",
|
|
2026
|
+
confidence: 0.76,
|
|
2027
|
+
backlinks,
|
|
2028
|
+
schemaHash: input.schemaHash,
|
|
2029
|
+
sourceHashes: {},
|
|
2030
|
+
relatedPageIds,
|
|
2031
|
+
relatedNodeIds,
|
|
2032
|
+
relatedSourceIds,
|
|
2033
|
+
origin: "explore",
|
|
2034
|
+
question: input.question
|
|
2035
|
+
},
|
|
2036
|
+
content: matter2.stringify(
|
|
2037
|
+
[
|
|
2038
|
+
`# ${title}`,
|
|
2039
|
+
"",
|
|
2040
|
+
"## Root Question",
|
|
2041
|
+
"",
|
|
2042
|
+
input.question,
|
|
2043
|
+
"",
|
|
2044
|
+
"## Steps",
|
|
2045
|
+
"",
|
|
2046
|
+
...input.stepPages.length ? input.stepPages.map((page) => `- ${pageLink(page)}`) : ["- No steps recorded."],
|
|
2047
|
+
"",
|
|
2048
|
+
"## Follow-Up Questions",
|
|
2049
|
+
"",
|
|
2050
|
+
...input.followUpQuestions.length ? input.followUpQuestions.map((question) => `- ${question}`) : ["- No follow-up questions generated."],
|
|
2051
|
+
"",
|
|
2052
|
+
"## Citations",
|
|
2053
|
+
"",
|
|
2054
|
+
...relatedSourceIds.map((citation) => `- [source:${citation}]`),
|
|
2055
|
+
""
|
|
2056
|
+
].join("\n"),
|
|
2057
|
+
frontmatter
|
|
2058
|
+
)
|
|
2059
|
+
};
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
// src/outputs.ts
|
|
2063
|
+
import fs9 from "fs/promises";
|
|
2064
|
+
import path11 from "path";
|
|
2065
|
+
import matter3 from "gray-matter";
|
|
2066
|
+
function normalizeStringArray(value) {
|
|
2067
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1561
2068
|
}
|
|
1562
|
-
function
|
|
1563
|
-
|
|
2069
|
+
function normalizeSourceHashes(value) {
|
|
2070
|
+
if (!value || typeof value !== "object") {
|
|
2071
|
+
return {};
|
|
2072
|
+
}
|
|
2073
|
+
return Object.fromEntries(
|
|
2074
|
+
Object.entries(value).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
|
|
2075
|
+
);
|
|
1564
2076
|
}
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
baseUrl: config.baseUrl ?? "https://api.openai.com/v1",
|
|
1572
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1573
|
-
headers: config.headers,
|
|
1574
|
-
apiStyle: config.apiStyle ?? "responses",
|
|
1575
|
-
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming"])
|
|
1576
|
-
});
|
|
1577
|
-
case "ollama":
|
|
1578
|
-
return new OpenAiCompatibleProviderAdapter(id, "ollama", config.model, {
|
|
1579
|
-
baseUrl: config.baseUrl ?? "http://localhost:11434/v1",
|
|
1580
|
-
apiKey: envOrUndefined(config.apiKeyEnv) ?? "ollama",
|
|
1581
|
-
headers: config.headers,
|
|
1582
|
-
apiStyle: config.apiStyle ?? "responses",
|
|
1583
|
-
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming", "local"])
|
|
1584
|
-
});
|
|
1585
|
-
case "openai-compatible":
|
|
1586
|
-
return new OpenAiCompatibleProviderAdapter(id, "openai-compatible", config.model, {
|
|
1587
|
-
baseUrl: config.baseUrl ?? "http://localhost:8000/v1",
|
|
1588
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1589
|
-
headers: config.headers,
|
|
1590
|
-
apiStyle: config.apiStyle ?? "responses",
|
|
1591
|
-
capabilities: resolveCapabilities(config, ["chat", "structured"])
|
|
1592
|
-
});
|
|
1593
|
-
case "anthropic":
|
|
1594
|
-
return new AnthropicProviderAdapter(id, config.model, {
|
|
1595
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1596
|
-
headers: config.headers,
|
|
1597
|
-
baseUrl: config.baseUrl
|
|
1598
|
-
});
|
|
1599
|
-
case "gemini":
|
|
1600
|
-
return new GeminiProviderAdapter(id, config.model, {
|
|
1601
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1602
|
-
baseUrl: config.baseUrl
|
|
1603
|
-
});
|
|
1604
|
-
case "custom": {
|
|
1605
|
-
if (!config.module) {
|
|
1606
|
-
throw new Error(`Provider ${id} is type "custom" but no module path was configured.`);
|
|
1607
|
-
}
|
|
1608
|
-
const resolvedModule = path7.isAbsolute(config.module) ? config.module : path7.resolve(rootDir, config.module);
|
|
1609
|
-
const loaded = await import(pathToFileURL(resolvedModule).href);
|
|
1610
|
-
const parsed = customModuleSchema.parse(loaded);
|
|
1611
|
-
return parsed.createAdapter(id, config, rootDir);
|
|
1612
|
-
}
|
|
1613
|
-
default:
|
|
1614
|
-
throw new Error(`Unsupported provider type ${String(config.type)}`);
|
|
2077
|
+
function relationRank(outputPage, targetPage) {
|
|
2078
|
+
if (outputPage.relatedPageIds.includes(targetPage.id)) {
|
|
2079
|
+
return 3;
|
|
2080
|
+
}
|
|
2081
|
+
if (outputPage.relatedNodeIds.some((nodeId) => targetPage.nodeIds.includes(nodeId))) {
|
|
2082
|
+
return 2;
|
|
1615
2083
|
}
|
|
2084
|
+
if (outputPage.relatedSourceIds.some((sourceId) => targetPage.sourceIds.includes(sourceId))) {
|
|
2085
|
+
return 1;
|
|
2086
|
+
}
|
|
2087
|
+
return 0;
|
|
1616
2088
|
}
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
2089
|
+
function relatedOutputsForPage(targetPage, outputPages) {
|
|
2090
|
+
return outputPages.map((page) => ({ page, rank: relationRank(page, targetPage) })).filter((item) => item.rank > 0).sort((left, right) => right.rank - left.rank || left.page.title.localeCompare(right.page.title)).map((item) => item.page);
|
|
2091
|
+
}
|
|
2092
|
+
async function resolveUniqueOutputSlug(wikiDir, baseSlug) {
|
|
2093
|
+
const outputsDir = path11.join(wikiDir, "outputs");
|
|
2094
|
+
const root = baseSlug || "output";
|
|
2095
|
+
let candidate = root;
|
|
2096
|
+
let counter = 2;
|
|
2097
|
+
while (await fileExists(path11.join(outputsDir, `${candidate}.md`))) {
|
|
2098
|
+
candidate = `${root}-${counter}`;
|
|
2099
|
+
counter++;
|
|
1623
2100
|
}
|
|
1624
|
-
return
|
|
2101
|
+
return candidate;
|
|
1625
2102
|
}
|
|
1626
|
-
function
|
|
1627
|
-
|
|
1628
|
-
|
|
2103
|
+
async function loadSavedOutputPages(wikiDir) {
|
|
2104
|
+
const outputsDir = path11.join(wikiDir, "outputs");
|
|
2105
|
+
const entries = await fs9.readdir(outputsDir, { withFileTypes: true }).catch(() => []);
|
|
2106
|
+
const outputs = [];
|
|
2107
|
+
for (const entry of entries) {
|
|
2108
|
+
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
|
2109
|
+
continue;
|
|
2110
|
+
}
|
|
2111
|
+
const relativePath = path11.posix.join("outputs", entry.name);
|
|
2112
|
+
const absolutePath = path11.join(outputsDir, entry.name);
|
|
2113
|
+
const content = await fs9.readFile(absolutePath, "utf8");
|
|
2114
|
+
const parsed = matter3(content);
|
|
2115
|
+
const slug = entry.name.replace(/\.md$/, "");
|
|
2116
|
+
const title = typeof parsed.data.title === "string" ? parsed.data.title : slug;
|
|
2117
|
+
const pageId = typeof parsed.data.page_id === "string" ? parsed.data.page_id : `output:${slug}`;
|
|
2118
|
+
const sourceIds = normalizeStringArray(parsed.data.source_ids);
|
|
2119
|
+
const nodeIds = normalizeStringArray(parsed.data.node_ids);
|
|
2120
|
+
const relatedPageIds = normalizeStringArray(parsed.data.related_page_ids);
|
|
2121
|
+
const relatedNodeIds = normalizeStringArray(parsed.data.related_node_ids);
|
|
2122
|
+
const relatedSourceIds = normalizeStringArray(parsed.data.related_source_ids);
|
|
2123
|
+
const backlinks = normalizeStringArray(parsed.data.backlinks);
|
|
2124
|
+
outputs.push({
|
|
2125
|
+
page: {
|
|
2126
|
+
id: pageId,
|
|
2127
|
+
path: relativePath,
|
|
2128
|
+
title,
|
|
2129
|
+
kind: "output",
|
|
2130
|
+
sourceIds,
|
|
2131
|
+
nodeIds,
|
|
2132
|
+
freshness: parsed.data.freshness === "stale" ? "stale" : "fresh",
|
|
2133
|
+
confidence: typeof parsed.data.confidence === "number" ? parsed.data.confidence : 0.74,
|
|
2134
|
+
backlinks,
|
|
2135
|
+
schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
|
|
2136
|
+
sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
|
|
2137
|
+
relatedPageIds,
|
|
2138
|
+
relatedNodeIds,
|
|
2139
|
+
relatedSourceIds,
|
|
2140
|
+
origin: typeof parsed.data.origin === "string" ? parsed.data.origin : void 0,
|
|
2141
|
+
question: typeof parsed.data.question === "string" ? parsed.data.question : void 0
|
|
2142
|
+
},
|
|
2143
|
+
content,
|
|
2144
|
+
contentHash: sha256(content)
|
|
2145
|
+
});
|
|
1629
2146
|
}
|
|
2147
|
+
return outputs.sort((left, right) => left.page.title.localeCompare(right.page.title));
|
|
1630
2148
|
}
|
|
1631
2149
|
|
|
1632
2150
|
// src/search.ts
|
|
1633
|
-
import
|
|
1634
|
-
import
|
|
1635
|
-
import
|
|
2151
|
+
import fs10 from "fs/promises";
|
|
2152
|
+
import path12 from "path";
|
|
2153
|
+
import matter4 from "gray-matter";
|
|
1636
2154
|
function getDatabaseSync() {
|
|
1637
2155
|
const builtin = process.getBuiltinModule?.("node:sqlite");
|
|
1638
2156
|
if (!builtin?.DatabaseSync) {
|
|
@@ -1645,7 +2163,7 @@ function toFtsQuery(query) {
|
|
|
1645
2163
|
return tokens.join(" OR ");
|
|
1646
2164
|
}
|
|
1647
2165
|
async function rebuildSearchIndex(dbPath, pages, wikiDir) {
|
|
1648
|
-
await ensureDir(
|
|
2166
|
+
await ensureDir(path12.dirname(dbPath));
|
|
1649
2167
|
const DatabaseSync = getDatabaseSync();
|
|
1650
2168
|
const db = new DatabaseSync(dbPath);
|
|
1651
2169
|
db.exec("PRAGMA journal_mode = WAL;");
|
|
@@ -1667,9 +2185,9 @@ async function rebuildSearchIndex(dbPath, pages, wikiDir) {
|
|
|
1667
2185
|
`);
|
|
1668
2186
|
const insertPage = db.prepare("INSERT INTO pages (id, path, title, body) VALUES (?, ?, ?, ?)");
|
|
1669
2187
|
for (const page of pages) {
|
|
1670
|
-
const absolutePath =
|
|
1671
|
-
const content = await
|
|
1672
|
-
const parsed =
|
|
2188
|
+
const absolutePath = path12.join(wikiDir, page.path);
|
|
2189
|
+
const content = await fs10.readFile(absolutePath, "utf8");
|
|
2190
|
+
const parsed = matter4(content);
|
|
1673
2191
|
insertPage.run(page.id, page.path, page.title, parsed.content);
|
|
1674
2192
|
}
|
|
1675
2193
|
db.exec("INSERT INTO page_search (rowid, title, body) SELECT rowid, title, body FROM pages;");
|
|
@@ -1706,32 +2224,6 @@ function searchPages(dbPath, query, limit = 5) {
|
|
|
1706
2224
|
}));
|
|
1707
2225
|
}
|
|
1708
2226
|
|
|
1709
|
-
// src/schema.ts
|
|
1710
|
-
import fs8 from "fs/promises";
|
|
1711
|
-
import path9 from "path";
|
|
1712
|
-
async function loadVaultSchema(rootDir) {
|
|
1713
|
-
const { paths } = await loadVaultConfig(rootDir);
|
|
1714
|
-
const schemaPath = paths.schemaPath;
|
|
1715
|
-
const content = await fileExists(schemaPath) ? await fs8.readFile(schemaPath, "utf8") : defaultVaultSchema();
|
|
1716
|
-
const normalized = content.trim() ? content.trim() : defaultVaultSchema().trim();
|
|
1717
|
-
return {
|
|
1718
|
-
path: schemaPath,
|
|
1719
|
-
content: normalized,
|
|
1720
|
-
hash: sha256(normalized),
|
|
1721
|
-
isLegacyPath: path9.basename(schemaPath) === LEGACY_SCHEMA_FILENAME && path9.basename(schemaPath) !== PRIMARY_SCHEMA_FILENAME
|
|
1722
|
-
};
|
|
1723
|
-
}
|
|
1724
|
-
function buildSchemaPrompt(schema, instruction) {
|
|
1725
|
-
return [
|
|
1726
|
-
instruction,
|
|
1727
|
-
"",
|
|
1728
|
-
`Vault schema path: ${schema.path}`,
|
|
1729
|
-
"",
|
|
1730
|
-
"Vault schema instructions:",
|
|
1731
|
-
schema.content
|
|
1732
|
-
].join("\n");
|
|
1733
|
-
}
|
|
1734
|
-
|
|
1735
2227
|
// src/vault.ts
|
|
1736
2228
|
function buildGraph(manifests, analyses, pages) {
|
|
1737
2229
|
const sourceNodes = manifests.map((manifest) => ({
|
|
@@ -1749,14 +2241,15 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1749
2241
|
for (const analysis of analyses) {
|
|
1750
2242
|
for (const concept of analysis.concepts) {
|
|
1751
2243
|
const existing = conceptMap.get(concept.id);
|
|
2244
|
+
const sourceIds = [.../* @__PURE__ */ new Set([...existing?.sourceIds ?? [], analysis.sourceId])];
|
|
1752
2245
|
conceptMap.set(concept.id, {
|
|
1753
2246
|
id: concept.id,
|
|
1754
2247
|
type: "concept",
|
|
1755
2248
|
label: concept.name,
|
|
1756
2249
|
pageId: `concept:${slugify(concept.name)}`,
|
|
1757
2250
|
freshness: "fresh",
|
|
1758
|
-
confidence:
|
|
1759
|
-
sourceIds
|
|
2251
|
+
confidence: nodeConfidence(sourceIds.length),
|
|
2252
|
+
sourceIds
|
|
1760
2253
|
});
|
|
1761
2254
|
edges.push({
|
|
1762
2255
|
id: `${analysis.sourceId}->${concept.id}`,
|
|
@@ -1764,20 +2257,21 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1764
2257
|
target: concept.id,
|
|
1765
2258
|
relation: "mentions",
|
|
1766
2259
|
status: "extracted",
|
|
1767
|
-
confidence:
|
|
2260
|
+
confidence: edgeConfidence(analysis.claims, concept.name),
|
|
1768
2261
|
provenance: [analysis.sourceId]
|
|
1769
2262
|
});
|
|
1770
2263
|
}
|
|
1771
2264
|
for (const entity of analysis.entities) {
|
|
1772
2265
|
const existing = entityMap.get(entity.id);
|
|
2266
|
+
const sourceIds = [.../* @__PURE__ */ new Set([...existing?.sourceIds ?? [], analysis.sourceId])];
|
|
1773
2267
|
entityMap.set(entity.id, {
|
|
1774
2268
|
id: entity.id,
|
|
1775
2269
|
type: "entity",
|
|
1776
2270
|
label: entity.name,
|
|
1777
2271
|
pageId: `entity:${slugify(entity.name)}`,
|
|
1778
2272
|
freshness: "fresh",
|
|
1779
|
-
confidence:
|
|
1780
|
-
sourceIds
|
|
2273
|
+
confidence: nodeConfidence(sourceIds.length),
|
|
2274
|
+
sourceIds
|
|
1781
2275
|
});
|
|
1782
2276
|
edges.push({
|
|
1783
2277
|
id: `${analysis.sourceId}->${entity.id}`,
|
|
@@ -1785,22 +2279,46 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1785
2279
|
target: entity.id,
|
|
1786
2280
|
relation: "mentions",
|
|
1787
2281
|
status: "extracted",
|
|
1788
|
-
confidence:
|
|
2282
|
+
confidence: edgeConfidence(analysis.claims, entity.name),
|
|
1789
2283
|
provenance: [analysis.sourceId]
|
|
1790
2284
|
});
|
|
1791
2285
|
}
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
2286
|
+
}
|
|
2287
|
+
const conceptClaims = /* @__PURE__ */ new Map();
|
|
2288
|
+
for (const analysis of analyses) {
|
|
2289
|
+
for (const claim of analysis.claims) {
|
|
2290
|
+
for (const concept of analysis.concepts) {
|
|
2291
|
+
if (claim.text.toLowerCase().includes(concept.name.toLowerCase())) {
|
|
2292
|
+
const key = concept.id;
|
|
2293
|
+
const list = conceptClaims.get(key) ?? [];
|
|
2294
|
+
list.push({ claim, sourceId: analysis.sourceId });
|
|
2295
|
+
conceptClaims.set(key, list);
|
|
2296
|
+
}
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2300
|
+
const conflictEdgeKeys = /* @__PURE__ */ new Set();
|
|
2301
|
+
for (const [, claimsForConcept] of conceptClaims) {
|
|
2302
|
+
const positive = claimsForConcept.filter((item) => item.claim.polarity === "positive");
|
|
2303
|
+
const negative = claimsForConcept.filter((item) => item.claim.polarity === "negative");
|
|
2304
|
+
for (const positiveClaim of positive) {
|
|
2305
|
+
for (const negativeClaim of negative) {
|
|
2306
|
+
if (positiveClaim.sourceId === negativeClaim.sourceId) {
|
|
2307
|
+
continue;
|
|
2308
|
+
}
|
|
2309
|
+
const edgeKey = [positiveClaim.sourceId, negativeClaim.sourceId].sort().join("|");
|
|
2310
|
+
if (conflictEdgeKeys.has(edgeKey)) {
|
|
2311
|
+
continue;
|
|
2312
|
+
}
|
|
2313
|
+
conflictEdgeKeys.add(edgeKey);
|
|
1796
2314
|
edges.push({
|
|
1797
|
-
id:
|
|
1798
|
-
source: `source:${
|
|
1799
|
-
target: `source:${
|
|
2315
|
+
id: `conflict:${positiveClaim.claim.id}->${negativeClaim.claim.id}`,
|
|
2316
|
+
source: `source:${positiveClaim.sourceId}`,
|
|
2317
|
+
target: `source:${negativeClaim.sourceId}`,
|
|
1800
2318
|
relation: "conflicted_with",
|
|
1801
2319
|
status: "conflicted",
|
|
1802
|
-
confidence:
|
|
1803
|
-
provenance: [
|
|
2320
|
+
confidence: conflictConfidence(positiveClaim.claim, negativeClaim.claim),
|
|
2321
|
+
provenance: [positiveClaim.sourceId, negativeClaim.sourceId]
|
|
1804
2322
|
});
|
|
1805
2323
|
}
|
|
1806
2324
|
}
|
|
@@ -1814,7 +2332,7 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1814
2332
|
};
|
|
1815
2333
|
}
|
|
1816
2334
|
async function writePage(wikiDir, relativePath, content, changedPages) {
|
|
1817
|
-
const absolutePath =
|
|
2335
|
+
const absolutePath = path13.resolve(wikiDir, relativePath);
|
|
1818
2336
|
const changed = await writeFileIfChanged(absolutePath, content);
|
|
1819
2337
|
if (changed) {
|
|
1820
2338
|
changedPages.push(relativePath);
|
|
@@ -1839,6 +2357,216 @@ function aggregateItems(analyses, kind) {
|
|
|
1839
2357
|
}
|
|
1840
2358
|
return [...grouped.values()];
|
|
1841
2359
|
}
|
|
2360
|
+
function emptyGraphPage(input) {
|
|
2361
|
+
return {
|
|
2362
|
+
id: input.id,
|
|
2363
|
+
path: input.path,
|
|
2364
|
+
title: input.title,
|
|
2365
|
+
kind: input.kind,
|
|
2366
|
+
sourceIds: input.sourceIds,
|
|
2367
|
+
nodeIds: input.nodeIds,
|
|
2368
|
+
freshness: "fresh",
|
|
2369
|
+
confidence: input.confidence,
|
|
2370
|
+
backlinks: [],
|
|
2371
|
+
schemaHash: input.schemaHash,
|
|
2372
|
+
sourceHashes: input.sourceHashes,
|
|
2373
|
+
relatedPageIds: [],
|
|
2374
|
+
relatedNodeIds: [],
|
|
2375
|
+
relatedSourceIds: []
|
|
2376
|
+
};
|
|
2377
|
+
}
|
|
2378
|
+
function outputHashes(outputPages) {
|
|
2379
|
+
return Object.fromEntries(outputPages.map((page) => [page.page.id, page.contentHash]));
|
|
2380
|
+
}
|
|
2381
|
+
function recordsEqual(left, right) {
|
|
2382
|
+
const leftKeys = Object.keys(left);
|
|
2383
|
+
const rightKeys = Object.keys(right);
|
|
2384
|
+
if (leftKeys.length !== rightKeys.length) {
|
|
2385
|
+
return false;
|
|
2386
|
+
}
|
|
2387
|
+
return leftKeys.every((key) => left[key] === right[key]);
|
|
2388
|
+
}
|
|
2389
|
+
async function requiredCompileArtifactsExist(paths) {
|
|
2390
|
+
const requiredPaths = [
|
|
2391
|
+
paths.graphPath,
|
|
2392
|
+
paths.searchDbPath,
|
|
2393
|
+
path13.join(paths.wikiDir, "index.md"),
|
|
2394
|
+
path13.join(paths.wikiDir, "sources", "index.md"),
|
|
2395
|
+
path13.join(paths.wikiDir, "concepts", "index.md"),
|
|
2396
|
+
path13.join(paths.wikiDir, "entities", "index.md"),
|
|
2397
|
+
path13.join(paths.wikiDir, "outputs", "index.md")
|
|
2398
|
+
];
|
|
2399
|
+
const checks = await Promise.all(requiredPaths.map((filePath) => fileExists(filePath)));
|
|
2400
|
+
return checks.every(Boolean);
|
|
2401
|
+
}
|
|
2402
|
+
async function refreshIndexesAndSearch(rootDir, schemaHash, pages) {
|
|
2403
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2404
|
+
await ensureDir(path13.join(paths.wikiDir, "outputs"));
|
|
2405
|
+
await writeFileIfChanged(path13.join(paths.wikiDir, "index.md"), buildIndexPage(pages, schemaHash));
|
|
2406
|
+
await writeFileIfChanged(
|
|
2407
|
+
path13.join(paths.wikiDir, "outputs", "index.md"),
|
|
2408
|
+
buildSectionIndex(
|
|
2409
|
+
"outputs",
|
|
2410
|
+
pages.filter((page) => page.kind === "output"),
|
|
2411
|
+
schemaHash
|
|
2412
|
+
)
|
|
2413
|
+
);
|
|
2414
|
+
await rebuildSearchIndex(paths.searchDbPath, pages, paths.wikiDir);
|
|
2415
|
+
}
|
|
2416
|
+
async function upsertGraphPages(rootDir, pages) {
|
|
2417
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2418
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2419
|
+
const manifests = await listManifests(rootDir);
|
|
2420
|
+
const nonOutputPages = graph?.pages.filter((page) => page.kind !== "output") ?? [];
|
|
2421
|
+
const nextGraph = {
|
|
2422
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2423
|
+
nodes: graph?.nodes ?? [],
|
|
2424
|
+
edges: graph?.edges ?? [],
|
|
2425
|
+
sources: graph?.sources ?? manifests,
|
|
2426
|
+
pages: [...nonOutputPages, ...pages]
|
|
2427
|
+
};
|
|
2428
|
+
await writeJsonFile(paths.graphPath, nextGraph);
|
|
2429
|
+
}
|
|
2430
|
+
async function persistOutputPage(rootDir, input) {
|
|
2431
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2432
|
+
const slug = await resolveUniqueOutputSlug(paths.wikiDir, input.slug ?? slugify(input.question));
|
|
2433
|
+
const output = buildOutputPage({ ...input, slug });
|
|
2434
|
+
const absolutePath = path13.join(paths.wikiDir, output.page.path);
|
|
2435
|
+
await ensureDir(path13.dirname(absolutePath));
|
|
2436
|
+
await fs11.writeFile(absolutePath, output.content, "utf8");
|
|
2437
|
+
const storedOutputs = await loadSavedOutputPages(paths.wikiDir);
|
|
2438
|
+
const outputPages = storedOutputs.map((page) => page.page);
|
|
2439
|
+
await upsertGraphPages(rootDir, outputPages);
|
|
2440
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2441
|
+
await refreshIndexesAndSearch(rootDir, input.schemaHash, graph?.pages ?? outputPages);
|
|
2442
|
+
return { page: output.page, savedTo: absolutePath };
|
|
2443
|
+
}
|
|
2444
|
+
async function persistExploreHub(rootDir, input) {
|
|
2445
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2446
|
+
const slug = await resolveUniqueOutputSlug(paths.wikiDir, input.slug ?? `explore-${slugify(input.question)}`);
|
|
2447
|
+
const hub = buildExploreHubPage({ ...input, slug });
|
|
2448
|
+
const absolutePath = path13.join(paths.wikiDir, hub.page.path);
|
|
2449
|
+
await ensureDir(path13.dirname(absolutePath));
|
|
2450
|
+
await fs11.writeFile(absolutePath, hub.content, "utf8");
|
|
2451
|
+
const storedOutputs = await loadSavedOutputPages(paths.wikiDir);
|
|
2452
|
+
const outputPages = storedOutputs.map((page) => page.page);
|
|
2453
|
+
await upsertGraphPages(rootDir, outputPages);
|
|
2454
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2455
|
+
await refreshIndexesAndSearch(rootDir, input.schemaHash, graph?.pages ?? outputPages);
|
|
2456
|
+
return { page: hub.page, savedTo: absolutePath };
|
|
2457
|
+
}
|
|
2458
|
+
async function executeQuery(rootDir, question) {
|
|
2459
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2460
|
+
const schema = await loadVaultSchema(rootDir);
|
|
2461
|
+
const provider = await getProviderForTask(rootDir, "queryProvider");
|
|
2462
|
+
if (!await fileExists(paths.searchDbPath) || !await fileExists(paths.graphPath)) {
|
|
2463
|
+
await compileVault(rootDir);
|
|
2464
|
+
}
|
|
2465
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2466
|
+
const pageMap = new Map((graph?.pages ?? []).map((page) => [page.id, page]));
|
|
2467
|
+
const searchResults = searchPages(paths.searchDbPath, question, 5);
|
|
2468
|
+
const excerpts = await Promise.all(
|
|
2469
|
+
searchResults.map(async (result) => {
|
|
2470
|
+
const absolutePath = path13.join(paths.wikiDir, result.path);
|
|
2471
|
+
try {
|
|
2472
|
+
const content = await fs11.readFile(absolutePath, "utf8");
|
|
2473
|
+
const parsed = matter5(content);
|
|
2474
|
+
return `# ${result.title}
|
|
2475
|
+
${truncate(normalizeWhitespace(parsed.content), 1200)}`;
|
|
2476
|
+
} catch {
|
|
2477
|
+
return `# ${result.title}
|
|
2478
|
+
${result.snippet}`;
|
|
2479
|
+
}
|
|
2480
|
+
})
|
|
2481
|
+
);
|
|
2482
|
+
const relatedPageIds = uniqueBy(
|
|
2483
|
+
searchResults.map((result) => result.pageId),
|
|
2484
|
+
(item) => item
|
|
2485
|
+
);
|
|
2486
|
+
const relatedNodeIds = uniqueBy(
|
|
2487
|
+
relatedPageIds.flatMap((pageId) => pageMap.get(pageId)?.nodeIds ?? []),
|
|
2488
|
+
(item) => item
|
|
2489
|
+
);
|
|
2490
|
+
const relatedSourceIds = uniqueBy(
|
|
2491
|
+
relatedPageIds.flatMap((pageId) => pageMap.get(pageId)?.sourceIds ?? []),
|
|
2492
|
+
(item) => item
|
|
2493
|
+
);
|
|
2494
|
+
const manifests = await listManifests(rootDir);
|
|
2495
|
+
const rawExcerpts = [];
|
|
2496
|
+
for (const sourceId of relatedSourceIds.slice(0, 5)) {
|
|
2497
|
+
const manifest = manifests.find((item) => item.sourceId === sourceId);
|
|
2498
|
+
if (!manifest) {
|
|
2499
|
+
continue;
|
|
2500
|
+
}
|
|
2501
|
+
const text = await readExtractedText(rootDir, manifest);
|
|
2502
|
+
if (text) {
|
|
2503
|
+
rawExcerpts.push(`# [source:${sourceId}] ${manifest.title}
|
|
2504
|
+
${truncate(normalizeWhitespace(text), 800)}`);
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
let answer;
|
|
2508
|
+
if (provider.type === "heuristic") {
|
|
2509
|
+
answer = [
|
|
2510
|
+
`Question: ${question}`,
|
|
2511
|
+
"",
|
|
2512
|
+
"Relevant pages:",
|
|
2513
|
+
...searchResults.map((result) => `- ${result.title} (${result.path})`),
|
|
2514
|
+
"",
|
|
2515
|
+
excerpts.length ? excerpts.join("\n\n") : "No relevant pages found yet.",
|
|
2516
|
+
...rawExcerpts.length ? ["", "Raw source material:", "", ...rawExcerpts] : []
|
|
2517
|
+
].join("\n");
|
|
2518
|
+
} else {
|
|
2519
|
+
const context = [
|
|
2520
|
+
"Wiki context:",
|
|
2521
|
+
excerpts.join("\n\n---\n\n"),
|
|
2522
|
+
...rawExcerpts.length ? ["", "Raw source material:", rawExcerpts.join("\n\n---\n\n")] : []
|
|
2523
|
+
].join("\n\n");
|
|
2524
|
+
const response = await provider.generateText({
|
|
2525
|
+
system: buildSchemaPrompt(
|
|
2526
|
+
schema,
|
|
2527
|
+
"Answer using the provided context. Prefer raw source material over wiki summaries when they differ. Cite source IDs."
|
|
2528
|
+
),
|
|
2529
|
+
prompt: `Question: ${question}
|
|
2530
|
+
|
|
2531
|
+
${context}`
|
|
2532
|
+
});
|
|
2533
|
+
answer = response.text;
|
|
2534
|
+
}
|
|
2535
|
+
return {
|
|
2536
|
+
answer,
|
|
2537
|
+
citations: relatedSourceIds,
|
|
2538
|
+
relatedPageIds,
|
|
2539
|
+
relatedNodeIds,
|
|
2540
|
+
relatedSourceIds
|
|
2541
|
+
};
|
|
2542
|
+
}
|
|
2543
|
+
async function generateFollowUpQuestions(rootDir, question, answer) {
|
|
2544
|
+
const provider = await getProviderForTask(rootDir, "queryProvider");
|
|
2545
|
+
const schema = await loadVaultSchema(rootDir);
|
|
2546
|
+
if (provider.type === "heuristic") {
|
|
2547
|
+
return uniqueBy(
|
|
2548
|
+
[
|
|
2549
|
+
`What evidence best supports ${question}?`,
|
|
2550
|
+
`What contradicts ${question}?`,
|
|
2551
|
+
`Which sources should be added to answer ${question} better?`
|
|
2552
|
+
],
|
|
2553
|
+
(item) => item
|
|
2554
|
+
).slice(0, 3);
|
|
2555
|
+
}
|
|
2556
|
+
const response = await provider.generateStructured(
|
|
2557
|
+
{
|
|
2558
|
+
system: buildSchemaPrompt(schema, "Propose concise follow-up research questions for the vault. Return only useful next questions."),
|
|
2559
|
+
prompt: `Root question: ${question}
|
|
2560
|
+
|
|
2561
|
+
Current answer:
|
|
2562
|
+
${answer}`
|
|
2563
|
+
},
|
|
2564
|
+
z8.object({
|
|
2565
|
+
questions: z8.array(z8.string().min(1)).max(5)
|
|
2566
|
+
})
|
|
2567
|
+
);
|
|
2568
|
+
return uniqueBy(response.questions, (item) => item).filter((item) => item !== question);
|
|
2569
|
+
}
|
|
1842
2570
|
async function initVault(rootDir) {
|
|
1843
2571
|
await initWorkspace(rootDir);
|
|
1844
2572
|
await installConfiguredAgents(rootDir);
|
|
@@ -1848,107 +2576,306 @@ async function compileVault(rootDir) {
|
|
|
1848
2576
|
const schema = await loadVaultSchema(rootDir);
|
|
1849
2577
|
const provider = await getProviderForTask(rootDir, "compileProvider");
|
|
1850
2578
|
const manifests = await listManifests(rootDir);
|
|
1851
|
-
const
|
|
1852
|
-
|
|
1853
|
-
);
|
|
2579
|
+
const storedOutputPages = await loadSavedOutputPages(paths.wikiDir);
|
|
2580
|
+
const outputPages = storedOutputPages.map((page) => page.page);
|
|
2581
|
+
const currentOutputHashes = outputHashes(storedOutputPages);
|
|
2582
|
+
const previousState = await readJsonFile(paths.compileStatePath);
|
|
2583
|
+
const schemaChanged = !previousState || previousState.schemaHash !== schema.hash;
|
|
2584
|
+
const previousSourceHashes = previousState?.sourceHashes ?? {};
|
|
2585
|
+
const previousAnalyses = previousState?.analyses ?? {};
|
|
2586
|
+
const previousOutputHashes = previousState?.outputHashes ?? {};
|
|
2587
|
+
const currentSourceIds = new Set(manifests.map((item) => item.sourceId));
|
|
2588
|
+
const previousSourceIds = new Set(Object.keys(previousSourceHashes));
|
|
2589
|
+
const sourcesChanged = currentSourceIds.size !== previousSourceIds.size || [...currentSourceIds].some((sourceId) => !previousSourceIds.has(sourceId));
|
|
2590
|
+
const outputsChanged = !recordsEqual(currentOutputHashes, previousOutputHashes);
|
|
2591
|
+
const artifactsExist = await requiredCompileArtifactsExist(paths);
|
|
2592
|
+
const dirty = [];
|
|
2593
|
+
const clean = [];
|
|
2594
|
+
for (const manifest of manifests) {
|
|
2595
|
+
const hashChanged = previousSourceHashes[manifest.sourceId] !== manifest.contentHash;
|
|
2596
|
+
const noAnalysis = !previousAnalyses[manifest.sourceId];
|
|
2597
|
+
if (schemaChanged || hashChanged || noAnalysis) {
|
|
2598
|
+
dirty.push(manifest);
|
|
2599
|
+
} else {
|
|
2600
|
+
clean.push(manifest);
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
if (dirty.length === 0 && !schemaChanged && !sourcesChanged && !outputsChanged && artifactsExist) {
|
|
2604
|
+
const graph2 = await readJsonFile(paths.graphPath);
|
|
2605
|
+
return {
|
|
2606
|
+
graphPath: paths.graphPath,
|
|
2607
|
+
pageCount: graph2?.pages.length ?? outputPages.length,
|
|
2608
|
+
changedPages: [],
|
|
2609
|
+
sourceCount: manifests.length
|
|
2610
|
+
};
|
|
2611
|
+
}
|
|
2612
|
+
const [dirtyAnalyses, cleanAnalyses] = await Promise.all([
|
|
2613
|
+
Promise.all(
|
|
2614
|
+
dirty.map(async (manifest) => analyzeSource(manifest, await readExtractedText(rootDir, manifest), provider, paths, schema))
|
|
2615
|
+
),
|
|
2616
|
+
Promise.all(
|
|
2617
|
+
clean.map(async (manifest) => {
|
|
2618
|
+
const cached = await readJsonFile(path13.join(paths.analysesDir, `${manifest.sourceId}.json`));
|
|
2619
|
+
if (cached) {
|
|
2620
|
+
return cached;
|
|
2621
|
+
}
|
|
2622
|
+
return analyzeSource(manifest, await readExtractedText(rootDir, manifest), provider, paths, schema);
|
|
2623
|
+
})
|
|
2624
|
+
)
|
|
2625
|
+
]);
|
|
2626
|
+
const analyses = [...dirtyAnalyses, ...cleanAnalyses];
|
|
1854
2627
|
const changedPages = [];
|
|
1855
|
-
const
|
|
2628
|
+
const compiledPages = [];
|
|
1856
2629
|
await Promise.all([
|
|
1857
|
-
ensureDir(
|
|
1858
|
-
ensureDir(
|
|
1859
|
-
ensureDir(
|
|
1860
|
-
ensureDir(
|
|
2630
|
+
ensureDir(path13.join(paths.wikiDir, "sources")),
|
|
2631
|
+
ensureDir(path13.join(paths.wikiDir, "concepts")),
|
|
2632
|
+
ensureDir(path13.join(paths.wikiDir, "entities")),
|
|
2633
|
+
ensureDir(path13.join(paths.wikiDir, "outputs"))
|
|
1861
2634
|
]);
|
|
1862
2635
|
for (const manifest of manifests) {
|
|
1863
2636
|
const analysis = analyses.find((item) => item.sourceId === manifest.sourceId);
|
|
1864
2637
|
if (!analysis) {
|
|
1865
2638
|
continue;
|
|
1866
2639
|
}
|
|
1867
|
-
const
|
|
1868
|
-
|
|
2640
|
+
const preview = emptyGraphPage({
|
|
2641
|
+
id: `source:${manifest.sourceId}`,
|
|
2642
|
+
path: `sources/${manifest.sourceId}.md`,
|
|
2643
|
+
title: analysis.title,
|
|
2644
|
+
kind: "source",
|
|
2645
|
+
sourceIds: [manifest.sourceId],
|
|
2646
|
+
nodeIds: [`source:${manifest.sourceId}`, ...analysis.concepts.map((item) => item.id), ...analysis.entities.map((item) => item.id)],
|
|
2647
|
+
schemaHash: schema.hash,
|
|
2648
|
+
sourceHashes: { [manifest.sourceId]: manifest.contentHash },
|
|
2649
|
+
confidence: 1
|
|
2650
|
+
});
|
|
2651
|
+
const sourcePage = buildSourcePage(manifest, analysis, schema.hash, 1, relatedOutputsForPage(preview, outputPages));
|
|
2652
|
+
compiledPages.push(sourcePage.page);
|
|
1869
2653
|
await writePage(paths.wikiDir, sourcePage.page.path, sourcePage.content, changedPages);
|
|
1870
2654
|
}
|
|
1871
2655
|
for (const aggregate of aggregateItems(analyses, "concepts")) {
|
|
1872
|
-
const
|
|
1873
|
-
|
|
2656
|
+
const confidence = nodeConfidence(aggregate.sourceAnalyses.length);
|
|
2657
|
+
const preview = emptyGraphPage({
|
|
2658
|
+
id: `concept:${slugify(aggregate.name)}`,
|
|
2659
|
+
path: `concepts/${slugify(aggregate.name)}.md`,
|
|
2660
|
+
title: aggregate.name,
|
|
2661
|
+
kind: "concept",
|
|
2662
|
+
sourceIds: aggregate.sourceAnalyses.map((item) => item.sourceId),
|
|
2663
|
+
nodeIds: [`concept:${slugify(aggregate.name)}`],
|
|
2664
|
+
schemaHash: schema.hash,
|
|
2665
|
+
sourceHashes: aggregate.sourceHashes,
|
|
2666
|
+
confidence
|
|
2667
|
+
});
|
|
2668
|
+
const page = buildAggregatePage(
|
|
2669
|
+
"concept",
|
|
2670
|
+
aggregate.name,
|
|
2671
|
+
aggregate.descriptions,
|
|
2672
|
+
aggregate.sourceAnalyses,
|
|
2673
|
+
aggregate.sourceHashes,
|
|
2674
|
+
schema.hash,
|
|
2675
|
+
confidence,
|
|
2676
|
+
relatedOutputsForPage(preview, outputPages)
|
|
2677
|
+
);
|
|
2678
|
+
compiledPages.push(page.page);
|
|
1874
2679
|
await writePage(paths.wikiDir, page.page.path, page.content, changedPages);
|
|
1875
2680
|
}
|
|
1876
2681
|
for (const aggregate of aggregateItems(analyses, "entities")) {
|
|
1877
|
-
const
|
|
1878
|
-
|
|
2682
|
+
const confidence = nodeConfidence(aggregate.sourceAnalyses.length);
|
|
2683
|
+
const preview = emptyGraphPage({
|
|
2684
|
+
id: `entity:${slugify(aggregate.name)}`,
|
|
2685
|
+
path: `entities/${slugify(aggregate.name)}.md`,
|
|
2686
|
+
title: aggregate.name,
|
|
2687
|
+
kind: "entity",
|
|
2688
|
+
sourceIds: aggregate.sourceAnalyses.map((item) => item.sourceId),
|
|
2689
|
+
nodeIds: [`entity:${slugify(aggregate.name)}`],
|
|
2690
|
+
schemaHash: schema.hash,
|
|
2691
|
+
sourceHashes: aggregate.sourceHashes,
|
|
2692
|
+
confidence
|
|
2693
|
+
});
|
|
2694
|
+
const page = buildAggregatePage(
|
|
2695
|
+
"entity",
|
|
2696
|
+
aggregate.name,
|
|
2697
|
+
aggregate.descriptions,
|
|
2698
|
+
aggregate.sourceAnalyses,
|
|
2699
|
+
aggregate.sourceHashes,
|
|
2700
|
+
schema.hash,
|
|
2701
|
+
confidence,
|
|
2702
|
+
relatedOutputsForPage(preview, outputPages)
|
|
2703
|
+
);
|
|
2704
|
+
compiledPages.push(page.page);
|
|
1879
2705
|
await writePage(paths.wikiDir, page.page.path, page.content, changedPages);
|
|
1880
2706
|
}
|
|
1881
|
-
const
|
|
2707
|
+
const allPages = [...compiledPages, ...outputPages];
|
|
2708
|
+
const graph = buildGraph(manifests, analyses, allPages);
|
|
1882
2709
|
await writeJsonFile(paths.graphPath, graph);
|
|
1883
2710
|
await writeJsonFile(paths.compileStatePath, {
|
|
1884
2711
|
generatedAt: graph.generatedAt,
|
|
1885
2712
|
schemaHash: schema.hash,
|
|
1886
|
-
analyses: Object.fromEntries(analyses.map((analysis) => [analysis.sourceId, analysisSignature(analysis)]))
|
|
2713
|
+
analyses: Object.fromEntries(analyses.map((analysis) => [analysis.sourceId, analysisSignature(analysis)])),
|
|
2714
|
+
sourceHashes: Object.fromEntries(manifests.map((manifest) => [manifest.sourceId, manifest.contentHash])),
|
|
2715
|
+
outputHashes: currentOutputHashes
|
|
1887
2716
|
});
|
|
1888
|
-
await writePage(paths.wikiDir, "index.md", buildIndexPage(
|
|
1889
|
-
await writePage(
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
2717
|
+
await writePage(paths.wikiDir, "index.md", buildIndexPage(allPages, schema.hash), changedPages);
|
|
2718
|
+
await writePage(
|
|
2719
|
+
paths.wikiDir,
|
|
2720
|
+
"sources/index.md",
|
|
2721
|
+
buildSectionIndex(
|
|
2722
|
+
"sources",
|
|
2723
|
+
allPages.filter((page) => page.kind === "source"),
|
|
2724
|
+
schema.hash
|
|
2725
|
+
),
|
|
2726
|
+
changedPages
|
|
2727
|
+
);
|
|
2728
|
+
await writePage(
|
|
2729
|
+
paths.wikiDir,
|
|
2730
|
+
"concepts/index.md",
|
|
2731
|
+
buildSectionIndex(
|
|
2732
|
+
"concepts",
|
|
2733
|
+
allPages.filter((page) => page.kind === "concept"),
|
|
2734
|
+
schema.hash
|
|
2735
|
+
),
|
|
2736
|
+
changedPages
|
|
2737
|
+
);
|
|
2738
|
+
await writePage(
|
|
2739
|
+
paths.wikiDir,
|
|
2740
|
+
"entities/index.md",
|
|
2741
|
+
buildSectionIndex(
|
|
2742
|
+
"entities",
|
|
2743
|
+
allPages.filter((page) => page.kind === "entity"),
|
|
2744
|
+
schema.hash
|
|
2745
|
+
),
|
|
2746
|
+
changedPages
|
|
2747
|
+
);
|
|
2748
|
+
await writePage(
|
|
2749
|
+
paths.wikiDir,
|
|
2750
|
+
"outputs/index.md",
|
|
2751
|
+
buildSectionIndex(
|
|
2752
|
+
"outputs",
|
|
2753
|
+
allPages.filter((page) => page.kind === "output"),
|
|
2754
|
+
schema.hash
|
|
2755
|
+
),
|
|
2756
|
+
changedPages
|
|
2757
|
+
);
|
|
2758
|
+
if (changedPages.length > 0 || outputsChanged || !artifactsExist) {
|
|
2759
|
+
await rebuildSearchIndex(paths.searchDbPath, allPages, paths.wikiDir);
|
|
2760
|
+
}
|
|
2761
|
+
await appendLogEntry(rootDir, "compile", `Compiled ${manifests.length} source(s)`, [
|
|
2762
|
+
`provider=${provider.id}`,
|
|
2763
|
+
`pages=${allPages.length}`,
|
|
2764
|
+
`dirty=${dirty.length}`,
|
|
2765
|
+
`clean=${clean.length}`,
|
|
2766
|
+
`outputs=${outputPages.length}`,
|
|
2767
|
+
`schema=${schema.hash.slice(0, 12)}`
|
|
2768
|
+
]);
|
|
1894
2769
|
return {
|
|
1895
2770
|
graphPath: paths.graphPath,
|
|
1896
|
-
pageCount:
|
|
2771
|
+
pageCount: allPages.length,
|
|
1897
2772
|
changedPages,
|
|
1898
2773
|
sourceCount: manifests.length
|
|
1899
2774
|
};
|
|
1900
2775
|
}
|
|
1901
2776
|
async function queryVault(rootDir, question, save = false) {
|
|
1902
|
-
const { paths } = await loadVaultConfig(rootDir);
|
|
1903
2777
|
const schema = await loadVaultSchema(rootDir);
|
|
1904
|
-
const
|
|
1905
|
-
|
|
1906
|
-
|
|
2778
|
+
const query = await executeQuery(rootDir, question);
|
|
2779
|
+
let savedTo;
|
|
2780
|
+
let savedPageId;
|
|
2781
|
+
if (save) {
|
|
2782
|
+
const saved = await persistOutputPage(rootDir, {
|
|
2783
|
+
question,
|
|
2784
|
+
answer: query.answer,
|
|
2785
|
+
citations: query.citations,
|
|
2786
|
+
schemaHash: schema.hash,
|
|
2787
|
+
relatedPageIds: query.relatedPageIds,
|
|
2788
|
+
relatedNodeIds: query.relatedNodeIds,
|
|
2789
|
+
relatedSourceIds: query.relatedSourceIds,
|
|
2790
|
+
origin: "query"
|
|
2791
|
+
});
|
|
2792
|
+
savedTo = saved.savedTo;
|
|
2793
|
+
savedPageId = saved.page.id;
|
|
1907
2794
|
}
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
2795
|
+
await appendLogEntry(rootDir, "query", question, [
|
|
2796
|
+
`citations=${query.citations.join(",") || "none"}`,
|
|
2797
|
+
`saved=${Boolean(savedTo)}`,
|
|
2798
|
+
`rawSources=${query.relatedSourceIds.length}`
|
|
2799
|
+
]);
|
|
2800
|
+
return {
|
|
2801
|
+
answer: query.answer,
|
|
2802
|
+
savedTo,
|
|
2803
|
+
savedPageId,
|
|
2804
|
+
citations: query.citations,
|
|
2805
|
+
relatedPageIds: query.relatedPageIds,
|
|
2806
|
+
relatedNodeIds: query.relatedNodeIds,
|
|
2807
|
+
relatedSourceIds: query.relatedSourceIds
|
|
2808
|
+
};
|
|
2809
|
+
}
|
|
2810
|
+
async function exploreVault(rootDir, question, steps = 3) {
|
|
2811
|
+
const schema = await loadVaultSchema(rootDir);
|
|
2812
|
+
const stepResults = [];
|
|
2813
|
+
const stepPages = [];
|
|
2814
|
+
const visited = /* @__PURE__ */ new Set();
|
|
2815
|
+
const suggestedQuestions = [];
|
|
2816
|
+
let currentQuestion = question;
|
|
2817
|
+
for (let step = 1; step <= Math.max(1, steps); step++) {
|
|
2818
|
+
const normalizedQuestion = normalizeWhitespace(currentQuestion).toLowerCase();
|
|
2819
|
+
if (!normalizedQuestion || visited.has(normalizedQuestion)) {
|
|
2820
|
+
break;
|
|
2821
|
+
}
|
|
2822
|
+
visited.add(normalizedQuestion);
|
|
2823
|
+
const query = await executeQuery(rootDir, currentQuestion);
|
|
2824
|
+
const saved = await persistOutputPage(rootDir, {
|
|
2825
|
+
title: `Explore Step ${step}: ${currentQuestion}`,
|
|
2826
|
+
question: currentQuestion,
|
|
2827
|
+
answer: query.answer,
|
|
2828
|
+
citations: query.citations,
|
|
2829
|
+
schemaHash: schema.hash,
|
|
2830
|
+
relatedPageIds: query.relatedPageIds,
|
|
2831
|
+
relatedNodeIds: query.relatedNodeIds,
|
|
2832
|
+
relatedSourceIds: query.relatedSourceIds,
|
|
2833
|
+
origin: "explore",
|
|
2834
|
+
slug: `explore-${slugify(question)}-step-${step}`
|
|
1935
2835
|
});
|
|
1936
|
-
|
|
2836
|
+
const followUpQuestions = await generateFollowUpQuestions(rootDir, currentQuestion, query.answer);
|
|
2837
|
+
stepResults.push({
|
|
2838
|
+
step,
|
|
2839
|
+
question: currentQuestion,
|
|
2840
|
+
answer: query.answer,
|
|
2841
|
+
savedTo: saved.savedTo,
|
|
2842
|
+
savedPageId: saved.page.id,
|
|
2843
|
+
citations: query.citations,
|
|
2844
|
+
followUpQuestions
|
|
2845
|
+
});
|
|
2846
|
+
stepPages.push(saved.page);
|
|
2847
|
+
suggestedQuestions.push(...followUpQuestions);
|
|
2848
|
+
const nextQuestion = followUpQuestions.find((item) => !visited.has(normalizeWhitespace(item).toLowerCase()));
|
|
2849
|
+
if (!nextQuestion) {
|
|
2850
|
+
break;
|
|
2851
|
+
}
|
|
2852
|
+
currentQuestion = nextQuestion;
|
|
1937
2853
|
}
|
|
1938
|
-
const
|
|
1939
|
-
|
|
2854
|
+
const allCitations = uniqueBy(
|
|
2855
|
+
stepResults.flatMap((step) => step.citations),
|
|
1940
2856
|
(item) => item
|
|
1941
2857
|
);
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
}
|
|
1950
|
-
await appendLogEntry(rootDir, "
|
|
1951
|
-
|
|
2858
|
+
const hub = await persistExploreHub(rootDir, {
|
|
2859
|
+
question,
|
|
2860
|
+
stepPages,
|
|
2861
|
+
followUpQuestions: uniqueBy(suggestedQuestions, (item) => item),
|
|
2862
|
+
citations: allCitations,
|
|
2863
|
+
schemaHash: schema.hash,
|
|
2864
|
+
slug: `explore-${slugify(question)}`
|
|
2865
|
+
});
|
|
2866
|
+
await appendLogEntry(rootDir, "explore", question, [
|
|
2867
|
+
`steps=${stepResults.length}`,
|
|
2868
|
+
`hub=${hub.page.id}`,
|
|
2869
|
+
`citations=${allCitations.join(",") || "none"}`
|
|
2870
|
+
]);
|
|
2871
|
+
return {
|
|
2872
|
+
rootQuestion: question,
|
|
2873
|
+
hubPath: hub.savedTo,
|
|
2874
|
+
hubPageId: hub.page.id,
|
|
2875
|
+
stepCount: stepResults.length,
|
|
2876
|
+
steps: stepResults,
|
|
2877
|
+
suggestedQuestions: uniqueBy(suggestedQuestions, (item) => item)
|
|
2878
|
+
};
|
|
1952
2879
|
}
|
|
1953
2880
|
async function searchVault(rootDir, query, limit = 5) {
|
|
1954
2881
|
const { paths } = await loadVaultConfig(rootDir);
|
|
@@ -1964,15 +2891,15 @@ async function listPages(rootDir) {
|
|
|
1964
2891
|
}
|
|
1965
2892
|
async function readPage(rootDir, relativePath) {
|
|
1966
2893
|
const { paths } = await loadVaultConfig(rootDir);
|
|
1967
|
-
const absolutePath =
|
|
2894
|
+
const absolutePath = path13.resolve(paths.wikiDir, relativePath);
|
|
1968
2895
|
if (!absolutePath.startsWith(paths.wikiDir) || !await fileExists(absolutePath)) {
|
|
1969
2896
|
return null;
|
|
1970
2897
|
}
|
|
1971
|
-
const raw = await
|
|
1972
|
-
const parsed =
|
|
2898
|
+
const raw = await fs11.readFile(absolutePath, "utf8");
|
|
2899
|
+
const parsed = matter5(raw);
|
|
1973
2900
|
return {
|
|
1974
2901
|
path: relativePath,
|
|
1975
|
-
title: typeof parsed.data.title === "string" ? parsed.data.title :
|
|
2902
|
+
title: typeof parsed.data.title === "string" ? parsed.data.title : path13.basename(relativePath, path13.extname(relativePath)),
|
|
1976
2903
|
frontmatter: parsed.data,
|
|
1977
2904
|
content: parsed.content
|
|
1978
2905
|
};
|
|
@@ -1994,12 +2921,70 @@ async function getWorkspaceInfo(rootDir) {
|
|
|
1994
2921
|
pageCount: pages.length
|
|
1995
2922
|
};
|
|
1996
2923
|
}
|
|
1997
|
-
|
|
2924
|
+
function structuralLintFindings(_rootDir, paths, graph, schemaHash, manifests) {
|
|
2925
|
+
const manifestMap = new Map(manifests.map((manifest) => [manifest.sourceId, manifest]));
|
|
2926
|
+
return Promise.all(
|
|
2927
|
+
graph.pages.map(async (page) => {
|
|
2928
|
+
const findings = [];
|
|
2929
|
+
if (page.schemaHash !== schemaHash) {
|
|
2930
|
+
findings.push({
|
|
2931
|
+
severity: "warning",
|
|
2932
|
+
code: "stale_page",
|
|
2933
|
+
message: `Page ${page.title} is stale because the vault schema changed.`,
|
|
2934
|
+
pagePath: path13.join(paths.wikiDir, page.path),
|
|
2935
|
+
relatedPageIds: [page.id]
|
|
2936
|
+
});
|
|
2937
|
+
}
|
|
2938
|
+
for (const [sourceId, knownHash] of Object.entries(page.sourceHashes)) {
|
|
2939
|
+
const manifest = manifestMap.get(sourceId);
|
|
2940
|
+
if (manifest && manifest.contentHash !== knownHash) {
|
|
2941
|
+
findings.push({
|
|
2942
|
+
severity: "warning",
|
|
2943
|
+
code: "stale_page",
|
|
2944
|
+
message: `Page ${page.title} is stale because source ${sourceId} changed.`,
|
|
2945
|
+
pagePath: path13.join(paths.wikiDir, page.path),
|
|
2946
|
+
relatedSourceIds: [sourceId],
|
|
2947
|
+
relatedPageIds: [page.id]
|
|
2948
|
+
});
|
|
2949
|
+
}
|
|
2950
|
+
}
|
|
2951
|
+
if (page.kind !== "index" && page.backlinks.length === 0) {
|
|
2952
|
+
findings.push({
|
|
2953
|
+
severity: "info",
|
|
2954
|
+
code: "orphan_page",
|
|
2955
|
+
message: `Page ${page.title} has no backlinks.`,
|
|
2956
|
+
pagePath: path13.join(paths.wikiDir, page.path),
|
|
2957
|
+
relatedPageIds: [page.id]
|
|
2958
|
+
});
|
|
2959
|
+
}
|
|
2960
|
+
const absolutePath = path13.join(paths.wikiDir, page.path);
|
|
2961
|
+
if (await fileExists(absolutePath)) {
|
|
2962
|
+
const content = await fs11.readFile(absolutePath, "utf8");
|
|
2963
|
+
if (content.includes("## Claims")) {
|
|
2964
|
+
const uncited = content.split("\n").filter((line) => line.startsWith("- ") && !line.includes("[source:"));
|
|
2965
|
+
if (uncited.length) {
|
|
2966
|
+
findings.push({
|
|
2967
|
+
severity: "warning",
|
|
2968
|
+
code: "uncited_claims",
|
|
2969
|
+
message: `Page ${page.title} contains uncited claim bullets.`,
|
|
2970
|
+
pagePath: absolutePath,
|
|
2971
|
+
relatedPageIds: [page.id]
|
|
2972
|
+
});
|
|
2973
|
+
}
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
return findings;
|
|
2977
|
+
})
|
|
2978
|
+
).then((results) => results.flat());
|
|
2979
|
+
}
|
|
2980
|
+
async function lintVault(rootDir, options = {}) {
|
|
2981
|
+
if (options.web && !options.deep) {
|
|
2982
|
+
throw new Error("`--web` can only be used together with `--deep`.");
|
|
2983
|
+
}
|
|
1998
2984
|
const { paths } = await loadVaultConfig(rootDir);
|
|
1999
2985
|
const schema = await loadVaultSchema(rootDir);
|
|
2000
2986
|
const manifests = await listManifests(rootDir);
|
|
2001
2987
|
const graph = await readJsonFile(paths.graphPath);
|
|
2002
|
-
const findings = [];
|
|
2003
2988
|
if (!graph) {
|
|
2004
2989
|
return [
|
|
2005
2990
|
{
|
|
@@ -2009,52 +2994,15 @@ async function lintVault(rootDir) {
|
|
|
2009
2994
|
}
|
|
2010
2995
|
];
|
|
2011
2996
|
}
|
|
2012
|
-
const
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
findings.push({
|
|
2016
|
-
severity: "warning",
|
|
2017
|
-
code: "stale_page",
|
|
2018
|
-
message: `Page ${page.title} is stale because the vault schema changed.`,
|
|
2019
|
-
pagePath: path10.join(paths.wikiDir, page.path)
|
|
2020
|
-
});
|
|
2021
|
-
}
|
|
2022
|
-
for (const [sourceId, knownHash] of Object.entries(page.sourceHashes)) {
|
|
2023
|
-
const manifest = manifestMap.get(sourceId);
|
|
2024
|
-
if (manifest && manifest.contentHash !== knownHash) {
|
|
2025
|
-
findings.push({
|
|
2026
|
-
severity: "warning",
|
|
2027
|
-
code: "stale_page",
|
|
2028
|
-
message: `Page ${page.title} is stale because source ${sourceId} changed.`,
|
|
2029
|
-
pagePath: path10.join(paths.wikiDir, page.path)
|
|
2030
|
-
});
|
|
2031
|
-
}
|
|
2032
|
-
}
|
|
2033
|
-
if (page.kind !== "index" && page.backlinks.length === 0) {
|
|
2034
|
-
findings.push({
|
|
2035
|
-
severity: "info",
|
|
2036
|
-
code: "orphan_page",
|
|
2037
|
-
message: `Page ${page.title} has no backlinks.`,
|
|
2038
|
-
pagePath: path10.join(paths.wikiDir, page.path)
|
|
2039
|
-
});
|
|
2040
|
-
}
|
|
2041
|
-
const absolutePath = path10.join(paths.wikiDir, page.path);
|
|
2042
|
-
if (await fileExists(absolutePath)) {
|
|
2043
|
-
const content = await fs9.readFile(absolutePath, "utf8");
|
|
2044
|
-
if (content.includes("## Claims")) {
|
|
2045
|
-
const uncited = content.split("\n").filter((line) => line.startsWith("- ") && !line.includes("[source:"));
|
|
2046
|
-
if (uncited.length) {
|
|
2047
|
-
findings.push({
|
|
2048
|
-
severity: "warning",
|
|
2049
|
-
code: "uncited_claims",
|
|
2050
|
-
message: `Page ${page.title} contains uncited claim bullets.`,
|
|
2051
|
-
pagePath: absolutePath
|
|
2052
|
-
});
|
|
2053
|
-
}
|
|
2054
|
-
}
|
|
2055
|
-
}
|
|
2997
|
+
const findings = await structuralLintFindings(rootDir, paths, graph, schema.hash, manifests);
|
|
2998
|
+
if (options.deep) {
|
|
2999
|
+
findings.push(...await runDeepLint(rootDir, findings, { web: options.web }));
|
|
2056
3000
|
}
|
|
2057
|
-
await appendLogEntry(rootDir, "lint", `Linted ${graph.pages.length} page(s)`, [
|
|
3001
|
+
await appendLogEntry(rootDir, "lint", `Linted ${graph.pages.length} page(s)`, [
|
|
3002
|
+
`findings=${findings.length}`,
|
|
3003
|
+
`deep=${Boolean(options.deep)}`,
|
|
3004
|
+
`web=${Boolean(options.web)}`
|
|
3005
|
+
]);
|
|
2058
3006
|
return findings;
|
|
2059
3007
|
}
|
|
2060
3008
|
async function bootstrapDemo(rootDir, input) {
|
|
@@ -2070,174 +3018,170 @@ async function bootstrapDemo(rootDir, input) {
|
|
|
2070
3018
|
};
|
|
2071
3019
|
}
|
|
2072
3020
|
|
|
2073
|
-
// src/viewer.ts
|
|
2074
|
-
import fs10 from "fs/promises";
|
|
2075
|
-
import http from "http";
|
|
2076
|
-
import path11 from "path";
|
|
2077
|
-
import mime2 from "mime-types";
|
|
2078
|
-
async function startGraphServer(rootDir, port) {
|
|
2079
|
-
const { config, paths } = await loadVaultConfig(rootDir);
|
|
2080
|
-
const effectivePort = port ?? config.viewer.port;
|
|
2081
|
-
const server = http.createServer(async (request, response) => {
|
|
2082
|
-
const url = new URL(request.url ?? "/", `http://${request.headers.host ?? `localhost:${effectivePort}`}`);
|
|
2083
|
-
if (url.pathname === "/api/graph") {
|
|
2084
|
-
if (!await fileExists(paths.graphPath)) {
|
|
2085
|
-
response.writeHead(404, { "content-type": "application/json" });
|
|
2086
|
-
response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
|
|
2087
|
-
return;
|
|
2088
|
-
}
|
|
2089
|
-
response.writeHead(200, { "content-type": "application/json" });
|
|
2090
|
-
response.end(await fs10.readFile(paths.graphPath, "utf8"));
|
|
2091
|
-
return;
|
|
2092
|
-
}
|
|
2093
|
-
const relativePath = url.pathname === "/" ? "index.html" : url.pathname.slice(1);
|
|
2094
|
-
const target = path11.join(paths.viewerDistDir, relativePath);
|
|
2095
|
-
const fallback = path11.join(paths.viewerDistDir, "index.html");
|
|
2096
|
-
const filePath = await fileExists(target) ? target : fallback;
|
|
2097
|
-
if (!await fileExists(filePath)) {
|
|
2098
|
-
response.writeHead(503, { "content-type": "text/plain" });
|
|
2099
|
-
response.end("Viewer build not found. Run `pnpm build` first.");
|
|
2100
|
-
return;
|
|
2101
|
-
}
|
|
2102
|
-
response.writeHead(200, { "content-type": mime2.lookup(filePath) || "text/plain" });
|
|
2103
|
-
response.end(await fs10.readFile(filePath));
|
|
2104
|
-
});
|
|
2105
|
-
await new Promise((resolve) => {
|
|
2106
|
-
server.listen(effectivePort, resolve);
|
|
2107
|
-
});
|
|
2108
|
-
return {
|
|
2109
|
-
port: effectivePort,
|
|
2110
|
-
close: async () => {
|
|
2111
|
-
await new Promise((resolve, reject) => {
|
|
2112
|
-
server.close((error) => {
|
|
2113
|
-
if (error) {
|
|
2114
|
-
reject(error);
|
|
2115
|
-
return;
|
|
2116
|
-
}
|
|
2117
|
-
resolve();
|
|
2118
|
-
});
|
|
2119
|
-
});
|
|
2120
|
-
}
|
|
2121
|
-
};
|
|
2122
|
-
}
|
|
2123
|
-
|
|
2124
3021
|
// src/mcp.ts
|
|
2125
|
-
|
|
2126
|
-
import path12 from "path";
|
|
2127
|
-
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2128
|
-
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
2129
|
-
import { z as z6 } from "zod";
|
|
2130
|
-
var SERVER_VERSION = "0.1.4";
|
|
3022
|
+
var SERVER_VERSION = "0.1.5";
|
|
2131
3023
|
async function createMcpServer(rootDir) {
|
|
2132
3024
|
const server = new McpServer({
|
|
2133
3025
|
name: "swarmvault",
|
|
2134
3026
|
version: SERVER_VERSION,
|
|
2135
3027
|
websiteUrl: "https://www.swarmvault.ai"
|
|
2136
3028
|
});
|
|
2137
|
-
server.registerTool(
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
inputSchema: {
|
|
2146
|
-
query: z6.string().min(1).describe("Search query"),
|
|
2147
|
-
limit: z6.number().int().min(1).max(25).optional().describe("Maximum number of results")
|
|
2148
|
-
}
|
|
2149
|
-
}, async ({ query, limit }) => {
|
|
2150
|
-
const results = await searchVault(rootDir, query, limit ?? 5);
|
|
2151
|
-
return asToolText(results);
|
|
2152
|
-
});
|
|
2153
|
-
server.registerTool("read_page", {
|
|
2154
|
-
description: "Read a generated wiki page by its path relative to wiki/.",
|
|
2155
|
-
inputSchema: {
|
|
2156
|
-
path: z6.string().min(1).describe("Path relative to wiki/, for example sources/example.md")
|
|
3029
|
+
server.registerTool(
|
|
3030
|
+
"workspace_info",
|
|
3031
|
+
{
|
|
3032
|
+
description: "Return the current SwarmVault workspace paths and high-level counts."
|
|
3033
|
+
},
|
|
3034
|
+
async () => {
|
|
3035
|
+
const info = await getWorkspaceInfo(rootDir);
|
|
3036
|
+
return asToolText(info);
|
|
2157
3037
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
3038
|
+
);
|
|
3039
|
+
server.registerTool(
|
|
3040
|
+
"search_pages",
|
|
3041
|
+
{
|
|
3042
|
+
description: "Search compiled wiki pages using the local full-text index.",
|
|
3043
|
+
inputSchema: {
|
|
3044
|
+
query: z9.string().min(1).describe("Search query"),
|
|
3045
|
+
limit: z9.number().int().min(1).max(25).optional().describe("Maximum number of results")
|
|
3046
|
+
}
|
|
3047
|
+
},
|
|
3048
|
+
async ({ query, limit }) => {
|
|
3049
|
+
const results = await searchVault(rootDir, query, limit ?? 5);
|
|
3050
|
+
return asToolText(results);
|
|
2162
3051
|
}
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
}
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
server.registerTool(
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
3052
|
+
);
|
|
3053
|
+
server.registerTool(
|
|
3054
|
+
"read_page",
|
|
3055
|
+
{
|
|
3056
|
+
description: "Read a generated wiki page by its path relative to wiki/.",
|
|
3057
|
+
inputSchema: {
|
|
3058
|
+
path: z9.string().min(1).describe("Path relative to wiki/, for example sources/example.md")
|
|
3059
|
+
}
|
|
3060
|
+
},
|
|
3061
|
+
async ({ path: relativePath }) => {
|
|
3062
|
+
const page = await readPage(rootDir, relativePath);
|
|
3063
|
+
if (!page) {
|
|
3064
|
+
return asToolError(`Page not found: ${relativePath}`);
|
|
3065
|
+
}
|
|
3066
|
+
return asToolText(page);
|
|
3067
|
+
}
|
|
3068
|
+
);
|
|
3069
|
+
server.registerTool(
|
|
3070
|
+
"list_sources",
|
|
3071
|
+
{
|
|
3072
|
+
description: "List source manifests in the current workspace.",
|
|
3073
|
+
inputSchema: {
|
|
3074
|
+
limit: z9.number().int().min(1).max(100).optional().describe("Maximum number of manifests to return")
|
|
3075
|
+
}
|
|
3076
|
+
},
|
|
3077
|
+
async ({ limit }) => {
|
|
3078
|
+
const manifests = await listManifests(rootDir);
|
|
3079
|
+
return asToolText(limit ? manifests.slice(0, limit) : manifests);
|
|
3080
|
+
}
|
|
3081
|
+
);
|
|
3082
|
+
server.registerTool(
|
|
3083
|
+
"query_vault",
|
|
3084
|
+
{
|
|
3085
|
+
description: "Ask a question against the compiled vault and optionally save the answer.",
|
|
3086
|
+
inputSchema: {
|
|
3087
|
+
question: z9.string().min(1).describe("Question to ask the vault"),
|
|
3088
|
+
save: z9.boolean().optional().describe("Persist the answer to wiki/outputs")
|
|
3089
|
+
}
|
|
3090
|
+
},
|
|
3091
|
+
async ({ question, save }) => {
|
|
3092
|
+
const result = await queryVault(rootDir, question, save ?? false);
|
|
3093
|
+
return asToolText(result);
|
|
3094
|
+
}
|
|
3095
|
+
);
|
|
3096
|
+
server.registerTool(
|
|
3097
|
+
"ingest_input",
|
|
3098
|
+
{
|
|
3099
|
+
description: "Ingest a local file path or URL into the SwarmVault workspace.",
|
|
3100
|
+
inputSchema: {
|
|
3101
|
+
input: z9.string().min(1).describe("Local path or URL to ingest")
|
|
3102
|
+
}
|
|
3103
|
+
},
|
|
3104
|
+
async ({ input }) => {
|
|
3105
|
+
const manifest = await ingestInput(rootDir, input);
|
|
3106
|
+
return asToolText(manifest);
|
|
3107
|
+
}
|
|
3108
|
+
);
|
|
3109
|
+
server.registerTool(
|
|
3110
|
+
"compile_vault",
|
|
3111
|
+
{
|
|
3112
|
+
description: "Compile source manifests into wiki pages, graph data, and search index."
|
|
3113
|
+
},
|
|
3114
|
+
async () => {
|
|
3115
|
+
const result = await compileVault(rootDir);
|
|
3116
|
+
return asToolText(result);
|
|
3117
|
+
}
|
|
3118
|
+
);
|
|
3119
|
+
server.registerTool(
|
|
3120
|
+
"lint_vault",
|
|
3121
|
+
{
|
|
3122
|
+
description: "Run anti-drift and vault health checks."
|
|
3123
|
+
},
|
|
3124
|
+
async () => {
|
|
3125
|
+
const findings = await lintVault(rootDir);
|
|
3126
|
+
return asToolText(findings);
|
|
3127
|
+
}
|
|
3128
|
+
);
|
|
3129
|
+
server.registerResource(
|
|
3130
|
+
"swarmvault-config",
|
|
3131
|
+
"swarmvault://config",
|
|
3132
|
+
{
|
|
3133
|
+
title: "SwarmVault Config",
|
|
3134
|
+
description: "The resolved SwarmVault config file.",
|
|
3135
|
+
mimeType: "application/json"
|
|
3136
|
+
},
|
|
3137
|
+
async () => {
|
|
3138
|
+
const { config } = await loadVaultConfig(rootDir);
|
|
3139
|
+
return asTextResource("swarmvault://config", JSON.stringify(config, null, 2));
|
|
3140
|
+
}
|
|
3141
|
+
);
|
|
3142
|
+
server.registerResource(
|
|
3143
|
+
"swarmvault-graph",
|
|
3144
|
+
"swarmvault://graph",
|
|
3145
|
+
{
|
|
3146
|
+
title: "SwarmVault Graph",
|
|
3147
|
+
description: "The compiled graph artifact for the current workspace.",
|
|
3148
|
+
mimeType: "application/json"
|
|
3149
|
+
},
|
|
3150
|
+
async () => {
|
|
3151
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
3152
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
3153
|
+
return asTextResource(
|
|
3154
|
+
"swarmvault://graph",
|
|
3155
|
+
JSON.stringify(graph ?? { error: "Graph artifact not found. Run `swarmvault compile` first." }, null, 2)
|
|
3156
|
+
);
|
|
3157
|
+
}
|
|
3158
|
+
);
|
|
3159
|
+
server.registerResource(
|
|
3160
|
+
"swarmvault-manifests",
|
|
3161
|
+
"swarmvault://manifests",
|
|
3162
|
+
{
|
|
3163
|
+
title: "SwarmVault Manifests",
|
|
3164
|
+
description: "All source manifests in the workspace.",
|
|
3165
|
+
mimeType: "application/json"
|
|
3166
|
+
},
|
|
3167
|
+
async () => {
|
|
3168
|
+
const manifests = await listManifests(rootDir);
|
|
3169
|
+
return asTextResource("swarmvault://manifests", JSON.stringify(manifests, null, 2));
|
|
3170
|
+
}
|
|
3171
|
+
);
|
|
3172
|
+
server.registerResource(
|
|
3173
|
+
"swarmvault-schema",
|
|
3174
|
+
"swarmvault://schema",
|
|
3175
|
+
{
|
|
3176
|
+
title: "SwarmVault Schema",
|
|
3177
|
+
description: "The vault schema file that guides compile and query behavior.",
|
|
3178
|
+
mimeType: "text/markdown"
|
|
3179
|
+
},
|
|
3180
|
+
async () => {
|
|
3181
|
+
const schema = await loadVaultSchema(rootDir);
|
|
3182
|
+
return asTextResource("swarmvault://schema", schema.content);
|
|
3183
|
+
}
|
|
3184
|
+
);
|
|
2241
3185
|
server.registerResource(
|
|
2242
3186
|
"swarmvault-pages",
|
|
2243
3187
|
new ResourceTemplate("swarmvault://pages/{path}", {
|
|
@@ -2267,8 +3211,8 @@ async function createMcpServer(rootDir) {
|
|
|
2267
3211
|
return asTextResource(`swarmvault://pages/${encodedPath}`, `Page not found: ${relativePath}`);
|
|
2268
3212
|
}
|
|
2269
3213
|
const { paths } = await loadVaultConfig(rootDir);
|
|
2270
|
-
const absolutePath =
|
|
2271
|
-
return asTextResource(`swarmvault://pages/${encodedPath}`, await
|
|
3214
|
+
const absolutePath = path14.resolve(paths.wikiDir, relativePath);
|
|
3215
|
+
return asTextResource(`swarmvault://pages/${encodedPath}`, await fs12.readFile(absolutePath, "utf8"));
|
|
2272
3216
|
}
|
|
2273
3217
|
);
|
|
2274
3218
|
return server;
|
|
@@ -2315,21 +3259,78 @@ function asTextResource(uri, text) {
|
|
|
2315
3259
|
};
|
|
2316
3260
|
}
|
|
2317
3261
|
|
|
3262
|
+
// src/viewer.ts
|
|
3263
|
+
import fs13 from "fs/promises";
|
|
3264
|
+
import http from "http";
|
|
3265
|
+
import path15 from "path";
|
|
3266
|
+
import mime2 from "mime-types";
|
|
3267
|
+
async function startGraphServer(rootDir, port) {
|
|
3268
|
+
const { config, paths } = await loadVaultConfig(rootDir);
|
|
3269
|
+
const effectivePort = port ?? config.viewer.port;
|
|
3270
|
+
const server = http.createServer(async (request, response) => {
|
|
3271
|
+
const url = new URL(request.url ?? "/", `http://${request.headers.host ?? `localhost:${effectivePort}`}`);
|
|
3272
|
+
if (url.pathname === "/api/graph") {
|
|
3273
|
+
if (!await fileExists(paths.graphPath)) {
|
|
3274
|
+
response.writeHead(404, { "content-type": "application/json" });
|
|
3275
|
+
response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
|
|
3276
|
+
return;
|
|
3277
|
+
}
|
|
3278
|
+
response.writeHead(200, { "content-type": "application/json" });
|
|
3279
|
+
response.end(await fs13.readFile(paths.graphPath, "utf8"));
|
|
3280
|
+
return;
|
|
3281
|
+
}
|
|
3282
|
+
const relativePath = url.pathname === "/" ? "index.html" : url.pathname.slice(1);
|
|
3283
|
+
const target = path15.join(paths.viewerDistDir, relativePath);
|
|
3284
|
+
const fallback = path15.join(paths.viewerDistDir, "index.html");
|
|
3285
|
+
const filePath = await fileExists(target) ? target : fallback;
|
|
3286
|
+
if (!await fileExists(filePath)) {
|
|
3287
|
+
response.writeHead(503, { "content-type": "text/plain" });
|
|
3288
|
+
response.end("Viewer build not found. Run `pnpm build` first.");
|
|
3289
|
+
return;
|
|
3290
|
+
}
|
|
3291
|
+
response.writeHead(200, { "content-type": mime2.lookup(filePath) || "text/plain" });
|
|
3292
|
+
response.end(await fs13.readFile(filePath));
|
|
3293
|
+
});
|
|
3294
|
+
await new Promise((resolve) => {
|
|
3295
|
+
server.listen(effectivePort, resolve);
|
|
3296
|
+
});
|
|
3297
|
+
return {
|
|
3298
|
+
port: effectivePort,
|
|
3299
|
+
close: async () => {
|
|
3300
|
+
await new Promise((resolve, reject) => {
|
|
3301
|
+
server.close((error) => {
|
|
3302
|
+
if (error) {
|
|
3303
|
+
reject(error);
|
|
3304
|
+
return;
|
|
3305
|
+
}
|
|
3306
|
+
resolve();
|
|
3307
|
+
});
|
|
3308
|
+
});
|
|
3309
|
+
}
|
|
3310
|
+
};
|
|
3311
|
+
}
|
|
3312
|
+
|
|
2318
3313
|
// src/watch.ts
|
|
2319
|
-
import
|
|
3314
|
+
import path16 from "path";
|
|
3315
|
+
import process2 from "process";
|
|
2320
3316
|
import chokidar from "chokidar";
|
|
3317
|
+
var MAX_BACKOFF_MS = 3e4;
|
|
3318
|
+
var BACKOFF_THRESHOLD = 3;
|
|
3319
|
+
var CRITICAL_THRESHOLD = 10;
|
|
2321
3320
|
async function watchVault(rootDir, options = {}) {
|
|
2322
3321
|
const { paths } = await initWorkspace(rootDir);
|
|
2323
|
-
const
|
|
3322
|
+
const baseDebounceMs = options.debounceMs ?? 900;
|
|
2324
3323
|
let timer;
|
|
2325
3324
|
let running = false;
|
|
2326
3325
|
let pending = false;
|
|
2327
3326
|
let closed = false;
|
|
3327
|
+
let consecutiveFailures = 0;
|
|
3328
|
+
let currentDebounceMs = baseDebounceMs;
|
|
2328
3329
|
const reasons = /* @__PURE__ */ new Set();
|
|
2329
3330
|
const watcher = chokidar.watch(paths.inboxDir, {
|
|
2330
3331
|
ignoreInitial: true,
|
|
2331
3332
|
awaitWriteFinish: {
|
|
2332
|
-
stabilityThreshold: Math.max(250, Math.floor(
|
|
3333
|
+
stabilityThreshold: Math.max(250, Math.floor(baseDebounceMs / 2)),
|
|
2333
3334
|
pollInterval: 100
|
|
2334
3335
|
}
|
|
2335
3336
|
});
|
|
@@ -2344,7 +3345,7 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2344
3345
|
}
|
|
2345
3346
|
timer = setTimeout(() => {
|
|
2346
3347
|
void runCycle();
|
|
2347
|
-
},
|
|
3348
|
+
}, currentDebounceMs);
|
|
2348
3349
|
};
|
|
2349
3350
|
const runCycle = async () => {
|
|
2350
3351
|
if (running || closed || !pending) {
|
|
@@ -2373,9 +3374,23 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2373
3374
|
const findings = await lintVault(rootDir);
|
|
2374
3375
|
lintFindingCount = findings.length;
|
|
2375
3376
|
}
|
|
3377
|
+
consecutiveFailures = 0;
|
|
3378
|
+
currentDebounceMs = baseDebounceMs;
|
|
2376
3379
|
} catch (caught) {
|
|
2377
3380
|
success = false;
|
|
2378
3381
|
error = caught instanceof Error ? caught.message : String(caught);
|
|
3382
|
+
consecutiveFailures++;
|
|
3383
|
+
pending = true;
|
|
3384
|
+
if (consecutiveFailures >= CRITICAL_THRESHOLD) {
|
|
3385
|
+
process2.stderr.write(
|
|
3386
|
+
`[swarmvault watch] ${consecutiveFailures} consecutive failures. Check vault state. Continuing at max backoff.
|
|
3387
|
+
`
|
|
3388
|
+
);
|
|
3389
|
+
}
|
|
3390
|
+
if (consecutiveFailures >= BACKOFF_THRESHOLD) {
|
|
3391
|
+
const multiplier = 2 ** (consecutiveFailures - BACKOFF_THRESHOLD);
|
|
3392
|
+
currentDebounceMs = Math.min(baseDebounceMs * multiplier, MAX_BACKOFF_MS);
|
|
3393
|
+
}
|
|
2379
3394
|
} finally {
|
|
2380
3395
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
2381
3396
|
await appendWatchRun(rootDir, {
|
|
@@ -2399,6 +3414,18 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2399
3414
|
}
|
|
2400
3415
|
};
|
|
2401
3416
|
watcher.on("add", (filePath) => schedule(`add:${toWatchReason(paths.inboxDir, filePath)}`)).on("change", (filePath) => schedule(`change:${toWatchReason(paths.inboxDir, filePath)}`)).on("unlink", (filePath) => schedule(`unlink:${toWatchReason(paths.inboxDir, filePath)}`)).on("addDir", (dirPath) => schedule(`addDir:${toWatchReason(paths.inboxDir, dirPath)}`)).on("unlinkDir", (dirPath) => schedule(`unlinkDir:${toWatchReason(paths.inboxDir, dirPath)}`)).on("error", (caught) => schedule(`error:${caught instanceof Error ? caught.message : String(caught)}`));
|
|
3417
|
+
await new Promise((resolve, reject) => {
|
|
3418
|
+
const handleReady = () => {
|
|
3419
|
+
watcher.off("error", handleError);
|
|
3420
|
+
resolve();
|
|
3421
|
+
};
|
|
3422
|
+
const handleError = (caught) => {
|
|
3423
|
+
watcher.off("ready", handleReady);
|
|
3424
|
+
reject(caught);
|
|
3425
|
+
};
|
|
3426
|
+
watcher.once("ready", handleReady);
|
|
3427
|
+
watcher.once("error", handleError);
|
|
3428
|
+
});
|
|
2402
3429
|
return {
|
|
2403
3430
|
close: async () => {
|
|
2404
3431
|
closed = true;
|
|
@@ -2410,7 +3437,7 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2410
3437
|
};
|
|
2411
3438
|
}
|
|
2412
3439
|
function toWatchReason(baseDir, targetPath) {
|
|
2413
|
-
return
|
|
3440
|
+
return path16.relative(baseDir, targetPath) || ".";
|
|
2414
3441
|
}
|
|
2415
3442
|
export {
|
|
2416
3443
|
assertProviderCapability,
|
|
@@ -2418,9 +3445,12 @@ export {
|
|
|
2418
3445
|
compileVault,
|
|
2419
3446
|
createMcpServer,
|
|
2420
3447
|
createProvider,
|
|
3448
|
+
createWebSearchAdapter,
|
|
2421
3449
|
defaultVaultConfig,
|
|
2422
3450
|
defaultVaultSchema,
|
|
3451
|
+
exploreVault,
|
|
2423
3452
|
getProviderForTask,
|
|
3453
|
+
getWebSearchAdapterForTask,
|
|
2424
3454
|
getWorkspaceInfo,
|
|
2425
3455
|
importInbox,
|
|
2426
3456
|
ingestInput,
|