@swarmvaultai/engine 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -8
- package/dist/index.d.ts +110 -14
- package/dist/index.js +2101 -917
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,8 +1,19 @@
|
|
|
1
|
+
// src/agents.ts
|
|
2
|
+
import fs3 from "fs/promises";
|
|
3
|
+
import path3 from "path";
|
|
4
|
+
|
|
1
5
|
// src/config.ts
|
|
6
|
+
import fs2 from "fs/promises";
|
|
2
7
|
import path2 from "path";
|
|
3
8
|
import { fileURLToPath } from "url";
|
|
4
9
|
import { z as z2 } from "zod";
|
|
5
10
|
|
|
11
|
+
// src/types.ts
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
var providerCapabilitySchema = z.enum(["responses", "chat", "structured", "tools", "vision", "embeddings", "streaming", "local"]);
|
|
14
|
+
var providerTypeSchema = z.enum(["heuristic", "openai", "ollama", "anthropic", "gemini", "openai-compatible", "custom"]);
|
|
15
|
+
var webSearchProviderTypeSchema = z.enum(["http-json", "custom"]);
|
|
16
|
+
|
|
6
17
|
// src/utils.ts
|
|
7
18
|
import crypto from "crypto";
|
|
8
19
|
import fs from "fs/promises";
|
|
@@ -109,31 +120,11 @@ async function listFilesRecursive(rootDir) {
|
|
|
109
120
|
return files;
|
|
110
121
|
}
|
|
111
122
|
|
|
112
|
-
// src/types.ts
|
|
113
|
-
import { z } from "zod";
|
|
114
|
-
var providerCapabilitySchema = z.enum([
|
|
115
|
-
"responses",
|
|
116
|
-
"chat",
|
|
117
|
-
"structured",
|
|
118
|
-
"tools",
|
|
119
|
-
"vision",
|
|
120
|
-
"embeddings",
|
|
121
|
-
"streaming",
|
|
122
|
-
"local"
|
|
123
|
-
]);
|
|
124
|
-
var providerTypeSchema = z.enum([
|
|
125
|
-
"heuristic",
|
|
126
|
-
"openai",
|
|
127
|
-
"ollama",
|
|
128
|
-
"anthropic",
|
|
129
|
-
"gemini",
|
|
130
|
-
"openai-compatible",
|
|
131
|
-
"custom"
|
|
132
|
-
]);
|
|
133
|
-
|
|
134
123
|
// src/config.ts
|
|
135
124
|
var PRIMARY_CONFIG_FILENAME = "swarmvault.config.json";
|
|
136
125
|
var LEGACY_CONFIG_FILENAME = "vault.config.json";
|
|
126
|
+
var PRIMARY_SCHEMA_FILENAME = "swarmvault.schema.md";
|
|
127
|
+
var LEGACY_SCHEMA_FILENAME = "schema.md";
|
|
137
128
|
var moduleDir = path2.dirname(fileURLToPath(import.meta.url));
|
|
138
129
|
var providerConfigSchema = z2.object({
|
|
139
130
|
type: providerTypeSchema,
|
|
@@ -145,6 +136,22 @@ var providerConfigSchema = z2.object({
|
|
|
145
136
|
capabilities: z2.array(providerCapabilitySchema).optional(),
|
|
146
137
|
apiStyle: z2.enum(["responses", "chat"]).optional()
|
|
147
138
|
});
|
|
139
|
+
var webSearchProviderConfigSchema = z2.object({
|
|
140
|
+
type: webSearchProviderTypeSchema,
|
|
141
|
+
endpoint: z2.string().url().optional(),
|
|
142
|
+
method: z2.enum(["GET", "POST"]).optional(),
|
|
143
|
+
apiKeyEnv: z2.string().min(1).optional(),
|
|
144
|
+
apiKeyHeader: z2.string().min(1).optional(),
|
|
145
|
+
apiKeyPrefix: z2.string().optional(),
|
|
146
|
+
headers: z2.record(z2.string(), z2.string()).optional(),
|
|
147
|
+
queryParam: z2.string().min(1).optional(),
|
|
148
|
+
limitParam: z2.string().min(1).optional(),
|
|
149
|
+
resultsPath: z2.string().min(1).optional(),
|
|
150
|
+
titleField: z2.string().min(1).optional(),
|
|
151
|
+
urlField: z2.string().min(1).optional(),
|
|
152
|
+
snippetField: z2.string().min(1).optional(),
|
|
153
|
+
module: z2.string().min(1).optional()
|
|
154
|
+
});
|
|
148
155
|
var vaultConfigSchema = z2.object({
|
|
149
156
|
workspace: z2.object({
|
|
150
157
|
rawDir: z2.string().min(1),
|
|
@@ -163,7 +170,13 @@ var vaultConfigSchema = z2.object({
|
|
|
163
170
|
viewer: z2.object({
|
|
164
171
|
port: z2.number().int().positive()
|
|
165
172
|
}),
|
|
166
|
-
agents: z2.array(z2.enum(["codex", "claude", "cursor"])).default(["codex", "claude", "cursor"])
|
|
173
|
+
agents: z2.array(z2.enum(["codex", "claude", "cursor"])).default(["codex", "claude", "cursor"]),
|
|
174
|
+
webSearch: z2.object({
|
|
175
|
+
providers: z2.record(z2.string(), webSearchProviderConfigSchema),
|
|
176
|
+
tasks: z2.object({
|
|
177
|
+
deepLintProvider: z2.string().min(1)
|
|
178
|
+
})
|
|
179
|
+
}).optional()
|
|
167
180
|
});
|
|
168
181
|
function defaultVaultConfig() {
|
|
169
182
|
return {
|
|
@@ -193,6 +206,52 @@ function defaultVaultConfig() {
|
|
|
193
206
|
agents: ["codex", "claude", "cursor"]
|
|
194
207
|
};
|
|
195
208
|
}
|
|
209
|
+
function defaultVaultSchema() {
|
|
210
|
+
return [
|
|
211
|
+
"# SwarmVault Schema",
|
|
212
|
+
"",
|
|
213
|
+
"Edit this file to teach SwarmVault how this vault should be organized and maintained.",
|
|
214
|
+
"",
|
|
215
|
+
"## Vault Purpose",
|
|
216
|
+
"",
|
|
217
|
+
"- Describe the domain this vault covers.",
|
|
218
|
+
"- Note the intended audience and the kinds of questions the vault should answer well.",
|
|
219
|
+
"",
|
|
220
|
+
"## Naming Conventions",
|
|
221
|
+
"",
|
|
222
|
+
"- Prefer stable, descriptive page titles.",
|
|
223
|
+
"- Keep concept and entity names specific to the domain.",
|
|
224
|
+
"",
|
|
225
|
+
"## Page Structure Rules",
|
|
226
|
+
"",
|
|
227
|
+
"- Source pages should stay grounded in the original material.",
|
|
228
|
+
"- Concept and entity pages should aggregate source-backed claims instead of inventing new ones.",
|
|
229
|
+
"- Preserve contradictions instead of smoothing them away.",
|
|
230
|
+
"",
|
|
231
|
+
"## Categories",
|
|
232
|
+
"",
|
|
233
|
+
"- List domain-specific concept categories here.",
|
|
234
|
+
"- List important entity types here.",
|
|
235
|
+
"",
|
|
236
|
+
"## Relationship Types",
|
|
237
|
+
"",
|
|
238
|
+
"- Mentions",
|
|
239
|
+
"- Supports",
|
|
240
|
+
"- Contradicts",
|
|
241
|
+
"- Depends on",
|
|
242
|
+
"",
|
|
243
|
+
"## Grounding Rules",
|
|
244
|
+
"",
|
|
245
|
+
"- Prefer raw sources over summaries.",
|
|
246
|
+
"- Cite source ids whenever claims are stated.",
|
|
247
|
+
"- Do not treat the wiki as a source of truth when the raw material disagrees.",
|
|
248
|
+
"",
|
|
249
|
+
"## Exclusions",
|
|
250
|
+
"",
|
|
251
|
+
"- List topics, claims, or page types the compiler should avoid generating.",
|
|
252
|
+
""
|
|
253
|
+
].join("\n");
|
|
254
|
+
}
|
|
196
255
|
async function findConfigPath(rootDir) {
|
|
197
256
|
const primaryPath = path2.join(rootDir, PRIMARY_CONFIG_FILENAME);
|
|
198
257
|
if (await fileExists(primaryPath)) {
|
|
@@ -204,7 +263,18 @@ async function findConfigPath(rootDir) {
|
|
|
204
263
|
}
|
|
205
264
|
return primaryPath;
|
|
206
265
|
}
|
|
207
|
-
function
|
|
266
|
+
async function findSchemaPath(rootDir) {
|
|
267
|
+
const primaryPath = path2.join(rootDir, PRIMARY_SCHEMA_FILENAME);
|
|
268
|
+
if (await fileExists(primaryPath)) {
|
|
269
|
+
return primaryPath;
|
|
270
|
+
}
|
|
271
|
+
const legacyPath = path2.join(rootDir, LEGACY_SCHEMA_FILENAME);
|
|
272
|
+
if (await fileExists(legacyPath)) {
|
|
273
|
+
return legacyPath;
|
|
274
|
+
}
|
|
275
|
+
return primaryPath;
|
|
276
|
+
}
|
|
277
|
+
function resolvePaths(rootDir, config, configPath = path2.join(rootDir, PRIMARY_CONFIG_FILENAME), schemaPath = path2.join(rootDir, PRIMARY_SCHEMA_FILENAME)) {
|
|
208
278
|
const effective = config ?? defaultVaultConfig();
|
|
209
279
|
const rawDir = path2.resolve(rootDir, effective.workspace.rawDir);
|
|
210
280
|
const rawSourcesDir = path2.join(rawDir, "sources");
|
|
@@ -215,6 +285,7 @@ function resolvePaths(rootDir, config, configPath = path2.join(rootDir, PRIMARY_
|
|
|
215
285
|
const inboxDir = path2.resolve(rootDir, effective.workspace.inboxDir);
|
|
216
286
|
return {
|
|
217
287
|
rootDir,
|
|
288
|
+
schemaPath,
|
|
218
289
|
rawDir,
|
|
219
290
|
rawSourcesDir,
|
|
220
291
|
rawAssetsDir,
|
|
@@ -235,17 +306,21 @@ function resolvePaths(rootDir, config, configPath = path2.join(rootDir, PRIMARY_
|
|
|
235
306
|
}
|
|
236
307
|
async function loadVaultConfig(rootDir) {
|
|
237
308
|
const configPath = await findConfigPath(rootDir);
|
|
309
|
+
const schemaPath = await findSchemaPath(rootDir);
|
|
238
310
|
const raw = await readJsonFile(configPath);
|
|
239
311
|
const parsed = vaultConfigSchema.parse(raw ?? defaultVaultConfig());
|
|
240
312
|
return {
|
|
241
313
|
config: parsed,
|
|
242
|
-
paths: resolvePaths(rootDir, parsed, configPath)
|
|
314
|
+
paths: resolvePaths(rootDir, parsed, configPath, schemaPath)
|
|
243
315
|
};
|
|
244
316
|
}
|
|
245
317
|
async function initWorkspace(rootDir) {
|
|
246
318
|
const configPath = await findConfigPath(rootDir);
|
|
319
|
+
const schemaPath = await findSchemaPath(rootDir);
|
|
247
320
|
const config = await fileExists(configPath) ? (await loadVaultConfig(rootDir)).config : defaultVaultConfig();
|
|
248
|
-
const paths = resolvePaths(rootDir, config, configPath);
|
|
321
|
+
const paths = resolvePaths(rootDir, config, configPath, schemaPath);
|
|
322
|
+
const primarySchemaPath = path2.join(rootDir, PRIMARY_SCHEMA_FILENAME);
|
|
323
|
+
const legacySchemaPath = path2.join(rootDir, LEGACY_SCHEMA_FILENAME);
|
|
249
324
|
await Promise.all([
|
|
250
325
|
ensureDir(paths.rawDir),
|
|
251
326
|
ensureDir(paths.wikiDir),
|
|
@@ -261,28 +336,108 @@ async function initWorkspace(rootDir) {
|
|
|
261
336
|
if (!await fileExists(configPath)) {
|
|
262
337
|
await writeJsonFile(configPath, config);
|
|
263
338
|
}
|
|
339
|
+
if (!await fileExists(primarySchemaPath) && !await fileExists(legacySchemaPath)) {
|
|
340
|
+
await ensureDir(path2.dirname(primarySchemaPath));
|
|
341
|
+
await fs2.writeFile(primarySchemaPath, defaultVaultSchema(), "utf8");
|
|
342
|
+
}
|
|
264
343
|
return { config, paths };
|
|
265
344
|
}
|
|
266
345
|
|
|
346
|
+
// src/agents.ts
|
|
347
|
+
var managedStart = "<!-- swarmvault:managed:start -->";
|
|
348
|
+
var managedEnd = "<!-- swarmvault:managed:end -->";
|
|
349
|
+
var legacyManagedStart = "<!-- vault:managed:start -->";
|
|
350
|
+
var legacyManagedEnd = "<!-- vault:managed:end -->";
|
|
351
|
+
function buildManagedBlock(agent) {
|
|
352
|
+
const body = [
|
|
353
|
+
managedStart,
|
|
354
|
+
`# SwarmVault Rules (${agent})`,
|
|
355
|
+
"",
|
|
356
|
+
"- Read `swarmvault.schema.md` before compile or query style work. If only `schema.md` exists, treat it as the legacy schema path.",
|
|
357
|
+
"- Treat `raw/` as immutable source input.",
|
|
358
|
+
"- Treat `wiki/` as generated markdown owned by the agent and compiler workflow.",
|
|
359
|
+
"- Read `wiki/index.md` before broad file searching when answering SwarmVault questions.",
|
|
360
|
+
"- Preserve frontmatter fields including `page_id`, `source_ids`, `node_ids`, `freshness`, and `source_hashes`.",
|
|
361
|
+
"- Save high-value answers back into `wiki/outputs/` instead of leaving them only in chat.",
|
|
362
|
+
"- Prefer `swarmvault ingest`, `swarmvault compile`, `swarmvault query`, and `swarmvault lint` for SwarmVault maintenance tasks.",
|
|
363
|
+
managedEnd,
|
|
364
|
+
""
|
|
365
|
+
].join("\n");
|
|
366
|
+
if (agent === "cursor") {
|
|
367
|
+
return body;
|
|
368
|
+
}
|
|
369
|
+
return body;
|
|
370
|
+
}
|
|
371
|
+
async function upsertManagedBlock(filePath, block) {
|
|
372
|
+
const existing = await fileExists(filePath) ? await fs3.readFile(filePath, "utf8") : "";
|
|
373
|
+
if (!existing) {
|
|
374
|
+
await ensureDir(path3.dirname(filePath));
|
|
375
|
+
await fs3.writeFile(filePath, `${block}
|
|
376
|
+
`, "utf8");
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
const startIndex = existing.includes(managedStart) ? existing.indexOf(managedStart) : existing.indexOf(legacyManagedStart);
|
|
380
|
+
const endIndex = existing.includes(managedEnd) ? existing.indexOf(managedEnd) : existing.indexOf(legacyManagedEnd);
|
|
381
|
+
if (startIndex !== -1 && endIndex !== -1) {
|
|
382
|
+
const next = `${existing.slice(0, startIndex)}${block}${existing.slice(endIndex + managedEnd.length)}`;
|
|
383
|
+
await fs3.writeFile(filePath, next, "utf8");
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
await fs3.writeFile(filePath, `${existing.trimEnd()}
|
|
387
|
+
|
|
388
|
+
${block}
|
|
389
|
+
`, "utf8");
|
|
390
|
+
}
|
|
391
|
+
async function installAgent(rootDir, agent) {
|
|
392
|
+
await initWorkspace(rootDir);
|
|
393
|
+
const block = buildManagedBlock(agent);
|
|
394
|
+
switch (agent) {
|
|
395
|
+
case "codex": {
|
|
396
|
+
const target = path3.join(rootDir, "AGENTS.md");
|
|
397
|
+
await upsertManagedBlock(target, block);
|
|
398
|
+
return target;
|
|
399
|
+
}
|
|
400
|
+
case "claude": {
|
|
401
|
+
const target = path3.join(rootDir, "CLAUDE.md");
|
|
402
|
+
await upsertManagedBlock(target, block);
|
|
403
|
+
return target;
|
|
404
|
+
}
|
|
405
|
+
case "cursor": {
|
|
406
|
+
const rulesDir = path3.join(rootDir, ".cursor", "rules");
|
|
407
|
+
await ensureDir(rulesDir);
|
|
408
|
+
const target = path3.join(rulesDir, "swarmvault.mdc");
|
|
409
|
+
await fs3.writeFile(target, `${block}
|
|
410
|
+
`, "utf8");
|
|
411
|
+
return target;
|
|
412
|
+
}
|
|
413
|
+
default:
|
|
414
|
+
throw new Error(`Unsupported agent ${String(agent)}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
async function installConfiguredAgents(rootDir) {
|
|
418
|
+
const { config } = await initWorkspace(rootDir);
|
|
419
|
+
return Promise.all(config.agents.map((agent) => installAgent(rootDir, agent)));
|
|
420
|
+
}
|
|
421
|
+
|
|
267
422
|
// src/ingest.ts
|
|
268
|
-
import
|
|
269
|
-
import
|
|
270
|
-
import { JSDOM } from "jsdom";
|
|
271
|
-
import TurndownService from "turndown";
|
|
423
|
+
import fs5 from "fs/promises";
|
|
424
|
+
import path5 from "path";
|
|
272
425
|
import { Readability } from "@mozilla/readability";
|
|
426
|
+
import { JSDOM } from "jsdom";
|
|
273
427
|
import mime from "mime-types";
|
|
428
|
+
import TurndownService from "turndown";
|
|
274
429
|
|
|
275
430
|
// src/logs.ts
|
|
276
|
-
import
|
|
277
|
-
import
|
|
431
|
+
import fs4 from "fs/promises";
|
|
432
|
+
import path4 from "path";
|
|
278
433
|
async function appendLogEntry(rootDir, action, title, lines = []) {
|
|
279
434
|
const { paths } = await initWorkspace(rootDir);
|
|
280
435
|
await ensureDir(paths.wikiDir);
|
|
281
|
-
const logPath =
|
|
436
|
+
const logPath = path4.join(paths.wikiDir, "log.md");
|
|
282
437
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().slice(0, 19).replace("T", " ");
|
|
283
438
|
const entry = [`## [${timestamp}] ${action} | ${title}`, ...lines.map((line) => `- ${line}`), ""].join("\n");
|
|
284
|
-
const existing = await fileExists(logPath) ? await
|
|
285
|
-
await
|
|
439
|
+
const existing = await fileExists(logPath) ? await fs4.readFile(logPath, "utf8") : "# Log\n\n";
|
|
440
|
+
await fs4.writeFile(logPath, `${existing}${entry}
|
|
286
441
|
`, "utf8");
|
|
287
442
|
}
|
|
288
443
|
async function appendWatchRun(rootDir, run) {
|
|
@@ -324,7 +479,7 @@ function buildCompositeHash(payloadBytes, attachments = []) {
|
|
|
324
479
|
return sha256(`${sha256(payloadBytes)}|${attachmentSignature}`);
|
|
325
480
|
}
|
|
326
481
|
function sanitizeAssetRelativePath(value) {
|
|
327
|
-
const normalized =
|
|
482
|
+
const normalized = path5.posix.normalize(value.replace(/\\/g, "/"));
|
|
328
483
|
const segments = normalized.split("/").filter(Boolean).map((segment) => {
|
|
329
484
|
if (segment === ".") {
|
|
330
485
|
return "";
|
|
@@ -344,7 +499,7 @@ function normalizeLocalReference(value) {
|
|
|
344
499
|
return null;
|
|
345
500
|
}
|
|
346
501
|
const lowered = candidate.toLowerCase();
|
|
347
|
-
if (lowered.startsWith("http://") || lowered.startsWith("https://") || lowered.startsWith("data:") || lowered.startsWith("mailto:") || lowered.startsWith("#") ||
|
|
502
|
+
if (lowered.startsWith("http://") || lowered.startsWith("https://") || lowered.startsWith("data:") || lowered.startsWith("mailto:") || lowered.startsWith("#") || path5.isAbsolute(candidate)) {
|
|
348
503
|
return null;
|
|
349
504
|
}
|
|
350
505
|
return candidate.replace(/\\/g, "/");
|
|
@@ -372,12 +527,12 @@ async function convertHtmlToMarkdown(html, url) {
|
|
|
372
527
|
};
|
|
373
528
|
}
|
|
374
529
|
async function readManifestByHash(manifestsDir, contentHash) {
|
|
375
|
-
const entries = await
|
|
530
|
+
const entries = await fs5.readdir(manifestsDir, { withFileTypes: true }).catch(() => []);
|
|
376
531
|
for (const entry of entries) {
|
|
377
532
|
if (!entry.isFile() || !entry.name.endsWith(".json")) {
|
|
378
533
|
continue;
|
|
379
534
|
}
|
|
380
|
-
const manifest = await readJsonFile(
|
|
535
|
+
const manifest = await readJsonFile(path5.join(manifestsDir, entry.name));
|
|
381
536
|
if (manifest?.contentHash === contentHash) {
|
|
382
537
|
return manifest;
|
|
383
538
|
}
|
|
@@ -397,20 +552,20 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
397
552
|
}
|
|
398
553
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
399
554
|
const sourceId = `${slugify(prepared.title)}-${contentHash.slice(0, 8)}`;
|
|
400
|
-
const storedPath =
|
|
401
|
-
await
|
|
555
|
+
const storedPath = path5.join(paths.rawSourcesDir, `${sourceId}${prepared.storedExtension}`);
|
|
556
|
+
await fs5.writeFile(storedPath, prepared.payloadBytes);
|
|
402
557
|
let extractedTextPath;
|
|
403
558
|
if (prepared.extractedText) {
|
|
404
|
-
extractedTextPath =
|
|
405
|
-
await
|
|
559
|
+
extractedTextPath = path5.join(paths.extractsDir, `${sourceId}.md`);
|
|
560
|
+
await fs5.writeFile(extractedTextPath, prepared.extractedText, "utf8");
|
|
406
561
|
}
|
|
407
562
|
const manifestAttachments = [];
|
|
408
563
|
for (const attachment of attachments) {
|
|
409
|
-
const absoluteAttachmentPath =
|
|
410
|
-
await ensureDir(
|
|
411
|
-
await
|
|
564
|
+
const absoluteAttachmentPath = path5.join(paths.rawAssetsDir, sourceId, attachment.relativePath);
|
|
565
|
+
await ensureDir(path5.dirname(absoluteAttachmentPath));
|
|
566
|
+
await fs5.writeFile(absoluteAttachmentPath, attachment.bytes);
|
|
412
567
|
manifestAttachments.push({
|
|
413
|
-
path: toPosix(
|
|
568
|
+
path: toPosix(path5.relative(rootDir, absoluteAttachmentPath)),
|
|
414
569
|
mimeType: attachment.mimeType,
|
|
415
570
|
originalPath: attachment.originalPath
|
|
416
571
|
});
|
|
@@ -422,15 +577,15 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
422
577
|
sourceKind: prepared.sourceKind,
|
|
423
578
|
originalPath: prepared.originalPath,
|
|
424
579
|
url: prepared.url,
|
|
425
|
-
storedPath: toPosix(
|
|
426
|
-
extractedTextPath: extractedTextPath ? toPosix(
|
|
580
|
+
storedPath: toPosix(path5.relative(rootDir, storedPath)),
|
|
581
|
+
extractedTextPath: extractedTextPath ? toPosix(path5.relative(rootDir, extractedTextPath)) : void 0,
|
|
427
582
|
mimeType: prepared.mimeType,
|
|
428
583
|
contentHash,
|
|
429
584
|
createdAt: now,
|
|
430
585
|
updatedAt: now,
|
|
431
586
|
attachments: manifestAttachments.length ? manifestAttachments : void 0
|
|
432
587
|
};
|
|
433
|
-
await writeJsonFile(
|
|
588
|
+
await writeJsonFile(path5.join(paths.manifestsDir, `${sourceId}.json`), manifest);
|
|
434
589
|
await appendLogEntry(rootDir, "ingest", prepared.title, [
|
|
435
590
|
`source_id=${sourceId}`,
|
|
436
591
|
`kind=${prepared.sourceKind}`,
|
|
@@ -438,18 +593,18 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
438
593
|
]);
|
|
439
594
|
return { manifest, isNew: true };
|
|
440
595
|
}
|
|
441
|
-
async function prepareFileInput(
|
|
442
|
-
const payloadBytes = await
|
|
596
|
+
async function prepareFileInput(_rootDir, absoluteInput) {
|
|
597
|
+
const payloadBytes = await fs5.readFile(absoluteInput);
|
|
443
598
|
const mimeType = guessMimeType(absoluteInput);
|
|
444
599
|
const sourceKind = inferKind(mimeType, absoluteInput);
|
|
445
|
-
const storedExtension =
|
|
600
|
+
const storedExtension = path5.extname(absoluteInput) || `.${mime.extension(mimeType) || "bin"}`;
|
|
446
601
|
let title;
|
|
447
602
|
let extractedText;
|
|
448
603
|
if (sourceKind === "markdown" || sourceKind === "text") {
|
|
449
604
|
extractedText = payloadBytes.toString("utf8");
|
|
450
|
-
title = titleFromText(
|
|
605
|
+
title = titleFromText(path5.basename(absoluteInput, path5.extname(absoluteInput)), extractedText);
|
|
451
606
|
} else {
|
|
452
|
-
title =
|
|
607
|
+
title = path5.basename(absoluteInput, path5.extname(absoluteInput));
|
|
453
608
|
}
|
|
454
609
|
return {
|
|
455
610
|
title,
|
|
@@ -483,7 +638,7 @@ async function prepareUrlInput(input) {
|
|
|
483
638
|
sourceKind = "markdown";
|
|
484
639
|
storedExtension = ".md";
|
|
485
640
|
} else {
|
|
486
|
-
const extension =
|
|
641
|
+
const extension = path5.extname(new URL(input).pathname);
|
|
487
642
|
storedExtension = extension || `.${mime.extension(mimeType) || "bin"}`;
|
|
488
643
|
if (sourceKind === "markdown" || sourceKind === "text") {
|
|
489
644
|
extractedText = payloadBytes.toString("utf8");
|
|
@@ -509,14 +664,14 @@ async function collectInboxAttachmentRefs(inputDir, files) {
|
|
|
509
664
|
if (sourceKind !== "markdown") {
|
|
510
665
|
continue;
|
|
511
666
|
}
|
|
512
|
-
const content = await
|
|
667
|
+
const content = await fs5.readFile(absolutePath, "utf8");
|
|
513
668
|
const refs = extractMarkdownReferences(content);
|
|
514
669
|
if (!refs.length) {
|
|
515
670
|
continue;
|
|
516
671
|
}
|
|
517
672
|
const sourceRefs = [];
|
|
518
673
|
for (const ref of refs) {
|
|
519
|
-
const resolved =
|
|
674
|
+
const resolved = path5.resolve(path5.dirname(absolutePath), ref);
|
|
520
675
|
if (!resolved.startsWith(inputDir) || !await fileExists(resolved)) {
|
|
521
676
|
continue;
|
|
522
677
|
}
|
|
@@ -550,12 +705,12 @@ function rewriteMarkdownReferences(content, replacements) {
|
|
|
550
705
|
});
|
|
551
706
|
}
|
|
552
707
|
async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
|
|
553
|
-
const originalBytes = await
|
|
708
|
+
const originalBytes = await fs5.readFile(absolutePath);
|
|
554
709
|
const originalText = originalBytes.toString("utf8");
|
|
555
|
-
const title = titleFromText(
|
|
710
|
+
const title = titleFromText(path5.basename(absolutePath, path5.extname(absolutePath)), originalText);
|
|
556
711
|
const attachments = [];
|
|
557
712
|
for (const attachmentRef of attachmentRefs) {
|
|
558
|
-
const bytes = await
|
|
713
|
+
const bytes = await fs5.readFile(attachmentRef.absolutePath);
|
|
559
714
|
attachments.push({
|
|
560
715
|
relativePath: sanitizeAssetRelativePath(attachmentRef.relativeRef),
|
|
561
716
|
mimeType: guessMimeType(attachmentRef.absolutePath),
|
|
@@ -578,7 +733,7 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
|
|
|
578
733
|
sourceKind: "markdown",
|
|
579
734
|
originalPath: toPosix(absolutePath),
|
|
580
735
|
mimeType: "text/markdown",
|
|
581
|
-
storedExtension:
|
|
736
|
+
storedExtension: path5.extname(absolutePath) || ".md",
|
|
582
737
|
payloadBytes: Buffer.from(rewrittenText, "utf8"),
|
|
583
738
|
extractedText: rewrittenText,
|
|
584
739
|
attachments,
|
|
@@ -590,50 +745,48 @@ function isSupportedInboxKind(sourceKind) {
|
|
|
590
745
|
}
|
|
591
746
|
async function ingestInput(rootDir, input) {
|
|
592
747
|
const { paths } = await initWorkspace(rootDir);
|
|
593
|
-
const prepared = /^https?:\/\//i.test(input) ? await prepareUrlInput(input) : await prepareFileInput(rootDir,
|
|
748
|
+
const prepared = /^https?:\/\//i.test(input) ? await prepareUrlInput(input) : await prepareFileInput(rootDir, path5.resolve(rootDir, input));
|
|
594
749
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
595
750
|
return result.manifest;
|
|
596
751
|
}
|
|
597
752
|
async function importInbox(rootDir, inputDir) {
|
|
598
753
|
const { paths } = await initWorkspace(rootDir);
|
|
599
|
-
const effectiveInputDir =
|
|
754
|
+
const effectiveInputDir = path5.resolve(rootDir, inputDir ?? paths.inboxDir);
|
|
600
755
|
if (!await fileExists(effectiveInputDir)) {
|
|
601
756
|
throw new Error(`Inbox directory not found: ${effectiveInputDir}`);
|
|
602
757
|
}
|
|
603
758
|
const files = (await listFilesRecursive(effectiveInputDir)).sort();
|
|
604
759
|
const refsBySource = await collectInboxAttachmentRefs(effectiveInputDir, files);
|
|
605
|
-
const claimedAttachments = new Set(
|
|
606
|
-
[...refsBySource.values()].flatMap((refs) => refs.map((ref) => ref.absolutePath))
|
|
607
|
-
);
|
|
760
|
+
const claimedAttachments = new Set([...refsBySource.values()].flatMap((refs) => refs.map((ref) => ref.absolutePath)));
|
|
608
761
|
const imported = [];
|
|
609
762
|
const skipped = [];
|
|
610
763
|
let attachmentCount = 0;
|
|
611
764
|
for (const absolutePath of files) {
|
|
612
|
-
const basename =
|
|
765
|
+
const basename = path5.basename(absolutePath);
|
|
613
766
|
if (basename.startsWith(".")) {
|
|
614
|
-
skipped.push({ path: toPosix(
|
|
767
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: "hidden_file" });
|
|
615
768
|
continue;
|
|
616
769
|
}
|
|
617
770
|
if (claimedAttachments.has(absolutePath)) {
|
|
618
|
-
skipped.push({ path: toPosix(
|
|
771
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: "referenced_attachment" });
|
|
619
772
|
continue;
|
|
620
773
|
}
|
|
621
774
|
const mimeType = guessMimeType(absolutePath);
|
|
622
775
|
const sourceKind = inferKind(mimeType, absolutePath);
|
|
623
776
|
if (!isSupportedInboxKind(sourceKind)) {
|
|
624
|
-
skipped.push({ path: toPosix(
|
|
777
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: `unsupported_kind:${sourceKind}` });
|
|
625
778
|
continue;
|
|
626
779
|
}
|
|
627
780
|
const prepared = sourceKind === "markdown" && refsBySource.has(absolutePath) ? await prepareInboxMarkdownInput(absolutePath, refsBySource.get(absolutePath) ?? []) : await prepareFileInput(rootDir, absolutePath);
|
|
628
781
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
629
782
|
if (!result.isNew) {
|
|
630
|
-
skipped.push({ path: toPosix(
|
|
783
|
+
skipped.push({ path: toPosix(path5.relative(rootDir, absolutePath)), reason: "duplicate_content" });
|
|
631
784
|
continue;
|
|
632
785
|
}
|
|
633
786
|
attachmentCount += result.manifest.attachments?.length ?? 0;
|
|
634
787
|
imported.push(result.manifest);
|
|
635
788
|
}
|
|
636
|
-
await appendLogEntry(rootDir, "inbox_import", toPosix(
|
|
789
|
+
await appendLogEntry(rootDir, "inbox_import", toPosix(path5.relative(rootDir, effectiveInputDir)) || ".", [
|
|
637
790
|
`scanned=${files.length}`,
|
|
638
791
|
`imported=${imported.length}`,
|
|
639
792
|
`attachments=${attachmentCount}`,
|
|
@@ -652,9 +805,9 @@ async function listManifests(rootDir) {
|
|
|
652
805
|
if (!await fileExists(paths.manifestsDir)) {
|
|
653
806
|
return [];
|
|
654
807
|
}
|
|
655
|
-
const entries = await
|
|
808
|
+
const entries = await fs5.readdir(paths.manifestsDir);
|
|
656
809
|
const manifests = await Promise.all(
|
|
657
|
-
entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(
|
|
810
|
+
entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(path5.join(paths.manifestsDir, entry)))
|
|
658
811
|
);
|
|
659
812
|
return manifests.filter((manifest) => Boolean(manifest));
|
|
660
813
|
}
|
|
@@ -662,33 +815,62 @@ async function readExtractedText(rootDir, manifest) {
|
|
|
662
815
|
if (!manifest.extractedTextPath) {
|
|
663
816
|
return void 0;
|
|
664
817
|
}
|
|
665
|
-
const absolutePath =
|
|
818
|
+
const absolutePath = path5.resolve(rootDir, manifest.extractedTextPath);
|
|
666
819
|
if (!await fileExists(absolutePath)) {
|
|
667
820
|
return void 0;
|
|
668
821
|
}
|
|
669
|
-
return
|
|
822
|
+
return fs5.readFile(absolutePath, "utf8");
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
// src/mcp.ts
|
|
826
|
+
import fs12 from "fs/promises";
|
|
827
|
+
import path14 from "path";
|
|
828
|
+
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
829
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
830
|
+
import { z as z9 } from "zod";
|
|
831
|
+
|
|
832
|
+
// src/schema.ts
|
|
833
|
+
import fs6 from "fs/promises";
|
|
834
|
+
import path6 from "path";
|
|
835
|
+
async function loadVaultSchema(rootDir) {
|
|
836
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
837
|
+
const schemaPath = paths.schemaPath;
|
|
838
|
+
const content = await fileExists(schemaPath) ? await fs6.readFile(schemaPath, "utf8") : defaultVaultSchema();
|
|
839
|
+
const normalized = content.trim() ? content.trim() : defaultVaultSchema().trim();
|
|
840
|
+
return {
|
|
841
|
+
path: schemaPath,
|
|
842
|
+
content: normalized,
|
|
843
|
+
hash: sha256(normalized),
|
|
844
|
+
isLegacyPath: path6.basename(schemaPath) === LEGACY_SCHEMA_FILENAME && path6.basename(schemaPath) !== PRIMARY_SCHEMA_FILENAME
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
function buildSchemaPrompt(schema, instruction) {
|
|
848
|
+
return [instruction, "", `Vault schema path: ${schema.path}`, "", "Vault schema instructions:", schema.content].join("\n");
|
|
670
849
|
}
|
|
671
850
|
|
|
672
851
|
// src/vault.ts
|
|
673
|
-
import
|
|
674
|
-
import
|
|
675
|
-
import
|
|
852
|
+
import fs11 from "fs/promises";
|
|
853
|
+
import path13 from "path";
|
|
854
|
+
import matter5 from "gray-matter";
|
|
855
|
+
import { z as z8 } from "zod";
|
|
676
856
|
|
|
677
857
|
// src/analysis.ts
|
|
678
|
-
import
|
|
858
|
+
import path7 from "path";
|
|
679
859
|
import { z as z3 } from "zod";
|
|
680
860
|
var sourceAnalysisSchema = z3.object({
|
|
681
861
|
title: z3.string().min(1),
|
|
682
862
|
summary: z3.string().min(1),
|
|
683
863
|
concepts: z3.array(z3.object({ name: z3.string().min(1), description: z3.string().default("") })).max(12).default([]),
|
|
684
864
|
entities: z3.array(z3.object({ name: z3.string().min(1), description: z3.string().default("") })).max(12).default([]),
|
|
685
|
-
claims: z3.array(
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
865
|
+
claims: z3.array(
|
|
866
|
+
z3.object({
|
|
867
|
+
text: z3.string().min(1),
|
|
868
|
+
confidence: z3.number().min(0).max(1).default(0.6),
|
|
869
|
+
status: z3.enum(["extracted", "inferred", "conflicted", "stale"]).default("extracted"),
|
|
870
|
+
polarity: z3.enum(["positive", "negative", "neutral"]).default("neutral"),
|
|
871
|
+
citation: z3.string().min(1)
|
|
872
|
+
})
|
|
873
|
+
).max(8).default([]),
|
|
692
874
|
questions: z3.array(z3.string()).max(6).default([])
|
|
693
875
|
});
|
|
694
876
|
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
@@ -743,7 +925,10 @@ function extractTopTerms(text, count) {
|
|
|
743
925
|
}
|
|
744
926
|
function extractEntities(text, count) {
|
|
745
927
|
const matches = text.match(/\b[A-Z][A-Za-z0-9-]+(?:\s+[A-Z][A-Za-z0-9-]+){0,2}\b/g) ?? [];
|
|
746
|
-
return uniqueBy(
|
|
928
|
+
return uniqueBy(
|
|
929
|
+
matches.map((value) => normalizeWhitespace(value)),
|
|
930
|
+
(value) => value.toLowerCase()
|
|
931
|
+
).slice(0, count);
|
|
747
932
|
}
|
|
748
933
|
function detectPolarity(text) {
|
|
749
934
|
if (/\b(no|not|never|cannot|can't|won't|without)\b/i.test(text)) {
|
|
@@ -758,7 +943,7 @@ function deriveTitle(manifest, text) {
|
|
|
758
943
|
const heading = text.match(/^#\s+(.+)$/m)?.[1]?.trim();
|
|
759
944
|
return heading || manifest.title;
|
|
760
945
|
}
|
|
761
|
-
function heuristicAnalysis(manifest, text) {
|
|
946
|
+
function heuristicAnalysis(manifest, text, schemaHash) {
|
|
762
947
|
const normalized = normalizeWhitespace(text);
|
|
763
948
|
const concepts = extractTopTerms(normalized, 6).map((term) => ({
|
|
764
949
|
id: `concept:${slugify(term)}`,
|
|
@@ -774,6 +959,7 @@ function heuristicAnalysis(manifest, text) {
|
|
|
774
959
|
return {
|
|
775
960
|
sourceId: manifest.sourceId,
|
|
776
961
|
sourceHash: manifest.contentHash,
|
|
962
|
+
schemaHash,
|
|
777
963
|
title: deriveTitle(manifest, text),
|
|
778
964
|
summary: firstSentences(normalized, 3) || truncate(normalized, 280) || `Imported ${manifest.sourceKind} source.`,
|
|
779
965
|
concepts,
|
|
@@ -790,10 +976,19 @@ function heuristicAnalysis(manifest, text) {
|
|
|
790
976
|
producedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
791
977
|
};
|
|
792
978
|
}
|
|
793
|
-
async function providerAnalysis(manifest, text, provider) {
|
|
979
|
+
async function providerAnalysis(manifest, text, provider, schema) {
|
|
794
980
|
const parsed = await provider.generateStructured(
|
|
795
981
|
{
|
|
796
|
-
system:
|
|
982
|
+
system: [
|
|
983
|
+
"You are compiling a durable markdown wiki and graph. Prefer grounded synthesis over creativity.",
|
|
984
|
+
"",
|
|
985
|
+
"Follow the vault schema when choosing titles, categories, relationships, and summaries.",
|
|
986
|
+
"",
|
|
987
|
+
`Vault schema path: ${schema.path}`,
|
|
988
|
+
"",
|
|
989
|
+
"Vault schema instructions:",
|
|
990
|
+
truncate(schema.content, 6e3)
|
|
991
|
+
].join("\n"),
|
|
797
992
|
prompt: `Analyze the following source and return structured JSON.
|
|
798
993
|
|
|
799
994
|
Source title: ${manifest.title}
|
|
@@ -808,6 +1003,7 @@ ${truncate(text, 18e3)}`
|
|
|
808
1003
|
return {
|
|
809
1004
|
sourceId: manifest.sourceId,
|
|
810
1005
|
sourceHash: manifest.contentHash,
|
|
1006
|
+
schemaHash: schema.hash,
|
|
811
1007
|
title: parsed.title,
|
|
812
1008
|
summary: parsed.summary,
|
|
813
1009
|
concepts: parsed.concepts.map((term) => ({
|
|
@@ -832,10 +1028,10 @@ ${truncate(text, 18e3)}`
|
|
|
832
1028
|
producedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
833
1029
|
};
|
|
834
1030
|
}
|
|
835
|
-
async function analyzeSource(manifest, extractedText, provider, paths) {
|
|
836
|
-
const cachePath =
|
|
1031
|
+
async function analyzeSource(manifest, extractedText, provider, paths, schema) {
|
|
1032
|
+
const cachePath = path7.join(paths.analysesDir, `${manifest.sourceId}.json`);
|
|
837
1033
|
const cached = await readJsonFile(cachePath);
|
|
838
|
-
if (cached && cached.sourceHash === manifest.contentHash) {
|
|
1034
|
+
if (cached && cached.sourceHash === manifest.contentHash && cached.schemaHash === schema.hash) {
|
|
839
1035
|
return cached;
|
|
840
1036
|
}
|
|
841
1037
|
const content = normalizeWhitespace(extractedText ?? "");
|
|
@@ -844,6 +1040,7 @@ async function analyzeSource(manifest, extractedText, provider, paths) {
|
|
|
844
1040
|
analysis = {
|
|
845
1041
|
sourceId: manifest.sourceId,
|
|
846
1042
|
sourceHash: manifest.contentHash,
|
|
1043
|
+
schemaHash: schema.hash,
|
|
847
1044
|
title: manifest.title,
|
|
848
1045
|
summary: `Imported ${manifest.sourceKind} source. Text extraction is not yet available for this source.`,
|
|
849
1046
|
concepts: [],
|
|
@@ -853,12 +1050,12 @@ async function analyzeSource(manifest, extractedText, provider, paths) {
|
|
|
853
1050
|
producedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
854
1051
|
};
|
|
855
1052
|
} else if (provider.type === "heuristic") {
|
|
856
|
-
analysis = heuristicAnalysis(manifest, content);
|
|
1053
|
+
analysis = heuristicAnalysis(manifest, content, schema.hash);
|
|
857
1054
|
} else {
|
|
858
1055
|
try {
|
|
859
|
-
analysis = await providerAnalysis(manifest, content, provider);
|
|
1056
|
+
analysis = await providerAnalysis(manifest, content, provider, schema);
|
|
860
1057
|
} catch {
|
|
861
|
-
analysis = heuristicAnalysis(manifest, content);
|
|
1058
|
+
analysis = heuristicAnalysis(manifest, content, schema.hash);
|
|
862
1059
|
}
|
|
863
1060
|
}
|
|
864
1061
|
await writeJsonFile(cachePath, analysis);
|
|
@@ -868,328 +1065,42 @@ function analysisSignature(analysis) {
|
|
|
868
1065
|
return sha256(JSON.stringify(analysis));
|
|
869
1066
|
}
|
|
870
1067
|
|
|
871
|
-
// src/
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
var managedStart = "<!-- swarmvault:managed:start -->";
|
|
875
|
-
var managedEnd = "<!-- swarmvault:managed:end -->";
|
|
876
|
-
var legacyManagedStart = "<!-- vault:managed:start -->";
|
|
877
|
-
var legacyManagedEnd = "<!-- vault:managed:end -->";
|
|
878
|
-
function buildManagedBlock(agent) {
|
|
879
|
-
const body = [
|
|
880
|
-
managedStart,
|
|
881
|
-
`# SwarmVault Rules (${agent})`,
|
|
882
|
-
"",
|
|
883
|
-
"- Treat `raw/` as immutable source input.",
|
|
884
|
-
"- Treat `wiki/` as generated markdown owned by the agent and compiler workflow.",
|
|
885
|
-
"- Read `wiki/index.md` before broad file searching when answering SwarmVault questions.",
|
|
886
|
-
"- Preserve frontmatter fields including `page_id`, `source_ids`, `node_ids`, `freshness`, and `source_hashes`.",
|
|
887
|
-
"- Save high-value answers back into `wiki/outputs/` instead of leaving them only in chat.",
|
|
888
|
-
"- Prefer `swarmvault ingest`, `swarmvault compile`, `swarmvault query`, and `swarmvault lint` for SwarmVault maintenance tasks.",
|
|
889
|
-
managedEnd,
|
|
890
|
-
""
|
|
891
|
-
].join("\n");
|
|
892
|
-
if (agent === "cursor") {
|
|
893
|
-
return body;
|
|
894
|
-
}
|
|
895
|
-
return body;
|
|
896
|
-
}
|
|
897
|
-
async function upsertManagedBlock(filePath, block) {
|
|
898
|
-
const existing = await fileExists(filePath) ? await fs4.readFile(filePath, "utf8") : "";
|
|
899
|
-
if (!existing) {
|
|
900
|
-
await ensureDir(path6.dirname(filePath));
|
|
901
|
-
await fs4.writeFile(filePath, `${block}
|
|
902
|
-
`, "utf8");
|
|
903
|
-
return;
|
|
904
|
-
}
|
|
905
|
-
const startIndex = existing.includes(managedStart) ? existing.indexOf(managedStart) : existing.indexOf(legacyManagedStart);
|
|
906
|
-
const endIndex = existing.includes(managedEnd) ? existing.indexOf(managedEnd) : existing.indexOf(legacyManagedEnd);
|
|
907
|
-
if (startIndex !== -1 && endIndex !== -1) {
|
|
908
|
-
const next = `${existing.slice(0, startIndex)}${block}${existing.slice(endIndex + managedEnd.length)}`;
|
|
909
|
-
await fs4.writeFile(filePath, next, "utf8");
|
|
910
|
-
return;
|
|
911
|
-
}
|
|
912
|
-
await fs4.writeFile(filePath, `${existing.trimEnd()}
|
|
913
|
-
|
|
914
|
-
${block}
|
|
915
|
-
`, "utf8");
|
|
1068
|
+
// src/confidence.ts
|
|
1069
|
+
function nodeConfidence(sourceCount) {
|
|
1070
|
+
return Math.min(0.5 + sourceCount * 0.15, 0.95);
|
|
916
1071
|
}
|
|
917
|
-
|
|
918
|
-
const
|
|
919
|
-
const
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
const target = path6.join(rootDir, "AGENTS.md");
|
|
923
|
-
await upsertManagedBlock(target, block);
|
|
924
|
-
return target;
|
|
925
|
-
}
|
|
926
|
-
case "claude": {
|
|
927
|
-
const target = path6.join(rootDir, "CLAUDE.md");
|
|
928
|
-
await upsertManagedBlock(target, block);
|
|
929
|
-
return target;
|
|
930
|
-
}
|
|
931
|
-
case "cursor": {
|
|
932
|
-
const rulesDir = path6.join(rootDir, ".cursor", "rules");
|
|
933
|
-
await ensureDir(rulesDir);
|
|
934
|
-
const target = path6.join(rulesDir, "swarmvault.mdc");
|
|
935
|
-
await fs4.writeFile(target, `${block}
|
|
936
|
-
`, "utf8");
|
|
937
|
-
return target;
|
|
938
|
-
}
|
|
939
|
-
default:
|
|
940
|
-
throw new Error(`Unsupported agent ${String(agent)}`);
|
|
1072
|
+
function edgeConfidence(claims, conceptName) {
|
|
1073
|
+
const lower = conceptName.toLowerCase();
|
|
1074
|
+
const relevant = claims.filter((c) => c.text.toLowerCase().includes(lower));
|
|
1075
|
+
if (!relevant.length) {
|
|
1076
|
+
return 0.5;
|
|
941
1077
|
}
|
|
1078
|
+
return relevant.reduce((sum, c) => sum + c.confidence, 0) / relevant.length;
|
|
942
1079
|
}
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
return Promise.all(config.agents.map((agent) => installAgent(rootDir, agent)));
|
|
1080
|
+
function conflictConfidence(claimA, claimB) {
|
|
1081
|
+
return Math.min(claimA.confidence, claimB.confidence);
|
|
946
1082
|
}
|
|
947
1083
|
|
|
948
|
-
// src/
|
|
1084
|
+
// src/deep-lint.ts
|
|
1085
|
+
import fs8 from "fs/promises";
|
|
1086
|
+
import path10 from "path";
|
|
949
1087
|
import matter from "gray-matter";
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
const pageId = `source:${manifest.sourceId}`;
|
|
967
|
-
const nodeIds = [
|
|
968
|
-
`source:${manifest.sourceId}`,
|
|
969
|
-
...analysis.concepts.map((item) => item.id),
|
|
970
|
-
...analysis.entities.map((item) => item.id)
|
|
971
|
-
];
|
|
972
|
-
const backlinks = [
|
|
973
|
-
...analysis.concepts.map((item) => `concept:${slugify(item.name)}`),
|
|
974
|
-
...analysis.entities.map((item) => `entity:${slugify(item.name)}`)
|
|
975
|
-
];
|
|
976
|
-
const frontmatter = {
|
|
977
|
-
page_id: pageId,
|
|
978
|
-
kind: "source",
|
|
979
|
-
title: analysis.title,
|
|
980
|
-
tags: ["source"],
|
|
981
|
-
source_ids: [manifest.sourceId],
|
|
982
|
-
node_ids: nodeIds,
|
|
983
|
-
freshness: "fresh",
|
|
984
|
-
confidence: 0.8,
|
|
985
|
-
updated_at: analysis.producedAt,
|
|
986
|
-
backlinks,
|
|
987
|
-
source_hashes: {
|
|
988
|
-
[manifest.sourceId]: manifest.contentHash
|
|
989
|
-
}
|
|
990
|
-
};
|
|
991
|
-
const body = [
|
|
992
|
-
`# ${analysis.title}`,
|
|
993
|
-
"",
|
|
994
|
-
`Source ID: \`${manifest.sourceId}\``,
|
|
995
|
-
manifest.url ? `Source URL: ${manifest.url}` : `Source Path: \`${manifest.originalPath ?? manifest.storedPath}\``,
|
|
996
|
-
"",
|
|
997
|
-
"## Summary",
|
|
998
|
-
"",
|
|
999
|
-
analysis.summary,
|
|
1000
|
-
"",
|
|
1001
|
-
"## Concepts",
|
|
1002
|
-
"",
|
|
1003
|
-
...analysis.concepts.length ? analysis.concepts.map((item) => `- [[${pagePathFor("concept", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`) : ["- None detected."],
|
|
1004
|
-
"",
|
|
1005
|
-
"## Entities",
|
|
1006
|
-
"",
|
|
1007
|
-
...analysis.entities.length ? analysis.entities.map((item) => `- [[${pagePathFor("entity", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`) : ["- None detected."],
|
|
1008
|
-
"",
|
|
1009
|
-
"## Claims",
|
|
1010
|
-
"",
|
|
1011
|
-
...analysis.claims.length ? analysis.claims.map((claim) => `- ${claim.text} [source:${claim.citation}]`) : ["- No claims extracted."],
|
|
1012
|
-
"",
|
|
1013
|
-
"## Questions",
|
|
1014
|
-
"",
|
|
1015
|
-
...analysis.questions.length ? analysis.questions.map((question) => `- ${question}`) : ["- No follow-up questions yet."],
|
|
1016
|
-
""
|
|
1017
|
-
].join("\n");
|
|
1018
|
-
return {
|
|
1019
|
-
page: {
|
|
1020
|
-
id: pageId,
|
|
1021
|
-
path: relativePath,
|
|
1022
|
-
title: analysis.title,
|
|
1023
|
-
kind: "source",
|
|
1024
|
-
sourceIds: [manifest.sourceId],
|
|
1025
|
-
nodeIds,
|
|
1026
|
-
freshness: "fresh",
|
|
1027
|
-
confidence: 0.8,
|
|
1028
|
-
backlinks,
|
|
1029
|
-
sourceHashes: { [manifest.sourceId]: manifest.contentHash }
|
|
1030
|
-
},
|
|
1031
|
-
content: matter.stringify(body, frontmatter)
|
|
1032
|
-
};
|
|
1033
|
-
}
|
|
1034
|
-
function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes) {
|
|
1035
|
-
const slug = slugify(name);
|
|
1036
|
-
const relativePath = pagePathFor(kind, slug);
|
|
1037
|
-
const pageId = `${kind}:${slug}`;
|
|
1038
|
-
const sourceIds = sourceAnalyses.map((item) => item.sourceId);
|
|
1039
|
-
const otherPages = sourceAnalyses.map((item) => `source:${item.sourceId}`);
|
|
1040
|
-
const summary = descriptions.find(Boolean) ?? `${kind} aggregated from ${sourceIds.length} source(s).`;
|
|
1041
|
-
const frontmatter = {
|
|
1042
|
-
page_id: pageId,
|
|
1043
|
-
kind,
|
|
1044
|
-
title: name,
|
|
1045
|
-
tags: [kind],
|
|
1046
|
-
source_ids: sourceIds,
|
|
1047
|
-
node_ids: [pageId],
|
|
1048
|
-
freshness: "fresh",
|
|
1049
|
-
confidence: 0.72,
|
|
1050
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1051
|
-
backlinks: otherPages,
|
|
1052
|
-
source_hashes: sourceHashes
|
|
1053
|
-
};
|
|
1054
|
-
const body = [
|
|
1055
|
-
`# ${name}`,
|
|
1056
|
-
"",
|
|
1057
|
-
"## Summary",
|
|
1058
|
-
"",
|
|
1059
|
-
summary,
|
|
1060
|
-
"",
|
|
1061
|
-
"## Seen In",
|
|
1062
|
-
"",
|
|
1063
|
-
...sourceAnalyses.map((item) => `- [[${pagePathFor("source", item.sourceId).replace(/\.md$/, "")}|${item.title}]]`),
|
|
1064
|
-
"",
|
|
1065
|
-
"## Source Claims",
|
|
1066
|
-
"",
|
|
1067
|
-
...sourceAnalyses.flatMap(
|
|
1068
|
-
(item) => item.claims.filter((claim) => claim.text.toLowerCase().includes(name.toLowerCase())).map((claim) => `- ${claim.text} [source:${claim.citation}]`)
|
|
1069
|
-
),
|
|
1070
|
-
""
|
|
1071
|
-
].join("\n");
|
|
1072
|
-
return {
|
|
1073
|
-
page: {
|
|
1074
|
-
id: pageId,
|
|
1075
|
-
path: relativePath,
|
|
1076
|
-
title: name,
|
|
1077
|
-
kind,
|
|
1078
|
-
sourceIds,
|
|
1079
|
-
nodeIds: [pageId],
|
|
1080
|
-
freshness: "fresh",
|
|
1081
|
-
confidence: 0.72,
|
|
1082
|
-
backlinks: otherPages,
|
|
1083
|
-
sourceHashes
|
|
1084
|
-
},
|
|
1085
|
-
content: matter.stringify(body, frontmatter)
|
|
1086
|
-
};
|
|
1087
|
-
}
|
|
1088
|
-
function buildIndexPage(pages) {
|
|
1089
|
-
const sources = pages.filter((page) => page.kind === "source");
|
|
1090
|
-
const concepts = pages.filter((page) => page.kind === "concept");
|
|
1091
|
-
const entities = pages.filter((page) => page.kind === "entity");
|
|
1092
|
-
return [
|
|
1093
|
-
"---",
|
|
1094
|
-
"page_id: index",
|
|
1095
|
-
"kind: index",
|
|
1096
|
-
"title: SwarmVault Index",
|
|
1097
|
-
"tags:",
|
|
1098
|
-
" - index",
|
|
1099
|
-
"source_ids: []",
|
|
1100
|
-
"node_ids: []",
|
|
1101
|
-
"freshness: fresh",
|
|
1102
|
-
"confidence: 1",
|
|
1103
|
-
`updated_at: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
1104
|
-
"backlinks: []",
|
|
1105
|
-
"source_hashes: {}",
|
|
1106
|
-
"---",
|
|
1107
|
-
"",
|
|
1108
|
-
"# SwarmVault Index",
|
|
1109
|
-
"",
|
|
1110
|
-
"## Sources",
|
|
1111
|
-
"",
|
|
1112
|
-
...sources.length ? sources.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No sources yet."],
|
|
1113
|
-
"",
|
|
1114
|
-
"## Concepts",
|
|
1115
|
-
"",
|
|
1116
|
-
...concepts.length ? concepts.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No concepts yet."],
|
|
1117
|
-
"",
|
|
1118
|
-
"## Entities",
|
|
1119
|
-
"",
|
|
1120
|
-
...entities.length ? entities.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No entities yet."],
|
|
1121
|
-
""
|
|
1122
|
-
].join("\n");
|
|
1123
|
-
}
|
|
1124
|
-
function buildSectionIndex(kind, pages) {
|
|
1125
|
-
const title = kind.charAt(0).toUpperCase() + kind.slice(1);
|
|
1126
|
-
return [
|
|
1127
|
-
`# ${title}`,
|
|
1128
|
-
"",
|
|
1129
|
-
...pages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`),
|
|
1130
|
-
""
|
|
1131
|
-
].join("\n");
|
|
1132
|
-
}
|
|
1133
|
-
function buildOutputPage(question, answer, citations) {
|
|
1134
|
-
const slug = slugify(question);
|
|
1135
|
-
const pageId = `output:${slug}`;
|
|
1136
|
-
const pathValue = pagePathFor("output", slug);
|
|
1137
|
-
const frontmatter = {
|
|
1138
|
-
page_id: pageId,
|
|
1139
|
-
kind: "output",
|
|
1140
|
-
title: question,
|
|
1141
|
-
tags: ["output"],
|
|
1142
|
-
source_ids: citations,
|
|
1143
|
-
node_ids: [],
|
|
1144
|
-
freshness: "fresh",
|
|
1145
|
-
confidence: 0.74,
|
|
1146
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1147
|
-
backlinks: citations.map((sourceId) => `source:${sourceId}`),
|
|
1148
|
-
source_hashes: {}
|
|
1149
|
-
};
|
|
1150
|
-
return {
|
|
1151
|
-
page: {
|
|
1152
|
-
id: pageId,
|
|
1153
|
-
path: pathValue,
|
|
1154
|
-
title: question,
|
|
1155
|
-
kind: "output",
|
|
1156
|
-
sourceIds: citations,
|
|
1157
|
-
nodeIds: [],
|
|
1158
|
-
freshness: "fresh",
|
|
1159
|
-
confidence: 0.74,
|
|
1160
|
-
backlinks: citations.map((sourceId) => `source:${sourceId}`),
|
|
1161
|
-
sourceHashes: {}
|
|
1162
|
-
},
|
|
1163
|
-
content: matter.stringify(
|
|
1164
|
-
[
|
|
1165
|
-
`# ${question}`,
|
|
1166
|
-
"",
|
|
1167
|
-
answer,
|
|
1168
|
-
"",
|
|
1169
|
-
"## Citations",
|
|
1170
|
-
"",
|
|
1171
|
-
...citations.map((citation) => `- [source:${citation}]`),
|
|
1172
|
-
""
|
|
1173
|
-
].join("\n"),
|
|
1174
|
-
frontmatter
|
|
1175
|
-
)
|
|
1176
|
-
};
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
// src/providers/registry.ts
|
|
1180
|
-
import path7 from "path";
|
|
1181
|
-
import { pathToFileURL } from "url";
|
|
1182
|
-
import { z as z5 } from "zod";
|
|
1183
|
-
|
|
1184
|
-
// src/providers/base.ts
|
|
1185
|
-
import fs5 from "fs/promises";
|
|
1186
|
-
import { z as z4 } from "zod";
|
|
1187
|
-
var BaseProviderAdapter = class {
|
|
1188
|
-
constructor(id, type, model, capabilities) {
|
|
1189
|
-
this.id = id;
|
|
1190
|
-
this.type = type;
|
|
1191
|
-
this.model = model;
|
|
1192
|
-
this.capabilities = new Set(capabilities);
|
|
1088
|
+
import { z as z7 } from "zod";
|
|
1089
|
+
|
|
1090
|
+
// src/providers/registry.ts
|
|
1091
|
+
import path8 from "path";
|
|
1092
|
+
import { pathToFileURL } from "url";
|
|
1093
|
+
import { z as z5 } from "zod";
|
|
1094
|
+
|
|
1095
|
+
// src/providers/base.ts
|
|
1096
|
+
import fs7 from "fs/promises";
|
|
1097
|
+
import { z as z4 } from "zod";
|
|
1098
|
+
var BaseProviderAdapter = class {
|
|
1099
|
+
constructor(id, type, model, capabilities) {
|
|
1100
|
+
this.id = id;
|
|
1101
|
+
this.type = type;
|
|
1102
|
+
this.model = model;
|
|
1103
|
+
this.capabilities = new Set(capabilities);
|
|
1193
1104
|
}
|
|
1194
1105
|
id;
|
|
1195
1106
|
type;
|
|
@@ -1211,80 +1122,54 @@ ${schemaDescription}`
|
|
|
1211
1122
|
return Promise.all(
|
|
1212
1123
|
attachments.map(async (attachment) => ({
|
|
1213
1124
|
mimeType: attachment.mimeType,
|
|
1214
|
-
base64: await
|
|
1125
|
+
base64: await fs7.readFile(attachment.filePath, "base64")
|
|
1215
1126
|
}))
|
|
1216
1127
|
);
|
|
1217
1128
|
}
|
|
1218
1129
|
};
|
|
1219
1130
|
|
|
1220
|
-
// src/providers/
|
|
1221
|
-
|
|
1222
|
-
const cleaned = normalizeWhitespace(prompt);
|
|
1223
|
-
if (!cleaned) {
|
|
1224
|
-
return "No prompt content provided.";
|
|
1225
|
-
}
|
|
1226
|
-
return firstSentences(cleaned, 2) || cleaned.slice(0, 280);
|
|
1227
|
-
}
|
|
1228
|
-
var HeuristicProviderAdapter = class extends BaseProviderAdapter {
|
|
1229
|
-
constructor(id, model) {
|
|
1230
|
-
super(id, "heuristic", model, ["chat", "structured", "vision", "local"]);
|
|
1231
|
-
}
|
|
1232
|
-
async generateText(request) {
|
|
1233
|
-
const attachmentHint = request.attachments?.length ? ` Attachments: ${request.attachments.length}.` : "";
|
|
1234
|
-
return {
|
|
1235
|
-
text: `Heuristic provider response.${attachmentHint} ${summarizePrompt(request.prompt)}`.trim()
|
|
1236
|
-
};
|
|
1237
|
-
}
|
|
1238
|
-
};
|
|
1239
|
-
|
|
1240
|
-
// src/providers/openai-compatible.ts
|
|
1241
|
-
function buildAuthHeaders(apiKey) {
|
|
1242
|
-
return apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
|
|
1243
|
-
}
|
|
1244
|
-
var OpenAiCompatibleProviderAdapter = class extends BaseProviderAdapter {
|
|
1245
|
-
baseUrl;
|
|
1131
|
+
// src/providers/anthropic.ts
|
|
1132
|
+
var AnthropicProviderAdapter = class extends BaseProviderAdapter {
|
|
1246
1133
|
apiKey;
|
|
1247
1134
|
headers;
|
|
1248
|
-
|
|
1249
|
-
constructor(id,
|
|
1250
|
-
super(id,
|
|
1251
|
-
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
|
1135
|
+
baseUrl;
|
|
1136
|
+
constructor(id, model, options) {
|
|
1137
|
+
super(id, "anthropic", model, ["chat", "structured", "tools", "vision", "streaming"]);
|
|
1252
1138
|
this.apiKey = options.apiKey;
|
|
1253
1139
|
this.headers = options.headers;
|
|
1254
|
-
this.
|
|
1140
|
+
this.baseUrl = (options.baseUrl ?? "https://api.anthropic.com").replace(/\/+$/, "");
|
|
1255
1141
|
}
|
|
1256
1142
|
async generateText(request) {
|
|
1257
|
-
if (this.apiStyle === "chat") {
|
|
1258
|
-
return this.generateViaChatCompletions(request);
|
|
1259
|
-
}
|
|
1260
|
-
return this.generateViaResponses(request);
|
|
1261
|
-
}
|
|
1262
|
-
async generateViaResponses(request) {
|
|
1263
1143
|
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1264
|
-
const
|
|
1265
|
-
{
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
const response = await fetch(`${this.baseUrl}/responses`, {
|
|
1144
|
+
const content = [
|
|
1145
|
+
{ type: "text", text: request.prompt },
|
|
1146
|
+
...encodedAttachments.map((item) => ({
|
|
1147
|
+
type: "image",
|
|
1148
|
+
source: {
|
|
1149
|
+
type: "base64",
|
|
1150
|
+
media_type: item.mimeType,
|
|
1151
|
+
data: item.base64
|
|
1152
|
+
}
|
|
1153
|
+
}))
|
|
1154
|
+
];
|
|
1155
|
+
const response = await fetch(`${this.baseUrl}/v1/messages`, {
|
|
1277
1156
|
method: "POST",
|
|
1278
1157
|
headers: {
|
|
1279
1158
|
"content-type": "application/json",
|
|
1280
|
-
|
|
1159
|
+
"anthropic-version": "2023-06-01",
|
|
1160
|
+
...this.apiKey ? { "x-api-key": this.apiKey } : {},
|
|
1281
1161
|
...this.headers
|
|
1282
1162
|
},
|
|
1283
1163
|
body: JSON.stringify({
|
|
1284
1164
|
model: this.model,
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1165
|
+
max_tokens: request.maxOutputTokens ?? 1200,
|
|
1166
|
+
system: request.system,
|
|
1167
|
+
messages: [
|
|
1168
|
+
{
|
|
1169
|
+
role: "user",
|
|
1170
|
+
content
|
|
1171
|
+
}
|
|
1172
|
+
]
|
|
1288
1173
|
})
|
|
1289
1174
|
});
|
|
1290
1175
|
if (!response.ok) {
|
|
@@ -1292,93 +1177,132 @@ var OpenAiCompatibleProviderAdapter = class extends BaseProviderAdapter {
|
|
|
1292
1177
|
}
|
|
1293
1178
|
const payload = await response.json();
|
|
1294
1179
|
return {
|
|
1295
|
-
text: payload.
|
|
1180
|
+
text: payload.content?.filter((item) => item.type === "text").map((item) => item.text ?? "").join("\n") ?? "",
|
|
1296
1181
|
usage: payload.usage ? { inputTokens: payload.usage.input_tokens, outputTokens: payload.usage.output_tokens } : void 0
|
|
1297
1182
|
};
|
|
1298
1183
|
}
|
|
1299
|
-
|
|
1184
|
+
};
|
|
1185
|
+
|
|
1186
|
+
// src/providers/gemini.ts
|
|
1187
|
+
var GeminiProviderAdapter = class extends BaseProviderAdapter {
|
|
1188
|
+
apiKey;
|
|
1189
|
+
baseUrl;
|
|
1190
|
+
constructor(id, model, options) {
|
|
1191
|
+
super(id, "gemini", model, ["chat", "structured", "vision", "tools", "streaming"]);
|
|
1192
|
+
this.apiKey = options.apiKey;
|
|
1193
|
+
this.baseUrl = (options.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta").replace(/\/+$/, "");
|
|
1194
|
+
}
|
|
1195
|
+
async generateText(request) {
|
|
1300
1196
|
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1301
|
-
const
|
|
1302
|
-
|
|
1197
|
+
const parts = [
|
|
1198
|
+
...request.system ? [{ text: `System instructions:
|
|
1199
|
+
${request.system}` }] : [],
|
|
1200
|
+
{ text: request.prompt },
|
|
1303
1201
|
...encodedAttachments.map((item) => ({
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1202
|
+
inline_data: {
|
|
1203
|
+
mime_type: item.mimeType,
|
|
1204
|
+
data: item.base64
|
|
1307
1205
|
}
|
|
1308
1206
|
}))
|
|
1309
|
-
] : request.prompt;
|
|
1310
|
-
const messages = [
|
|
1311
|
-
...request.system ? [{ role: "system", content: request.system }] : [],
|
|
1312
|
-
{ role: "user", content }
|
|
1313
1207
|
];
|
|
1314
|
-
const response = await fetch(`${this.baseUrl}/
|
|
1208
|
+
const response = await fetch(`${this.baseUrl}/models/${this.model}:generateContent`, {
|
|
1315
1209
|
method: "POST",
|
|
1316
1210
|
headers: {
|
|
1317
1211
|
"content-type": "application/json",
|
|
1318
|
-
...
|
|
1319
|
-
...this.headers
|
|
1212
|
+
...this.apiKey ? { "x-goog-api-key": this.apiKey } : {}
|
|
1320
1213
|
},
|
|
1321
1214
|
body: JSON.stringify({
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1215
|
+
contents: [
|
|
1216
|
+
{
|
|
1217
|
+
role: "user",
|
|
1218
|
+
parts
|
|
1219
|
+
}
|
|
1220
|
+
],
|
|
1221
|
+
generationConfig: {
|
|
1222
|
+
maxOutputTokens: request.maxOutputTokens ?? 1200
|
|
1223
|
+
}
|
|
1325
1224
|
})
|
|
1326
1225
|
});
|
|
1327
1226
|
if (!response.ok) {
|
|
1328
1227
|
throw new Error(`Provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1329
1228
|
}
|
|
1330
1229
|
const payload = await response.json();
|
|
1331
|
-
const
|
|
1332
|
-
const text = Array.isArray(contentValue) ? contentValue.map((item) => item.text ?? "").join("\n") : contentValue ?? "";
|
|
1230
|
+
const text = payload.candidates?.[0]?.content?.parts?.map((part) => part.text ?? "").join("\n") ?? "";
|
|
1333
1231
|
return {
|
|
1334
1232
|
text,
|
|
1335
|
-
usage: payload.
|
|
1233
|
+
usage: payload.usageMetadata ? { inputTokens: payload.usageMetadata.promptTokenCount, outputTokens: payload.usageMetadata.candidatesTokenCount } : void 0
|
|
1336
1234
|
};
|
|
1337
1235
|
}
|
|
1338
1236
|
};
|
|
1339
1237
|
|
|
1340
|
-
// src/providers/
|
|
1341
|
-
|
|
1238
|
+
// src/providers/heuristic.ts
|
|
1239
|
+
function summarizePrompt(prompt) {
|
|
1240
|
+
const cleaned = normalizeWhitespace(prompt);
|
|
1241
|
+
if (!cleaned) {
|
|
1242
|
+
return "No prompt content provided.";
|
|
1243
|
+
}
|
|
1244
|
+
return firstSentences(cleaned, 2) || cleaned.slice(0, 280);
|
|
1245
|
+
}
|
|
1246
|
+
var HeuristicProviderAdapter = class extends BaseProviderAdapter {
|
|
1247
|
+
constructor(id, model) {
|
|
1248
|
+
super(id, "heuristic", model, ["chat", "structured", "vision", "local"]);
|
|
1249
|
+
}
|
|
1250
|
+
async generateText(request) {
|
|
1251
|
+
const attachmentHint = request.attachments?.length ? ` Attachments: ${request.attachments.length}.` : "";
|
|
1252
|
+
return {
|
|
1253
|
+
text: `Heuristic provider response.${attachmentHint} ${summarizePrompt(request.prompt)}`.trim()
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
};
|
|
1257
|
+
|
|
1258
|
+
// src/providers/openai-compatible.ts
|
|
1259
|
+
function buildAuthHeaders(apiKey) {
|
|
1260
|
+
return apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
|
|
1261
|
+
}
|
|
1262
|
+
var OpenAiCompatibleProviderAdapter = class extends BaseProviderAdapter {
|
|
1263
|
+
baseUrl;
|
|
1342
1264
|
apiKey;
|
|
1343
1265
|
headers;
|
|
1344
|
-
|
|
1345
|
-
constructor(id, model, options) {
|
|
1346
|
-
super(id,
|
|
1266
|
+
apiStyle;
|
|
1267
|
+
constructor(id, type, model, options) {
|
|
1268
|
+
super(id, type, model, options.capabilities);
|
|
1269
|
+
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
|
1347
1270
|
this.apiKey = options.apiKey;
|
|
1348
1271
|
this.headers = options.headers;
|
|
1349
|
-
this.
|
|
1272
|
+
this.apiStyle = options.apiStyle ?? "responses";
|
|
1350
1273
|
}
|
|
1351
1274
|
async generateText(request) {
|
|
1275
|
+
if (this.apiStyle === "chat") {
|
|
1276
|
+
return this.generateViaChatCompletions(request);
|
|
1277
|
+
}
|
|
1278
|
+
return this.generateViaResponses(request);
|
|
1279
|
+
}
|
|
1280
|
+
async generateViaResponses(request) {
|
|
1352
1281
|
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1353
|
-
const
|
|
1354
|
-
{
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1282
|
+
const input = encodedAttachments.length ? [
|
|
1283
|
+
{
|
|
1284
|
+
role: "user",
|
|
1285
|
+
content: [
|
|
1286
|
+
{ type: "input_text", text: request.prompt },
|
|
1287
|
+
...encodedAttachments.map((item) => ({
|
|
1288
|
+
type: "input_image",
|
|
1289
|
+
image_url: `data:${item.mimeType};base64,${item.base64}`
|
|
1290
|
+
}))
|
|
1291
|
+
]
|
|
1292
|
+
}
|
|
1293
|
+
] : request.prompt;
|
|
1294
|
+
const response = await fetch(`${this.baseUrl}/responses`, {
|
|
1365
1295
|
method: "POST",
|
|
1366
1296
|
headers: {
|
|
1367
1297
|
"content-type": "application/json",
|
|
1368
|
-
|
|
1369
|
-
...this.apiKey ? { "x-api-key": this.apiKey } : {},
|
|
1298
|
+
...buildAuthHeaders(this.apiKey),
|
|
1370
1299
|
...this.headers
|
|
1371
1300
|
},
|
|
1372
1301
|
body: JSON.stringify({
|
|
1373
1302
|
model: this.model,
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
{
|
|
1378
|
-
role: "user",
|
|
1379
|
-
content
|
|
1380
|
-
}
|
|
1381
|
-
]
|
|
1303
|
+
input,
|
|
1304
|
+
instructions: request.system,
|
|
1305
|
+
max_output_tokens: request.maxOutputTokens
|
|
1382
1306
|
})
|
|
1383
1307
|
});
|
|
1384
1308
|
if (!response.ok) {
|
|
@@ -1386,148 +1310,847 @@ var AnthropicProviderAdapter = class extends BaseProviderAdapter {
|
|
|
1386
1310
|
}
|
|
1387
1311
|
const payload = await response.json();
|
|
1388
1312
|
return {
|
|
1389
|
-
text: payload.
|
|
1313
|
+
text: payload.output_text ?? "",
|
|
1390
1314
|
usage: payload.usage ? { inputTokens: payload.usage.input_tokens, outputTokens: payload.usage.output_tokens } : void 0
|
|
1391
1315
|
};
|
|
1392
1316
|
}
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
// src/providers/gemini.ts
|
|
1396
|
-
var GeminiProviderAdapter = class extends BaseProviderAdapter {
|
|
1397
|
-
apiKey;
|
|
1398
|
-
baseUrl;
|
|
1399
|
-
constructor(id, model, options) {
|
|
1400
|
-
super(id, "gemini", model, ["chat", "structured", "vision", "tools", "streaming"]);
|
|
1401
|
-
this.apiKey = options.apiKey;
|
|
1402
|
-
this.baseUrl = (options.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta").replace(/\/+$/, "");
|
|
1403
|
-
}
|
|
1404
|
-
async generateText(request) {
|
|
1317
|
+
async generateViaChatCompletions(request) {
|
|
1405
1318
|
const encodedAttachments = await this.encodeAttachments(request.attachments);
|
|
1406
|
-
const
|
|
1407
|
-
|
|
1408
|
-
${request.system}` }] : [],
|
|
1409
|
-
{ text: request.prompt },
|
|
1319
|
+
const content = encodedAttachments.length ? [
|
|
1320
|
+
{ type: "text", text: request.prompt },
|
|
1410
1321
|
...encodedAttachments.map((item) => ({
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1322
|
+
type: "image_url",
|
|
1323
|
+
image_url: {
|
|
1324
|
+
url: `data:${item.mimeType};base64,${item.base64}`
|
|
1414
1325
|
}
|
|
1415
1326
|
}))
|
|
1416
|
-
];
|
|
1417
|
-
const
|
|
1327
|
+
] : request.prompt;
|
|
1328
|
+
const messages = [...request.system ? [{ role: "system", content: request.system }] : [], { role: "user", content }];
|
|
1329
|
+
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
1418
1330
|
method: "POST",
|
|
1419
1331
|
headers: {
|
|
1420
1332
|
"content-type": "application/json",
|
|
1421
|
-
...this.apiKey
|
|
1333
|
+
...buildAuthHeaders(this.apiKey),
|
|
1334
|
+
...this.headers
|
|
1422
1335
|
},
|
|
1423
1336
|
body: JSON.stringify({
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
parts
|
|
1428
|
-
}
|
|
1429
|
-
],
|
|
1430
|
-
generationConfig: {
|
|
1431
|
-
maxOutputTokens: request.maxOutputTokens ?? 1200
|
|
1432
|
-
}
|
|
1337
|
+
model: this.model,
|
|
1338
|
+
messages,
|
|
1339
|
+
max_tokens: request.maxOutputTokens
|
|
1433
1340
|
})
|
|
1434
1341
|
});
|
|
1435
1342
|
if (!response.ok) {
|
|
1436
1343
|
throw new Error(`Provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1437
1344
|
}
|
|
1438
1345
|
const payload = await response.json();
|
|
1439
|
-
const
|
|
1346
|
+
const contentValue = payload.choices?.[0]?.message?.content;
|
|
1347
|
+
const text = Array.isArray(contentValue) ? contentValue.map((item) => item.text ?? "").join("\n") : contentValue ?? "";
|
|
1440
1348
|
return {
|
|
1441
1349
|
text,
|
|
1442
|
-
usage: payload.
|
|
1350
|
+
usage: payload.usage ? { inputTokens: payload.usage.prompt_tokens, outputTokens: payload.usage.completion_tokens } : void 0
|
|
1443
1351
|
};
|
|
1444
1352
|
}
|
|
1445
1353
|
};
|
|
1446
1354
|
|
|
1447
|
-
// src/providers/registry.ts
|
|
1448
|
-
var customModuleSchema = z5.object({
|
|
1449
|
-
createAdapter: z5.function({
|
|
1450
|
-
input: [z5.string(), z5.custom(), z5.string()],
|
|
1451
|
-
output: z5.promise(z5.custom())
|
|
1452
|
-
})
|
|
1453
|
-
});
|
|
1454
|
-
function resolveCapabilities(config, fallback) {
|
|
1455
|
-
return config.capabilities?.length ? config.capabilities : fallback;
|
|
1355
|
+
// src/providers/registry.ts
|
|
1356
|
+
var customModuleSchema = z5.object({
|
|
1357
|
+
createAdapter: z5.function({
|
|
1358
|
+
input: [z5.string(), z5.custom(), z5.string()],
|
|
1359
|
+
output: z5.promise(z5.custom())
|
|
1360
|
+
})
|
|
1361
|
+
});
|
|
1362
|
+
function resolveCapabilities(config, fallback) {
|
|
1363
|
+
return config.capabilities?.length ? config.capabilities : fallback;
|
|
1364
|
+
}
|
|
1365
|
+
function envOrUndefined(name) {
|
|
1366
|
+
return name ? process.env[name] : void 0;
|
|
1367
|
+
}
|
|
1368
|
+
async function createProvider(id, config, rootDir) {
|
|
1369
|
+
switch (config.type) {
|
|
1370
|
+
case "heuristic":
|
|
1371
|
+
return new HeuristicProviderAdapter(id, config.model);
|
|
1372
|
+
case "openai":
|
|
1373
|
+
return new OpenAiCompatibleProviderAdapter(id, "openai", config.model, {
|
|
1374
|
+
baseUrl: config.baseUrl ?? "https://api.openai.com/v1",
|
|
1375
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1376
|
+
headers: config.headers,
|
|
1377
|
+
apiStyle: config.apiStyle ?? "responses",
|
|
1378
|
+
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming"])
|
|
1379
|
+
});
|
|
1380
|
+
case "ollama":
|
|
1381
|
+
return new OpenAiCompatibleProviderAdapter(id, "ollama", config.model, {
|
|
1382
|
+
baseUrl: config.baseUrl ?? "http://localhost:11434/v1",
|
|
1383
|
+
apiKey: envOrUndefined(config.apiKeyEnv) ?? "ollama",
|
|
1384
|
+
headers: config.headers,
|
|
1385
|
+
apiStyle: config.apiStyle ?? "responses",
|
|
1386
|
+
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming", "local"])
|
|
1387
|
+
});
|
|
1388
|
+
case "openai-compatible":
|
|
1389
|
+
return new OpenAiCompatibleProviderAdapter(id, "openai-compatible", config.model, {
|
|
1390
|
+
baseUrl: config.baseUrl ?? "http://localhost:8000/v1",
|
|
1391
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1392
|
+
headers: config.headers,
|
|
1393
|
+
apiStyle: config.apiStyle ?? "responses",
|
|
1394
|
+
capabilities: resolveCapabilities(config, ["chat", "structured"])
|
|
1395
|
+
});
|
|
1396
|
+
case "anthropic":
|
|
1397
|
+
return new AnthropicProviderAdapter(id, config.model, {
|
|
1398
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1399
|
+
headers: config.headers,
|
|
1400
|
+
baseUrl: config.baseUrl
|
|
1401
|
+
});
|
|
1402
|
+
case "gemini":
|
|
1403
|
+
return new GeminiProviderAdapter(id, config.model, {
|
|
1404
|
+
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1405
|
+
baseUrl: config.baseUrl
|
|
1406
|
+
});
|
|
1407
|
+
case "custom": {
|
|
1408
|
+
if (!config.module) {
|
|
1409
|
+
throw new Error(`Provider ${id} is type "custom" but no module path was configured.`);
|
|
1410
|
+
}
|
|
1411
|
+
const resolvedModule = path8.isAbsolute(config.module) ? config.module : path8.resolve(rootDir, config.module);
|
|
1412
|
+
const loaded = await import(pathToFileURL(resolvedModule).href);
|
|
1413
|
+
const parsed = customModuleSchema.parse(loaded);
|
|
1414
|
+
return parsed.createAdapter(id, config, rootDir);
|
|
1415
|
+
}
|
|
1416
|
+
default:
|
|
1417
|
+
throw new Error(`Unsupported provider type ${String(config.type)}`);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
async function getProviderForTask(rootDir, task) {
|
|
1421
|
+
const { config } = await loadVaultConfig(rootDir);
|
|
1422
|
+
const providerId = config.tasks[task];
|
|
1423
|
+
const providerConfig = config.providers[providerId];
|
|
1424
|
+
if (!providerConfig) {
|
|
1425
|
+
throw new Error(`No provider configured with id "${providerId}" for task "${task}".`);
|
|
1426
|
+
}
|
|
1427
|
+
return createProvider(providerId, providerConfig, rootDir);
|
|
1428
|
+
}
|
|
1429
|
+
function assertProviderCapability(provider, capability) {
|
|
1430
|
+
if (!provider.capabilities.has(capability)) {
|
|
1431
|
+
throw new Error(`Provider ${provider.id} does not support required capability "${capability}".`);
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
// src/web-search/registry.ts
|
|
1436
|
+
import path9 from "path";
|
|
1437
|
+
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
1438
|
+
import { z as z6 } from "zod";
|
|
1439
|
+
|
|
1440
|
+
// src/web-search/http-json.ts
|
|
1441
|
+
function deepGet(value, pathValue) {
|
|
1442
|
+
if (!pathValue) {
|
|
1443
|
+
return value;
|
|
1444
|
+
}
|
|
1445
|
+
return pathValue.split(".").filter(Boolean).reduce((current, segment) => {
|
|
1446
|
+
if (current && typeof current === "object" && segment in current) {
|
|
1447
|
+
return current[segment];
|
|
1448
|
+
}
|
|
1449
|
+
return void 0;
|
|
1450
|
+
}, value);
|
|
1451
|
+
}
|
|
1452
|
+
function envOrUndefined2(name) {
|
|
1453
|
+
return name ? process.env[name] : void 0;
|
|
1454
|
+
}
|
|
1455
|
+
var HttpJsonWebSearchAdapter = class {
|
|
1456
|
+
constructor(id, config) {
|
|
1457
|
+
this.id = id;
|
|
1458
|
+
this.config = config;
|
|
1459
|
+
}
|
|
1460
|
+
id;
|
|
1461
|
+
config;
|
|
1462
|
+
type = "http-json";
|
|
1463
|
+
async search(query, limit = 5) {
|
|
1464
|
+
if (!this.config.endpoint) {
|
|
1465
|
+
throw new Error(`Web search provider ${this.id} is missing an endpoint.`);
|
|
1466
|
+
}
|
|
1467
|
+
const method = this.config.method ?? "GET";
|
|
1468
|
+
const queryParam = this.config.queryParam ?? "q";
|
|
1469
|
+
const limitParam = this.config.limitParam ?? "limit";
|
|
1470
|
+
const headers = {
|
|
1471
|
+
accept: "application/json",
|
|
1472
|
+
...this.config.headers
|
|
1473
|
+
};
|
|
1474
|
+
const apiKey = envOrUndefined2(this.config.apiKeyEnv);
|
|
1475
|
+
if (apiKey) {
|
|
1476
|
+
headers[this.config.apiKeyHeader ?? "Authorization"] = `${this.config.apiKeyPrefix ?? "Bearer "}${apiKey}`;
|
|
1477
|
+
}
|
|
1478
|
+
const endpoint = new URL(this.config.endpoint);
|
|
1479
|
+
let body;
|
|
1480
|
+
if (method === "GET") {
|
|
1481
|
+
endpoint.searchParams.set(queryParam, query);
|
|
1482
|
+
endpoint.searchParams.set(limitParam, String(limit));
|
|
1483
|
+
} else {
|
|
1484
|
+
headers["content-type"] = "application/json";
|
|
1485
|
+
body = JSON.stringify({
|
|
1486
|
+
[queryParam]: query,
|
|
1487
|
+
[limitParam]: limit
|
|
1488
|
+
});
|
|
1489
|
+
}
|
|
1490
|
+
const response = await fetch(endpoint, {
|
|
1491
|
+
method,
|
|
1492
|
+
headers,
|
|
1493
|
+
body
|
|
1494
|
+
});
|
|
1495
|
+
if (!response.ok) {
|
|
1496
|
+
throw new Error(`Web search provider ${this.id} failed: ${response.status} ${response.statusText}`);
|
|
1497
|
+
}
|
|
1498
|
+
const payload = await response.json();
|
|
1499
|
+
const rawResults = deepGet(payload, this.config.resultsPath ?? "results");
|
|
1500
|
+
if (!Array.isArray(rawResults)) {
|
|
1501
|
+
return [];
|
|
1502
|
+
}
|
|
1503
|
+
return rawResults.map((item) => {
|
|
1504
|
+
const title = deepGet(item, this.config.titleField ?? "title");
|
|
1505
|
+
const url = deepGet(item, this.config.urlField ?? "url");
|
|
1506
|
+
const snippet = deepGet(item, this.config.snippetField ?? "snippet");
|
|
1507
|
+
if (typeof title !== "string" || typeof url !== "string") {
|
|
1508
|
+
return null;
|
|
1509
|
+
}
|
|
1510
|
+
return {
|
|
1511
|
+
title,
|
|
1512
|
+
url,
|
|
1513
|
+
snippet: typeof snippet === "string" ? snippet : ""
|
|
1514
|
+
};
|
|
1515
|
+
}).filter((item) => item !== null);
|
|
1516
|
+
}
|
|
1517
|
+
};
|
|
1518
|
+
|
|
1519
|
+
// src/web-search/registry.ts
|
|
1520
|
+
var customWebSearchModuleSchema = z6.object({
|
|
1521
|
+
createAdapter: z6.function({
|
|
1522
|
+
input: [z6.string(), z6.custom(), z6.string()],
|
|
1523
|
+
output: z6.promise(z6.custom())
|
|
1524
|
+
})
|
|
1525
|
+
});
|
|
1526
|
+
async function createWebSearchAdapter(id, config, rootDir) {
|
|
1527
|
+
switch (config.type) {
|
|
1528
|
+
case "http-json":
|
|
1529
|
+
return new HttpJsonWebSearchAdapter(id, config);
|
|
1530
|
+
case "custom": {
|
|
1531
|
+
if (!config.module) {
|
|
1532
|
+
throw new Error(`Web search provider ${id} is type "custom" but no module path was configured.`);
|
|
1533
|
+
}
|
|
1534
|
+
const resolvedModule = path9.isAbsolute(config.module) ? config.module : path9.resolve(rootDir, config.module);
|
|
1535
|
+
const loaded = await import(pathToFileURL2(resolvedModule).href);
|
|
1536
|
+
const parsed = customWebSearchModuleSchema.parse(loaded);
|
|
1537
|
+
return parsed.createAdapter(id, config, rootDir);
|
|
1538
|
+
}
|
|
1539
|
+
default:
|
|
1540
|
+
throw new Error(`Unsupported web search provider type ${String(config.type)}`);
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
async function getWebSearchAdapterForTask(rootDir, task) {
|
|
1544
|
+
const { config } = await loadVaultConfig(rootDir);
|
|
1545
|
+
const webSearchConfig = config.webSearch;
|
|
1546
|
+
if (!webSearchConfig) {
|
|
1547
|
+
throw new Error("No web search providers are configured. Add a webSearch block to swarmvault.config.json.");
|
|
1548
|
+
}
|
|
1549
|
+
const providerId = webSearchConfig.tasks[task];
|
|
1550
|
+
const providerConfig = webSearchConfig.providers[providerId];
|
|
1551
|
+
if (!providerConfig) {
|
|
1552
|
+
throw new Error(`No web search provider configured with id "${providerId}" for task "${task}".`);
|
|
1553
|
+
}
|
|
1554
|
+
return createWebSearchAdapter(providerId, providerConfig, rootDir);
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
// src/deep-lint.ts
|
|
1558
|
+
var deepLintResponseSchema = z7.object({
|
|
1559
|
+
findings: z7.array(
|
|
1560
|
+
z7.object({
|
|
1561
|
+
severity: z7.enum(["error", "warning", "info"]).default("info"),
|
|
1562
|
+
code: z7.enum(["coverage_gap", "contradiction_candidate", "missing_citation", "candidate_page", "follow_up_question"]),
|
|
1563
|
+
message: z7.string().min(1),
|
|
1564
|
+
relatedSourceIds: z7.array(z7.string()).default([]),
|
|
1565
|
+
relatedPageIds: z7.array(z7.string()).default([]),
|
|
1566
|
+
suggestedQuery: z7.string().optional()
|
|
1567
|
+
})
|
|
1568
|
+
).max(20)
|
|
1569
|
+
});
|
|
1570
|
+
async function loadContextPages(rootDir, graph) {
|
|
1571
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
1572
|
+
const contextPages = graph.pages.filter(
|
|
1573
|
+
(page) => page.kind === "source" || page.kind === "concept" || page.kind === "entity"
|
|
1574
|
+
);
|
|
1575
|
+
return Promise.all(
|
|
1576
|
+
contextPages.slice(0, 18).map(async (page) => {
|
|
1577
|
+
const absolutePath = path10.join(paths.wikiDir, page.path);
|
|
1578
|
+
const raw = await fs8.readFile(absolutePath, "utf8").catch(() => "");
|
|
1579
|
+
const parsed = matter(raw);
|
|
1580
|
+
return {
|
|
1581
|
+
id: page.id,
|
|
1582
|
+
title: page.title,
|
|
1583
|
+
path: page.path,
|
|
1584
|
+
kind: page.kind,
|
|
1585
|
+
sourceIds: page.sourceIds,
|
|
1586
|
+
excerpt: truncate(normalizeWhitespace(parsed.content), 1400)
|
|
1587
|
+
};
|
|
1588
|
+
})
|
|
1589
|
+
);
|
|
1590
|
+
}
|
|
1591
|
+
function heuristicDeepFindings(contextPages, structuralFindings) {
|
|
1592
|
+
const findings = [];
|
|
1593
|
+
for (const page of contextPages) {
|
|
1594
|
+
if (page.excerpt.includes("No claims extracted.")) {
|
|
1595
|
+
findings.push({
|
|
1596
|
+
severity: "warning",
|
|
1597
|
+
code: "coverage_gap",
|
|
1598
|
+
message: `Page ${page.title} has no extracted claims yet.`,
|
|
1599
|
+
pagePath: page.path,
|
|
1600
|
+
relatedSourceIds: page.sourceIds,
|
|
1601
|
+
relatedPageIds: [page.id],
|
|
1602
|
+
suggestedQuery: `What evidence or claims should ${page.title} contain?`
|
|
1603
|
+
});
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
for (const finding of structuralFindings.filter((item) => item.code === "uncited_claims").slice(0, 5)) {
|
|
1607
|
+
findings.push({
|
|
1608
|
+
severity: "warning",
|
|
1609
|
+
code: "missing_citation",
|
|
1610
|
+
message: finding.message,
|
|
1611
|
+
pagePath: finding.pagePath,
|
|
1612
|
+
suggestedQuery: finding.pagePath ? `Which sources support the claims in ${path10.basename(finding.pagePath, ".md")}?` : void 0
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1615
|
+
for (const page of contextPages.filter((item) => item.kind === "source").slice(0, 3)) {
|
|
1616
|
+
findings.push({
|
|
1617
|
+
severity: "info",
|
|
1618
|
+
code: "follow_up_question",
|
|
1619
|
+
message: `Investigate what broader implications ${page.title} has for the rest of the vault.`,
|
|
1620
|
+
pagePath: page.path,
|
|
1621
|
+
relatedSourceIds: page.sourceIds,
|
|
1622
|
+
relatedPageIds: [page.id],
|
|
1623
|
+
suggestedQuery: `What broader implications does ${page.title} have?`
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
return uniqueBy(findings, (item) => `${item.code}:${item.message}`);
|
|
1627
|
+
}
|
|
1628
|
+
async function runDeepLint(rootDir, structuralFindings, options = {}) {
|
|
1629
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
1630
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
1631
|
+
if (!graph) {
|
|
1632
|
+
return [];
|
|
1633
|
+
}
|
|
1634
|
+
const schema = await loadVaultSchema(rootDir);
|
|
1635
|
+
const provider = await getProviderForTask(rootDir, "lintProvider");
|
|
1636
|
+
const manifests = await listManifests(rootDir);
|
|
1637
|
+
const contextPages = await loadContextPages(rootDir, graph);
|
|
1638
|
+
let findings;
|
|
1639
|
+
if (provider.type === "heuristic") {
|
|
1640
|
+
findings = heuristicDeepFindings(contextPages, structuralFindings);
|
|
1641
|
+
} else {
|
|
1642
|
+
const response = await provider.generateStructured(
|
|
1643
|
+
{
|
|
1644
|
+
system: "You are an auditor for a local-first LLM knowledge vault. Return advisory findings only. Do not propose direct file edits.",
|
|
1645
|
+
prompt: [
|
|
1646
|
+
"Review this SwarmVault state and return high-signal advisory findings.",
|
|
1647
|
+
"",
|
|
1648
|
+
"Schema:",
|
|
1649
|
+
schema.content,
|
|
1650
|
+
"",
|
|
1651
|
+
"Vault summary:",
|
|
1652
|
+
`- sources: ${manifests.length}`,
|
|
1653
|
+
`- pages: ${graph.pages.length}`,
|
|
1654
|
+
`- structural_findings: ${structuralFindings.length}`,
|
|
1655
|
+
"",
|
|
1656
|
+
"Structural findings:",
|
|
1657
|
+
structuralFindings.map((item) => `- [${item.severity}] ${item.code}: ${item.message}`).join("\n") || "- none",
|
|
1658
|
+
"",
|
|
1659
|
+
"Page context:",
|
|
1660
|
+
contextPages.map(
|
|
1661
|
+
(page) => [
|
|
1662
|
+
`## ${page.title}`,
|
|
1663
|
+
`page_id: ${page.id}`,
|
|
1664
|
+
`path: ${page.path}`,
|
|
1665
|
+
`kind: ${page.kind}`,
|
|
1666
|
+
`source_ids: ${page.sourceIds.join(",") || "none"}`,
|
|
1667
|
+
page.excerpt
|
|
1668
|
+
].join("\n")
|
|
1669
|
+
).join("\n\n---\n\n")
|
|
1670
|
+
].join("\n")
|
|
1671
|
+
},
|
|
1672
|
+
deepLintResponseSchema
|
|
1673
|
+
);
|
|
1674
|
+
findings = response.findings.map((item) => ({
|
|
1675
|
+
severity: item.severity,
|
|
1676
|
+
code: item.code,
|
|
1677
|
+
message: item.message,
|
|
1678
|
+
relatedSourceIds: item.relatedSourceIds,
|
|
1679
|
+
relatedPageIds: item.relatedPageIds,
|
|
1680
|
+
suggestedQuery: item.suggestedQuery
|
|
1681
|
+
}));
|
|
1682
|
+
}
|
|
1683
|
+
if (!options.web) {
|
|
1684
|
+
return findings;
|
|
1685
|
+
}
|
|
1686
|
+
const webSearch = await getWebSearchAdapterForTask(rootDir, "deepLintProvider");
|
|
1687
|
+
const queryCache = /* @__PURE__ */ new Map();
|
|
1688
|
+
for (const finding of findings) {
|
|
1689
|
+
const query = finding.suggestedQuery ?? finding.message;
|
|
1690
|
+
if (!queryCache.has(query)) {
|
|
1691
|
+
queryCache.set(query, await webSearch.search(query, 3));
|
|
1692
|
+
}
|
|
1693
|
+
finding.evidence = queryCache.get(query);
|
|
1694
|
+
}
|
|
1695
|
+
return findings;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
// src/markdown.ts
|
|
1699
|
+
import matter2 from "gray-matter";
|
|
1700
|
+
function pagePathFor(kind, slug) {
|
|
1701
|
+
switch (kind) {
|
|
1702
|
+
case "source":
|
|
1703
|
+
return `sources/${slug}.md`;
|
|
1704
|
+
case "concept":
|
|
1705
|
+
return `concepts/${slug}.md`;
|
|
1706
|
+
case "entity":
|
|
1707
|
+
return `entities/${slug}.md`;
|
|
1708
|
+
case "output":
|
|
1709
|
+
return `outputs/${slug}.md`;
|
|
1710
|
+
default:
|
|
1711
|
+
return `${slug}.md`;
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
function pageLink(page) {
|
|
1715
|
+
return `[[${page.path.replace(/\.md$/, "")}|${page.title}]]`;
|
|
1716
|
+
}
|
|
1717
|
+
function relatedOutputsSection(relatedOutputs) {
|
|
1718
|
+
if (!relatedOutputs.length) {
|
|
1719
|
+
return [];
|
|
1720
|
+
}
|
|
1721
|
+
return ["## Related Outputs", "", ...relatedOutputs.map((page) => `- ${pageLink(page)}`), ""];
|
|
1722
|
+
}
|
|
1723
|
+
function buildSourcePage(manifest, analysis, schemaHash, confidence = 1, relatedOutputs = []) {
|
|
1724
|
+
const relativePath = pagePathFor("source", manifest.sourceId);
|
|
1725
|
+
const pageId = `source:${manifest.sourceId}`;
|
|
1726
|
+
const nodeIds = [`source:${manifest.sourceId}`, ...analysis.concepts.map((item) => item.id), ...analysis.entities.map((item) => item.id)];
|
|
1727
|
+
const backlinks = [
|
|
1728
|
+
...analysis.concepts.map((item) => `concept:${slugify(item.name)}`),
|
|
1729
|
+
...analysis.entities.map((item) => `entity:${slugify(item.name)}`),
|
|
1730
|
+
...relatedOutputs.map((page) => page.id)
|
|
1731
|
+
];
|
|
1732
|
+
const frontmatter = {
|
|
1733
|
+
page_id: pageId,
|
|
1734
|
+
kind: "source",
|
|
1735
|
+
title: analysis.title,
|
|
1736
|
+
tags: ["source"],
|
|
1737
|
+
source_ids: [manifest.sourceId],
|
|
1738
|
+
node_ids: nodeIds,
|
|
1739
|
+
freshness: "fresh",
|
|
1740
|
+
confidence,
|
|
1741
|
+
updated_at: analysis.producedAt,
|
|
1742
|
+
backlinks,
|
|
1743
|
+
schema_hash: schemaHash,
|
|
1744
|
+
source_hashes: {
|
|
1745
|
+
[manifest.sourceId]: manifest.contentHash
|
|
1746
|
+
}
|
|
1747
|
+
};
|
|
1748
|
+
const body = [
|
|
1749
|
+
`# ${analysis.title}`,
|
|
1750
|
+
"",
|
|
1751
|
+
`Source ID: \`${manifest.sourceId}\``,
|
|
1752
|
+
manifest.url ? `Source URL: ${manifest.url}` : `Source Path: \`${manifest.originalPath ?? manifest.storedPath}\``,
|
|
1753
|
+
"",
|
|
1754
|
+
"## Summary",
|
|
1755
|
+
"",
|
|
1756
|
+
analysis.summary,
|
|
1757
|
+
"",
|
|
1758
|
+
"## Concepts",
|
|
1759
|
+
"",
|
|
1760
|
+
...analysis.concepts.length ? analysis.concepts.map(
|
|
1761
|
+
(item) => `- [[${pagePathFor("concept", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`
|
|
1762
|
+
) : ["- None detected."],
|
|
1763
|
+
"",
|
|
1764
|
+
"## Entities",
|
|
1765
|
+
"",
|
|
1766
|
+
...analysis.entities.length ? analysis.entities.map(
|
|
1767
|
+
(item) => `- [[${pagePathFor("entity", slugify(item.name)).replace(/\.md$/, "")}|${item.name}]]: ${item.description}`
|
|
1768
|
+
) : ["- None detected."],
|
|
1769
|
+
"",
|
|
1770
|
+
"## Claims",
|
|
1771
|
+
"",
|
|
1772
|
+
...analysis.claims.length ? analysis.claims.map((claim) => `- ${claim.text} [source:${claim.citation}]`) : ["- No claims extracted."],
|
|
1773
|
+
"",
|
|
1774
|
+
"## Questions",
|
|
1775
|
+
"",
|
|
1776
|
+
...analysis.questions.length ? analysis.questions.map((question) => `- ${question}`) : ["- No follow-up questions yet."],
|
|
1777
|
+
"",
|
|
1778
|
+
...relatedOutputsSection(relatedOutputs),
|
|
1779
|
+
""
|
|
1780
|
+
].join("\n");
|
|
1781
|
+
return {
|
|
1782
|
+
page: {
|
|
1783
|
+
id: pageId,
|
|
1784
|
+
path: relativePath,
|
|
1785
|
+
title: analysis.title,
|
|
1786
|
+
kind: "source",
|
|
1787
|
+
sourceIds: [manifest.sourceId],
|
|
1788
|
+
nodeIds,
|
|
1789
|
+
freshness: "fresh",
|
|
1790
|
+
confidence,
|
|
1791
|
+
backlinks,
|
|
1792
|
+
schemaHash,
|
|
1793
|
+
sourceHashes: { [manifest.sourceId]: manifest.contentHash },
|
|
1794
|
+
relatedPageIds: relatedOutputs.map((page) => page.id),
|
|
1795
|
+
relatedNodeIds: [],
|
|
1796
|
+
relatedSourceIds: []
|
|
1797
|
+
},
|
|
1798
|
+
content: matter2.stringify(body, frontmatter)
|
|
1799
|
+
};
|
|
1800
|
+
}
|
|
1801
|
+
function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes, schemaHash, confidence = 0.72, relatedOutputs = []) {
|
|
1802
|
+
const slug = slugify(name);
|
|
1803
|
+
const relativePath = pagePathFor(kind, slug);
|
|
1804
|
+
const pageId = `${kind}:${slug}`;
|
|
1805
|
+
const sourceIds = sourceAnalyses.map((item) => item.sourceId);
|
|
1806
|
+
const otherPages = [...sourceAnalyses.map((item) => `source:${item.sourceId}`), ...relatedOutputs.map((page) => page.id)];
|
|
1807
|
+
const summary = descriptions.find(Boolean) ?? `${kind} aggregated from ${sourceIds.length} source(s).`;
|
|
1808
|
+
const frontmatter = {
|
|
1809
|
+
page_id: pageId,
|
|
1810
|
+
kind,
|
|
1811
|
+
title: name,
|
|
1812
|
+
tags: [kind],
|
|
1813
|
+
source_ids: sourceIds,
|
|
1814
|
+
node_ids: [pageId],
|
|
1815
|
+
freshness: "fresh",
|
|
1816
|
+
confidence,
|
|
1817
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1818
|
+
backlinks: otherPages,
|
|
1819
|
+
schema_hash: schemaHash,
|
|
1820
|
+
source_hashes: sourceHashes
|
|
1821
|
+
};
|
|
1822
|
+
const body = [
|
|
1823
|
+
`# ${name}`,
|
|
1824
|
+
"",
|
|
1825
|
+
"## Summary",
|
|
1826
|
+
"",
|
|
1827
|
+
summary,
|
|
1828
|
+
"",
|
|
1829
|
+
"## Seen In",
|
|
1830
|
+
"",
|
|
1831
|
+
...sourceAnalyses.map((item) => `- [[${pagePathFor("source", item.sourceId).replace(/\.md$/, "")}|${item.title}]]`),
|
|
1832
|
+
"",
|
|
1833
|
+
"## Source Claims",
|
|
1834
|
+
"",
|
|
1835
|
+
...sourceAnalyses.flatMap(
|
|
1836
|
+
(item) => item.claims.filter((claim) => claim.text.toLowerCase().includes(name.toLowerCase())).map((claim) => `- ${claim.text} [source:${claim.citation}]`)
|
|
1837
|
+
),
|
|
1838
|
+
"",
|
|
1839
|
+
...relatedOutputsSection(relatedOutputs),
|
|
1840
|
+
""
|
|
1841
|
+
].join("\n");
|
|
1842
|
+
return {
|
|
1843
|
+
page: {
|
|
1844
|
+
id: pageId,
|
|
1845
|
+
path: relativePath,
|
|
1846
|
+
title: name,
|
|
1847
|
+
kind,
|
|
1848
|
+
sourceIds,
|
|
1849
|
+
nodeIds: [pageId],
|
|
1850
|
+
freshness: "fresh",
|
|
1851
|
+
confidence,
|
|
1852
|
+
backlinks: otherPages,
|
|
1853
|
+
schemaHash,
|
|
1854
|
+
sourceHashes,
|
|
1855
|
+
relatedPageIds: relatedOutputs.map((page) => page.id),
|
|
1856
|
+
relatedNodeIds: [],
|
|
1857
|
+
relatedSourceIds: []
|
|
1858
|
+
},
|
|
1859
|
+
content: matter2.stringify(body, frontmatter)
|
|
1860
|
+
};
|
|
1861
|
+
}
|
|
1862
|
+
function buildIndexPage(pages, schemaHash) {
|
|
1863
|
+
const sources = pages.filter((page) => page.kind === "source");
|
|
1864
|
+
const concepts = pages.filter((page) => page.kind === "concept");
|
|
1865
|
+
const entities = pages.filter((page) => page.kind === "entity");
|
|
1866
|
+
const outputs = pages.filter((page) => page.kind === "output");
|
|
1867
|
+
return [
|
|
1868
|
+
"---",
|
|
1869
|
+
"page_id: index",
|
|
1870
|
+
"kind: index",
|
|
1871
|
+
"title: SwarmVault Index",
|
|
1872
|
+
"tags:",
|
|
1873
|
+
" - index",
|
|
1874
|
+
"source_ids: []",
|
|
1875
|
+
"node_ids: []",
|
|
1876
|
+
"freshness: fresh",
|
|
1877
|
+
"confidence: 1",
|
|
1878
|
+
`updated_at: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
1879
|
+
"backlinks: []",
|
|
1880
|
+
`schema_hash: ${schemaHash}`,
|
|
1881
|
+
"source_hashes: {}",
|
|
1882
|
+
"---",
|
|
1883
|
+
"",
|
|
1884
|
+
"# SwarmVault Index",
|
|
1885
|
+
"",
|
|
1886
|
+
"## Sources",
|
|
1887
|
+
"",
|
|
1888
|
+
...sources.length ? sources.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No sources yet."],
|
|
1889
|
+
"",
|
|
1890
|
+
"## Concepts",
|
|
1891
|
+
"",
|
|
1892
|
+
...concepts.length ? concepts.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No concepts yet."],
|
|
1893
|
+
"",
|
|
1894
|
+
"## Entities",
|
|
1895
|
+
"",
|
|
1896
|
+
...entities.length ? entities.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No entities yet."],
|
|
1897
|
+
"",
|
|
1898
|
+
"## Outputs",
|
|
1899
|
+
"",
|
|
1900
|
+
...outputs.length ? outputs.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No saved outputs yet."],
|
|
1901
|
+
""
|
|
1902
|
+
].join("\n");
|
|
1903
|
+
}
|
|
1904
|
+
function buildSectionIndex(kind, pages, schemaHash) {
|
|
1905
|
+
const title = kind.charAt(0).toUpperCase() + kind.slice(1);
|
|
1906
|
+
return matter2.stringify(
|
|
1907
|
+
[`# ${title}`, "", ...pages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`), ""].join("\n"),
|
|
1908
|
+
{
|
|
1909
|
+
page_id: `${kind}:index`,
|
|
1910
|
+
kind: "index",
|
|
1911
|
+
title,
|
|
1912
|
+
tags: ["index", kind],
|
|
1913
|
+
source_ids: [],
|
|
1914
|
+
node_ids: [],
|
|
1915
|
+
freshness: "fresh",
|
|
1916
|
+
confidence: 1,
|
|
1917
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1918
|
+
backlinks: [],
|
|
1919
|
+
schema_hash: schemaHash,
|
|
1920
|
+
source_hashes: {}
|
|
1921
|
+
}
|
|
1922
|
+
);
|
|
1923
|
+
}
|
|
1924
|
+
function buildOutputPage(input) {
|
|
1925
|
+
const slug = input.slug ?? slugify(input.question);
|
|
1926
|
+
const pageId = `output:${slug}`;
|
|
1927
|
+
const pathValue = pagePathFor("output", slug);
|
|
1928
|
+
const relatedPageIds = input.relatedPageIds ?? [];
|
|
1929
|
+
const relatedNodeIds = input.relatedNodeIds ?? [];
|
|
1930
|
+
const relatedSourceIds = input.relatedSourceIds ?? input.citations;
|
|
1931
|
+
const backlinks = [.../* @__PURE__ */ new Set([...relatedPageIds, ...relatedSourceIds.map((sourceId) => `source:${sourceId}`)])];
|
|
1932
|
+
const frontmatter = {
|
|
1933
|
+
page_id: pageId,
|
|
1934
|
+
kind: "output",
|
|
1935
|
+
title: input.title ?? input.question,
|
|
1936
|
+
tags: ["output"],
|
|
1937
|
+
source_ids: input.citations,
|
|
1938
|
+
node_ids: relatedNodeIds,
|
|
1939
|
+
freshness: "fresh",
|
|
1940
|
+
confidence: 0.74,
|
|
1941
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1942
|
+
backlinks,
|
|
1943
|
+
schema_hash: input.schemaHash,
|
|
1944
|
+
source_hashes: {},
|
|
1945
|
+
related_page_ids: relatedPageIds,
|
|
1946
|
+
related_node_ids: relatedNodeIds,
|
|
1947
|
+
related_source_ids: relatedSourceIds,
|
|
1948
|
+
origin: input.origin,
|
|
1949
|
+
question: input.question
|
|
1950
|
+
};
|
|
1951
|
+
return {
|
|
1952
|
+
page: {
|
|
1953
|
+
id: pageId,
|
|
1954
|
+
path: pathValue,
|
|
1955
|
+
title: input.title ?? input.question,
|
|
1956
|
+
kind: "output",
|
|
1957
|
+
sourceIds: input.citations,
|
|
1958
|
+
nodeIds: relatedNodeIds,
|
|
1959
|
+
freshness: "fresh",
|
|
1960
|
+
confidence: 0.74,
|
|
1961
|
+
backlinks,
|
|
1962
|
+
schemaHash: input.schemaHash,
|
|
1963
|
+
sourceHashes: {},
|
|
1964
|
+
relatedPageIds,
|
|
1965
|
+
relatedNodeIds,
|
|
1966
|
+
relatedSourceIds,
|
|
1967
|
+
origin: input.origin,
|
|
1968
|
+
question: input.question
|
|
1969
|
+
},
|
|
1970
|
+
content: matter2.stringify(
|
|
1971
|
+
[
|
|
1972
|
+
`# ${input.title ?? input.question}`,
|
|
1973
|
+
"",
|
|
1974
|
+
input.answer,
|
|
1975
|
+
"",
|
|
1976
|
+
"## Related Pages",
|
|
1977
|
+
"",
|
|
1978
|
+
...relatedPageIds.length ? relatedPageIds.map((pageId2) => `- \`${pageId2}\``) : ["- None recorded."],
|
|
1979
|
+
"",
|
|
1980
|
+
"## Citations",
|
|
1981
|
+
"",
|
|
1982
|
+
...input.citations.map((citation) => `- [source:${citation}]`),
|
|
1983
|
+
""
|
|
1984
|
+
].join("\n"),
|
|
1985
|
+
frontmatter
|
|
1986
|
+
)
|
|
1987
|
+
};
|
|
1988
|
+
}
|
|
1989
|
+
function buildExploreHubPage(input) {
|
|
1990
|
+
const slug = input.slug ?? `explore-${slugify(input.question)}`;
|
|
1991
|
+
const pageId = `output:${slug}`;
|
|
1992
|
+
const pathValue = pagePathFor("output", slug);
|
|
1993
|
+
const relatedPageIds = input.stepPages.map((page) => page.id);
|
|
1994
|
+
const relatedSourceIds = [...new Set(input.citations)];
|
|
1995
|
+
const relatedNodeIds = [...new Set(input.stepPages.flatMap((page) => page.nodeIds))];
|
|
1996
|
+
const backlinks = [.../* @__PURE__ */ new Set([...relatedPageIds, ...relatedSourceIds.map((sourceId) => `source:${sourceId}`)])];
|
|
1997
|
+
const title = `Explore: ${input.question}`;
|
|
1998
|
+
const frontmatter = {
|
|
1999
|
+
page_id: pageId,
|
|
2000
|
+
kind: "output",
|
|
2001
|
+
title,
|
|
2002
|
+
tags: ["output", "explore"],
|
|
2003
|
+
source_ids: relatedSourceIds,
|
|
2004
|
+
node_ids: relatedNodeIds,
|
|
2005
|
+
freshness: "fresh",
|
|
2006
|
+
confidence: 0.76,
|
|
2007
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2008
|
+
backlinks,
|
|
2009
|
+
schema_hash: input.schemaHash,
|
|
2010
|
+
source_hashes: {},
|
|
2011
|
+
related_page_ids: relatedPageIds,
|
|
2012
|
+
related_node_ids: relatedNodeIds,
|
|
2013
|
+
related_source_ids: relatedSourceIds,
|
|
2014
|
+
origin: "explore",
|
|
2015
|
+
question: input.question
|
|
2016
|
+
};
|
|
2017
|
+
return {
|
|
2018
|
+
page: {
|
|
2019
|
+
id: pageId,
|
|
2020
|
+
path: pathValue,
|
|
2021
|
+
title,
|
|
2022
|
+
kind: "output",
|
|
2023
|
+
sourceIds: relatedSourceIds,
|
|
2024
|
+
nodeIds: relatedNodeIds,
|
|
2025
|
+
freshness: "fresh",
|
|
2026
|
+
confidence: 0.76,
|
|
2027
|
+
backlinks,
|
|
2028
|
+
schemaHash: input.schemaHash,
|
|
2029
|
+
sourceHashes: {},
|
|
2030
|
+
relatedPageIds,
|
|
2031
|
+
relatedNodeIds,
|
|
2032
|
+
relatedSourceIds,
|
|
2033
|
+
origin: "explore",
|
|
2034
|
+
question: input.question
|
|
2035
|
+
},
|
|
2036
|
+
content: matter2.stringify(
|
|
2037
|
+
[
|
|
2038
|
+
`# ${title}`,
|
|
2039
|
+
"",
|
|
2040
|
+
"## Root Question",
|
|
2041
|
+
"",
|
|
2042
|
+
input.question,
|
|
2043
|
+
"",
|
|
2044
|
+
"## Steps",
|
|
2045
|
+
"",
|
|
2046
|
+
...input.stepPages.length ? input.stepPages.map((page) => `- ${pageLink(page)}`) : ["- No steps recorded."],
|
|
2047
|
+
"",
|
|
2048
|
+
"## Follow-Up Questions",
|
|
2049
|
+
"",
|
|
2050
|
+
...input.followUpQuestions.length ? input.followUpQuestions.map((question) => `- ${question}`) : ["- No follow-up questions generated."],
|
|
2051
|
+
"",
|
|
2052
|
+
"## Citations",
|
|
2053
|
+
"",
|
|
2054
|
+
...relatedSourceIds.map((citation) => `- [source:${citation}]`),
|
|
2055
|
+
""
|
|
2056
|
+
].join("\n"),
|
|
2057
|
+
frontmatter
|
|
2058
|
+
)
|
|
2059
|
+
};
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
// src/outputs.ts
|
|
2063
|
+
import fs9 from "fs/promises";
|
|
2064
|
+
import path11 from "path";
|
|
2065
|
+
import matter3 from "gray-matter";
|
|
2066
|
+
function normalizeStringArray(value) {
|
|
2067
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1456
2068
|
}
|
|
1457
|
-
function
|
|
1458
|
-
|
|
2069
|
+
function normalizeSourceHashes(value) {
|
|
2070
|
+
if (!value || typeof value !== "object") {
|
|
2071
|
+
return {};
|
|
2072
|
+
}
|
|
2073
|
+
return Object.fromEntries(
|
|
2074
|
+
Object.entries(value).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
|
|
2075
|
+
);
|
|
1459
2076
|
}
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
return new HeuristicProviderAdapter(id, config.model);
|
|
1464
|
-
case "openai":
|
|
1465
|
-
return new OpenAiCompatibleProviderAdapter(id, "openai", config.model, {
|
|
1466
|
-
baseUrl: config.baseUrl ?? "https://api.openai.com/v1",
|
|
1467
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1468
|
-
headers: config.headers,
|
|
1469
|
-
apiStyle: config.apiStyle ?? "responses",
|
|
1470
|
-
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming"])
|
|
1471
|
-
});
|
|
1472
|
-
case "ollama":
|
|
1473
|
-
return new OpenAiCompatibleProviderAdapter(id, "ollama", config.model, {
|
|
1474
|
-
baseUrl: config.baseUrl ?? "http://localhost:11434/v1",
|
|
1475
|
-
apiKey: envOrUndefined(config.apiKeyEnv) ?? "ollama",
|
|
1476
|
-
headers: config.headers,
|
|
1477
|
-
apiStyle: config.apiStyle ?? "responses",
|
|
1478
|
-
capabilities: resolveCapabilities(config, ["responses", "chat", "structured", "tools", "vision", "streaming", "local"])
|
|
1479
|
-
});
|
|
1480
|
-
case "openai-compatible":
|
|
1481
|
-
return new OpenAiCompatibleProviderAdapter(id, "openai-compatible", config.model, {
|
|
1482
|
-
baseUrl: config.baseUrl ?? "http://localhost:8000/v1",
|
|
1483
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1484
|
-
headers: config.headers,
|
|
1485
|
-
apiStyle: config.apiStyle ?? "responses",
|
|
1486
|
-
capabilities: resolveCapabilities(config, ["chat", "structured"])
|
|
1487
|
-
});
|
|
1488
|
-
case "anthropic":
|
|
1489
|
-
return new AnthropicProviderAdapter(id, config.model, {
|
|
1490
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1491
|
-
headers: config.headers,
|
|
1492
|
-
baseUrl: config.baseUrl
|
|
1493
|
-
});
|
|
1494
|
-
case "gemini":
|
|
1495
|
-
return new GeminiProviderAdapter(id, config.model, {
|
|
1496
|
-
apiKey: envOrUndefined(config.apiKeyEnv),
|
|
1497
|
-
baseUrl: config.baseUrl
|
|
1498
|
-
});
|
|
1499
|
-
case "custom": {
|
|
1500
|
-
if (!config.module) {
|
|
1501
|
-
throw new Error(`Provider ${id} is type "custom" but no module path was configured.`);
|
|
1502
|
-
}
|
|
1503
|
-
const resolvedModule = path7.isAbsolute(config.module) ? config.module : path7.resolve(rootDir, config.module);
|
|
1504
|
-
const loaded = await import(pathToFileURL(resolvedModule).href);
|
|
1505
|
-
const parsed = customModuleSchema.parse(loaded);
|
|
1506
|
-
return parsed.createAdapter(id, config, rootDir);
|
|
1507
|
-
}
|
|
1508
|
-
default:
|
|
1509
|
-
throw new Error(`Unsupported provider type ${String(config.type)}`);
|
|
2077
|
+
function relationRank(outputPage, targetPage) {
|
|
2078
|
+
if (outputPage.relatedPageIds.includes(targetPage.id)) {
|
|
2079
|
+
return 3;
|
|
1510
2080
|
}
|
|
2081
|
+
if (outputPage.relatedNodeIds.some((nodeId) => targetPage.nodeIds.includes(nodeId))) {
|
|
2082
|
+
return 2;
|
|
2083
|
+
}
|
|
2084
|
+
if (outputPage.relatedSourceIds.some((sourceId) => targetPage.sourceIds.includes(sourceId))) {
|
|
2085
|
+
return 1;
|
|
2086
|
+
}
|
|
2087
|
+
return 0;
|
|
1511
2088
|
}
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
2089
|
+
function relatedOutputsForPage(targetPage, outputPages) {
|
|
2090
|
+
return outputPages.map((page) => ({ page, rank: relationRank(page, targetPage) })).filter((item) => item.rank > 0).sort((left, right) => right.rank - left.rank || left.page.title.localeCompare(right.page.title)).map((item) => item.page);
|
|
2091
|
+
}
|
|
2092
|
+
async function resolveUniqueOutputSlug(wikiDir, baseSlug) {
|
|
2093
|
+
const outputsDir = path11.join(wikiDir, "outputs");
|
|
2094
|
+
const root = baseSlug || "output";
|
|
2095
|
+
let candidate = root;
|
|
2096
|
+
let counter = 2;
|
|
2097
|
+
while (await fileExists(path11.join(outputsDir, `${candidate}.md`))) {
|
|
2098
|
+
candidate = `${root}-${counter}`;
|
|
2099
|
+
counter++;
|
|
1518
2100
|
}
|
|
1519
|
-
return
|
|
2101
|
+
return candidate;
|
|
1520
2102
|
}
|
|
1521
|
-
function
|
|
1522
|
-
|
|
1523
|
-
|
|
2103
|
+
async function loadSavedOutputPages(wikiDir) {
|
|
2104
|
+
const outputsDir = path11.join(wikiDir, "outputs");
|
|
2105
|
+
const entries = await fs9.readdir(outputsDir, { withFileTypes: true }).catch(() => []);
|
|
2106
|
+
const outputs = [];
|
|
2107
|
+
for (const entry of entries) {
|
|
2108
|
+
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
|
2109
|
+
continue;
|
|
2110
|
+
}
|
|
2111
|
+
const relativePath = path11.posix.join("outputs", entry.name);
|
|
2112
|
+
const absolutePath = path11.join(outputsDir, entry.name);
|
|
2113
|
+
const content = await fs9.readFile(absolutePath, "utf8");
|
|
2114
|
+
const parsed = matter3(content);
|
|
2115
|
+
const slug = entry.name.replace(/\.md$/, "");
|
|
2116
|
+
const title = typeof parsed.data.title === "string" ? parsed.data.title : slug;
|
|
2117
|
+
const pageId = typeof parsed.data.page_id === "string" ? parsed.data.page_id : `output:${slug}`;
|
|
2118
|
+
const sourceIds = normalizeStringArray(parsed.data.source_ids);
|
|
2119
|
+
const nodeIds = normalizeStringArray(parsed.data.node_ids);
|
|
2120
|
+
const relatedPageIds = normalizeStringArray(parsed.data.related_page_ids);
|
|
2121
|
+
const relatedNodeIds = normalizeStringArray(parsed.data.related_node_ids);
|
|
2122
|
+
const relatedSourceIds = normalizeStringArray(parsed.data.related_source_ids);
|
|
2123
|
+
const backlinks = normalizeStringArray(parsed.data.backlinks);
|
|
2124
|
+
outputs.push({
|
|
2125
|
+
page: {
|
|
2126
|
+
id: pageId,
|
|
2127
|
+
path: relativePath,
|
|
2128
|
+
title,
|
|
2129
|
+
kind: "output",
|
|
2130
|
+
sourceIds,
|
|
2131
|
+
nodeIds,
|
|
2132
|
+
freshness: parsed.data.freshness === "stale" ? "stale" : "fresh",
|
|
2133
|
+
confidence: typeof parsed.data.confidence === "number" ? parsed.data.confidence : 0.74,
|
|
2134
|
+
backlinks,
|
|
2135
|
+
schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
|
|
2136
|
+
sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
|
|
2137
|
+
relatedPageIds,
|
|
2138
|
+
relatedNodeIds,
|
|
2139
|
+
relatedSourceIds,
|
|
2140
|
+
origin: typeof parsed.data.origin === "string" ? parsed.data.origin : void 0,
|
|
2141
|
+
question: typeof parsed.data.question === "string" ? parsed.data.question : void 0
|
|
2142
|
+
},
|
|
2143
|
+
content,
|
|
2144
|
+
contentHash: sha256(content)
|
|
2145
|
+
});
|
|
1524
2146
|
}
|
|
2147
|
+
return outputs.sort((left, right) => left.page.title.localeCompare(right.page.title));
|
|
1525
2148
|
}
|
|
1526
2149
|
|
|
1527
2150
|
// src/search.ts
|
|
1528
|
-
import
|
|
1529
|
-
import
|
|
1530
|
-
import
|
|
2151
|
+
import fs10 from "fs/promises";
|
|
2152
|
+
import path12 from "path";
|
|
2153
|
+
import matter4 from "gray-matter";
|
|
1531
2154
|
function getDatabaseSync() {
|
|
1532
2155
|
const builtin = process.getBuiltinModule?.("node:sqlite");
|
|
1533
2156
|
if (!builtin?.DatabaseSync) {
|
|
@@ -1540,7 +2163,7 @@ function toFtsQuery(query) {
|
|
|
1540
2163
|
return tokens.join(" OR ");
|
|
1541
2164
|
}
|
|
1542
2165
|
async function rebuildSearchIndex(dbPath, pages, wikiDir) {
|
|
1543
|
-
await ensureDir(
|
|
2166
|
+
await ensureDir(path12.dirname(dbPath));
|
|
1544
2167
|
const DatabaseSync = getDatabaseSync();
|
|
1545
2168
|
const db = new DatabaseSync(dbPath);
|
|
1546
2169
|
db.exec("PRAGMA journal_mode = WAL;");
|
|
@@ -1562,9 +2185,9 @@ async function rebuildSearchIndex(dbPath, pages, wikiDir) {
|
|
|
1562
2185
|
`);
|
|
1563
2186
|
const insertPage = db.prepare("INSERT INTO pages (id, path, title, body) VALUES (?, ?, ?, ?)");
|
|
1564
2187
|
for (const page of pages) {
|
|
1565
|
-
const absolutePath =
|
|
1566
|
-
const content = await
|
|
1567
|
-
const parsed =
|
|
2188
|
+
const absolutePath = path12.join(wikiDir, page.path);
|
|
2189
|
+
const content = await fs10.readFile(absolutePath, "utf8");
|
|
2190
|
+
const parsed = matter4(content);
|
|
1568
2191
|
insertPage.run(page.id, page.path, page.title, parsed.content);
|
|
1569
2192
|
}
|
|
1570
2193
|
db.exec("INSERT INTO page_search (rowid, title, body) SELECT rowid, title, body FROM pages;");
|
|
@@ -1618,14 +2241,15 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1618
2241
|
for (const analysis of analyses) {
|
|
1619
2242
|
for (const concept of analysis.concepts) {
|
|
1620
2243
|
const existing = conceptMap.get(concept.id);
|
|
2244
|
+
const sourceIds = [.../* @__PURE__ */ new Set([...existing?.sourceIds ?? [], analysis.sourceId])];
|
|
1621
2245
|
conceptMap.set(concept.id, {
|
|
1622
2246
|
id: concept.id,
|
|
1623
2247
|
type: "concept",
|
|
1624
2248
|
label: concept.name,
|
|
1625
2249
|
pageId: `concept:${slugify(concept.name)}`,
|
|
1626
2250
|
freshness: "fresh",
|
|
1627
|
-
confidence:
|
|
1628
|
-
sourceIds
|
|
2251
|
+
confidence: nodeConfidence(sourceIds.length),
|
|
2252
|
+
sourceIds
|
|
1629
2253
|
});
|
|
1630
2254
|
edges.push({
|
|
1631
2255
|
id: `${analysis.sourceId}->${concept.id}`,
|
|
@@ -1633,20 +2257,21 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1633
2257
|
target: concept.id,
|
|
1634
2258
|
relation: "mentions",
|
|
1635
2259
|
status: "extracted",
|
|
1636
|
-
confidence:
|
|
2260
|
+
confidence: edgeConfidence(analysis.claims, concept.name),
|
|
1637
2261
|
provenance: [analysis.sourceId]
|
|
1638
2262
|
});
|
|
1639
2263
|
}
|
|
1640
2264
|
for (const entity of analysis.entities) {
|
|
1641
2265
|
const existing = entityMap.get(entity.id);
|
|
2266
|
+
const sourceIds = [.../* @__PURE__ */ new Set([...existing?.sourceIds ?? [], analysis.sourceId])];
|
|
1642
2267
|
entityMap.set(entity.id, {
|
|
1643
2268
|
id: entity.id,
|
|
1644
2269
|
type: "entity",
|
|
1645
2270
|
label: entity.name,
|
|
1646
2271
|
pageId: `entity:${slugify(entity.name)}`,
|
|
1647
2272
|
freshness: "fresh",
|
|
1648
|
-
confidence:
|
|
1649
|
-
sourceIds
|
|
2273
|
+
confidence: nodeConfidence(sourceIds.length),
|
|
2274
|
+
sourceIds
|
|
1650
2275
|
});
|
|
1651
2276
|
edges.push({
|
|
1652
2277
|
id: `${analysis.sourceId}->${entity.id}`,
|
|
@@ -1654,22 +2279,46 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1654
2279
|
target: entity.id,
|
|
1655
2280
|
relation: "mentions",
|
|
1656
2281
|
status: "extracted",
|
|
1657
|
-
confidence:
|
|
2282
|
+
confidence: edgeConfidence(analysis.claims, entity.name),
|
|
1658
2283
|
provenance: [analysis.sourceId]
|
|
1659
2284
|
});
|
|
1660
2285
|
}
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
2286
|
+
}
|
|
2287
|
+
const conceptClaims = /* @__PURE__ */ new Map();
|
|
2288
|
+
for (const analysis of analyses) {
|
|
2289
|
+
for (const claim of analysis.claims) {
|
|
2290
|
+
for (const concept of analysis.concepts) {
|
|
2291
|
+
if (claim.text.toLowerCase().includes(concept.name.toLowerCase())) {
|
|
2292
|
+
const key = concept.id;
|
|
2293
|
+
const list = conceptClaims.get(key) ?? [];
|
|
2294
|
+
list.push({ claim, sourceId: analysis.sourceId });
|
|
2295
|
+
conceptClaims.set(key, list);
|
|
2296
|
+
}
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2300
|
+
const conflictEdgeKeys = /* @__PURE__ */ new Set();
|
|
2301
|
+
for (const [, claimsForConcept] of conceptClaims) {
|
|
2302
|
+
const positive = claimsForConcept.filter((item) => item.claim.polarity === "positive");
|
|
2303
|
+
const negative = claimsForConcept.filter((item) => item.claim.polarity === "negative");
|
|
2304
|
+
for (const positiveClaim of positive) {
|
|
2305
|
+
for (const negativeClaim of negative) {
|
|
2306
|
+
if (positiveClaim.sourceId === negativeClaim.sourceId) {
|
|
2307
|
+
continue;
|
|
2308
|
+
}
|
|
2309
|
+
const edgeKey = [positiveClaim.sourceId, negativeClaim.sourceId].sort().join("|");
|
|
2310
|
+
if (conflictEdgeKeys.has(edgeKey)) {
|
|
2311
|
+
continue;
|
|
2312
|
+
}
|
|
2313
|
+
conflictEdgeKeys.add(edgeKey);
|
|
1665
2314
|
edges.push({
|
|
1666
|
-
id:
|
|
1667
|
-
source: `source:${
|
|
1668
|
-
target: `source:${
|
|
2315
|
+
id: `conflict:${positiveClaim.claim.id}->${negativeClaim.claim.id}`,
|
|
2316
|
+
source: `source:${positiveClaim.sourceId}`,
|
|
2317
|
+
target: `source:${negativeClaim.sourceId}`,
|
|
1669
2318
|
relation: "conflicted_with",
|
|
1670
2319
|
status: "conflicted",
|
|
1671
|
-
confidence:
|
|
1672
|
-
provenance: [
|
|
2320
|
+
confidence: conflictConfidence(positiveClaim.claim, negativeClaim.claim),
|
|
2321
|
+
provenance: [positiveClaim.sourceId, negativeClaim.sourceId]
|
|
1673
2322
|
});
|
|
1674
2323
|
}
|
|
1675
2324
|
}
|
|
@@ -1682,8 +2331,8 @@ function buildGraph(manifests, analyses, pages) {
|
|
|
1682
2331
|
pages
|
|
1683
2332
|
};
|
|
1684
2333
|
}
|
|
1685
|
-
async function writePage(
|
|
1686
|
-
const absolutePath =
|
|
2334
|
+
async function writePage(wikiDir, relativePath, content, changedPages) {
|
|
2335
|
+
const absolutePath = path13.resolve(wikiDir, relativePath);
|
|
1687
2336
|
const changed = await writeFileIfChanged(absolutePath, content);
|
|
1688
2337
|
if (changed) {
|
|
1689
2338
|
changedPages.push(relativePath);
|
|
@@ -1708,113 +2357,525 @@ function aggregateItems(analyses, kind) {
|
|
|
1708
2357
|
}
|
|
1709
2358
|
return [...grouped.values()];
|
|
1710
2359
|
}
|
|
2360
|
+
function emptyGraphPage(input) {
|
|
2361
|
+
return {
|
|
2362
|
+
id: input.id,
|
|
2363
|
+
path: input.path,
|
|
2364
|
+
title: input.title,
|
|
2365
|
+
kind: input.kind,
|
|
2366
|
+
sourceIds: input.sourceIds,
|
|
2367
|
+
nodeIds: input.nodeIds,
|
|
2368
|
+
freshness: "fresh",
|
|
2369
|
+
confidence: input.confidence,
|
|
2370
|
+
backlinks: [],
|
|
2371
|
+
schemaHash: input.schemaHash,
|
|
2372
|
+
sourceHashes: input.sourceHashes,
|
|
2373
|
+
relatedPageIds: [],
|
|
2374
|
+
relatedNodeIds: [],
|
|
2375
|
+
relatedSourceIds: []
|
|
2376
|
+
};
|
|
2377
|
+
}
|
|
2378
|
+
function outputHashes(outputPages) {
|
|
2379
|
+
return Object.fromEntries(outputPages.map((page) => [page.page.id, page.contentHash]));
|
|
2380
|
+
}
|
|
2381
|
+
function recordsEqual(left, right) {
|
|
2382
|
+
const leftKeys = Object.keys(left);
|
|
2383
|
+
const rightKeys = Object.keys(right);
|
|
2384
|
+
if (leftKeys.length !== rightKeys.length) {
|
|
2385
|
+
return false;
|
|
2386
|
+
}
|
|
2387
|
+
return leftKeys.every((key) => left[key] === right[key]);
|
|
2388
|
+
}
|
|
2389
|
+
async function requiredCompileArtifactsExist(paths) {
|
|
2390
|
+
const requiredPaths = [
|
|
2391
|
+
paths.graphPath,
|
|
2392
|
+
paths.searchDbPath,
|
|
2393
|
+
path13.join(paths.wikiDir, "index.md"),
|
|
2394
|
+
path13.join(paths.wikiDir, "sources", "index.md"),
|
|
2395
|
+
path13.join(paths.wikiDir, "concepts", "index.md"),
|
|
2396
|
+
path13.join(paths.wikiDir, "entities", "index.md"),
|
|
2397
|
+
path13.join(paths.wikiDir, "outputs", "index.md")
|
|
2398
|
+
];
|
|
2399
|
+
const checks = await Promise.all(requiredPaths.map((filePath) => fileExists(filePath)));
|
|
2400
|
+
return checks.every(Boolean);
|
|
2401
|
+
}
|
|
2402
|
+
async function refreshIndexesAndSearch(rootDir, schemaHash, pages) {
|
|
2403
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2404
|
+
await ensureDir(path13.join(paths.wikiDir, "outputs"));
|
|
2405
|
+
await writeFileIfChanged(path13.join(paths.wikiDir, "index.md"), buildIndexPage(pages, schemaHash));
|
|
2406
|
+
await writeFileIfChanged(
|
|
2407
|
+
path13.join(paths.wikiDir, "outputs", "index.md"),
|
|
2408
|
+
buildSectionIndex(
|
|
2409
|
+
"outputs",
|
|
2410
|
+
pages.filter((page) => page.kind === "output"),
|
|
2411
|
+
schemaHash
|
|
2412
|
+
)
|
|
2413
|
+
);
|
|
2414
|
+
await rebuildSearchIndex(paths.searchDbPath, pages, paths.wikiDir);
|
|
2415
|
+
}
|
|
2416
|
+
async function upsertGraphPages(rootDir, pages) {
|
|
2417
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2418
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2419
|
+
const manifests = await listManifests(rootDir);
|
|
2420
|
+
const nonOutputPages = graph?.pages.filter((page) => page.kind !== "output") ?? [];
|
|
2421
|
+
const nextGraph = {
|
|
2422
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2423
|
+
nodes: graph?.nodes ?? [],
|
|
2424
|
+
edges: graph?.edges ?? [],
|
|
2425
|
+
sources: graph?.sources ?? manifests,
|
|
2426
|
+
pages: [...nonOutputPages, ...pages]
|
|
2427
|
+
};
|
|
2428
|
+
await writeJsonFile(paths.graphPath, nextGraph);
|
|
2429
|
+
}
|
|
2430
|
+
async function persistOutputPage(rootDir, input) {
|
|
2431
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2432
|
+
const slug = await resolveUniqueOutputSlug(paths.wikiDir, input.slug ?? slugify(input.question));
|
|
2433
|
+
const output = buildOutputPage({ ...input, slug });
|
|
2434
|
+
const absolutePath = path13.join(paths.wikiDir, output.page.path);
|
|
2435
|
+
await ensureDir(path13.dirname(absolutePath));
|
|
2436
|
+
await fs11.writeFile(absolutePath, output.content, "utf8");
|
|
2437
|
+
const storedOutputs = await loadSavedOutputPages(paths.wikiDir);
|
|
2438
|
+
const outputPages = storedOutputs.map((page) => page.page);
|
|
2439
|
+
await upsertGraphPages(rootDir, outputPages);
|
|
2440
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2441
|
+
await refreshIndexesAndSearch(rootDir, input.schemaHash, graph?.pages ?? outputPages);
|
|
2442
|
+
return { page: output.page, savedTo: absolutePath };
|
|
2443
|
+
}
|
|
2444
|
+
async function persistExploreHub(rootDir, input) {
|
|
2445
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2446
|
+
const slug = await resolveUniqueOutputSlug(paths.wikiDir, input.slug ?? `explore-${slugify(input.question)}`);
|
|
2447
|
+
const hub = buildExploreHubPage({ ...input, slug });
|
|
2448
|
+
const absolutePath = path13.join(paths.wikiDir, hub.page.path);
|
|
2449
|
+
await ensureDir(path13.dirname(absolutePath));
|
|
2450
|
+
await fs11.writeFile(absolutePath, hub.content, "utf8");
|
|
2451
|
+
const storedOutputs = await loadSavedOutputPages(paths.wikiDir);
|
|
2452
|
+
const outputPages = storedOutputs.map((page) => page.page);
|
|
2453
|
+
await upsertGraphPages(rootDir, outputPages);
|
|
2454
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2455
|
+
await refreshIndexesAndSearch(rootDir, input.schemaHash, graph?.pages ?? outputPages);
|
|
2456
|
+
return { page: hub.page, savedTo: absolutePath };
|
|
2457
|
+
}
|
|
2458
|
+
async function executeQuery(rootDir, question) {
|
|
2459
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
2460
|
+
const schema = await loadVaultSchema(rootDir);
|
|
2461
|
+
const provider = await getProviderForTask(rootDir, "queryProvider");
|
|
2462
|
+
if (!await fileExists(paths.searchDbPath) || !await fileExists(paths.graphPath)) {
|
|
2463
|
+
await compileVault(rootDir);
|
|
2464
|
+
}
|
|
2465
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
2466
|
+
const pageMap = new Map((graph?.pages ?? []).map((page) => [page.id, page]));
|
|
2467
|
+
const searchResults = searchPages(paths.searchDbPath, question, 5);
|
|
2468
|
+
const excerpts = await Promise.all(
|
|
2469
|
+
searchResults.map(async (result) => {
|
|
2470
|
+
const absolutePath = path13.join(paths.wikiDir, result.path);
|
|
2471
|
+
try {
|
|
2472
|
+
const content = await fs11.readFile(absolutePath, "utf8");
|
|
2473
|
+
const parsed = matter5(content);
|
|
2474
|
+
return `# ${result.title}
|
|
2475
|
+
${truncate(normalizeWhitespace(parsed.content), 1200)}`;
|
|
2476
|
+
} catch {
|
|
2477
|
+
return `# ${result.title}
|
|
2478
|
+
${result.snippet}`;
|
|
2479
|
+
}
|
|
2480
|
+
})
|
|
2481
|
+
);
|
|
2482
|
+
const relatedPageIds = uniqueBy(
|
|
2483
|
+
searchResults.map((result) => result.pageId),
|
|
2484
|
+
(item) => item
|
|
2485
|
+
);
|
|
2486
|
+
const relatedNodeIds = uniqueBy(
|
|
2487
|
+
relatedPageIds.flatMap((pageId) => pageMap.get(pageId)?.nodeIds ?? []),
|
|
2488
|
+
(item) => item
|
|
2489
|
+
);
|
|
2490
|
+
const relatedSourceIds = uniqueBy(
|
|
2491
|
+
relatedPageIds.flatMap((pageId) => pageMap.get(pageId)?.sourceIds ?? []),
|
|
2492
|
+
(item) => item
|
|
2493
|
+
);
|
|
2494
|
+
const manifests = await listManifests(rootDir);
|
|
2495
|
+
const rawExcerpts = [];
|
|
2496
|
+
for (const sourceId of relatedSourceIds.slice(0, 5)) {
|
|
2497
|
+
const manifest = manifests.find((item) => item.sourceId === sourceId);
|
|
2498
|
+
if (!manifest) {
|
|
2499
|
+
continue;
|
|
2500
|
+
}
|
|
2501
|
+
const text = await readExtractedText(rootDir, manifest);
|
|
2502
|
+
if (text) {
|
|
2503
|
+
rawExcerpts.push(`# [source:${sourceId}] ${manifest.title}
|
|
2504
|
+
${truncate(normalizeWhitespace(text), 800)}`);
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
let answer;
|
|
2508
|
+
if (provider.type === "heuristic") {
|
|
2509
|
+
answer = [
|
|
2510
|
+
`Question: ${question}`,
|
|
2511
|
+
"",
|
|
2512
|
+
"Relevant pages:",
|
|
2513
|
+
...searchResults.map((result) => `- ${result.title} (${result.path})`),
|
|
2514
|
+
"",
|
|
2515
|
+
excerpts.length ? excerpts.join("\n\n") : "No relevant pages found yet.",
|
|
2516
|
+
...rawExcerpts.length ? ["", "Raw source material:", "", ...rawExcerpts] : []
|
|
2517
|
+
].join("\n");
|
|
2518
|
+
} else {
|
|
2519
|
+
const context = [
|
|
2520
|
+
"Wiki context:",
|
|
2521
|
+
excerpts.join("\n\n---\n\n"),
|
|
2522
|
+
...rawExcerpts.length ? ["", "Raw source material:", rawExcerpts.join("\n\n---\n\n")] : []
|
|
2523
|
+
].join("\n\n");
|
|
2524
|
+
const response = await provider.generateText({
|
|
2525
|
+
system: buildSchemaPrompt(
|
|
2526
|
+
schema,
|
|
2527
|
+
"Answer using the provided context. Prefer raw source material over wiki summaries when they differ. Cite source IDs."
|
|
2528
|
+
),
|
|
2529
|
+
prompt: `Question: ${question}
|
|
2530
|
+
|
|
2531
|
+
${context}`
|
|
2532
|
+
});
|
|
2533
|
+
answer = response.text;
|
|
2534
|
+
}
|
|
2535
|
+
return {
|
|
2536
|
+
answer,
|
|
2537
|
+
citations: relatedSourceIds,
|
|
2538
|
+
relatedPageIds,
|
|
2539
|
+
relatedNodeIds,
|
|
2540
|
+
relatedSourceIds
|
|
2541
|
+
};
|
|
2542
|
+
}
|
|
2543
|
+
async function generateFollowUpQuestions(rootDir, question, answer) {
|
|
2544
|
+
const provider = await getProviderForTask(rootDir, "queryProvider");
|
|
2545
|
+
const schema = await loadVaultSchema(rootDir);
|
|
2546
|
+
if (provider.type === "heuristic") {
|
|
2547
|
+
return uniqueBy(
|
|
2548
|
+
[
|
|
2549
|
+
`What evidence best supports ${question}?`,
|
|
2550
|
+
`What contradicts ${question}?`,
|
|
2551
|
+
`Which sources should be added to answer ${question} better?`
|
|
2552
|
+
],
|
|
2553
|
+
(item) => item
|
|
2554
|
+
).slice(0, 3);
|
|
2555
|
+
}
|
|
2556
|
+
const response = await provider.generateStructured(
|
|
2557
|
+
{
|
|
2558
|
+
system: buildSchemaPrompt(schema, "Propose concise follow-up research questions for the vault. Return only useful next questions."),
|
|
2559
|
+
prompt: `Root question: ${question}
|
|
2560
|
+
|
|
2561
|
+
Current answer:
|
|
2562
|
+
${answer}`
|
|
2563
|
+
},
|
|
2564
|
+
z8.object({
|
|
2565
|
+
questions: z8.array(z8.string().min(1)).max(5)
|
|
2566
|
+
})
|
|
2567
|
+
);
|
|
2568
|
+
return uniqueBy(response.questions, (item) => item).filter((item) => item !== question);
|
|
2569
|
+
}
|
|
1711
2570
|
async function initVault(rootDir) {
|
|
1712
2571
|
await initWorkspace(rootDir);
|
|
1713
2572
|
await installConfiguredAgents(rootDir);
|
|
1714
2573
|
}
|
|
1715
2574
|
async function compileVault(rootDir) {
|
|
1716
2575
|
const { paths } = await initWorkspace(rootDir);
|
|
2576
|
+
const schema = await loadVaultSchema(rootDir);
|
|
1717
2577
|
const provider = await getProviderForTask(rootDir, "compileProvider");
|
|
1718
2578
|
const manifests = await listManifests(rootDir);
|
|
1719
|
-
const
|
|
1720
|
-
|
|
1721
|
-
);
|
|
2579
|
+
const storedOutputPages = await loadSavedOutputPages(paths.wikiDir);
|
|
2580
|
+
const outputPages = storedOutputPages.map((page) => page.page);
|
|
2581
|
+
const currentOutputHashes = outputHashes(storedOutputPages);
|
|
2582
|
+
const previousState = await readJsonFile(paths.compileStatePath);
|
|
2583
|
+
const schemaChanged = !previousState || previousState.schemaHash !== schema.hash;
|
|
2584
|
+
const previousSourceHashes = previousState?.sourceHashes ?? {};
|
|
2585
|
+
const previousAnalyses = previousState?.analyses ?? {};
|
|
2586
|
+
const previousOutputHashes = previousState?.outputHashes ?? {};
|
|
2587
|
+
const currentSourceIds = new Set(manifests.map((item) => item.sourceId));
|
|
2588
|
+
const previousSourceIds = new Set(Object.keys(previousSourceHashes));
|
|
2589
|
+
const sourcesChanged = currentSourceIds.size !== previousSourceIds.size || [...currentSourceIds].some((sourceId) => !previousSourceIds.has(sourceId));
|
|
2590
|
+
const outputsChanged = !recordsEqual(currentOutputHashes, previousOutputHashes);
|
|
2591
|
+
const artifactsExist = await requiredCompileArtifactsExist(paths);
|
|
2592
|
+
const dirty = [];
|
|
2593
|
+
const clean = [];
|
|
2594
|
+
for (const manifest of manifests) {
|
|
2595
|
+
const hashChanged = previousSourceHashes[manifest.sourceId] !== manifest.contentHash;
|
|
2596
|
+
const noAnalysis = !previousAnalyses[manifest.sourceId];
|
|
2597
|
+
if (schemaChanged || hashChanged || noAnalysis) {
|
|
2598
|
+
dirty.push(manifest);
|
|
2599
|
+
} else {
|
|
2600
|
+
clean.push(manifest);
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
if (dirty.length === 0 && !schemaChanged && !sourcesChanged && !outputsChanged && artifactsExist) {
|
|
2604
|
+
const graph2 = await readJsonFile(paths.graphPath);
|
|
2605
|
+
return {
|
|
2606
|
+
graphPath: paths.graphPath,
|
|
2607
|
+
pageCount: graph2?.pages.length ?? outputPages.length,
|
|
2608
|
+
changedPages: [],
|
|
2609
|
+
sourceCount: manifests.length
|
|
2610
|
+
};
|
|
2611
|
+
}
|
|
2612
|
+
const [dirtyAnalyses, cleanAnalyses] = await Promise.all([
|
|
2613
|
+
Promise.all(
|
|
2614
|
+
dirty.map(async (manifest) => analyzeSource(manifest, await readExtractedText(rootDir, manifest), provider, paths, schema))
|
|
2615
|
+
),
|
|
2616
|
+
Promise.all(
|
|
2617
|
+
clean.map(async (manifest) => {
|
|
2618
|
+
const cached = await readJsonFile(path13.join(paths.analysesDir, `${manifest.sourceId}.json`));
|
|
2619
|
+
if (cached) {
|
|
2620
|
+
return cached;
|
|
2621
|
+
}
|
|
2622
|
+
return analyzeSource(manifest, await readExtractedText(rootDir, manifest), provider, paths, schema);
|
|
2623
|
+
})
|
|
2624
|
+
)
|
|
2625
|
+
]);
|
|
2626
|
+
const analyses = [...dirtyAnalyses, ...cleanAnalyses];
|
|
1722
2627
|
const changedPages = [];
|
|
1723
|
-
const
|
|
2628
|
+
const compiledPages = [];
|
|
1724
2629
|
await Promise.all([
|
|
1725
|
-
ensureDir(
|
|
1726
|
-
ensureDir(
|
|
1727
|
-
ensureDir(
|
|
1728
|
-
ensureDir(
|
|
2630
|
+
ensureDir(path13.join(paths.wikiDir, "sources")),
|
|
2631
|
+
ensureDir(path13.join(paths.wikiDir, "concepts")),
|
|
2632
|
+
ensureDir(path13.join(paths.wikiDir, "entities")),
|
|
2633
|
+
ensureDir(path13.join(paths.wikiDir, "outputs"))
|
|
1729
2634
|
]);
|
|
1730
2635
|
for (const manifest of manifests) {
|
|
1731
2636
|
const analysis = analyses.find((item) => item.sourceId === manifest.sourceId);
|
|
1732
2637
|
if (!analysis) {
|
|
1733
2638
|
continue;
|
|
1734
2639
|
}
|
|
1735
|
-
const
|
|
1736
|
-
|
|
1737
|
-
|
|
2640
|
+
const preview = emptyGraphPage({
|
|
2641
|
+
id: `source:${manifest.sourceId}`,
|
|
2642
|
+
path: `sources/${manifest.sourceId}.md`,
|
|
2643
|
+
title: analysis.title,
|
|
2644
|
+
kind: "source",
|
|
2645
|
+
sourceIds: [manifest.sourceId],
|
|
2646
|
+
nodeIds: [`source:${manifest.sourceId}`, ...analysis.concepts.map((item) => item.id), ...analysis.entities.map((item) => item.id)],
|
|
2647
|
+
schemaHash: schema.hash,
|
|
2648
|
+
sourceHashes: { [manifest.sourceId]: manifest.contentHash },
|
|
2649
|
+
confidence: 1
|
|
2650
|
+
});
|
|
2651
|
+
const sourcePage = buildSourcePage(manifest, analysis, schema.hash, 1, relatedOutputsForPage(preview, outputPages));
|
|
2652
|
+
compiledPages.push(sourcePage.page);
|
|
2653
|
+
await writePage(paths.wikiDir, sourcePage.page.path, sourcePage.content, changedPages);
|
|
1738
2654
|
}
|
|
1739
2655
|
for (const aggregate of aggregateItems(analyses, "concepts")) {
|
|
1740
|
-
const
|
|
1741
|
-
|
|
1742
|
-
|
|
2656
|
+
const confidence = nodeConfidence(aggregate.sourceAnalyses.length);
|
|
2657
|
+
const preview = emptyGraphPage({
|
|
2658
|
+
id: `concept:${slugify(aggregate.name)}`,
|
|
2659
|
+
path: `concepts/${slugify(aggregate.name)}.md`,
|
|
2660
|
+
title: aggregate.name,
|
|
2661
|
+
kind: "concept",
|
|
2662
|
+
sourceIds: aggregate.sourceAnalyses.map((item) => item.sourceId),
|
|
2663
|
+
nodeIds: [`concept:${slugify(aggregate.name)}`],
|
|
2664
|
+
schemaHash: schema.hash,
|
|
2665
|
+
sourceHashes: aggregate.sourceHashes,
|
|
2666
|
+
confidence
|
|
2667
|
+
});
|
|
2668
|
+
const page = buildAggregatePage(
|
|
2669
|
+
"concept",
|
|
2670
|
+
aggregate.name,
|
|
2671
|
+
aggregate.descriptions,
|
|
2672
|
+
aggregate.sourceAnalyses,
|
|
2673
|
+
aggregate.sourceHashes,
|
|
2674
|
+
schema.hash,
|
|
2675
|
+
confidence,
|
|
2676
|
+
relatedOutputsForPage(preview, outputPages)
|
|
2677
|
+
);
|
|
2678
|
+
compiledPages.push(page.page);
|
|
2679
|
+
await writePage(paths.wikiDir, page.page.path, page.content, changedPages);
|
|
1743
2680
|
}
|
|
1744
2681
|
for (const aggregate of aggregateItems(analyses, "entities")) {
|
|
1745
|
-
const
|
|
1746
|
-
|
|
1747
|
-
|
|
2682
|
+
const confidence = nodeConfidence(aggregate.sourceAnalyses.length);
|
|
2683
|
+
const preview = emptyGraphPage({
|
|
2684
|
+
id: `entity:${slugify(aggregate.name)}`,
|
|
2685
|
+
path: `entities/${slugify(aggregate.name)}.md`,
|
|
2686
|
+
title: aggregate.name,
|
|
2687
|
+
kind: "entity",
|
|
2688
|
+
sourceIds: aggregate.sourceAnalyses.map((item) => item.sourceId),
|
|
2689
|
+
nodeIds: [`entity:${slugify(aggregate.name)}`],
|
|
2690
|
+
schemaHash: schema.hash,
|
|
2691
|
+
sourceHashes: aggregate.sourceHashes,
|
|
2692
|
+
confidence
|
|
2693
|
+
});
|
|
2694
|
+
const page = buildAggregatePage(
|
|
2695
|
+
"entity",
|
|
2696
|
+
aggregate.name,
|
|
2697
|
+
aggregate.descriptions,
|
|
2698
|
+
aggregate.sourceAnalyses,
|
|
2699
|
+
aggregate.sourceHashes,
|
|
2700
|
+
schema.hash,
|
|
2701
|
+
confidence,
|
|
2702
|
+
relatedOutputsForPage(preview, outputPages)
|
|
2703
|
+
);
|
|
2704
|
+
compiledPages.push(page.page);
|
|
2705
|
+
await writePage(paths.wikiDir, page.page.path, page.content, changedPages);
|
|
1748
2706
|
}
|
|
1749
|
-
const
|
|
2707
|
+
const allPages = [...compiledPages, ...outputPages];
|
|
2708
|
+
const graph = buildGraph(manifests, analyses, allPages);
|
|
1750
2709
|
await writeJsonFile(paths.graphPath, graph);
|
|
1751
2710
|
await writeJsonFile(paths.compileStatePath, {
|
|
1752
2711
|
generatedAt: graph.generatedAt,
|
|
1753
|
-
|
|
2712
|
+
schemaHash: schema.hash,
|
|
2713
|
+
analyses: Object.fromEntries(analyses.map((analysis) => [analysis.sourceId, analysisSignature(analysis)])),
|
|
2714
|
+
sourceHashes: Object.fromEntries(manifests.map((manifest) => [manifest.sourceId, manifest.contentHash])),
|
|
2715
|
+
outputHashes: currentOutputHashes
|
|
1754
2716
|
});
|
|
1755
|
-
await writePage(
|
|
1756
|
-
await writePage(
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
2717
|
+
await writePage(paths.wikiDir, "index.md", buildIndexPage(allPages, schema.hash), changedPages);
|
|
2718
|
+
await writePage(
|
|
2719
|
+
paths.wikiDir,
|
|
2720
|
+
"sources/index.md",
|
|
2721
|
+
buildSectionIndex(
|
|
2722
|
+
"sources",
|
|
2723
|
+
allPages.filter((page) => page.kind === "source"),
|
|
2724
|
+
schema.hash
|
|
2725
|
+
),
|
|
2726
|
+
changedPages
|
|
2727
|
+
);
|
|
2728
|
+
await writePage(
|
|
2729
|
+
paths.wikiDir,
|
|
2730
|
+
"concepts/index.md",
|
|
2731
|
+
buildSectionIndex(
|
|
2732
|
+
"concepts",
|
|
2733
|
+
allPages.filter((page) => page.kind === "concept"),
|
|
2734
|
+
schema.hash
|
|
2735
|
+
),
|
|
2736
|
+
changedPages
|
|
2737
|
+
);
|
|
2738
|
+
await writePage(
|
|
2739
|
+
paths.wikiDir,
|
|
2740
|
+
"entities/index.md",
|
|
2741
|
+
buildSectionIndex(
|
|
2742
|
+
"entities",
|
|
2743
|
+
allPages.filter((page) => page.kind === "entity"),
|
|
2744
|
+
schema.hash
|
|
2745
|
+
),
|
|
2746
|
+
changedPages
|
|
2747
|
+
);
|
|
2748
|
+
await writePage(
|
|
2749
|
+
paths.wikiDir,
|
|
2750
|
+
"outputs/index.md",
|
|
2751
|
+
buildSectionIndex(
|
|
2752
|
+
"outputs",
|
|
2753
|
+
allPages.filter((page) => page.kind === "output"),
|
|
2754
|
+
schema.hash
|
|
2755
|
+
),
|
|
2756
|
+
changedPages
|
|
2757
|
+
);
|
|
2758
|
+
if (changedPages.length > 0 || outputsChanged || !artifactsExist) {
|
|
2759
|
+
await rebuildSearchIndex(paths.searchDbPath, allPages, paths.wikiDir);
|
|
2760
|
+
}
|
|
2761
|
+
await appendLogEntry(rootDir, "compile", `Compiled ${manifests.length} source(s)`, [
|
|
2762
|
+
`provider=${provider.id}`,
|
|
2763
|
+
`pages=${allPages.length}`,
|
|
2764
|
+
`dirty=${dirty.length}`,
|
|
2765
|
+
`clean=${clean.length}`,
|
|
2766
|
+
`outputs=${outputPages.length}`,
|
|
2767
|
+
`schema=${schema.hash.slice(0, 12)}`
|
|
2768
|
+
]);
|
|
1761
2769
|
return {
|
|
1762
2770
|
graphPath: paths.graphPath,
|
|
1763
|
-
pageCount:
|
|
2771
|
+
pageCount: allPages.length,
|
|
1764
2772
|
changedPages,
|
|
1765
2773
|
sourceCount: manifests.length
|
|
1766
2774
|
};
|
|
1767
2775
|
}
|
|
1768
2776
|
async function queryVault(rootDir, question, save = false) {
|
|
1769
|
-
const
|
|
1770
|
-
const
|
|
1771
|
-
|
|
1772
|
-
|
|
2777
|
+
const schema = await loadVaultSchema(rootDir);
|
|
2778
|
+
const query = await executeQuery(rootDir, question);
|
|
2779
|
+
let savedTo;
|
|
2780
|
+
let savedPageId;
|
|
2781
|
+
if (save) {
|
|
2782
|
+
const saved = await persistOutputPage(rootDir, {
|
|
2783
|
+
question,
|
|
2784
|
+
answer: query.answer,
|
|
2785
|
+
citations: query.citations,
|
|
2786
|
+
schemaHash: schema.hash,
|
|
2787
|
+
relatedPageIds: query.relatedPageIds,
|
|
2788
|
+
relatedNodeIds: query.relatedNodeIds,
|
|
2789
|
+
relatedSourceIds: query.relatedSourceIds,
|
|
2790
|
+
origin: "query"
|
|
2791
|
+
});
|
|
2792
|
+
savedTo = saved.savedTo;
|
|
2793
|
+
savedPageId = saved.page.id;
|
|
1773
2794
|
}
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
2795
|
+
await appendLogEntry(rootDir, "query", question, [
|
|
2796
|
+
`citations=${query.citations.join(",") || "none"}`,
|
|
2797
|
+
`saved=${Boolean(savedTo)}`,
|
|
2798
|
+
`rawSources=${query.relatedSourceIds.length}`
|
|
2799
|
+
]);
|
|
2800
|
+
return {
|
|
2801
|
+
answer: query.answer,
|
|
2802
|
+
savedTo,
|
|
2803
|
+
savedPageId,
|
|
2804
|
+
citations: query.citations,
|
|
2805
|
+
relatedPageIds: query.relatedPageIds,
|
|
2806
|
+
relatedNodeIds: query.relatedNodeIds,
|
|
2807
|
+
relatedSourceIds: query.relatedSourceIds
|
|
2808
|
+
};
|
|
2809
|
+
}
|
|
2810
|
+
async function exploreVault(rootDir, question, steps = 3) {
|
|
2811
|
+
const schema = await loadVaultSchema(rootDir);
|
|
2812
|
+
const stepResults = [];
|
|
2813
|
+
const stepPages = [];
|
|
2814
|
+
const visited = /* @__PURE__ */ new Set();
|
|
2815
|
+
const suggestedQuestions = [];
|
|
2816
|
+
let currentQuestion = question;
|
|
2817
|
+
for (let step = 1; step <= Math.max(1, steps); step++) {
|
|
2818
|
+
const normalizedQuestion = normalizeWhitespace(currentQuestion).toLowerCase();
|
|
2819
|
+
if (!normalizedQuestion || visited.has(normalizedQuestion)) {
|
|
2820
|
+
break;
|
|
2821
|
+
}
|
|
2822
|
+
visited.add(normalizedQuestion);
|
|
2823
|
+
const query = await executeQuery(rootDir, currentQuestion);
|
|
2824
|
+
const saved = await persistOutputPage(rootDir, {
|
|
2825
|
+
title: `Explore Step ${step}: ${currentQuestion}`,
|
|
2826
|
+
question: currentQuestion,
|
|
2827
|
+
answer: query.answer,
|
|
2828
|
+
citations: query.citations,
|
|
2829
|
+
schemaHash: schema.hash,
|
|
2830
|
+
relatedPageIds: query.relatedPageIds,
|
|
2831
|
+
relatedNodeIds: query.relatedNodeIds,
|
|
2832
|
+
relatedSourceIds: query.relatedSourceIds,
|
|
2833
|
+
origin: "explore",
|
|
2834
|
+
slug: `explore-${slugify(question)}-step-${step}`
|
|
1801
2835
|
});
|
|
1802
|
-
|
|
2836
|
+
const followUpQuestions = await generateFollowUpQuestions(rootDir, currentQuestion, query.answer);
|
|
2837
|
+
stepResults.push({
|
|
2838
|
+
step,
|
|
2839
|
+
question: currentQuestion,
|
|
2840
|
+
answer: query.answer,
|
|
2841
|
+
savedTo: saved.savedTo,
|
|
2842
|
+
savedPageId: saved.page.id,
|
|
2843
|
+
citations: query.citations,
|
|
2844
|
+
followUpQuestions
|
|
2845
|
+
});
|
|
2846
|
+
stepPages.push(saved.page);
|
|
2847
|
+
suggestedQuestions.push(...followUpQuestions);
|
|
2848
|
+
const nextQuestion = followUpQuestions.find((item) => !visited.has(normalizeWhitespace(item).toLowerCase()));
|
|
2849
|
+
if (!nextQuestion) {
|
|
2850
|
+
break;
|
|
2851
|
+
}
|
|
2852
|
+
currentQuestion = nextQuestion;
|
|
1803
2853
|
}
|
|
1804
|
-
const
|
|
1805
|
-
|
|
2854
|
+
const allCitations = uniqueBy(
|
|
2855
|
+
stepResults.flatMap((step) => step.citations),
|
|
1806
2856
|
(item) => item
|
|
1807
2857
|
);
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
}
|
|
1816
|
-
await appendLogEntry(rootDir, "
|
|
1817
|
-
|
|
2858
|
+
const hub = await persistExploreHub(rootDir, {
|
|
2859
|
+
question,
|
|
2860
|
+
stepPages,
|
|
2861
|
+
followUpQuestions: uniqueBy(suggestedQuestions, (item) => item),
|
|
2862
|
+
citations: allCitations,
|
|
2863
|
+
schemaHash: schema.hash,
|
|
2864
|
+
slug: `explore-${slugify(question)}`
|
|
2865
|
+
});
|
|
2866
|
+
await appendLogEntry(rootDir, "explore", question, [
|
|
2867
|
+
`steps=${stepResults.length}`,
|
|
2868
|
+
`hub=${hub.page.id}`,
|
|
2869
|
+
`citations=${allCitations.join(",") || "none"}`
|
|
2870
|
+
]);
|
|
2871
|
+
return {
|
|
2872
|
+
rootQuestion: question,
|
|
2873
|
+
hubPath: hub.savedTo,
|
|
2874
|
+
hubPageId: hub.page.id,
|
|
2875
|
+
stepCount: stepResults.length,
|
|
2876
|
+
steps: stepResults,
|
|
2877
|
+
suggestedQuestions: uniqueBy(suggestedQuestions, (item) => item)
|
|
2878
|
+
};
|
|
1818
2879
|
}
|
|
1819
2880
|
async function searchVault(rootDir, query, limit = 5) {
|
|
1820
2881
|
const { paths } = await loadVaultConfig(rootDir);
|
|
@@ -1830,15 +2891,15 @@ async function listPages(rootDir) {
|
|
|
1830
2891
|
}
|
|
1831
2892
|
async function readPage(rootDir, relativePath) {
|
|
1832
2893
|
const { paths } = await loadVaultConfig(rootDir);
|
|
1833
|
-
const absolutePath =
|
|
2894
|
+
const absolutePath = path13.resolve(paths.wikiDir, relativePath);
|
|
1834
2895
|
if (!absolutePath.startsWith(paths.wikiDir) || !await fileExists(absolutePath)) {
|
|
1835
2896
|
return null;
|
|
1836
2897
|
}
|
|
1837
|
-
const raw = await
|
|
1838
|
-
const parsed =
|
|
2898
|
+
const raw = await fs11.readFile(absolutePath, "utf8");
|
|
2899
|
+
const parsed = matter5(raw);
|
|
1839
2900
|
return {
|
|
1840
2901
|
path: relativePath,
|
|
1841
|
-
title: typeof parsed.data.title === "string" ? parsed.data.title :
|
|
2902
|
+
title: typeof parsed.data.title === "string" ? parsed.data.title : path13.basename(relativePath, path13.extname(relativePath)),
|
|
1842
2903
|
frontmatter: parsed.data,
|
|
1843
2904
|
content: parsed.content
|
|
1844
2905
|
};
|
|
@@ -1850,6 +2911,7 @@ async function getWorkspaceInfo(rootDir) {
|
|
|
1850
2911
|
return {
|
|
1851
2912
|
rootDir,
|
|
1852
2913
|
configPath: paths.configPath,
|
|
2914
|
+
schemaPath: paths.schemaPath,
|
|
1853
2915
|
rawDir: paths.rawDir,
|
|
1854
2916
|
wikiDir: paths.wikiDir,
|
|
1855
2917
|
stateDir: paths.stateDir,
|
|
@@ -1859,11 +2921,70 @@ async function getWorkspaceInfo(rootDir) {
|
|
|
1859
2921
|
pageCount: pages.length
|
|
1860
2922
|
};
|
|
1861
2923
|
}
|
|
1862
|
-
|
|
2924
|
+
function structuralLintFindings(_rootDir, paths, graph, schemaHash, manifests) {
|
|
2925
|
+
const manifestMap = new Map(manifests.map((manifest) => [manifest.sourceId, manifest]));
|
|
2926
|
+
return Promise.all(
|
|
2927
|
+
graph.pages.map(async (page) => {
|
|
2928
|
+
const findings = [];
|
|
2929
|
+
if (page.schemaHash !== schemaHash) {
|
|
2930
|
+
findings.push({
|
|
2931
|
+
severity: "warning",
|
|
2932
|
+
code: "stale_page",
|
|
2933
|
+
message: `Page ${page.title} is stale because the vault schema changed.`,
|
|
2934
|
+
pagePath: path13.join(paths.wikiDir, page.path),
|
|
2935
|
+
relatedPageIds: [page.id]
|
|
2936
|
+
});
|
|
2937
|
+
}
|
|
2938
|
+
for (const [sourceId, knownHash] of Object.entries(page.sourceHashes)) {
|
|
2939
|
+
const manifest = manifestMap.get(sourceId);
|
|
2940
|
+
if (manifest && manifest.contentHash !== knownHash) {
|
|
2941
|
+
findings.push({
|
|
2942
|
+
severity: "warning",
|
|
2943
|
+
code: "stale_page",
|
|
2944
|
+
message: `Page ${page.title} is stale because source ${sourceId} changed.`,
|
|
2945
|
+
pagePath: path13.join(paths.wikiDir, page.path),
|
|
2946
|
+
relatedSourceIds: [sourceId],
|
|
2947
|
+
relatedPageIds: [page.id]
|
|
2948
|
+
});
|
|
2949
|
+
}
|
|
2950
|
+
}
|
|
2951
|
+
if (page.kind !== "index" && page.backlinks.length === 0) {
|
|
2952
|
+
findings.push({
|
|
2953
|
+
severity: "info",
|
|
2954
|
+
code: "orphan_page",
|
|
2955
|
+
message: `Page ${page.title} has no backlinks.`,
|
|
2956
|
+
pagePath: path13.join(paths.wikiDir, page.path),
|
|
2957
|
+
relatedPageIds: [page.id]
|
|
2958
|
+
});
|
|
2959
|
+
}
|
|
2960
|
+
const absolutePath = path13.join(paths.wikiDir, page.path);
|
|
2961
|
+
if (await fileExists(absolutePath)) {
|
|
2962
|
+
const content = await fs11.readFile(absolutePath, "utf8");
|
|
2963
|
+
if (content.includes("## Claims")) {
|
|
2964
|
+
const uncited = content.split("\n").filter((line) => line.startsWith("- ") && !line.includes("[source:"));
|
|
2965
|
+
if (uncited.length) {
|
|
2966
|
+
findings.push({
|
|
2967
|
+
severity: "warning",
|
|
2968
|
+
code: "uncited_claims",
|
|
2969
|
+
message: `Page ${page.title} contains uncited claim bullets.`,
|
|
2970
|
+
pagePath: absolutePath,
|
|
2971
|
+
relatedPageIds: [page.id]
|
|
2972
|
+
});
|
|
2973
|
+
}
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
return findings;
|
|
2977
|
+
})
|
|
2978
|
+
).then((results) => results.flat());
|
|
2979
|
+
}
|
|
2980
|
+
async function lintVault(rootDir, options = {}) {
|
|
2981
|
+
if (options.web && !options.deep) {
|
|
2982
|
+
throw new Error("`--web` can only be used together with `--deep`.");
|
|
2983
|
+
}
|
|
1863
2984
|
const { paths } = await loadVaultConfig(rootDir);
|
|
2985
|
+
const schema = await loadVaultSchema(rootDir);
|
|
1864
2986
|
const manifests = await listManifests(rootDir);
|
|
1865
2987
|
const graph = await readJsonFile(paths.graphPath);
|
|
1866
|
-
const findings = [];
|
|
1867
2988
|
if (!graph) {
|
|
1868
2989
|
return [
|
|
1869
2990
|
{
|
|
@@ -1873,44 +2994,15 @@ async function lintVault(rootDir) {
|
|
|
1873
2994
|
}
|
|
1874
2995
|
];
|
|
1875
2996
|
}
|
|
1876
|
-
const
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
const manifest = manifestMap.get(sourceId);
|
|
1880
|
-
if (manifest && manifest.contentHash !== knownHash) {
|
|
1881
|
-
findings.push({
|
|
1882
|
-
severity: "warning",
|
|
1883
|
-
code: "stale_page",
|
|
1884
|
-
message: `Page ${page.title} is stale because source ${sourceId} changed.`,
|
|
1885
|
-
pagePath: path9.join(paths.wikiDir, page.path)
|
|
1886
|
-
});
|
|
1887
|
-
}
|
|
1888
|
-
}
|
|
1889
|
-
if (page.kind !== "index" && page.backlinks.length === 0) {
|
|
1890
|
-
findings.push({
|
|
1891
|
-
severity: "info",
|
|
1892
|
-
code: "orphan_page",
|
|
1893
|
-
message: `Page ${page.title} has no backlinks.`,
|
|
1894
|
-
pagePath: path9.join(paths.wikiDir, page.path)
|
|
1895
|
-
});
|
|
1896
|
-
}
|
|
1897
|
-
const absolutePath = path9.join(paths.wikiDir, page.path);
|
|
1898
|
-
if (await fileExists(absolutePath)) {
|
|
1899
|
-
const content = await fs7.readFile(absolutePath, "utf8");
|
|
1900
|
-
if (content.includes("## Claims")) {
|
|
1901
|
-
const uncited = content.split("\n").filter((line) => line.startsWith("- ") && !line.includes("[source:"));
|
|
1902
|
-
if (uncited.length) {
|
|
1903
|
-
findings.push({
|
|
1904
|
-
severity: "warning",
|
|
1905
|
-
code: "uncited_claims",
|
|
1906
|
-
message: `Page ${page.title} contains uncited claim bullets.`,
|
|
1907
|
-
pagePath: absolutePath
|
|
1908
|
-
});
|
|
1909
|
-
}
|
|
1910
|
-
}
|
|
1911
|
-
}
|
|
2997
|
+
const findings = await structuralLintFindings(rootDir, paths, graph, schema.hash, manifests);
|
|
2998
|
+
if (options.deep) {
|
|
2999
|
+
findings.push(...await runDeepLint(rootDir, findings, { web: options.web }));
|
|
1912
3000
|
}
|
|
1913
|
-
await appendLogEntry(rootDir, "lint", `Linted ${graph.pages.length} page(s)`, [
|
|
3001
|
+
await appendLogEntry(rootDir, "lint", `Linted ${graph.pages.length} page(s)`, [
|
|
3002
|
+
`findings=${findings.length}`,
|
|
3003
|
+
`deep=${Boolean(options.deep)}`,
|
|
3004
|
+
`web=${Boolean(options.web)}`
|
|
3005
|
+
]);
|
|
1914
3006
|
return findings;
|
|
1915
3007
|
}
|
|
1916
3008
|
async function bootstrapDemo(rootDir, input) {
|
|
@@ -1926,166 +3018,170 @@ async function bootstrapDemo(rootDir, input) {
|
|
|
1926
3018
|
};
|
|
1927
3019
|
}
|
|
1928
3020
|
|
|
1929
|
-
// src/viewer.ts
|
|
1930
|
-
import fs8 from "fs/promises";
|
|
1931
|
-
import http from "http";
|
|
1932
|
-
import path10 from "path";
|
|
1933
|
-
import mime2 from "mime-types";
|
|
1934
|
-
async function startGraphServer(rootDir, port) {
|
|
1935
|
-
const { config, paths } = await loadVaultConfig(rootDir);
|
|
1936
|
-
const effectivePort = port ?? config.viewer.port;
|
|
1937
|
-
const server = http.createServer(async (request, response) => {
|
|
1938
|
-
const url = new URL(request.url ?? "/", `http://${request.headers.host ?? `localhost:${effectivePort}`}`);
|
|
1939
|
-
if (url.pathname === "/api/graph") {
|
|
1940
|
-
if (!await fileExists(paths.graphPath)) {
|
|
1941
|
-
response.writeHead(404, { "content-type": "application/json" });
|
|
1942
|
-
response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
|
|
1943
|
-
return;
|
|
1944
|
-
}
|
|
1945
|
-
response.writeHead(200, { "content-type": "application/json" });
|
|
1946
|
-
response.end(await fs8.readFile(paths.graphPath, "utf8"));
|
|
1947
|
-
return;
|
|
1948
|
-
}
|
|
1949
|
-
const relativePath = url.pathname === "/" ? "index.html" : url.pathname.slice(1);
|
|
1950
|
-
const target = path10.join(paths.viewerDistDir, relativePath);
|
|
1951
|
-
const fallback = path10.join(paths.viewerDistDir, "index.html");
|
|
1952
|
-
const filePath = await fileExists(target) ? target : fallback;
|
|
1953
|
-
if (!await fileExists(filePath)) {
|
|
1954
|
-
response.writeHead(503, { "content-type": "text/plain" });
|
|
1955
|
-
response.end("Viewer build not found. Run `pnpm build` first.");
|
|
1956
|
-
return;
|
|
1957
|
-
}
|
|
1958
|
-
response.writeHead(200, { "content-type": mime2.lookup(filePath) || "text/plain" });
|
|
1959
|
-
response.end(await fs8.readFile(filePath));
|
|
1960
|
-
});
|
|
1961
|
-
await new Promise((resolve) => {
|
|
1962
|
-
server.listen(effectivePort, resolve);
|
|
1963
|
-
});
|
|
1964
|
-
return {
|
|
1965
|
-
port: effectivePort,
|
|
1966
|
-
close: async () => {
|
|
1967
|
-
await new Promise((resolve, reject) => {
|
|
1968
|
-
server.close((error) => {
|
|
1969
|
-
if (error) {
|
|
1970
|
-
reject(error);
|
|
1971
|
-
return;
|
|
1972
|
-
}
|
|
1973
|
-
resolve();
|
|
1974
|
-
});
|
|
1975
|
-
});
|
|
1976
|
-
}
|
|
1977
|
-
};
|
|
1978
|
-
}
|
|
1979
|
-
|
|
1980
3021
|
// src/mcp.ts
|
|
1981
|
-
|
|
1982
|
-
import path11 from "path";
|
|
1983
|
-
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
1984
|
-
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
1985
|
-
import { z as z6 } from "zod";
|
|
1986
|
-
var SERVER_VERSION = "0.1.3";
|
|
3022
|
+
var SERVER_VERSION = "0.1.5";
|
|
1987
3023
|
async function createMcpServer(rootDir) {
|
|
1988
3024
|
const server = new McpServer({
|
|
1989
3025
|
name: "swarmvault",
|
|
1990
3026
|
version: SERVER_VERSION,
|
|
1991
3027
|
websiteUrl: "https://www.swarmvault.ai"
|
|
1992
3028
|
});
|
|
1993
|
-
server.registerTool(
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
inputSchema: {
|
|
2002
|
-
query: z6.string().min(1).describe("Search query"),
|
|
2003
|
-
limit: z6.number().int().min(1).max(25).optional().describe("Maximum number of results")
|
|
2004
|
-
}
|
|
2005
|
-
}, async ({ query, limit }) => {
|
|
2006
|
-
const results = await searchVault(rootDir, query, limit ?? 5);
|
|
2007
|
-
return asToolText(results);
|
|
2008
|
-
});
|
|
2009
|
-
server.registerTool("read_page", {
|
|
2010
|
-
description: "Read a generated wiki page by its path relative to wiki/.",
|
|
2011
|
-
inputSchema: {
|
|
2012
|
-
path: z6.string().min(1).describe("Path relative to wiki/, for example sources/example.md")
|
|
3029
|
+
server.registerTool(
|
|
3030
|
+
"workspace_info",
|
|
3031
|
+
{
|
|
3032
|
+
description: "Return the current SwarmVault workspace paths and high-level counts."
|
|
3033
|
+
},
|
|
3034
|
+
async () => {
|
|
3035
|
+
const info = await getWorkspaceInfo(rootDir);
|
|
3036
|
+
return asToolText(info);
|
|
2013
3037
|
}
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
3038
|
+
);
|
|
3039
|
+
server.registerTool(
|
|
3040
|
+
"search_pages",
|
|
3041
|
+
{
|
|
3042
|
+
description: "Search compiled wiki pages using the local full-text index.",
|
|
3043
|
+
inputSchema: {
|
|
3044
|
+
query: z9.string().min(1).describe("Search query"),
|
|
3045
|
+
limit: z9.number().int().min(1).max(25).optional().describe("Maximum number of results")
|
|
3046
|
+
}
|
|
3047
|
+
},
|
|
3048
|
+
async ({ query, limit }) => {
|
|
3049
|
+
const results = await searchVault(rootDir, query, limit ?? 5);
|
|
3050
|
+
return asToolText(results);
|
|
2018
3051
|
}
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
}
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
server.registerTool(
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
3052
|
+
);
|
|
3053
|
+
server.registerTool(
|
|
3054
|
+
"read_page",
|
|
3055
|
+
{
|
|
3056
|
+
description: "Read a generated wiki page by its path relative to wiki/.",
|
|
3057
|
+
inputSchema: {
|
|
3058
|
+
path: z9.string().min(1).describe("Path relative to wiki/, for example sources/example.md")
|
|
3059
|
+
}
|
|
3060
|
+
},
|
|
3061
|
+
async ({ path: relativePath }) => {
|
|
3062
|
+
const page = await readPage(rootDir, relativePath);
|
|
3063
|
+
if (!page) {
|
|
3064
|
+
return asToolError(`Page not found: ${relativePath}`);
|
|
3065
|
+
}
|
|
3066
|
+
return asToolText(page);
|
|
3067
|
+
}
|
|
3068
|
+
);
|
|
3069
|
+
server.registerTool(
|
|
3070
|
+
"list_sources",
|
|
3071
|
+
{
|
|
3072
|
+
description: "List source manifests in the current workspace.",
|
|
3073
|
+
inputSchema: {
|
|
3074
|
+
limit: z9.number().int().min(1).max(100).optional().describe("Maximum number of manifests to return")
|
|
3075
|
+
}
|
|
3076
|
+
},
|
|
3077
|
+
async ({ limit }) => {
|
|
3078
|
+
const manifests = await listManifests(rootDir);
|
|
3079
|
+
return asToolText(limit ? manifests.slice(0, limit) : manifests);
|
|
3080
|
+
}
|
|
3081
|
+
);
|
|
3082
|
+
server.registerTool(
|
|
3083
|
+
"query_vault",
|
|
3084
|
+
{
|
|
3085
|
+
description: "Ask a question against the compiled vault and optionally save the answer.",
|
|
3086
|
+
inputSchema: {
|
|
3087
|
+
question: z9.string().min(1).describe("Question to ask the vault"),
|
|
3088
|
+
save: z9.boolean().optional().describe("Persist the answer to wiki/outputs")
|
|
3089
|
+
}
|
|
3090
|
+
},
|
|
3091
|
+
async ({ question, save }) => {
|
|
3092
|
+
const result = await queryVault(rootDir, question, save ?? false);
|
|
3093
|
+
return asToolText(result);
|
|
3094
|
+
}
|
|
3095
|
+
);
|
|
3096
|
+
server.registerTool(
|
|
3097
|
+
"ingest_input",
|
|
3098
|
+
{
|
|
3099
|
+
description: "Ingest a local file path or URL into the SwarmVault workspace.",
|
|
3100
|
+
inputSchema: {
|
|
3101
|
+
input: z9.string().min(1).describe("Local path or URL to ingest")
|
|
3102
|
+
}
|
|
3103
|
+
},
|
|
3104
|
+
async ({ input }) => {
|
|
3105
|
+
const manifest = await ingestInput(rootDir, input);
|
|
3106
|
+
return asToolText(manifest);
|
|
3107
|
+
}
|
|
3108
|
+
);
|
|
3109
|
+
server.registerTool(
|
|
3110
|
+
"compile_vault",
|
|
3111
|
+
{
|
|
3112
|
+
description: "Compile source manifests into wiki pages, graph data, and search index."
|
|
3113
|
+
},
|
|
3114
|
+
async () => {
|
|
3115
|
+
const result = await compileVault(rootDir);
|
|
3116
|
+
return asToolText(result);
|
|
3117
|
+
}
|
|
3118
|
+
);
|
|
3119
|
+
server.registerTool(
|
|
3120
|
+
"lint_vault",
|
|
3121
|
+
{
|
|
3122
|
+
description: "Run anti-drift and vault health checks."
|
|
3123
|
+
},
|
|
3124
|
+
async () => {
|
|
3125
|
+
const findings = await lintVault(rootDir);
|
|
3126
|
+
return asToolText(findings);
|
|
3127
|
+
}
|
|
3128
|
+
);
|
|
3129
|
+
server.registerResource(
|
|
3130
|
+
"swarmvault-config",
|
|
3131
|
+
"swarmvault://config",
|
|
3132
|
+
{
|
|
3133
|
+
title: "SwarmVault Config",
|
|
3134
|
+
description: "The resolved SwarmVault config file.",
|
|
3135
|
+
mimeType: "application/json"
|
|
3136
|
+
},
|
|
3137
|
+
async () => {
|
|
3138
|
+
const { config } = await loadVaultConfig(rootDir);
|
|
3139
|
+
return asTextResource("swarmvault://config", JSON.stringify(config, null, 2));
|
|
3140
|
+
}
|
|
3141
|
+
);
|
|
3142
|
+
server.registerResource(
|
|
3143
|
+
"swarmvault-graph",
|
|
3144
|
+
"swarmvault://graph",
|
|
3145
|
+
{
|
|
3146
|
+
title: "SwarmVault Graph",
|
|
3147
|
+
description: "The compiled graph artifact for the current workspace.",
|
|
3148
|
+
mimeType: "application/json"
|
|
3149
|
+
},
|
|
3150
|
+
async () => {
|
|
3151
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
3152
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
3153
|
+
return asTextResource(
|
|
3154
|
+
"swarmvault://graph",
|
|
3155
|
+
JSON.stringify(graph ?? { error: "Graph artifact not found. Run `swarmvault compile` first." }, null, 2)
|
|
3156
|
+
);
|
|
3157
|
+
}
|
|
3158
|
+
);
|
|
3159
|
+
server.registerResource(
|
|
3160
|
+
"swarmvault-manifests",
|
|
3161
|
+
"swarmvault://manifests",
|
|
3162
|
+
{
|
|
3163
|
+
title: "SwarmVault Manifests",
|
|
3164
|
+
description: "All source manifests in the workspace.",
|
|
3165
|
+
mimeType: "application/json"
|
|
3166
|
+
},
|
|
3167
|
+
async () => {
|
|
3168
|
+
const manifests = await listManifests(rootDir);
|
|
3169
|
+
return asTextResource("swarmvault://manifests", JSON.stringify(manifests, null, 2));
|
|
3170
|
+
}
|
|
3171
|
+
);
|
|
3172
|
+
server.registerResource(
|
|
3173
|
+
"swarmvault-schema",
|
|
3174
|
+
"swarmvault://schema",
|
|
3175
|
+
{
|
|
3176
|
+
title: "SwarmVault Schema",
|
|
3177
|
+
description: "The vault schema file that guides compile and query behavior.",
|
|
3178
|
+
mimeType: "text/markdown"
|
|
3179
|
+
},
|
|
3180
|
+
async () => {
|
|
3181
|
+
const schema = await loadVaultSchema(rootDir);
|
|
3182
|
+
return asTextResource("swarmvault://schema", schema.content);
|
|
3183
|
+
}
|
|
3184
|
+
);
|
|
2089
3185
|
server.registerResource(
|
|
2090
3186
|
"swarmvault-pages",
|
|
2091
3187
|
new ResourceTemplate("swarmvault://pages/{path}", {
|
|
@@ -2115,8 +3211,8 @@ async function createMcpServer(rootDir) {
|
|
|
2115
3211
|
return asTextResource(`swarmvault://pages/${encodedPath}`, `Page not found: ${relativePath}`);
|
|
2116
3212
|
}
|
|
2117
3213
|
const { paths } = await loadVaultConfig(rootDir);
|
|
2118
|
-
const absolutePath =
|
|
2119
|
-
return asTextResource(`swarmvault://pages/${encodedPath}`, await
|
|
3214
|
+
const absolutePath = path14.resolve(paths.wikiDir, relativePath);
|
|
3215
|
+
return asTextResource(`swarmvault://pages/${encodedPath}`, await fs12.readFile(absolutePath, "utf8"));
|
|
2120
3216
|
}
|
|
2121
3217
|
);
|
|
2122
3218
|
return server;
|
|
@@ -2163,21 +3259,78 @@ function asTextResource(uri, text) {
|
|
|
2163
3259
|
};
|
|
2164
3260
|
}
|
|
2165
3261
|
|
|
3262
|
+
// src/viewer.ts
|
|
3263
|
+
import fs13 from "fs/promises";
|
|
3264
|
+
import http from "http";
|
|
3265
|
+
import path15 from "path";
|
|
3266
|
+
import mime2 from "mime-types";
|
|
3267
|
+
async function startGraphServer(rootDir, port) {
|
|
3268
|
+
const { config, paths } = await loadVaultConfig(rootDir);
|
|
3269
|
+
const effectivePort = port ?? config.viewer.port;
|
|
3270
|
+
const server = http.createServer(async (request, response) => {
|
|
3271
|
+
const url = new URL(request.url ?? "/", `http://${request.headers.host ?? `localhost:${effectivePort}`}`);
|
|
3272
|
+
if (url.pathname === "/api/graph") {
|
|
3273
|
+
if (!await fileExists(paths.graphPath)) {
|
|
3274
|
+
response.writeHead(404, { "content-type": "application/json" });
|
|
3275
|
+
response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
|
|
3276
|
+
return;
|
|
3277
|
+
}
|
|
3278
|
+
response.writeHead(200, { "content-type": "application/json" });
|
|
3279
|
+
response.end(await fs13.readFile(paths.graphPath, "utf8"));
|
|
3280
|
+
return;
|
|
3281
|
+
}
|
|
3282
|
+
const relativePath = url.pathname === "/" ? "index.html" : url.pathname.slice(1);
|
|
3283
|
+
const target = path15.join(paths.viewerDistDir, relativePath);
|
|
3284
|
+
const fallback = path15.join(paths.viewerDistDir, "index.html");
|
|
3285
|
+
const filePath = await fileExists(target) ? target : fallback;
|
|
3286
|
+
if (!await fileExists(filePath)) {
|
|
3287
|
+
response.writeHead(503, { "content-type": "text/plain" });
|
|
3288
|
+
response.end("Viewer build not found. Run `pnpm build` first.");
|
|
3289
|
+
return;
|
|
3290
|
+
}
|
|
3291
|
+
response.writeHead(200, { "content-type": mime2.lookup(filePath) || "text/plain" });
|
|
3292
|
+
response.end(await fs13.readFile(filePath));
|
|
3293
|
+
});
|
|
3294
|
+
await new Promise((resolve) => {
|
|
3295
|
+
server.listen(effectivePort, resolve);
|
|
3296
|
+
});
|
|
3297
|
+
return {
|
|
3298
|
+
port: effectivePort,
|
|
3299
|
+
close: async () => {
|
|
3300
|
+
await new Promise((resolve, reject) => {
|
|
3301
|
+
server.close((error) => {
|
|
3302
|
+
if (error) {
|
|
3303
|
+
reject(error);
|
|
3304
|
+
return;
|
|
3305
|
+
}
|
|
3306
|
+
resolve();
|
|
3307
|
+
});
|
|
3308
|
+
});
|
|
3309
|
+
}
|
|
3310
|
+
};
|
|
3311
|
+
}
|
|
3312
|
+
|
|
2166
3313
|
// src/watch.ts
|
|
2167
|
-
import
|
|
3314
|
+
import path16 from "path";
|
|
3315
|
+
import process2 from "process";
|
|
2168
3316
|
import chokidar from "chokidar";
|
|
3317
|
+
var MAX_BACKOFF_MS = 3e4;
|
|
3318
|
+
var BACKOFF_THRESHOLD = 3;
|
|
3319
|
+
var CRITICAL_THRESHOLD = 10;
|
|
2169
3320
|
async function watchVault(rootDir, options = {}) {
|
|
2170
3321
|
const { paths } = await initWorkspace(rootDir);
|
|
2171
|
-
const
|
|
3322
|
+
const baseDebounceMs = options.debounceMs ?? 900;
|
|
2172
3323
|
let timer;
|
|
2173
3324
|
let running = false;
|
|
2174
3325
|
let pending = false;
|
|
2175
3326
|
let closed = false;
|
|
3327
|
+
let consecutiveFailures = 0;
|
|
3328
|
+
let currentDebounceMs = baseDebounceMs;
|
|
2176
3329
|
const reasons = /* @__PURE__ */ new Set();
|
|
2177
3330
|
const watcher = chokidar.watch(paths.inboxDir, {
|
|
2178
3331
|
ignoreInitial: true,
|
|
2179
3332
|
awaitWriteFinish: {
|
|
2180
|
-
stabilityThreshold: Math.max(250, Math.floor(
|
|
3333
|
+
stabilityThreshold: Math.max(250, Math.floor(baseDebounceMs / 2)),
|
|
2181
3334
|
pollInterval: 100
|
|
2182
3335
|
}
|
|
2183
3336
|
});
|
|
@@ -2192,7 +3345,7 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2192
3345
|
}
|
|
2193
3346
|
timer = setTimeout(() => {
|
|
2194
3347
|
void runCycle();
|
|
2195
|
-
},
|
|
3348
|
+
}, currentDebounceMs);
|
|
2196
3349
|
};
|
|
2197
3350
|
const runCycle = async () => {
|
|
2198
3351
|
if (running || closed || !pending) {
|
|
@@ -2221,9 +3374,23 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2221
3374
|
const findings = await lintVault(rootDir);
|
|
2222
3375
|
lintFindingCount = findings.length;
|
|
2223
3376
|
}
|
|
3377
|
+
consecutiveFailures = 0;
|
|
3378
|
+
currentDebounceMs = baseDebounceMs;
|
|
2224
3379
|
} catch (caught) {
|
|
2225
3380
|
success = false;
|
|
2226
3381
|
error = caught instanceof Error ? caught.message : String(caught);
|
|
3382
|
+
consecutiveFailures++;
|
|
3383
|
+
pending = true;
|
|
3384
|
+
if (consecutiveFailures >= CRITICAL_THRESHOLD) {
|
|
3385
|
+
process2.stderr.write(
|
|
3386
|
+
`[swarmvault watch] ${consecutiveFailures} consecutive failures. Check vault state. Continuing at max backoff.
|
|
3387
|
+
`
|
|
3388
|
+
);
|
|
3389
|
+
}
|
|
3390
|
+
if (consecutiveFailures >= BACKOFF_THRESHOLD) {
|
|
3391
|
+
const multiplier = 2 ** (consecutiveFailures - BACKOFF_THRESHOLD);
|
|
3392
|
+
currentDebounceMs = Math.min(baseDebounceMs * multiplier, MAX_BACKOFF_MS);
|
|
3393
|
+
}
|
|
2227
3394
|
} finally {
|
|
2228
3395
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
2229
3396
|
await appendWatchRun(rootDir, {
|
|
@@ -2247,6 +3414,18 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2247
3414
|
}
|
|
2248
3415
|
};
|
|
2249
3416
|
watcher.on("add", (filePath) => schedule(`add:${toWatchReason(paths.inboxDir, filePath)}`)).on("change", (filePath) => schedule(`change:${toWatchReason(paths.inboxDir, filePath)}`)).on("unlink", (filePath) => schedule(`unlink:${toWatchReason(paths.inboxDir, filePath)}`)).on("addDir", (dirPath) => schedule(`addDir:${toWatchReason(paths.inboxDir, dirPath)}`)).on("unlinkDir", (dirPath) => schedule(`unlinkDir:${toWatchReason(paths.inboxDir, dirPath)}`)).on("error", (caught) => schedule(`error:${caught instanceof Error ? caught.message : String(caught)}`));
|
|
3417
|
+
await new Promise((resolve, reject) => {
|
|
3418
|
+
const handleReady = () => {
|
|
3419
|
+
watcher.off("error", handleError);
|
|
3420
|
+
resolve();
|
|
3421
|
+
};
|
|
3422
|
+
const handleError = (caught) => {
|
|
3423
|
+
watcher.off("ready", handleReady);
|
|
3424
|
+
reject(caught);
|
|
3425
|
+
};
|
|
3426
|
+
watcher.once("ready", handleReady);
|
|
3427
|
+
watcher.once("error", handleError);
|
|
3428
|
+
});
|
|
2250
3429
|
return {
|
|
2251
3430
|
close: async () => {
|
|
2252
3431
|
closed = true;
|
|
@@ -2258,7 +3437,7 @@ async function watchVault(rootDir, options = {}) {
|
|
|
2258
3437
|
};
|
|
2259
3438
|
}
|
|
2260
3439
|
function toWatchReason(baseDir, targetPath) {
|
|
2261
|
-
return
|
|
3440
|
+
return path16.relative(baseDir, targetPath) || ".";
|
|
2262
3441
|
}
|
|
2263
3442
|
export {
|
|
2264
3443
|
assertProviderCapability,
|
|
@@ -2266,8 +3445,12 @@ export {
|
|
|
2266
3445
|
compileVault,
|
|
2267
3446
|
createMcpServer,
|
|
2268
3447
|
createProvider,
|
|
3448
|
+
createWebSearchAdapter,
|
|
2269
3449
|
defaultVaultConfig,
|
|
3450
|
+
defaultVaultSchema,
|
|
3451
|
+
exploreVault,
|
|
2270
3452
|
getProviderForTask,
|
|
3453
|
+
getWebSearchAdapterForTask,
|
|
2271
3454
|
getWorkspaceInfo,
|
|
2272
3455
|
importInbox,
|
|
2273
3456
|
ingestInput,
|
|
@@ -2279,6 +3462,7 @@ export {
|
|
|
2279
3462
|
listManifests,
|
|
2280
3463
|
listPages,
|
|
2281
3464
|
loadVaultConfig,
|
|
3465
|
+
loadVaultSchema,
|
|
2282
3466
|
queryVault,
|
|
2283
3467
|
readExtractedText,
|
|
2284
3468
|
readPage,
|