@swarmvaultai/engine 0.1.32 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +23 -1
- package/dist/chunk-CWLDFLH2.js +1163 -0
- package/dist/index.d.ts +68 -2
- package/dist/index.js +1924 -505
- package/dist/registry-2REAPKPO.js +12 -0
- package/package.json +8 -7
package/dist/index.js
CHANGED
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
uniqueBy,
|
|
22
22
|
writeFileIfChanged,
|
|
23
23
|
writeJsonFile
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-CWLDFLH2.js";
|
|
25
25
|
|
|
26
26
|
// src/agents.ts
|
|
27
27
|
import crypto from "crypto";
|
|
@@ -1721,8 +1721,8 @@ async function uninstallGitHooks(rootDir) {
|
|
|
1721
1721
|
}
|
|
1722
1722
|
|
|
1723
1723
|
// src/ingest.ts
|
|
1724
|
-
import
|
|
1725
|
-
import
|
|
1724
|
+
import fs11 from "fs/promises";
|
|
1725
|
+
import path12 from "path";
|
|
1726
1726
|
import { pathToFileURL } from "url";
|
|
1727
1727
|
import { Readability } from "@mozilla/readability";
|
|
1728
1728
|
import matter3 from "gray-matter";
|
|
@@ -1741,7 +1741,9 @@ import fs5 from "fs/promises";
|
|
|
1741
1741
|
import { createRequire } from "module";
|
|
1742
1742
|
import path5 from "path";
|
|
1743
1743
|
var require2 = createRequire(import.meta.url);
|
|
1744
|
-
var
|
|
1744
|
+
var TREE_SITTER_RUNTIME_PACKAGE = "@vscode/tree-sitter-wasm";
|
|
1745
|
+
var TREE_SITTER_EXTRA_GRAMMARS_PACKAGE = "tree-sitter-wasms";
|
|
1746
|
+
var packageRootCache = /* @__PURE__ */ new Map();
|
|
1745
1747
|
var RATIONALE_MARKERS = ["NOTE:", "IMPORTANT:", "HACK:", "WHY:", "RATIONALE:"];
|
|
1746
1748
|
function stripKnownCommentPrefix(line) {
|
|
1747
1749
|
let next = line.trim();
|
|
@@ -1755,21 +1757,41 @@ function stripKnownCommentPrefix(line) {
|
|
|
1755
1757
|
var treeSitterModulePromise;
|
|
1756
1758
|
var treeSitterInitPromise;
|
|
1757
1759
|
var languageCache = /* @__PURE__ */ new Map();
|
|
1758
|
-
var
|
|
1759
|
-
python: "tree-sitter-python.wasm",
|
|
1760
|
-
go: "tree-sitter-go.wasm",
|
|
1761
|
-
rust: "tree-sitter-rust.wasm",
|
|
1762
|
-
java: "tree-sitter-java.wasm",
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1760
|
+
var grammarAssetByLanguage = {
|
|
1761
|
+
python: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-python.wasm" },
|
|
1762
|
+
go: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-go.wasm" },
|
|
1763
|
+
rust: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-rust.wasm" },
|
|
1764
|
+
java: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-java.wasm" },
|
|
1765
|
+
kotlin: { packageName: TREE_SITTER_EXTRA_GRAMMARS_PACKAGE, relativePath: "out/tree-sitter-kotlin.wasm" },
|
|
1766
|
+
scala: { packageName: TREE_SITTER_EXTRA_GRAMMARS_PACKAGE, relativePath: "out/tree-sitter-scala.wasm" },
|
|
1767
|
+
csharp: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-c-sharp.wasm" },
|
|
1768
|
+
c: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-cpp.wasm" },
|
|
1769
|
+
cpp: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-cpp.wasm" },
|
|
1770
|
+
php: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-php.wasm" },
|
|
1771
|
+
ruby: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-ruby.wasm" },
|
|
1772
|
+
powershell: { packageName: TREE_SITTER_RUNTIME_PACKAGE, relativePath: "wasm/tree-sitter-powershell.wasm" }
|
|
1769
1773
|
};
|
|
1774
|
+
function resolvePackageRoot(packageName) {
|
|
1775
|
+
const cached = packageRootCache.get(packageName);
|
|
1776
|
+
if (cached) {
|
|
1777
|
+
return cached;
|
|
1778
|
+
}
|
|
1779
|
+
let resolved;
|
|
1780
|
+
try {
|
|
1781
|
+
resolved = path5.dirname(require2.resolve(`${packageName}/package.json`));
|
|
1782
|
+
} catch {
|
|
1783
|
+
resolved = path5.dirname(path5.dirname(require2.resolve(packageName)));
|
|
1784
|
+
}
|
|
1785
|
+
packageRootCache.set(packageName, resolved);
|
|
1786
|
+
return resolved;
|
|
1787
|
+
}
|
|
1788
|
+
function grammarAssetPath(language) {
|
|
1789
|
+
const asset = grammarAssetByLanguage[language];
|
|
1790
|
+
return path5.join(resolvePackageRoot(asset.packageName), asset.relativePath);
|
|
1791
|
+
}
|
|
1770
1792
|
async function getTreeSitterModule() {
|
|
1771
1793
|
if (!treeSitterModulePromise) {
|
|
1772
|
-
treeSitterModulePromise = import(require2.resolve(
|
|
1794
|
+
treeSitterModulePromise = import(require2.resolve(TREE_SITTER_RUNTIME_PACKAGE)).then(
|
|
1773
1795
|
(module) => module.default ?? module
|
|
1774
1796
|
);
|
|
1775
1797
|
}
|
|
@@ -1777,8 +1799,9 @@ async function getTreeSitterModule() {
|
|
|
1777
1799
|
}
|
|
1778
1800
|
async function ensureTreeSitterInit(module) {
|
|
1779
1801
|
if (!treeSitterInitPromise) {
|
|
1802
|
+
const runtimeRoot = resolvePackageRoot(TREE_SITTER_RUNTIME_PACKAGE);
|
|
1780
1803
|
treeSitterInitPromise = module.Parser.init({
|
|
1781
|
-
locateFile: () => path5.join(
|
|
1804
|
+
locateFile: () => path5.join(runtimeRoot, "wasm", "tree-sitter.wasm")
|
|
1782
1805
|
});
|
|
1783
1806
|
}
|
|
1784
1807
|
return treeSitterInitPromise;
|
|
@@ -1791,7 +1814,7 @@ async function loadLanguage(language) {
|
|
|
1791
1814
|
const loader = (async () => {
|
|
1792
1815
|
const module = await getTreeSitterModule();
|
|
1793
1816
|
await ensureTreeSitterInit(module);
|
|
1794
|
-
const bytes = await fs5.readFile(
|
|
1817
|
+
const bytes = await fs5.readFile(grammarAssetPath(language));
|
|
1795
1818
|
return module.Language.load(bytes);
|
|
1796
1819
|
})();
|
|
1797
1820
|
languageCache.set(language, loader);
|
|
@@ -1805,7 +1828,7 @@ function normalizeSymbolReference(value) {
|
|
|
1805
1828
|
return lastSegment.replace(/[,:;]+$/g, "").trim();
|
|
1806
1829
|
}
|
|
1807
1830
|
function stripCodeExtension(filePath) {
|
|
1808
|
-
return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
|
|
1831
|
+
return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|kt|kts|scala|sc|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
|
|
1809
1832
|
}
|
|
1810
1833
|
function manifestModuleName(manifest, language) {
|
|
1811
1834
|
const repoPath = manifest.repoRelativePath ?? path5.basename(manifest.originalPath ?? manifest.storedPath);
|
|
@@ -2044,6 +2067,7 @@ function extractIdentifier(node) {
|
|
|
2044
2067
|
}
|
|
2045
2068
|
if ([
|
|
2046
2069
|
"identifier",
|
|
2070
|
+
"simple_identifier",
|
|
2047
2071
|
"field_identifier",
|
|
2048
2072
|
"type_identifier",
|
|
2049
2073
|
"name",
|
|
@@ -2057,6 +2081,7 @@ function extractIdentifier(node) {
|
|
|
2057
2081
|
const preferred = node.childForFieldName("name") ?? node.namedChildren.find(
|
|
2058
2082
|
(child) => child && [
|
|
2059
2083
|
"identifier",
|
|
2084
|
+
"simple_identifier",
|
|
2060
2085
|
"field_identifier",
|
|
2061
2086
|
"type_identifier",
|
|
2062
2087
|
"name",
|
|
@@ -2119,6 +2144,25 @@ function diagnosticsFromTree(rootNode) {
|
|
|
2119
2144
|
visit(rootNode);
|
|
2120
2145
|
return diagnostics.slice(0, 20);
|
|
2121
2146
|
}
|
|
2147
|
+
function treeSitterCompatibilityMessage(language, error) {
|
|
2148
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2149
|
+
if (typeof error === "object" && error && "code" in error && error.code === "MODULE_NOT_FOUND") {
|
|
2150
|
+
return `Tree-sitter runtime support for ${language} is unavailable. Reinstall @swarmvaultai/engine so the packaged parser runtime is present.`;
|
|
2151
|
+
}
|
|
2152
|
+
if (typeof error === "object" && error && "code" in error && error.code === "ENOENT") {
|
|
2153
|
+
return `Missing tree-sitter grammar asset for ${language}. Reinstall @swarmvaultai/engine so the packaged grammar files are present.`;
|
|
2154
|
+
}
|
|
2155
|
+
return `Tree-sitter support for ${language} could not load: ${truncate(normalizeWhitespace(message), 220)}.`;
|
|
2156
|
+
}
|
|
2157
|
+
function treeSitterCompatibilityDiagnostic(language, error) {
|
|
2158
|
+
return {
|
|
2159
|
+
code: 9010,
|
|
2160
|
+
category: "error",
|
|
2161
|
+
message: treeSitterCompatibilityMessage(language, error),
|
|
2162
|
+
line: 1,
|
|
2163
|
+
column: 1
|
|
2164
|
+
};
|
|
2165
|
+
}
|
|
2122
2166
|
function parsePythonImportStatement(text) {
|
|
2123
2167
|
const match = text.trim().match(/^import\s+(.+)$/);
|
|
2124
2168
|
if (!match) {
|
|
@@ -2188,6 +2232,36 @@ function parseJavaImport(text) {
|
|
|
2188
2232
|
reExport: false
|
|
2189
2233
|
};
|
|
2190
2234
|
}
|
|
2235
|
+
function parseKotlinImport(text) {
|
|
2236
|
+
const cleaned = text.trim().replace(/^import\s+/, "");
|
|
2237
|
+
if (!cleaned) {
|
|
2238
|
+
return void 0;
|
|
2239
|
+
}
|
|
2240
|
+
const aliasMatch = cleaned.match(/^(.+?)\s+as\s+([A-Za-z_]\w*)$/);
|
|
2241
|
+
const specifier = (aliasMatch ? aliasMatch[1] : cleaned).trim();
|
|
2242
|
+
if (!specifier) {
|
|
2243
|
+
return void 0;
|
|
2244
|
+
}
|
|
2245
|
+
return {
|
|
2246
|
+
specifier,
|
|
2247
|
+
importedSymbols: [],
|
|
2248
|
+
namespaceImport: aliasMatch?.[2],
|
|
2249
|
+
isExternal: !specifier.startsWith("."),
|
|
2250
|
+
reExport: false
|
|
2251
|
+
};
|
|
2252
|
+
}
|
|
2253
|
+
function parseScalaImport(text) {
|
|
2254
|
+
const cleaned = text.trim().replace(/^import\s+/, "");
|
|
2255
|
+
if (!cleaned) {
|
|
2256
|
+
return [];
|
|
2257
|
+
}
|
|
2258
|
+
return cleaned.split(",").map((item) => item.trim()).filter(Boolean).map((item) => ({
|
|
2259
|
+
specifier: item.replace(/\s*=>\s*/g, " => "),
|
|
2260
|
+
importedSymbols: [],
|
|
2261
|
+
isExternal: !item.startsWith("."),
|
|
2262
|
+
reExport: false
|
|
2263
|
+
}));
|
|
2264
|
+
}
|
|
2191
2265
|
function parseCSharpUsing(text) {
|
|
2192
2266
|
const aliasMatch = text.trim().match(/^using\s+([A-Za-z_]\w*)\s*=\s*([^;]+);$/);
|
|
2193
2267
|
if (aliasMatch) {
|
|
@@ -2280,6 +2354,38 @@ function parsePowerShellImport(commandNode) {
|
|
|
2280
2354
|
}
|
|
2281
2355
|
return void 0;
|
|
2282
2356
|
}
|
|
2357
|
+
function keywordVisible(text, hiddenKeywords) {
|
|
2358
|
+
return !hiddenKeywords.some((keyword) => new RegExp(`\\b${keyword}\\b`).test(text));
|
|
2359
|
+
}
|
|
2360
|
+
function declarationVisible(node, hiddenKeywords) {
|
|
2361
|
+
const modifierText = nodeText(findNamedChild(node, "modifiers") ?? node.childForFieldName("modifiers"));
|
|
2362
|
+
return modifierText ? keywordVisible(modifierText, hiddenKeywords) : true;
|
|
2363
|
+
}
|
|
2364
|
+
function kotlinClassKind(text) {
|
|
2365
|
+
const trimmed = text.trimStart();
|
|
2366
|
+
if (trimmed.startsWith("interface ")) {
|
|
2367
|
+
return "interface";
|
|
2368
|
+
}
|
|
2369
|
+
if (trimmed.startsWith("enum class ")) {
|
|
2370
|
+
return "enum";
|
|
2371
|
+
}
|
|
2372
|
+
return "class";
|
|
2373
|
+
}
|
|
2374
|
+
function scalaDefinitionKind(node) {
|
|
2375
|
+
if (node.type === "trait_definition") {
|
|
2376
|
+
return "trait";
|
|
2377
|
+
}
|
|
2378
|
+
if (node.type === "class_definition") {
|
|
2379
|
+
return /\bcase\s+class\b/.test(node.text) ? "class" : "class";
|
|
2380
|
+
}
|
|
2381
|
+
if (node.type === "object_definition") {
|
|
2382
|
+
return "class";
|
|
2383
|
+
}
|
|
2384
|
+
if (node.type === "function_definition") {
|
|
2385
|
+
return "function";
|
|
2386
|
+
}
|
|
2387
|
+
return void 0;
|
|
2388
|
+
}
|
|
2283
2389
|
function pythonCodeAnalysis(manifest, rootNode, diagnostics) {
|
|
2284
2390
|
const imports = [];
|
|
2285
2391
|
const draftSymbols = [];
|
|
@@ -2533,6 +2639,181 @@ function javaCodeAnalysis(manifest, rootNode, diagnostics) {
|
|
|
2533
2639
|
namespace: packageName
|
|
2534
2640
|
});
|
|
2535
2641
|
}
|
|
2642
|
+
function kotlinCodeAnalysis(manifest, rootNode, diagnostics) {
|
|
2643
|
+
const imports = [];
|
|
2644
|
+
const draftSymbols = [];
|
|
2645
|
+
const exportLabels = [];
|
|
2646
|
+
let packageName;
|
|
2647
|
+
const pushBodyFunctions = (bodyNode, scopeName) => {
|
|
2648
|
+
if (!bodyNode) {
|
|
2649
|
+
return;
|
|
2650
|
+
}
|
|
2651
|
+
for (const child of bodyNode.namedChildren) {
|
|
2652
|
+
if (!child || child.type !== "function_declaration") {
|
|
2653
|
+
continue;
|
|
2654
|
+
}
|
|
2655
|
+
const functionName = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "simple_identifier"));
|
|
2656
|
+
if (!functionName) {
|
|
2657
|
+
continue;
|
|
2658
|
+
}
|
|
2659
|
+
const exported = declarationVisible(child, ["private", "internal", "protected"]);
|
|
2660
|
+
const symbolName = scopeName ? `${scopeName}.${functionName}` : functionName;
|
|
2661
|
+
draftSymbols.push({
|
|
2662
|
+
name: symbolName,
|
|
2663
|
+
kind: "function",
|
|
2664
|
+
signature: singleLineSignature(child.text),
|
|
2665
|
+
exported,
|
|
2666
|
+
callNames: [],
|
|
2667
|
+
extendsNames: [],
|
|
2668
|
+
implementsNames: [],
|
|
2669
|
+
bodyText: nodeText(child.childForFieldName("body") ?? findNamedChild(child, "function_body"))
|
|
2670
|
+
});
|
|
2671
|
+
if (exported) {
|
|
2672
|
+
exportLabels.push(symbolName);
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
};
|
|
2676
|
+
for (const child of rootNode.namedChildren) {
|
|
2677
|
+
if (!child) {
|
|
2678
|
+
continue;
|
|
2679
|
+
}
|
|
2680
|
+
if (child.type === "package_header") {
|
|
2681
|
+
packageName = nodeText(findNamedChild(child, "identifier") ?? child.namedChildren.at(0) ?? null) || packageName;
|
|
2682
|
+
continue;
|
|
2683
|
+
}
|
|
2684
|
+
if (child.type === "import_list") {
|
|
2685
|
+
for (const importNode of child.descendantsOfType("import_header").filter((item) => item !== null)) {
|
|
2686
|
+
const parsed = parseKotlinImport(importNode.text);
|
|
2687
|
+
if (parsed) {
|
|
2688
|
+
imports.push(parsed);
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2691
|
+
continue;
|
|
2692
|
+
}
|
|
2693
|
+
if (child.type === "function_declaration") {
|
|
2694
|
+
pushBodyFunctions({
|
|
2695
|
+
...child,
|
|
2696
|
+
namedChildren: [child]
|
|
2697
|
+
});
|
|
2698
|
+
continue;
|
|
2699
|
+
}
|
|
2700
|
+
if (child.type !== "class_declaration" && child.type !== "object_declaration") {
|
|
2701
|
+
continue;
|
|
2702
|
+
}
|
|
2703
|
+
const name = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "type_identifier"));
|
|
2704
|
+
if (!name) {
|
|
2705
|
+
continue;
|
|
2706
|
+
}
|
|
2707
|
+
const kind = child.type === "object_declaration" ? "class" : kotlinClassKind(child.text);
|
|
2708
|
+
const delegationNames = uniqueBy(
|
|
2709
|
+
child.namedChildren.filter((item) => item !== null && item.type === "delegation_specifier").flatMap((item) => descendantTypeNames(item)),
|
|
2710
|
+
(item) => item
|
|
2711
|
+
);
|
|
2712
|
+
const exported = declarationVisible(child, ["private", "internal"]);
|
|
2713
|
+
const bodyNode = findNamedChild(child, "class_body") ?? child.childForFieldName("body");
|
|
2714
|
+
draftSymbols.push({
|
|
2715
|
+
name,
|
|
2716
|
+
kind,
|
|
2717
|
+
signature: singleLineSignature(child.text),
|
|
2718
|
+
exported,
|
|
2719
|
+
callNames: [],
|
|
2720
|
+
extendsNames: kind === "interface" ? delegationNames : delegationNames.slice(0, 1),
|
|
2721
|
+
implementsNames: kind === "class" ? delegationNames.slice(1) : [],
|
|
2722
|
+
bodyText: nodeText(bodyNode) || child.text
|
|
2723
|
+
});
|
|
2724
|
+
if (exported) {
|
|
2725
|
+
exportLabels.push(name);
|
|
2726
|
+
}
|
|
2727
|
+
pushBodyFunctions(bodyNode, name);
|
|
2728
|
+
}
|
|
2729
|
+
return finalizeCodeAnalysis(manifest, "kotlin", imports, draftSymbols, exportLabels, diagnostics, {
|
|
2730
|
+
namespace: packageName
|
|
2731
|
+
});
|
|
2732
|
+
}
|
|
2733
|
+
function scalaCodeAnalysis(manifest, rootNode, diagnostics) {
|
|
2734
|
+
const imports = [];
|
|
2735
|
+
const draftSymbols = [];
|
|
2736
|
+
const exportLabels = [];
|
|
2737
|
+
let packageName;
|
|
2738
|
+
const pushTemplateFunctions = (bodyNode, scopeName) => {
|
|
2739
|
+
if (!bodyNode) {
|
|
2740
|
+
return;
|
|
2741
|
+
}
|
|
2742
|
+
for (const child of bodyNode.namedChildren) {
|
|
2743
|
+
if (!child || child.type !== "function_definition") {
|
|
2744
|
+
continue;
|
|
2745
|
+
}
|
|
2746
|
+
const functionName = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "identifier"));
|
|
2747
|
+
if (!functionName) {
|
|
2748
|
+
continue;
|
|
2749
|
+
}
|
|
2750
|
+
const exported = declarationVisible(child, ["private", "protected"]);
|
|
2751
|
+
const symbolName = scopeName ? `${scopeName}.${functionName}` : functionName;
|
|
2752
|
+
draftSymbols.push({
|
|
2753
|
+
name: symbolName,
|
|
2754
|
+
kind: "function",
|
|
2755
|
+
signature: singleLineSignature(child.text),
|
|
2756
|
+
exported,
|
|
2757
|
+
callNames: [],
|
|
2758
|
+
extendsNames: [],
|
|
2759
|
+
implementsNames: [],
|
|
2760
|
+
bodyText: child.text
|
|
2761
|
+
});
|
|
2762
|
+
if (exported) {
|
|
2763
|
+
exportLabels.push(symbolName);
|
|
2764
|
+
}
|
|
2765
|
+
}
|
|
2766
|
+
};
|
|
2767
|
+
for (const child of rootNode.namedChildren) {
|
|
2768
|
+
if (!child) {
|
|
2769
|
+
continue;
|
|
2770
|
+
}
|
|
2771
|
+
if (child.type === "package_clause") {
|
|
2772
|
+
packageName = nodeText(findNamedChild(child, "package_identifier") ?? child.namedChildren.at(0) ?? null) || packageName;
|
|
2773
|
+
continue;
|
|
2774
|
+
}
|
|
2775
|
+
if (child.type === "import_declaration") {
|
|
2776
|
+
imports.push(...parseScalaImport(child.text));
|
|
2777
|
+
continue;
|
|
2778
|
+
}
|
|
2779
|
+
if (child.type === "function_definition") {
|
|
2780
|
+
pushTemplateFunctions({
|
|
2781
|
+
...child,
|
|
2782
|
+
namedChildren: [child]
|
|
2783
|
+
});
|
|
2784
|
+
continue;
|
|
2785
|
+
}
|
|
2786
|
+
if (!["trait_definition", "class_definition", "object_definition"].includes(child.type)) {
|
|
2787
|
+
continue;
|
|
2788
|
+
}
|
|
2789
|
+
const name = extractIdentifier(child.childForFieldName("name") ?? findNamedChild(child, "identifier"));
|
|
2790
|
+
const kind = scalaDefinitionKind(child);
|
|
2791
|
+
if (!name || !kind) {
|
|
2792
|
+
continue;
|
|
2793
|
+
}
|
|
2794
|
+
const extendsClause = findNamedChild(child, "extends_clause") ?? child.childForFieldName("extends");
|
|
2795
|
+
const inheritance = uniqueBy(descendantTypeNames(extendsClause), (item) => item);
|
|
2796
|
+
const bodyNode = findNamedChild(child, "template_body") ?? child.childForFieldName("body");
|
|
2797
|
+
const exported = declarationVisible(child, ["private", "protected"]);
|
|
2798
|
+
draftSymbols.push({
|
|
2799
|
+
name,
|
|
2800
|
+
kind,
|
|
2801
|
+
signature: singleLineSignature(child.text),
|
|
2802
|
+
exported,
|
|
2803
|
+
callNames: [],
|
|
2804
|
+
extendsNames: kind === "trait" ? inheritance : inheritance.slice(0, 1),
|
|
2805
|
+
implementsNames: kind === "class" ? inheritance.slice(1) : [],
|
|
2806
|
+
bodyText: nodeText(bodyNode) || child.text
|
|
2807
|
+
});
|
|
2808
|
+
if (exported) {
|
|
2809
|
+
exportLabels.push(name);
|
|
2810
|
+
}
|
|
2811
|
+
pushTemplateFunctions(bodyNode, name);
|
|
2812
|
+
}
|
|
2813
|
+
return finalizeCodeAnalysis(manifest, "scala", imports, draftSymbols, exportLabels, diagnostics, {
|
|
2814
|
+
namespace: packageName
|
|
2815
|
+
});
|
|
2816
|
+
}
|
|
2536
2817
|
function csharpCodeAnalysis(manifest, rootNode, diagnostics) {
|
|
2537
2818
|
const imports = [];
|
|
2538
2819
|
const draftSymbols = [];
|
|
@@ -2915,11 +3196,19 @@ function cFamilyCodeAnalysis(manifest, language, rootNode, diagnostics) {
|
|
|
2915
3196
|
return finalizeCodeAnalysis(manifest, language, imports, draftSymbols, exportLabels, diagnostics);
|
|
2916
3197
|
}
|
|
2917
3198
|
async function analyzeTreeSitterCode(manifest, content, language) {
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
3199
|
+
let tree = null;
|
|
3200
|
+
try {
|
|
3201
|
+
const module = await getTreeSitterModule();
|
|
3202
|
+
await ensureTreeSitterInit(module);
|
|
3203
|
+
const parser = new module.Parser();
|
|
3204
|
+
parser.setLanguage(await loadLanguage(language));
|
|
3205
|
+
tree = parser.parse(content);
|
|
3206
|
+
} catch (error) {
|
|
3207
|
+
return {
|
|
3208
|
+
code: finalizeCodeAnalysis(manifest, language, [], [], [], [treeSitterCompatibilityDiagnostic(language, error)]),
|
|
3209
|
+
rationales: []
|
|
3210
|
+
};
|
|
3211
|
+
}
|
|
2923
3212
|
if (!tree) {
|
|
2924
3213
|
return {
|
|
2925
3214
|
code: finalizeCodeAnalysis(
|
|
@@ -2953,6 +3242,10 @@ async function analyzeTreeSitterCode(manifest, content, language) {
|
|
|
2953
3242
|
return { code: rustCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
|
|
2954
3243
|
case "java":
|
|
2955
3244
|
return { code: javaCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
|
|
3245
|
+
case "kotlin":
|
|
3246
|
+
return { code: kotlinCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
|
|
3247
|
+
case "scala":
|
|
3248
|
+
return { code: scalaCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
|
|
2956
3249
|
case "csharp":
|
|
2957
3250
|
return { code: csharpCodeAnalysis(manifest, tree.rootNode, diagnostics), rationales };
|
|
2958
3251
|
case "php":
|
|
@@ -2964,6 +3257,26 @@ async function analyzeTreeSitterCode(manifest, content, language) {
|
|
|
2964
3257
|
case "c":
|
|
2965
3258
|
case "cpp":
|
|
2966
3259
|
return { code: cFamilyCodeAnalysis(manifest, language, tree.rootNode, diagnostics), rationales };
|
|
3260
|
+
default:
|
|
3261
|
+
return {
|
|
3262
|
+
code: finalizeCodeAnalysis(
|
|
3263
|
+
manifest,
|
|
3264
|
+
language,
|
|
3265
|
+
[],
|
|
3266
|
+
[],
|
|
3267
|
+
[],
|
|
3268
|
+
[
|
|
3269
|
+
{
|
|
3270
|
+
code: 9011,
|
|
3271
|
+
category: "error",
|
|
3272
|
+
message: `No parser-backed analyzer is registered for ${language}.`,
|
|
3273
|
+
line: 1,
|
|
3274
|
+
column: 1
|
|
3275
|
+
}
|
|
3276
|
+
]
|
|
3277
|
+
),
|
|
3278
|
+
rationales
|
|
3279
|
+
};
|
|
2967
3280
|
}
|
|
2968
3281
|
} finally {
|
|
2969
3282
|
tree.delete();
|
|
@@ -3219,7 +3532,7 @@ function makeRationale2(manifest, index, text, kind, symbolName) {
|
|
|
3219
3532
|
};
|
|
3220
3533
|
}
|
|
3221
3534
|
function stripCodeExtension2(filePath) {
|
|
3222
|
-
return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
|
|
3535
|
+
return filePath.replace(/\.(?:[cm]?jsx?|tsx?|mts|cts|py|go|rs|java|kt|kts|scala|sc|cs|php|c|cc|cpp|cxx|h|hh|hpp|hxx)$/i, "");
|
|
3223
3536
|
}
|
|
3224
3537
|
function manifestModuleName2(manifest, language) {
|
|
3225
3538
|
const repoPath = manifest.repoRelativePath ?? path6.basename(manifest.originalPath ?? manifest.storedPath);
|
|
@@ -3550,6 +3863,12 @@ function inferCodeLanguage(filePath, mimeType = "") {
|
|
|
3550
3863
|
if (extension === ".java") {
|
|
3551
3864
|
return "java";
|
|
3552
3865
|
}
|
|
3866
|
+
if (extension === ".kt" || extension === ".kts") {
|
|
3867
|
+
return "kotlin";
|
|
3868
|
+
}
|
|
3869
|
+
if (extension === ".scala" || extension === ".sc") {
|
|
3870
|
+
return "scala";
|
|
3871
|
+
}
|
|
3553
3872
|
if (extension === ".cs") {
|
|
3554
3873
|
return "csharp";
|
|
3555
3874
|
}
|
|
@@ -3654,6 +3973,10 @@ function candidateExtensionsFor(language) {
|
|
|
3654
3973
|
return [".rs"];
|
|
3655
3974
|
case "java":
|
|
3656
3975
|
return [".java"];
|
|
3976
|
+
case "kotlin":
|
|
3977
|
+
return [".kt", ".kts"];
|
|
3978
|
+
case "scala":
|
|
3979
|
+
return [".scala", ".sc"];
|
|
3657
3980
|
case "csharp":
|
|
3658
3981
|
return [".cs"];
|
|
3659
3982
|
case "php":
|
|
@@ -3714,10 +4037,17 @@ async function buildCodeIndex(rootDir, manifests, analyses) {
|
|
|
3714
4037
|
break;
|
|
3715
4038
|
}
|
|
3716
4039
|
case "java":
|
|
4040
|
+
case "kotlin":
|
|
4041
|
+
case "scala":
|
|
3717
4042
|
case "csharp":
|
|
3718
4043
|
if (normalizedNamespace) {
|
|
3719
4044
|
recordAlias(aliases, `${normalizedNamespace}.${basename}`);
|
|
3720
4045
|
}
|
|
4046
|
+
if (normalizedNamespace) {
|
|
4047
|
+
for (const symbol of analysis.code.symbols) {
|
|
4048
|
+
recordAlias(aliases, `${normalizedNamespace}.${symbol.name}`);
|
|
4049
|
+
}
|
|
4050
|
+
}
|
|
3721
4051
|
break;
|
|
3722
4052
|
case "php":
|
|
3723
4053
|
if (normalizedNamespace) {
|
|
@@ -3824,6 +4154,8 @@ function findImportCandidates(manifest, codeImport, lookup) {
|
|
|
3824
4154
|
return aliasMatches(lookup, codeImport.specifier);
|
|
3825
4155
|
case "go":
|
|
3826
4156
|
case "java":
|
|
4157
|
+
case "kotlin":
|
|
4158
|
+
case "scala":
|
|
3827
4159
|
case "csharp":
|
|
3828
4160
|
return aliasMatches(lookup, codeImport.specifier);
|
|
3829
4161
|
case "php":
|
|
@@ -3870,6 +4202,8 @@ function importLooksLocal(manifest, codeImport, candidates) {
|
|
|
3870
4202
|
case "powershell":
|
|
3871
4203
|
case "c":
|
|
3872
4204
|
case "cpp":
|
|
4205
|
+
case "kotlin":
|
|
4206
|
+
case "scala":
|
|
3873
4207
|
return !codeImport.isExternal;
|
|
3874
4208
|
default:
|
|
3875
4209
|
return false;
|
|
@@ -4390,9 +4724,123 @@ function aggregateManifestSourceClass(manifests, sourceIds) {
|
|
|
4390
4724
|
return aggregateSourceClass(sourceIds.map((sourceId) => byId.get(sourceId)));
|
|
4391
4725
|
}
|
|
4392
4726
|
|
|
4393
|
-
// src/
|
|
4727
|
+
// src/source-registry.ts
|
|
4394
4728
|
import fs9 from "fs/promises";
|
|
4395
4729
|
import path10 from "path";
|
|
4730
|
+
var MANAGED_SOURCES_VERSION = 1;
|
|
4731
|
+
function repoRootFromManifest(manifest) {
|
|
4732
|
+
if (manifest.originType !== "file" || !manifest.originalPath || !manifest.repoRelativePath) {
|
|
4733
|
+
return null;
|
|
4734
|
+
}
|
|
4735
|
+
const repoDir = path10.posix.dirname(manifest.repoRelativePath);
|
|
4736
|
+
const fileDir = path10.dirname(path10.resolve(manifest.originalPath));
|
|
4737
|
+
if (repoDir === "." || !repoDir) {
|
|
4738
|
+
return fileDir;
|
|
4739
|
+
}
|
|
4740
|
+
const segments = repoDir.split("/").filter(Boolean);
|
|
4741
|
+
return path10.resolve(fileDir, ...segments.map(() => ".."));
|
|
4742
|
+
}
|
|
4743
|
+
async function loadManifestArtifacts(paths) {
|
|
4744
|
+
const entries = await fs9.readdir(paths.manifestsDir, { withFileTypes: true }).catch(() => []);
|
|
4745
|
+
const manifests = await Promise.all(
|
|
4746
|
+
entries.filter((entry) => entry.isFile() && entry.name.endsWith(".json")).map(async (entry) => await readJsonFile(path10.join(paths.manifestsDir, entry.name)))
|
|
4747
|
+
);
|
|
4748
|
+
return manifests.filter((manifest) => Boolean(manifest?.sourceId));
|
|
4749
|
+
}
|
|
4750
|
+
function buildLegacyDirectoryEntry(repoRoot, manifests) {
|
|
4751
|
+
const repoTitle = path10.basename(repoRoot) || repoRoot;
|
|
4752
|
+
const sourceIds = manifests.map((manifest) => manifest.sourceId).sort((left, right) => left.localeCompare(right));
|
|
4753
|
+
const createdAt = manifests.map((manifest) => manifest.createdAt).sort((left, right) => left.localeCompare(right))[0] ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
4754
|
+
const updatedAt = manifests.map((manifest) => manifest.updatedAt).sort((left, right) => right.localeCompare(left))[0] ?? createdAt;
|
|
4755
|
+
return {
|
|
4756
|
+
id: `directory-${slugify(repoTitle)}-${sha256(repoRoot).slice(0, 8)}`,
|
|
4757
|
+
kind: "directory",
|
|
4758
|
+
title: repoTitle,
|
|
4759
|
+
path: repoRoot,
|
|
4760
|
+
repoRoot,
|
|
4761
|
+
createdAt,
|
|
4762
|
+
updatedAt,
|
|
4763
|
+
status: "ready",
|
|
4764
|
+
sourceIds,
|
|
4765
|
+
lastSyncAt: updatedAt,
|
|
4766
|
+
lastSyncStatus: "success",
|
|
4767
|
+
lastSyncCounts: {
|
|
4768
|
+
scannedCount: manifests.length,
|
|
4769
|
+
importedCount: manifests.length,
|
|
4770
|
+
updatedCount: 0,
|
|
4771
|
+
removedCount: 0,
|
|
4772
|
+
skippedCount: 0
|
|
4773
|
+
}
|
|
4774
|
+
};
|
|
4775
|
+
}
|
|
4776
|
+
async function buildLegacyArtifact(paths) {
|
|
4777
|
+
const manifests = await loadManifestArtifacts(paths);
|
|
4778
|
+
const manifestsByRepoRoot = /* @__PURE__ */ new Map();
|
|
4779
|
+
for (const manifest of manifests) {
|
|
4780
|
+
const repoRoot = repoRootFromManifest(manifest);
|
|
4781
|
+
if (!repoRoot) {
|
|
4782
|
+
continue;
|
|
4783
|
+
}
|
|
4784
|
+
const key = path10.resolve(repoRoot);
|
|
4785
|
+
const bucket = manifestsByRepoRoot.get(key) ?? [];
|
|
4786
|
+
bucket.push(manifest);
|
|
4787
|
+
manifestsByRepoRoot.set(key, bucket);
|
|
4788
|
+
}
|
|
4789
|
+
const repoRoots = [...manifestsByRepoRoot.entries()].filter(([, repoManifests]) => {
|
|
4790
|
+
return repoManifests.length > 1 || repoManifests.some((manifest) => manifest.sourceKind === "code");
|
|
4791
|
+
});
|
|
4792
|
+
return {
|
|
4793
|
+
version: MANAGED_SOURCES_VERSION,
|
|
4794
|
+
sources: repoRoots.sort((left, right) => left[0].localeCompare(right[0])).map(([repoRoot, repoManifests]) => buildLegacyDirectoryEntry(repoRoot, repoManifests))
|
|
4795
|
+
};
|
|
4796
|
+
}
|
|
4797
|
+
async function ensureManagedSourcesArtifact(rootDir) {
|
|
4798
|
+
const { paths } = await initWorkspace(rootDir);
|
|
4799
|
+
if (await fileExists(paths.managedSourcesPath)) {
|
|
4800
|
+
const existing = await readJsonFile(paths.managedSourcesPath);
|
|
4801
|
+
if (existing?.version === MANAGED_SOURCES_VERSION && Array.isArray(existing.sources)) {
|
|
4802
|
+
return {
|
|
4803
|
+
version: MANAGED_SOURCES_VERSION,
|
|
4804
|
+
sources: [...existing.sources].sort(
|
|
4805
|
+
(left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id)
|
|
4806
|
+
)
|
|
4807
|
+
};
|
|
4808
|
+
}
|
|
4809
|
+
}
|
|
4810
|
+
const artifact = await buildLegacyArtifact(paths);
|
|
4811
|
+
await writeJsonFile(paths.managedSourcesPath, artifact);
|
|
4812
|
+
return artifact;
|
|
4813
|
+
}
|
|
4814
|
+
async function loadManagedSources(rootDir) {
|
|
4815
|
+
const artifact = await ensureManagedSourcesArtifact(rootDir);
|
|
4816
|
+
return artifact.sources;
|
|
4817
|
+
}
|
|
4818
|
+
async function readManagedSourcesIfPresent(rootDir) {
|
|
4819
|
+
const { paths } = await initWorkspace(rootDir);
|
|
4820
|
+
if (!await fileExists(paths.managedSourcesPath)) {
|
|
4821
|
+
return null;
|
|
4822
|
+
}
|
|
4823
|
+
const existing = await readJsonFile(paths.managedSourcesPath);
|
|
4824
|
+
if (!existing?.version || !Array.isArray(existing.sources)) {
|
|
4825
|
+
return null;
|
|
4826
|
+
}
|
|
4827
|
+
return existing.sources;
|
|
4828
|
+
}
|
|
4829
|
+
async function saveManagedSources(rootDir, sources) {
|
|
4830
|
+
const { paths } = await initWorkspace(rootDir);
|
|
4831
|
+
await writeJsonFile(paths.managedSourcesPath, {
|
|
4832
|
+
version: MANAGED_SOURCES_VERSION,
|
|
4833
|
+
sources: [...sources].sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id))
|
|
4834
|
+
});
|
|
4835
|
+
}
|
|
4836
|
+
async function managedSourceWorkingDir(rootDir, sourceId) {
|
|
4837
|
+
const { paths } = await initWorkspace(rootDir);
|
|
4838
|
+
return path10.join(paths.managedSourcesDir, sourceId);
|
|
4839
|
+
}
|
|
4840
|
+
|
|
4841
|
+
// src/watch-state.ts
|
|
4842
|
+
import fs10 from "fs/promises";
|
|
4843
|
+
import path11 from "path";
|
|
4396
4844
|
import matter2 from "gray-matter";
|
|
4397
4845
|
function pendingEntryKey(entry) {
|
|
4398
4846
|
return entry.path;
|
|
@@ -4406,7 +4854,7 @@ function normalizeRelativePath(rootDir, filePath) {
|
|
|
4406
4854
|
if (!filePath) {
|
|
4407
4855
|
return void 0;
|
|
4408
4856
|
}
|
|
4409
|
-
return toPosix(
|
|
4857
|
+
return toPosix(path11.relative(rootDir, path11.resolve(filePath)));
|
|
4410
4858
|
}
|
|
4411
4859
|
async function readPendingSemanticRefresh(rootDir) {
|
|
4412
4860
|
const { paths } = await initWorkspace(rootDir);
|
|
@@ -4500,11 +4948,11 @@ async function markPagesStaleForSources(rootDir, sourceIds) {
|
|
|
4500
4948
|
if (page.freshness !== "stale" || !page.sourceIds.some((sourceId) => affectedSourceIds.has(sourceId))) {
|
|
4501
4949
|
continue;
|
|
4502
4950
|
}
|
|
4503
|
-
const absolutePath =
|
|
4951
|
+
const absolutePath = path11.join(paths.wikiDir, page.path);
|
|
4504
4952
|
if (!await fileExists(absolutePath)) {
|
|
4505
4953
|
continue;
|
|
4506
4954
|
}
|
|
4507
|
-
const raw = await
|
|
4955
|
+
const raw = await fs10.readFile(absolutePath, "utf8");
|
|
4508
4956
|
const parsed = matter2(raw);
|
|
4509
4957
|
if (parsed.data.freshness === "stale") {
|
|
4510
4958
|
continue;
|
|
@@ -4520,6 +4968,9 @@ async function markPagesStaleForSources(rootDir, sourceIds) {
|
|
|
4520
4968
|
var DEFAULT_MAX_ASSET_SIZE = 10 * 1024 * 1024;
|
|
4521
4969
|
var DEFAULT_MAX_DIRECTORY_FILES = 5e3;
|
|
4522
4970
|
var HARD_REPO_IGNORES = /* @__PURE__ */ new Set([".git", ".venv"]);
|
|
4971
|
+
var PROGRESS_FILE_THRESHOLD = 150;
|
|
4972
|
+
var PROGRESS_UPDATE_INTERVAL = 100;
|
|
4973
|
+
var RST_HEADING_MARKERS = /* @__PURE__ */ new Set(["=", "-", "~", "^", '"', "#", "*", "+"]);
|
|
4523
4974
|
function uniqueStrings(values) {
|
|
4524
4975
|
return [...new Set(values.filter(Boolean))];
|
|
4525
4976
|
}
|
|
@@ -4527,6 +4978,9 @@ function inferKind(mimeType, filePath) {
|
|
|
4527
4978
|
if (inferCodeLanguage(filePath, mimeType)) {
|
|
4528
4979
|
return "code";
|
|
4529
4980
|
}
|
|
4981
|
+
if (isRstFilePath(filePath)) {
|
|
4982
|
+
return "text";
|
|
4983
|
+
}
|
|
4530
4984
|
if (mimeType.includes("markdown")) {
|
|
4531
4985
|
return "markdown";
|
|
4532
4986
|
}
|
|
@@ -4547,25 +5001,158 @@ function inferKind(mimeType, filePath) {
|
|
|
4547
5001
|
}
|
|
4548
5002
|
return "binary";
|
|
4549
5003
|
}
|
|
4550
|
-
function
|
|
5004
|
+
function isRstFilePath(filePath) {
|
|
5005
|
+
const extension = path12.extname(filePath).toLowerCase();
|
|
5006
|
+
return extension === ".rst" || extension === ".rest";
|
|
5007
|
+
}
|
|
5008
|
+
function titleFromText(fallback, content, filePath) {
|
|
5009
|
+
if (filePath && isRstFilePath(filePath)) {
|
|
5010
|
+
const rstTitle = titleFromRst(fallback, content);
|
|
5011
|
+
if (rstTitle) {
|
|
5012
|
+
return rstTitle;
|
|
5013
|
+
}
|
|
5014
|
+
}
|
|
4551
5015
|
const heading = content.match(/^#\s+(.+)$/m)?.[1]?.trim();
|
|
4552
5016
|
return heading || fallback;
|
|
4553
5017
|
}
|
|
4554
5018
|
function guessMimeType(target) {
|
|
5019
|
+
if (isRstFilePath(target)) {
|
|
5020
|
+
return "text/x-rst";
|
|
5021
|
+
}
|
|
4555
5022
|
return mime.lookup(target) || "application/octet-stream";
|
|
4556
5023
|
}
|
|
4557
|
-
function
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
5024
|
+
function rstAdornmentLine(line) {
|
|
5025
|
+
const trimmed = line.trim();
|
|
5026
|
+
if (trimmed.length < 3) {
|
|
5027
|
+
return void 0;
|
|
5028
|
+
}
|
|
5029
|
+
const marker = trimmed[0] ?? "";
|
|
5030
|
+
if (!RST_HEADING_MARKERS.has(marker) || ![...trimmed].every((char) => char === marker)) {
|
|
5031
|
+
return void 0;
|
|
5032
|
+
}
|
|
5033
|
+
return marker;
|
|
5034
|
+
}
|
|
5035
|
+
function rstHeadingLevel(marker) {
|
|
5036
|
+
switch (marker) {
|
|
5037
|
+
case "=":
|
|
5038
|
+
return "#";
|
|
5039
|
+
case "-":
|
|
5040
|
+
return "##";
|
|
5041
|
+
case "~":
|
|
5042
|
+
return "###";
|
|
5043
|
+
case "^":
|
|
5044
|
+
return "####";
|
|
5045
|
+
default:
|
|
5046
|
+
return "##";
|
|
5047
|
+
}
|
|
5048
|
+
}
|
|
5049
|
+
function normalizeRstDirective(line) {
|
|
5050
|
+
const trimmed = line.trim();
|
|
5051
|
+
if (!trimmed.startsWith(".. ")) {
|
|
5052
|
+
return void 0;
|
|
5053
|
+
}
|
|
5054
|
+
const directiveMatch = trimmed.match(/^\.\.\s+([A-Za-z][\w-]*)::\s*(.*)$/);
|
|
5055
|
+
if (!directiveMatch) {
|
|
5056
|
+
return "";
|
|
5057
|
+
}
|
|
5058
|
+
const label = directiveMatch[1].replace(/-/g, " ").split(/\s+/).filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(" ");
|
|
5059
|
+
const detail = directiveMatch[2]?.trim();
|
|
5060
|
+
return detail ? `${label}: ${detail}` : `${label}:`;
|
|
5061
|
+
}
|
|
5062
|
+
function normalizeRstExtractedText(content) {
|
|
5063
|
+
const lines = content.split(/\r?\n/);
|
|
5064
|
+
const normalized = [];
|
|
5065
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
5066
|
+
const current = lines[index] ?? "";
|
|
5067
|
+
const next = lines[index + 1] ?? "";
|
|
5068
|
+
const afterNext = lines[index + 2] ?? "";
|
|
5069
|
+
const trimmed = current.trim();
|
|
5070
|
+
if (!trimmed) {
|
|
5071
|
+
normalized.push("");
|
|
5072
|
+
continue;
|
|
5073
|
+
}
|
|
5074
|
+
const currentAdornment = rstAdornmentLine(current);
|
|
5075
|
+
const nextAdornment = rstAdornmentLine(next);
|
|
5076
|
+
const afterNextAdornment = rstAdornmentLine(afterNext);
|
|
5077
|
+
if (currentAdornment && next.trim() && afterNextAdornment && currentAdornment === afterNextAdornment) {
|
|
5078
|
+
normalized.push(`${rstHeadingLevel(currentAdornment)} ${next.trim()}`);
|
|
5079
|
+
normalized.push("");
|
|
5080
|
+
index += 2;
|
|
5081
|
+
continue;
|
|
5082
|
+
}
|
|
5083
|
+
if (nextAdornment && trimmed.length > 0) {
|
|
5084
|
+
normalized.push(`${rstHeadingLevel(nextAdornment)} ${trimmed}`);
|
|
5085
|
+
normalized.push("");
|
|
5086
|
+
index += 1;
|
|
5087
|
+
continue;
|
|
5088
|
+
}
|
|
5089
|
+
const directive = normalizeRstDirective(current);
|
|
5090
|
+
if (directive !== void 0) {
|
|
5091
|
+
if (directive) {
|
|
5092
|
+
normalized.push(directive);
|
|
5093
|
+
}
|
|
5094
|
+
continue;
|
|
5095
|
+
}
|
|
5096
|
+
normalized.push(current);
|
|
5097
|
+
}
|
|
5098
|
+
return normalized.join("\n").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
5099
|
+
}
|
|
5100
|
+
function titleFromRst(fallback, content) {
|
|
5101
|
+
const normalized = normalizeRstExtractedText(content);
|
|
5102
|
+
const heading = normalized.match(/^#+\s+(.+)$/m)?.[1]?.trim();
|
|
5103
|
+
return heading || fallback;
|
|
5104
|
+
}
|
|
5105
|
+
function extractedTextForPlainSource(filePath, sourceKind, content) {
|
|
5106
|
+
if (sourceKind === "text" && isRstFilePath(filePath)) {
|
|
5107
|
+
return normalizeRstExtractedText(content);
|
|
5108
|
+
}
|
|
5109
|
+
return content;
|
|
5110
|
+
}
|
|
5111
|
+
function shouldEmitProgress(totalItems) {
|
|
5112
|
+
return totalItems >= PROGRESS_FILE_THRESHOLD && Boolean(process.stderr?.isTTY);
|
|
5113
|
+
}
|
|
5114
|
+
function createProgressReporter(prefix, totalItems) {
|
|
5115
|
+
if (!shouldEmitProgress(totalItems)) {
|
|
5116
|
+
return {
|
|
5117
|
+
tick: () => {
|
|
5118
|
+
},
|
|
5119
|
+
finish: () => {
|
|
5120
|
+
}
|
|
5121
|
+
};
|
|
5122
|
+
}
|
|
5123
|
+
let completed = 0;
|
|
5124
|
+
let nextUpdate = Math.min(PROGRESS_UPDATE_INTERVAL, totalItems);
|
|
5125
|
+
process.stderr.write(`[swarmvault ${prefix}] starting ${totalItems} file(s)
|
|
5126
|
+
`);
|
|
5127
|
+
return {
|
|
5128
|
+
tick: () => {
|
|
5129
|
+
completed += 1;
|
|
5130
|
+
if (completed >= nextUpdate || completed === totalItems) {
|
|
5131
|
+
process.stderr.write(`[swarmvault ${prefix}] ${completed}/${totalItems}
|
|
5132
|
+
`);
|
|
5133
|
+
while (completed >= nextUpdate) {
|
|
5134
|
+
nextUpdate += PROGRESS_UPDATE_INTERVAL;
|
|
5135
|
+
}
|
|
5136
|
+
}
|
|
5137
|
+
},
|
|
5138
|
+
finish: (summary) => {
|
|
5139
|
+
process.stderr.write(`[swarmvault ${prefix}] finished ${totalItems} file(s)${summary ? ` (${summary})` : ""}
|
|
5140
|
+
`);
|
|
5141
|
+
}
|
|
5142
|
+
};
|
|
5143
|
+
}
|
|
5144
|
+
function normalizeIngestOptions(options) {
|
|
5145
|
+
return {
|
|
5146
|
+
includeAssets: options?.includeAssets ?? true,
|
|
5147
|
+
maxAssetSize: Math.max(0, Math.floor(options?.maxAssetSize ?? DEFAULT_MAX_ASSET_SIZE)),
|
|
5148
|
+
repoRoot: options?.repoRoot ? path12.resolve(options.repoRoot) : void 0,
|
|
5149
|
+
include: (options?.include ?? []).map((pattern) => pattern.trim()).filter(Boolean),
|
|
5150
|
+
exclude: (options?.exclude ?? []).map((pattern) => pattern.trim()).filter(Boolean),
|
|
5151
|
+
maxFiles: Math.max(1, Math.floor(options?.maxFiles ?? DEFAULT_MAX_DIRECTORY_FILES)),
|
|
5152
|
+
gitignore: options?.gitignore ?? true,
|
|
5153
|
+
extractClasses: options?.extractClasses ?? ["first_party"]
|
|
5154
|
+
};
|
|
5155
|
+
}
|
|
4569
5156
|
async function resolveRepoIngestOptions(rootDir, options) {
|
|
4570
5157
|
const normalized = normalizeIngestOptions(options);
|
|
4571
5158
|
const { config } = await loadVaultConfig(rootDir);
|
|
@@ -4578,27 +5165,27 @@ async function resolveRepoIngestOptions(rootDir, options) {
|
|
|
4578
5165
|
}
|
|
4579
5166
|
function matchesAnyGlob2(relativePath, patterns) {
|
|
4580
5167
|
return patterns.some(
|
|
4581
|
-
(pattern) =>
|
|
5168
|
+
(pattern) => path12.matchesGlob(relativePath, pattern) || path12.matchesGlob(path12.posix.basename(relativePath), pattern)
|
|
4582
5169
|
);
|
|
4583
5170
|
}
|
|
4584
5171
|
function supportedDirectoryKind(sourceKind) {
|
|
4585
5172
|
return sourceKind !== "binary";
|
|
4586
5173
|
}
|
|
4587
5174
|
async function findNearestGitRoot2(startPath) {
|
|
4588
|
-
let current =
|
|
5175
|
+
let current = path12.resolve(startPath);
|
|
4589
5176
|
try {
|
|
4590
|
-
const stat = await
|
|
5177
|
+
const stat = await fs11.stat(current);
|
|
4591
5178
|
if (!stat.isDirectory()) {
|
|
4592
|
-
current =
|
|
5179
|
+
current = path12.dirname(current);
|
|
4593
5180
|
}
|
|
4594
5181
|
} catch {
|
|
4595
|
-
current =
|
|
5182
|
+
current = path12.dirname(current);
|
|
4596
5183
|
}
|
|
4597
5184
|
while (true) {
|
|
4598
|
-
if (await fileExists(
|
|
5185
|
+
if (await fileExists(path12.join(current, ".git"))) {
|
|
4599
5186
|
return current;
|
|
4600
5187
|
}
|
|
4601
|
-
const parent =
|
|
5188
|
+
const parent = path12.dirname(current);
|
|
4602
5189
|
if (parent === current) {
|
|
4603
5190
|
return null;
|
|
4604
5191
|
}
|
|
@@ -4606,26 +5193,26 @@ async function findNearestGitRoot2(startPath) {
|
|
|
4606
5193
|
}
|
|
4607
5194
|
}
|
|
4608
5195
|
function withinRoot(rootPath, targetPath) {
|
|
4609
|
-
const relative =
|
|
4610
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
5196
|
+
const relative = path12.relative(rootPath, targetPath);
|
|
5197
|
+
return relative === "" || !relative.startsWith("..") && !path12.isAbsolute(relative);
|
|
4611
5198
|
}
|
|
4612
|
-
function
|
|
5199
|
+
function repoRootFromManifest2(manifest) {
|
|
4613
5200
|
if (manifest.originType !== "file" || !manifest.originalPath || !manifest.repoRelativePath) {
|
|
4614
5201
|
return null;
|
|
4615
5202
|
}
|
|
4616
|
-
const repoDir =
|
|
4617
|
-
const fileDir =
|
|
5203
|
+
const repoDir = path12.posix.dirname(manifest.repoRelativePath);
|
|
5204
|
+
const fileDir = path12.dirname(path12.resolve(manifest.originalPath));
|
|
4618
5205
|
if (repoDir === "." || !repoDir) {
|
|
4619
5206
|
return fileDir;
|
|
4620
5207
|
}
|
|
4621
5208
|
const segments = repoDir.split("/").filter(Boolean);
|
|
4622
|
-
return
|
|
5209
|
+
return path12.resolve(fileDir, ...segments.map(() => ".."));
|
|
4623
5210
|
}
|
|
4624
5211
|
function repoRelativePathFor(absolutePath, repoRoot) {
|
|
4625
5212
|
if (!repoRoot || !withinRoot(repoRoot, absolutePath)) {
|
|
4626
5213
|
return void 0;
|
|
4627
5214
|
}
|
|
4628
|
-
const relative = toPosix(
|
|
5215
|
+
const relative = toPosix(path12.relative(repoRoot, absolutePath));
|
|
4629
5216
|
return relative && !relative.startsWith("..") ? relative : void 0;
|
|
4630
5217
|
}
|
|
4631
5218
|
function normalizeOriginUrl(input) {
|
|
@@ -4957,7 +5544,7 @@ function buildCompositeHash(payloadBytes, attachments = []) {
|
|
|
4957
5544
|
return sha256(`${sha256(payloadBytes)}|${attachmentSignature}`);
|
|
4958
5545
|
}
|
|
4959
5546
|
function sanitizeAssetRelativePath(value) {
|
|
4960
|
-
const normalized =
|
|
5547
|
+
const normalized = path12.posix.normalize(value.replace(/\\/g, "/"));
|
|
4961
5548
|
const segments = normalized.split("/").filter(Boolean).map((segment) => {
|
|
4962
5549
|
if (segment === ".") {
|
|
4963
5550
|
return "";
|
|
@@ -4977,7 +5564,7 @@ function normalizeLocalReference(value) {
|
|
|
4977
5564
|
return null;
|
|
4978
5565
|
}
|
|
4979
5566
|
const lowered = candidate.toLowerCase();
|
|
4980
|
-
if (lowered.startsWith("http://") || lowered.startsWith("https://") || lowered.startsWith("data:") || lowered.startsWith("mailto:") || lowered.startsWith("#") ||
|
|
5567
|
+
if (lowered.startsWith("http://") || lowered.startsWith("https://") || lowered.startsWith("data:") || lowered.startsWith("mailto:") || lowered.startsWith("#") || path12.isAbsolute(candidate)) {
|
|
4981
5568
|
return null;
|
|
4982
5569
|
}
|
|
4983
5570
|
return candidate.replace(/\\/g, "/");
|
|
@@ -5055,12 +5642,12 @@ async function convertHtmlToMarkdown(html, url) {
|
|
|
5055
5642
|
};
|
|
5056
5643
|
}
|
|
5057
5644
|
async function readManifestByHash(manifestsDir, contentHash) {
|
|
5058
|
-
const entries = await
|
|
5645
|
+
const entries = await fs11.readdir(manifestsDir, { withFileTypes: true }).catch(() => []);
|
|
5059
5646
|
for (const entry of entries) {
|
|
5060
5647
|
if (!entry.isFile() || !entry.name.endsWith(".json")) {
|
|
5061
5648
|
continue;
|
|
5062
5649
|
}
|
|
5063
|
-
const manifest = await readJsonFile(
|
|
5650
|
+
const manifest = await readJsonFile(path12.join(manifestsDir, entry.name));
|
|
5064
5651
|
if (manifest?.contentHash === contentHash) {
|
|
5065
5652
|
return manifest;
|
|
5066
5653
|
}
|
|
@@ -5068,12 +5655,12 @@ async function readManifestByHash(manifestsDir, contentHash) {
|
|
|
5068
5655
|
return null;
|
|
5069
5656
|
}
|
|
5070
5657
|
async function readManifestByOrigin(manifestsDir, prepared) {
|
|
5071
|
-
const entries = await
|
|
5658
|
+
const entries = await fs11.readdir(manifestsDir, { withFileTypes: true }).catch(() => []);
|
|
5072
5659
|
for (const entry of entries) {
|
|
5073
5660
|
if (!entry.isFile() || !entry.name.endsWith(".json")) {
|
|
5074
5661
|
continue;
|
|
5075
5662
|
}
|
|
5076
|
-
const manifest = await readJsonFile(
|
|
5663
|
+
const manifest = await readJsonFile(path12.join(manifestsDir, entry.name));
|
|
5077
5664
|
if (manifest && manifestMatchesOrigin(manifest, prepared)) {
|
|
5078
5665
|
return manifest;
|
|
5079
5666
|
}
|
|
@@ -5084,12 +5671,12 @@ async function loadGitignoreMatcher(repoRoot, enabled) {
|
|
|
5084
5671
|
if (!enabled) {
|
|
5085
5672
|
return null;
|
|
5086
5673
|
}
|
|
5087
|
-
const gitignorePath =
|
|
5674
|
+
const gitignorePath = path12.join(repoRoot, ".gitignore");
|
|
5088
5675
|
if (!await fileExists(gitignorePath)) {
|
|
5089
5676
|
return null;
|
|
5090
5677
|
}
|
|
5091
5678
|
const matcher = ignore();
|
|
5092
|
-
matcher.add(await
|
|
5679
|
+
matcher.add(await fs11.readFile(gitignorePath, "utf8"));
|
|
5093
5680
|
return matcher;
|
|
5094
5681
|
}
|
|
5095
5682
|
function builtInIgnoreReason(relativePath) {
|
|
@@ -5113,23 +5700,23 @@ async function collectDirectoryFiles(rootDir, inputDir, repoRoot, options) {
|
|
|
5113
5700
|
if (!currentDir) {
|
|
5114
5701
|
continue;
|
|
5115
5702
|
}
|
|
5116
|
-
const entries = await
|
|
5703
|
+
const entries = await fs11.readdir(currentDir, { withFileTypes: true });
|
|
5117
5704
|
entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
5118
5705
|
for (const entry of entries) {
|
|
5119
|
-
const absolutePath =
|
|
5120
|
-
const relativeToRepo = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(
|
|
5706
|
+
const absolutePath = path12.join(currentDir, entry.name);
|
|
5707
|
+
const relativeToRepo = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path12.relative(inputDir, absolutePath));
|
|
5121
5708
|
const relativePath = relativeToRepo || entry.name;
|
|
5122
5709
|
const builtInReason = builtInIgnoreReason(relativePath);
|
|
5123
5710
|
if (builtInReason) {
|
|
5124
|
-
skipped.push({ path: toPosix(
|
|
5711
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: builtInReason });
|
|
5125
5712
|
continue;
|
|
5126
5713
|
}
|
|
5127
5714
|
if (matcher?.ignores(relativePath)) {
|
|
5128
|
-
skipped.push({ path: toPosix(
|
|
5715
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "gitignore" });
|
|
5129
5716
|
continue;
|
|
5130
5717
|
}
|
|
5131
5718
|
if (matchesAnyGlob2(relativePath, options.exclude)) {
|
|
5132
|
-
skipped.push({ path: toPosix(
|
|
5719
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "exclude_glob" });
|
|
5133
5720
|
continue;
|
|
5134
5721
|
}
|
|
5135
5722
|
if (entry.isDirectory()) {
|
|
@@ -5137,26 +5724,26 @@ async function collectDirectoryFiles(rootDir, inputDir, repoRoot, options) {
|
|
|
5137
5724
|
continue;
|
|
5138
5725
|
}
|
|
5139
5726
|
if (!entry.isFile()) {
|
|
5140
|
-
skipped.push({ path: toPosix(
|
|
5727
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "unsupported_entry" });
|
|
5141
5728
|
continue;
|
|
5142
5729
|
}
|
|
5143
5730
|
if (options.include.length > 0 && !matchesAnyGlob2(relativePath, options.include)) {
|
|
5144
|
-
skipped.push({ path: toPosix(
|
|
5731
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "include_glob" });
|
|
5145
5732
|
continue;
|
|
5146
5733
|
}
|
|
5147
5734
|
const mimeType = guessMimeType(absolutePath);
|
|
5148
5735
|
const sourceKind = inferKind(mimeType, absolutePath);
|
|
5149
5736
|
const sourceClass = sourceClassForRelativePath(relativePath, options);
|
|
5150
5737
|
if (!supportedDirectoryKind(sourceKind)) {
|
|
5151
|
-
skipped.push({ path: toPosix(
|
|
5738
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: `unsupported_kind:${sourceKind}` });
|
|
5152
5739
|
continue;
|
|
5153
5740
|
}
|
|
5154
5741
|
if (!options.extractClasses.includes(sourceClass)) {
|
|
5155
|
-
skipped.push({ path: toPosix(
|
|
5742
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: `source_class:${sourceClass}` });
|
|
5156
5743
|
continue;
|
|
5157
5744
|
}
|
|
5158
5745
|
if (files.length >= options.maxFiles) {
|
|
5159
|
-
skipped.push({ path: toPosix(
|
|
5746
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "max_files" });
|
|
5160
5747
|
continue;
|
|
5161
5748
|
}
|
|
5162
5749
|
files.push(absolutePath);
|
|
@@ -5178,12 +5765,12 @@ function resolveUrlMimeType(input, response) {
|
|
|
5178
5765
|
function buildRemoteAssetRelativePath(assetUrl, mimeType) {
|
|
5179
5766
|
const url = new URL(assetUrl);
|
|
5180
5767
|
const normalized = sanitizeAssetRelativePath(`${url.hostname}${url.pathname || "/asset"}`);
|
|
5181
|
-
const extension =
|
|
5182
|
-
const directory =
|
|
5183
|
-
const basename = extension ?
|
|
5768
|
+
const extension = path12.posix.extname(normalized);
|
|
5769
|
+
const directory = path12.posix.dirname(normalized);
|
|
5770
|
+
const basename = extension ? path12.posix.basename(normalized, extension) : path12.posix.basename(normalized);
|
|
5184
5771
|
const resolvedExtension = extension || `.${mime.extension(mimeType) || "bin"}`;
|
|
5185
5772
|
const hashedName = `${basename || "asset"}-${sha256(assetUrl).slice(0, 8)}${resolvedExtension}`;
|
|
5186
|
-
return directory === "." ? hashedName :
|
|
5773
|
+
return directory === "." ? hashedName : path12.posix.join(directory, hashedName);
|
|
5187
5774
|
}
|
|
5188
5775
|
async function readResponseBytesWithinLimit(response, maxBytes) {
|
|
5189
5776
|
const contentLength = Number.parseInt(response.headers.get("content-length") ?? "", 10);
|
|
@@ -5336,34 +5923,34 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
5336
5923
|
const previous = existingByOrigin ?? void 0;
|
|
5337
5924
|
const sourceId = previous?.sourceId ?? `${slugify(prepared.title)}-${contentHash.slice(0, 8)}`;
|
|
5338
5925
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5339
|
-
const storedPath =
|
|
5340
|
-
const extractedTextPath = prepared.extractedText ?
|
|
5341
|
-
const extractedMetadataPath = prepared.extractionArtifact ?
|
|
5342
|
-
const attachmentsDir =
|
|
5926
|
+
const storedPath = path12.join(paths.rawSourcesDir, `${sourceId}${prepared.storedExtension}`);
|
|
5927
|
+
const extractedTextPath = prepared.extractedText ? path12.join(paths.extractsDir, `${sourceId}.md`) : void 0;
|
|
5928
|
+
const extractedMetadataPath = prepared.extractionArtifact ? path12.join(paths.extractsDir, `${sourceId}.json`) : void 0;
|
|
5929
|
+
const attachmentsDir = path12.join(paths.rawAssetsDir, sourceId);
|
|
5343
5930
|
if (previous?.storedPath) {
|
|
5344
|
-
await
|
|
5931
|
+
await fs11.rm(path12.resolve(rootDir, previous.storedPath), { force: true });
|
|
5345
5932
|
}
|
|
5346
5933
|
if (previous?.extractedTextPath) {
|
|
5347
|
-
await
|
|
5934
|
+
await fs11.rm(path12.resolve(rootDir, previous.extractedTextPath), { force: true });
|
|
5348
5935
|
}
|
|
5349
5936
|
if (previous?.extractedMetadataPath) {
|
|
5350
|
-
await
|
|
5937
|
+
await fs11.rm(path12.resolve(rootDir, previous.extractedMetadataPath), { force: true });
|
|
5351
5938
|
}
|
|
5352
|
-
await
|
|
5353
|
-
await
|
|
5939
|
+
await fs11.rm(attachmentsDir, { recursive: true, force: true });
|
|
5940
|
+
await fs11.writeFile(storedPath, prepared.payloadBytes);
|
|
5354
5941
|
if (prepared.extractedText && extractedTextPath) {
|
|
5355
|
-
await
|
|
5942
|
+
await fs11.writeFile(extractedTextPath, prepared.extractedText, "utf8");
|
|
5356
5943
|
}
|
|
5357
5944
|
if (prepared.extractionArtifact && extractedMetadataPath) {
|
|
5358
5945
|
await writeJsonFile(extractedMetadataPath, prepared.extractionArtifact);
|
|
5359
5946
|
}
|
|
5360
5947
|
const manifestAttachments = [];
|
|
5361
5948
|
for (const attachment of attachments) {
|
|
5362
|
-
const absoluteAttachmentPath =
|
|
5363
|
-
await ensureDir(
|
|
5364
|
-
await
|
|
5949
|
+
const absoluteAttachmentPath = path12.join(attachmentsDir, attachment.relativePath);
|
|
5950
|
+
await ensureDir(path12.dirname(absoluteAttachmentPath));
|
|
5951
|
+
await fs11.writeFile(absoluteAttachmentPath, attachment.bytes);
|
|
5365
5952
|
manifestAttachments.push({
|
|
5366
|
-
path: toPosix(
|
|
5953
|
+
path: toPosix(path12.relative(rootDir, absoluteAttachmentPath)),
|
|
5367
5954
|
mimeType: attachment.mimeType,
|
|
5368
5955
|
originalPath: attachment.originalPath
|
|
5369
5956
|
});
|
|
@@ -5379,9 +5966,9 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
5379
5966
|
originalPath: prepared.originalPath,
|
|
5380
5967
|
repoRelativePath: prepared.repoRelativePath,
|
|
5381
5968
|
url: prepared.url,
|
|
5382
|
-
storedPath: toPosix(
|
|
5383
|
-
extractedTextPath: extractedTextPath ? toPosix(
|
|
5384
|
-
extractedMetadataPath: extractedMetadataPath ? toPosix(
|
|
5969
|
+
storedPath: toPosix(path12.relative(rootDir, storedPath)),
|
|
5970
|
+
extractedTextPath: extractedTextPath ? toPosix(path12.relative(rootDir, extractedTextPath)) : void 0,
|
|
5971
|
+
extractedMetadataPath: extractedMetadataPath ? toPosix(path12.relative(rootDir, extractedMetadataPath)) : void 0,
|
|
5385
5972
|
extractionHash,
|
|
5386
5973
|
mimeType: prepared.mimeType,
|
|
5387
5974
|
contentHash,
|
|
@@ -5389,7 +5976,7 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
5389
5976
|
updatedAt: now,
|
|
5390
5977
|
attachments: manifestAttachments.length ? manifestAttachments : void 0
|
|
5391
5978
|
};
|
|
5392
|
-
await writeJsonFile(
|
|
5979
|
+
await writeJsonFile(path12.join(paths.manifestsDir, `${sourceId}.json`), manifest);
|
|
5393
5980
|
await appendLogEntry(rootDir, "ingest", prepared.title, [
|
|
5394
5981
|
`source_id=${sourceId}`,
|
|
5395
5982
|
`kind=${prepared.sourceKind}`,
|
|
@@ -5407,16 +5994,16 @@ async function persistPreparedInput(rootDir, prepared, paths) {
|
|
|
5407
5994
|
return { manifest, isNew: !previous, wasUpdated: Boolean(previous) };
|
|
5408
5995
|
}
|
|
5409
5996
|
async function removeManifestArtifacts(rootDir, manifest, paths) {
|
|
5410
|
-
await
|
|
5411
|
-
await
|
|
5997
|
+
await fs11.rm(path12.join(paths.manifestsDir, `${manifest.sourceId}.json`), { force: true });
|
|
5998
|
+
await fs11.rm(path12.resolve(rootDir, manifest.storedPath), { force: true });
|
|
5412
5999
|
if (manifest.extractedTextPath) {
|
|
5413
|
-
await
|
|
6000
|
+
await fs11.rm(path12.resolve(rootDir, manifest.extractedTextPath), { force: true });
|
|
5414
6001
|
}
|
|
5415
6002
|
if (manifest.extractedMetadataPath) {
|
|
5416
|
-
await
|
|
6003
|
+
await fs11.rm(path12.resolve(rootDir, manifest.extractedMetadataPath), { force: true });
|
|
5417
6004
|
}
|
|
5418
|
-
await
|
|
5419
|
-
await
|
|
6005
|
+
await fs11.rm(path12.join(paths.rawAssetsDir, manifest.sourceId), { recursive: true, force: true });
|
|
6006
|
+
await fs11.rm(path12.join(paths.analysesDir, `${manifest.sourceId}.json`), { force: true });
|
|
5420
6007
|
}
|
|
5421
6008
|
function repoSyncWorkspaceIgnorePaths(rootDir, paths, repoRoot) {
|
|
5422
6009
|
const candidates = [
|
|
@@ -5425,11 +6012,11 @@ function repoSyncWorkspaceIgnorePaths(rootDir, paths, repoRoot) {
|
|
|
5425
6012
|
paths.stateDir,
|
|
5426
6013
|
paths.agentDir,
|
|
5427
6014
|
paths.inboxDir,
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
6015
|
+
path12.join(rootDir, ".claude"),
|
|
6016
|
+
path12.join(rootDir, ".cursor"),
|
|
6017
|
+
path12.join(rootDir, ".obsidian")
|
|
5431
6018
|
];
|
|
5432
|
-
return candidates.map((candidate) =>
|
|
6019
|
+
return candidates.map((candidate) => path12.resolve(candidate)).filter((candidate, index, items) => items.indexOf(candidate) === index).filter((candidate) => withinRoot(repoRoot, candidate));
|
|
5433
6020
|
}
|
|
5434
6021
|
function preparedMatchesManifest(manifest, prepared, contentHash) {
|
|
5435
6022
|
return manifest.contentHash === contentHash && manifest.extractionHash === (prepared.extractionHash ?? buildExtractionHash(prepared.extractedText, prepared.extractionArtifact)) && manifest.title === prepared.title && manifest.sourceKind === prepared.sourceKind && manifest.sourceType === prepared.sourceType && manifest.sourceClass === prepared.sourceClass && manifest.language === prepared.language && manifest.mimeType === prepared.mimeType && manifest.repoRelativePath === prepared.repoRelativePath;
|
|
@@ -5441,8 +6028,15 @@ function pendingSemanticRefreshId(changeType, repoRoot, relativePath) {
|
|
|
5441
6028
|
return `pending:${changeType}:${sha256(`${toPosix(repoRoot)}:${relativePath}`).slice(0, 12)}`;
|
|
5442
6029
|
}
|
|
5443
6030
|
async function listTrackedRepoRoots(rootDir) {
|
|
6031
|
+
const managedSources = await readManagedSourcesIfPresent(rootDir).catch(() => null);
|
|
6032
|
+
if (managedSources && managedSources.length > 0) {
|
|
6033
|
+
const directoryRoots = managedSources.filter((source) => source.kind === "directory" && source.path).map((source) => path12.resolve(source.path));
|
|
6034
|
+
if (directoryRoots.length > 0) {
|
|
6035
|
+
return [...new Set(directoryRoots)].sort((left, right) => left.localeCompare(right));
|
|
6036
|
+
}
|
|
6037
|
+
}
|
|
5444
6038
|
const manifests = await listManifests(rootDir);
|
|
5445
|
-
return [...new Set(manifests.map((manifest) =>
|
|
6039
|
+
return [...new Set(manifests.map((manifest) => repoRootFromManifest2(manifest)).filter((item) => Boolean(item)))].sort(
|
|
5446
6040
|
(left, right) => left.localeCompare(right)
|
|
5447
6041
|
);
|
|
5448
6042
|
}
|
|
@@ -5451,16 +6045,16 @@ async function syncTrackedRepos(rootDir, options, repoRoots) {
|
|
|
5451
6045
|
const normalizedOptions = await resolveRepoIngestOptions(rootDir, options);
|
|
5452
6046
|
const manifests = await listManifests(rootDir);
|
|
5453
6047
|
const trackedRoots = (repoRoots && repoRoots.length > 0 ? repoRoots : await listTrackedRepoRoots(rootDir)).map(
|
|
5454
|
-
(item) =>
|
|
6048
|
+
(item) => path12.resolve(item)
|
|
5455
6049
|
);
|
|
5456
6050
|
const uniqueRoots = [...new Set(trackedRoots)].sort((left, right) => left.localeCompare(right));
|
|
5457
6051
|
const manifestsByRepoRoot = /* @__PURE__ */ new Map();
|
|
5458
6052
|
for (const manifest of manifests) {
|
|
5459
|
-
const repoRoot =
|
|
5460
|
-
if (!repoRoot || !uniqueRoots.includes(
|
|
6053
|
+
const repoRoot = repoRootFromManifest2(manifest);
|
|
6054
|
+
if (!repoRoot || !uniqueRoots.includes(path12.resolve(repoRoot))) {
|
|
5461
6055
|
continue;
|
|
5462
6056
|
}
|
|
5463
|
-
const key =
|
|
6057
|
+
const key = path12.resolve(repoRoot);
|
|
5464
6058
|
const bucket = manifestsByRepoRoot.get(key) ?? [];
|
|
5465
6059
|
bucket.push(manifest);
|
|
5466
6060
|
manifestsByRepoRoot.set(key, bucket);
|
|
@@ -5485,14 +6079,15 @@ async function syncTrackedRepos(rootDir, options, repoRoots) {
|
|
|
5485
6079
|
skipped.push(
|
|
5486
6080
|
...collected.skipped,
|
|
5487
6081
|
...collected.files.filter((absolutePath) => ignoreRoots.some((ignoreRoot) => withinRoot(ignoreRoot, absolutePath))).map((absolutePath) => ({
|
|
5488
|
-
path: toPosix(
|
|
6082
|
+
path: toPosix(path12.relative(rootDir, absolutePath)),
|
|
5489
6083
|
reason: "workspace_generated"
|
|
5490
6084
|
}))
|
|
5491
6085
|
);
|
|
5492
6086
|
scannedCount += files.length;
|
|
5493
|
-
const
|
|
6087
|
+
const progress = createProgressReporter("sync", files.length);
|
|
6088
|
+
const currentPaths = new Set(files.map((absolutePath) => path12.resolve(absolutePath)));
|
|
5494
6089
|
for (const absolutePath of files) {
|
|
5495
|
-
const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(
|
|
6090
|
+
const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path12.relative(repoRoot, absolutePath));
|
|
5496
6091
|
const prepared = await prepareFileInput(rootDir, absolutePath, repoRoot, sourceClassForRelativePath(relativePath, normalizedOptions));
|
|
5497
6092
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
5498
6093
|
if (result.isNew) {
|
|
@@ -5500,9 +6095,11 @@ async function syncTrackedRepos(rootDir, options, repoRoots) {
|
|
|
5500
6095
|
} else if (result.wasUpdated) {
|
|
5501
6096
|
updated.push(result.manifest);
|
|
5502
6097
|
}
|
|
6098
|
+
progress.tick();
|
|
5503
6099
|
}
|
|
6100
|
+
progress.finish(`repo=${toPosix(path12.relative(rootDir, repoRoot)) || "."}`);
|
|
5504
6101
|
for (const manifest of repoManifests) {
|
|
5505
|
-
const originalPath = manifest.originalPath ?
|
|
6102
|
+
const originalPath = manifest.originalPath ? path12.resolve(manifest.originalPath) : null;
|
|
5506
6103
|
if (originalPath && !currentPaths.has(originalPath)) {
|
|
5507
6104
|
await removeManifestArtifacts(rootDir, manifest, paths);
|
|
5508
6105
|
removed.push(manifest);
|
|
@@ -5510,7 +6107,7 @@ async function syncTrackedRepos(rootDir, options, repoRoots) {
|
|
|
5510
6107
|
}
|
|
5511
6108
|
}
|
|
5512
6109
|
if (uniqueRoots.length > 0) {
|
|
5513
|
-
await appendLogEntry(rootDir, "sync_repo", uniqueRoots.map((repoRoot) => toPosix(
|
|
6110
|
+
await appendLogEntry(rootDir, "sync_repo", uniqueRoots.map((repoRoot) => toPosix(path12.relative(rootDir, repoRoot)) || ".").join(","), [
|
|
5514
6111
|
`repo_roots=${uniqueRoots.length}`,
|
|
5515
6112
|
`scanned=${scannedCount}`,
|
|
5516
6113
|
`imported=${imported.length}`,
|
|
@@ -5533,16 +6130,16 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5533
6130
|
const normalizedOptions = await resolveRepoIngestOptions(rootDir, options);
|
|
5534
6131
|
const manifests = await listManifests(rootDir);
|
|
5535
6132
|
const trackedRoots = (repoRoots && repoRoots.length > 0 ? repoRoots : await listTrackedRepoRoots(rootDir)).map(
|
|
5536
|
-
(item) =>
|
|
6133
|
+
(item) => path12.resolve(item)
|
|
5537
6134
|
);
|
|
5538
6135
|
const uniqueRoots = [...new Set(trackedRoots)].sort((left, right) => left.localeCompare(right));
|
|
5539
6136
|
const manifestsByRepoRoot = /* @__PURE__ */ new Map();
|
|
5540
6137
|
for (const manifest of manifests) {
|
|
5541
|
-
const repoRoot =
|
|
5542
|
-
if (!repoRoot || !uniqueRoots.includes(
|
|
6138
|
+
const repoRoot = repoRootFromManifest2(manifest);
|
|
6139
|
+
if (!repoRoot || !uniqueRoots.includes(path12.resolve(repoRoot))) {
|
|
5543
6140
|
continue;
|
|
5544
6141
|
}
|
|
5545
|
-
const key =
|
|
6142
|
+
const key = path12.resolve(repoRoot);
|
|
5546
6143
|
const bucket = manifestsByRepoRoot.get(key) ?? [];
|
|
5547
6144
|
bucket.push(manifest);
|
|
5548
6145
|
manifestsByRepoRoot.set(key, bucket);
|
|
@@ -5557,7 +6154,7 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5557
6154
|
for (const repoRoot of uniqueRoots) {
|
|
5558
6155
|
const repoManifests = manifestsByRepoRoot.get(repoRoot) ?? [];
|
|
5559
6156
|
const manifestsByOriginalPath = new Map(
|
|
5560
|
-
repoManifests.filter((manifest) => manifest.originalPath).map((manifest) => [
|
|
6157
|
+
repoManifests.filter((manifest) => manifest.originalPath).map((manifest) => [path12.resolve(manifest.originalPath), manifest])
|
|
5561
6158
|
);
|
|
5562
6159
|
if (!await fileExists(repoRoot)) {
|
|
5563
6160
|
for (const manifest of repoManifests) {
|
|
@@ -5565,7 +6162,7 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5565
6162
|
pendingSemanticRefresh.push({
|
|
5566
6163
|
id: pendingSemanticRefreshId("removed", repoRoot, manifest.repoRelativePath ?? manifest.storedPath),
|
|
5567
6164
|
repoRoot,
|
|
5568
|
-
path: toPosix(
|
|
6165
|
+
path: toPosix(path12.relative(rootDir, manifest.originalPath ?? manifest.storedPath)),
|
|
5569
6166
|
changeType: "removed",
|
|
5570
6167
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5571
6168
|
sourceId: manifest.sourceId,
|
|
@@ -5585,17 +6182,18 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5585
6182
|
skipped.push(
|
|
5586
6183
|
...collected.skipped,
|
|
5587
6184
|
...collected.files.filter((absolutePath) => ignoreRoots.some((ignoreRoot) => withinRoot(ignoreRoot, absolutePath))).map((absolutePath) => ({
|
|
5588
|
-
path: toPosix(
|
|
6185
|
+
path: toPosix(path12.relative(rootDir, absolutePath)),
|
|
5589
6186
|
reason: "workspace_generated"
|
|
5590
6187
|
}))
|
|
5591
6188
|
);
|
|
5592
6189
|
scannedCount += files.length;
|
|
5593
|
-
const
|
|
6190
|
+
const progress = createProgressReporter("sync-watch", files.length);
|
|
6191
|
+
const currentPaths = new Set(files.map((absolutePath) => path12.resolve(absolutePath)));
|
|
5594
6192
|
for (const absolutePath of files) {
|
|
5595
|
-
const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(
|
|
6193
|
+
const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path12.relative(repoRoot, absolutePath));
|
|
5596
6194
|
const prepared = await prepareFileInput(rootDir, absolutePath, repoRoot, sourceClassForRelativePath(relativePath, normalizedOptions));
|
|
5597
6195
|
if (shouldDeferWatchSemanticRefresh(prepared.sourceKind)) {
|
|
5598
|
-
const existing = manifestsByOriginalPath.get(
|
|
6196
|
+
const existing = manifestsByOriginalPath.get(path12.resolve(absolutePath));
|
|
5599
6197
|
const contentHash = buildCompositeHash(prepared.payloadBytes, prepared.attachments);
|
|
5600
6198
|
const changed = !existing || !preparedMatchesManifest(existing, prepared, contentHash);
|
|
5601
6199
|
if (changed) {
|
|
@@ -5603,10 +6201,10 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5603
6201
|
id: pendingSemanticRefreshId(
|
|
5604
6202
|
existing ? "modified" : "added",
|
|
5605
6203
|
repoRoot,
|
|
5606
|
-
prepared.repoRelativePath ?? toPosix(
|
|
6204
|
+
prepared.repoRelativePath ?? toPosix(path12.relative(repoRoot, absolutePath))
|
|
5607
6205
|
),
|
|
5608
6206
|
repoRoot,
|
|
5609
|
-
path: toPosix(
|
|
6207
|
+
path: toPosix(path12.relative(rootDir, absolutePath)),
|
|
5610
6208
|
changeType: existing ? "modified" : "added",
|
|
5611
6209
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5612
6210
|
sourceId: existing?.sourceId,
|
|
@@ -5616,6 +6214,7 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5616
6214
|
staleSourceIds.add(existing.sourceId);
|
|
5617
6215
|
}
|
|
5618
6216
|
}
|
|
6217
|
+
progress.tick();
|
|
5619
6218
|
continue;
|
|
5620
6219
|
}
|
|
5621
6220
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
@@ -5624,15 +6223,17 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5624
6223
|
} else if (result.wasUpdated) {
|
|
5625
6224
|
updated.push(result.manifest);
|
|
5626
6225
|
}
|
|
6226
|
+
progress.tick();
|
|
5627
6227
|
}
|
|
6228
|
+
progress.finish(`repo=${toPosix(path12.relative(rootDir, repoRoot)) || "."}`);
|
|
5628
6229
|
for (const manifest of repoManifests) {
|
|
5629
|
-
const originalPath = manifest.originalPath ?
|
|
6230
|
+
const originalPath = manifest.originalPath ? path12.resolve(manifest.originalPath) : null;
|
|
5630
6231
|
if (originalPath && !currentPaths.has(originalPath)) {
|
|
5631
6232
|
if (shouldDeferWatchSemanticRefresh(manifest.sourceKind)) {
|
|
5632
6233
|
pendingSemanticRefresh.push({
|
|
5633
|
-
id: pendingSemanticRefreshId("removed", repoRoot, manifest.repoRelativePath ?? toPosix(
|
|
6234
|
+
id: pendingSemanticRefreshId("removed", repoRoot, manifest.repoRelativePath ?? toPosix(path12.relative(repoRoot, originalPath))),
|
|
5634
6235
|
repoRoot,
|
|
5635
|
-
path: toPosix(
|
|
6236
|
+
path: toPosix(path12.relative(rootDir, originalPath)),
|
|
5636
6237
|
changeType: "removed",
|
|
5637
6238
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5638
6239
|
sourceId: manifest.sourceId,
|
|
@@ -5650,7 +6251,7 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5650
6251
|
await appendLogEntry(
|
|
5651
6252
|
rootDir,
|
|
5652
6253
|
"sync_repo_watch",
|
|
5653
|
-
uniqueRoots.map((repoRoot) => toPosix(
|
|
6254
|
+
uniqueRoots.map((repoRoot) => toPosix(path12.relative(rootDir, repoRoot)) || ".").join(","),
|
|
5654
6255
|
[
|
|
5655
6256
|
`repo_roots=${uniqueRoots.length}`,
|
|
5656
6257
|
`scanned=${scannedCount}`,
|
|
@@ -5676,17 +6277,17 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
|
|
|
5676
6277
|
};
|
|
5677
6278
|
}
|
|
5678
6279
|
async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
|
|
5679
|
-
const payloadBytes = await
|
|
6280
|
+
const payloadBytes = await fs11.readFile(absoluteInput);
|
|
5680
6281
|
const mimeType = guessMimeType(absoluteInput);
|
|
5681
6282
|
const sourceKind = inferKind(mimeType, absoluteInput);
|
|
5682
6283
|
const language = inferCodeLanguage(absoluteInput, mimeType);
|
|
5683
|
-
const storedExtension =
|
|
6284
|
+
const storedExtension = path12.extname(absoluteInput) || `.${mime.extension(mimeType) || "bin"}`;
|
|
5684
6285
|
let title;
|
|
5685
6286
|
let extractedText;
|
|
5686
6287
|
let extractionArtifact;
|
|
5687
6288
|
if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
|
|
5688
|
-
extractedText = payloadBytes.toString("utf8");
|
|
5689
|
-
title = titleFromText(
|
|
6289
|
+
extractedText = extractedTextForPlainSource(absoluteInput, sourceKind, payloadBytes.toString("utf8"));
|
|
6290
|
+
title = titleFromText(path12.basename(absoluteInput, path12.extname(absoluteInput)), extractedText, absoluteInput);
|
|
5690
6291
|
extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
|
|
5691
6292
|
} else if (sourceKind === "html") {
|
|
5692
6293
|
const html = payloadBytes.toString("utf8");
|
|
@@ -5695,18 +6296,18 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
|
|
|
5695
6296
|
extractedText = converted.markdown;
|
|
5696
6297
|
extractionArtifact = createHtmlReadabilityExtractionArtifact(sourceKind, mimeType);
|
|
5697
6298
|
} else if (sourceKind === "pdf") {
|
|
5698
|
-
title =
|
|
6299
|
+
title = path12.basename(absoluteInput, path12.extname(absoluteInput));
|
|
5699
6300
|
const extracted = await extractPdfText({ mimeType, bytes: payloadBytes });
|
|
5700
6301
|
extractedText = extracted.extractedText;
|
|
5701
6302
|
extractionArtifact = extracted.artifact;
|
|
5702
6303
|
} else if (sourceKind === "docx") {
|
|
5703
|
-
title =
|
|
6304
|
+
title = path12.basename(absoluteInput, path12.extname(absoluteInput));
|
|
5704
6305
|
const extracted = await extractDocxText({ mimeType, bytes: payloadBytes });
|
|
5705
6306
|
title = extracted.artifact.metadata?.title?.trim() || title;
|
|
5706
6307
|
extractedText = extracted.extractedText;
|
|
5707
6308
|
extractionArtifact = extracted.artifact;
|
|
5708
6309
|
} else if (sourceKind === "image") {
|
|
5709
|
-
title =
|
|
6310
|
+
title = path12.basename(absoluteInput, path12.extname(absoluteInput));
|
|
5710
6311
|
const extracted = await extractImageWithVision(rootDir, {
|
|
5711
6312
|
title,
|
|
5712
6313
|
mimeType,
|
|
@@ -5716,7 +6317,7 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
|
|
|
5716
6317
|
extractedText = extracted.extractedText;
|
|
5717
6318
|
extractionArtifact = extracted.artifact;
|
|
5718
6319
|
} else {
|
|
5719
|
-
title =
|
|
6320
|
+
title = path12.basename(absoluteInput, path12.extname(absoluteInput));
|
|
5720
6321
|
}
|
|
5721
6322
|
return {
|
|
5722
6323
|
title,
|
|
@@ -5793,11 +6394,11 @@ async function prepareUrlInput(rootDir, input, options) {
|
|
|
5793
6394
|
sourceKind = "markdown";
|
|
5794
6395
|
storedExtension = ".md";
|
|
5795
6396
|
} else {
|
|
5796
|
-
const extension =
|
|
6397
|
+
const extension = path12.extname(inputUrl.pathname);
|
|
5797
6398
|
storedExtension = extension || `.${mime.extension(mimeType) || "bin"}`;
|
|
5798
6399
|
if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
|
|
5799
|
-
extractedText = payloadBytes.toString("utf8");
|
|
5800
|
-
title = titleFromText(title || inputUrl.hostname, extractedText);
|
|
6400
|
+
extractedText = extractedTextForPlainSource(inputUrl.pathname, sourceKind, payloadBytes.toString("utf8"));
|
|
6401
|
+
title = titleFromText(title || inputUrl.hostname, extractedText, inputUrl.pathname);
|
|
5801
6402
|
extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
|
|
5802
6403
|
if (sourceKind === "markdown" && options.includeAssets) {
|
|
5803
6404
|
const { attachments: remoteAttachments, skippedCount } = await collectRemoteImageAttachments(
|
|
@@ -5864,14 +6465,14 @@ async function collectInboxAttachmentRefs(inputDir, files) {
|
|
|
5864
6465
|
if (sourceKind !== "markdown" && sourceKind !== "html") {
|
|
5865
6466
|
continue;
|
|
5866
6467
|
}
|
|
5867
|
-
const content = await
|
|
6468
|
+
const content = await fs11.readFile(absolutePath, "utf8");
|
|
5868
6469
|
const refs = sourceKind === "html" ? extractHtmlLocalReferences(content, pathToFileURL(absolutePath).toString()) : extractMarkdownReferences(content);
|
|
5869
6470
|
if (!refs.length) {
|
|
5870
6471
|
continue;
|
|
5871
6472
|
}
|
|
5872
6473
|
const sourceRefs = [];
|
|
5873
6474
|
for (const ref of refs) {
|
|
5874
|
-
const resolved =
|
|
6475
|
+
const resolved = path12.resolve(path12.dirname(absolutePath), ref);
|
|
5875
6476
|
if (!resolved.startsWith(inputDir) || !await fileExists(resolved)) {
|
|
5876
6477
|
continue;
|
|
5877
6478
|
}
|
|
@@ -5905,12 +6506,12 @@ function rewriteMarkdownReferences(content, replacements) {
|
|
|
5905
6506
|
});
|
|
5906
6507
|
}
|
|
5907
6508
|
async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
|
|
5908
|
-
const originalBytes = await
|
|
6509
|
+
const originalBytes = await fs11.readFile(absolutePath);
|
|
5909
6510
|
const originalText = originalBytes.toString("utf8");
|
|
5910
|
-
const title = titleFromText(
|
|
6511
|
+
const title = titleFromText(path12.basename(absolutePath, path12.extname(absolutePath)), originalText);
|
|
5911
6512
|
const attachments = [];
|
|
5912
6513
|
for (const attachmentRef of attachmentRefs) {
|
|
5913
|
-
const bytes = await
|
|
6514
|
+
const bytes = await fs11.readFile(attachmentRef.absolutePath);
|
|
5914
6515
|
attachments.push({
|
|
5915
6516
|
relativePath: sanitizeAssetRelativePath(attachmentRef.relativeRef),
|
|
5916
6517
|
mimeType: guessMimeType(attachmentRef.absolutePath),
|
|
@@ -5934,7 +6535,7 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
|
|
|
5934
6535
|
sourceKind: "markdown",
|
|
5935
6536
|
originalPath: toPosix(absolutePath),
|
|
5936
6537
|
mimeType: "text/markdown",
|
|
5937
|
-
storedExtension:
|
|
6538
|
+
storedExtension: path12.extname(absolutePath) || ".md",
|
|
5938
6539
|
payloadBytes: Buffer.from(rewrittenText, "utf8"),
|
|
5939
6540
|
extractedText: rewrittenText,
|
|
5940
6541
|
extractionArtifact,
|
|
@@ -5944,12 +6545,12 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
|
|
|
5944
6545
|
};
|
|
5945
6546
|
}
|
|
5946
6547
|
async function prepareInboxHtmlInput(absolutePath, attachmentRefs) {
|
|
5947
|
-
const originalBytes = await
|
|
6548
|
+
const originalBytes = await fs11.readFile(absolutePath);
|
|
5948
6549
|
const originalHtml = originalBytes.toString("utf8");
|
|
5949
6550
|
const initialConversion = await convertHtmlToMarkdown(originalHtml, pathToFileURL(absolutePath).toString());
|
|
5950
6551
|
const attachments = [];
|
|
5951
6552
|
for (const attachmentRef of attachmentRefs) {
|
|
5952
|
-
const bytes = await
|
|
6553
|
+
const bytes = await fs11.readFile(attachmentRef.absolutePath);
|
|
5953
6554
|
attachments.push({
|
|
5954
6555
|
relativePath: sanitizeAssetRelativePath(attachmentRef.relativeRef),
|
|
5955
6556
|
mimeType: guessMimeType(attachmentRef.absolutePath),
|
|
@@ -5958,7 +6559,7 @@ async function prepareInboxHtmlInput(absolutePath, attachmentRefs) {
|
|
|
5958
6559
|
});
|
|
5959
6560
|
}
|
|
5960
6561
|
const contentHash = buildCompositeHash(originalBytes, attachments);
|
|
5961
|
-
const fallbackTitle =
|
|
6562
|
+
const fallbackTitle = path12.basename(absolutePath, path12.extname(absolutePath));
|
|
5962
6563
|
const title = initialConversion.title || fallbackTitle;
|
|
5963
6564
|
const sourceId = `${slugify(title)}-${contentHash.slice(0, 8)}`;
|
|
5964
6565
|
const replacements = new Map(
|
|
@@ -5976,7 +6577,7 @@ async function prepareInboxHtmlInput(absolutePath, attachmentRefs) {
|
|
|
5976
6577
|
sourceKind: "html",
|
|
5977
6578
|
originalPath: toPosix(absolutePath),
|
|
5978
6579
|
mimeType: "text/html",
|
|
5979
|
-
storedExtension:
|
|
6580
|
+
storedExtension: path12.extname(absolutePath) || ".html",
|
|
5980
6581
|
payloadBytes: Buffer.from(rewrittenHtml, "utf8"),
|
|
5981
6582
|
extractedText: converted.markdown,
|
|
5982
6583
|
extractionArtifact,
|
|
@@ -5988,14 +6589,16 @@ async function prepareInboxHtmlInput(absolutePath, attachmentRefs) {
|
|
|
5988
6589
|
function isSupportedInboxKind(sourceKind) {
|
|
5989
6590
|
return ["markdown", "text", "html", "pdf", "docx", "image"].includes(sourceKind);
|
|
5990
6591
|
}
|
|
5991
|
-
async function
|
|
6592
|
+
async function ingestInputDetailed(rootDir, input, options) {
|
|
5992
6593
|
const { paths } = await initWorkspace(rootDir);
|
|
5993
6594
|
const normalizedOptions = normalizeIngestOptions(options);
|
|
5994
|
-
const absoluteInput =
|
|
5995
|
-
const repoRoot = isHttpUrl(input) || normalizedOptions.repoRoot ? normalizedOptions.repoRoot : await findNearestGitRoot2(absoluteInput).then((value) => value ??
|
|
6595
|
+
const absoluteInput = path12.resolve(rootDir, input);
|
|
6596
|
+
const repoRoot = isHttpUrl(input) || normalizedOptions.repoRoot ? normalizedOptions.repoRoot : await findNearestGitRoot2(absoluteInput).then((value) => value ?? path12.dirname(absoluteInput));
|
|
5996
6597
|
const prepared = isHttpUrl(input) ? await prepareUrlInput(rootDir, input, normalizedOptions) : await prepareFileInput(rootDir, absoluteInput, repoRoot);
|
|
5997
|
-
|
|
5998
|
-
|
|
6598
|
+
return await persistPreparedInput(rootDir, prepared, paths);
|
|
6599
|
+
}
|
|
6600
|
+
async function ingestInput(rootDir, input, options) {
|
|
6601
|
+
return (await ingestInputDetailed(rootDir, input, options)).manifest;
|
|
5999
6602
|
}
|
|
6000
6603
|
async function addInput(rootDir, input, options = {}) {
|
|
6001
6604
|
const { paths } = await initWorkspace(rootDir);
|
|
@@ -6082,7 +6685,7 @@ async function addInput(rootDir, input, options = {}) {
|
|
|
6082
6685
|
async function ingestDirectory(rootDir, inputDir, options) {
|
|
6083
6686
|
const { paths } = await initWorkspace(rootDir);
|
|
6084
6687
|
const normalizedOptions = await resolveRepoIngestOptions(rootDir, options);
|
|
6085
|
-
const absoluteInputDir =
|
|
6688
|
+
const absoluteInputDir = path12.resolve(rootDir, inputDir);
|
|
6086
6689
|
const repoRoot = normalizedOptions.repoRoot ?? await findNearestGitRoot2(absoluteInputDir) ?? absoluteInputDir;
|
|
6087
6690
|
if (!await fileExists(absoluteInputDir)) {
|
|
6088
6691
|
throw new Error(`Directory not found: ${absoluteInputDir}`);
|
|
@@ -6090,8 +6693,9 @@ async function ingestDirectory(rootDir, inputDir, options) {
|
|
|
6090
6693
|
const { files, skipped } = await collectDirectoryFiles(rootDir, absoluteInputDir, repoRoot, normalizedOptions);
|
|
6091
6694
|
const imported = [];
|
|
6092
6695
|
const updated = [];
|
|
6696
|
+
const progress = createProgressReporter("ingest", files.length);
|
|
6093
6697
|
for (const absolutePath of files) {
|
|
6094
|
-
const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(
|
|
6698
|
+
const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path12.relative(repoRoot, absolutePath));
|
|
6095
6699
|
const prepared = await prepareFileInput(rootDir, absolutePath, repoRoot, sourceClassForRelativePath(relativePath, normalizedOptions));
|
|
6096
6700
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
6097
6701
|
if (result.isNew) {
|
|
@@ -6099,11 +6703,13 @@ async function ingestDirectory(rootDir, inputDir, options) {
|
|
|
6099
6703
|
} else if (result.wasUpdated) {
|
|
6100
6704
|
updated.push(result.manifest);
|
|
6101
6705
|
} else {
|
|
6102
|
-
skipped.push({ path: toPosix(
|
|
6706
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "duplicate_content" });
|
|
6103
6707
|
}
|
|
6708
|
+
progress.tick();
|
|
6104
6709
|
}
|
|
6105
|
-
|
|
6106
|
-
|
|
6710
|
+
progress.finish(`imported=${imported.length}, updated=${updated.length}, skipped=${skipped.length}`);
|
|
6711
|
+
await appendLogEntry(rootDir, "ingest_directory", toPosix(path12.relative(rootDir, absoluteInputDir)) || ".", [
|
|
6712
|
+
`repo_root=${toPosix(path12.relative(rootDir, repoRoot)) || "."}`,
|
|
6107
6713
|
`scanned=${files.length}`,
|
|
6108
6714
|
`imported=${imported.length}`,
|
|
6109
6715
|
`updated=${updated.length}`,
|
|
@@ -6120,7 +6726,7 @@ async function ingestDirectory(rootDir, inputDir, options) {
|
|
|
6120
6726
|
}
|
|
6121
6727
|
async function importInbox(rootDir, inputDir) {
|
|
6122
6728
|
const { paths } = await initWorkspace(rootDir);
|
|
6123
|
-
const effectiveInputDir =
|
|
6729
|
+
const effectiveInputDir = path12.resolve(rootDir, inputDir ?? paths.inboxDir);
|
|
6124
6730
|
if (!await fileExists(effectiveInputDir)) {
|
|
6125
6731
|
throw new Error(`Inbox directory not found: ${effectiveInputDir}`);
|
|
6126
6732
|
}
|
|
@@ -6131,31 +6737,31 @@ async function importInbox(rootDir, inputDir) {
|
|
|
6131
6737
|
const skipped = [];
|
|
6132
6738
|
let attachmentCount = 0;
|
|
6133
6739
|
for (const absolutePath of files) {
|
|
6134
|
-
const basename =
|
|
6740
|
+
const basename = path12.basename(absolutePath);
|
|
6135
6741
|
if (basename.startsWith(".")) {
|
|
6136
|
-
skipped.push({ path: toPosix(
|
|
6742
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "hidden_file" });
|
|
6137
6743
|
continue;
|
|
6138
6744
|
}
|
|
6139
6745
|
if (claimedAttachments.has(absolutePath)) {
|
|
6140
|
-
skipped.push({ path: toPosix(
|
|
6746
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "referenced_attachment" });
|
|
6141
6747
|
continue;
|
|
6142
6748
|
}
|
|
6143
6749
|
const mimeType = guessMimeType(absolutePath);
|
|
6144
6750
|
const sourceKind = inferKind(mimeType, absolutePath);
|
|
6145
6751
|
if (!isSupportedInboxKind(sourceKind)) {
|
|
6146
|
-
skipped.push({ path: toPosix(
|
|
6752
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: `unsupported_kind:${sourceKind}` });
|
|
6147
6753
|
continue;
|
|
6148
6754
|
}
|
|
6149
6755
|
const prepared = sourceKind === "markdown" && refsBySource.has(absolutePath) ? await prepareInboxMarkdownInput(absolutePath, refsBySource.get(absolutePath) ?? []) : sourceKind === "html" && refsBySource.has(absolutePath) ? await prepareInboxHtmlInput(absolutePath, refsBySource.get(absolutePath) ?? []) : await prepareFileInput(rootDir, absolutePath);
|
|
6150
6756
|
const result = await persistPreparedInput(rootDir, prepared, paths);
|
|
6151
6757
|
if (!result.isNew) {
|
|
6152
|
-
skipped.push({ path: toPosix(
|
|
6758
|
+
skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "duplicate_content" });
|
|
6153
6759
|
continue;
|
|
6154
6760
|
}
|
|
6155
6761
|
attachmentCount += result.manifest.attachments?.length ?? 0;
|
|
6156
6762
|
imported.push(result.manifest);
|
|
6157
6763
|
}
|
|
6158
|
-
await appendLogEntry(rootDir, "inbox_import", toPosix(
|
|
6764
|
+
await appendLogEntry(rootDir, "inbox_import", toPosix(path12.relative(rootDir, effectiveInputDir)) || ".", [
|
|
6159
6765
|
`scanned=${files.length}`,
|
|
6160
6766
|
`imported=${imported.length}`,
|
|
6161
6767
|
`attachments=${attachmentCount}`,
|
|
@@ -6174,27 +6780,36 @@ async function listManifests(rootDir) {
|
|
|
6174
6780
|
if (!await fileExists(paths.manifestsDir)) {
|
|
6175
6781
|
return [];
|
|
6176
6782
|
}
|
|
6177
|
-
const entries = await
|
|
6783
|
+
const entries = await fs11.readdir(paths.manifestsDir);
|
|
6178
6784
|
const manifests = await Promise.all(
|
|
6179
|
-
entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(
|
|
6785
|
+
entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(path12.join(paths.manifestsDir, entry)))
|
|
6180
6786
|
);
|
|
6181
6787
|
return manifests.filter((manifest) => Boolean(manifest));
|
|
6182
6788
|
}
|
|
6789
|
+
async function removeManifestBySourceId(rootDir, sourceId) {
|
|
6790
|
+
const { paths } = await initWorkspace(rootDir);
|
|
6791
|
+
const manifest = await readJsonFile(path12.join(paths.manifestsDir, `${sourceId}.json`));
|
|
6792
|
+
if (!manifest) {
|
|
6793
|
+
return null;
|
|
6794
|
+
}
|
|
6795
|
+
await removeManifestArtifacts(rootDir, manifest, paths);
|
|
6796
|
+
return manifest;
|
|
6797
|
+
}
|
|
6183
6798
|
async function readExtractedText(rootDir, manifest) {
|
|
6184
6799
|
if (!manifest.extractedTextPath) {
|
|
6185
6800
|
return void 0;
|
|
6186
6801
|
}
|
|
6187
|
-
const absolutePath =
|
|
6802
|
+
const absolutePath = path12.resolve(rootDir, manifest.extractedTextPath);
|
|
6188
6803
|
if (!await fileExists(absolutePath)) {
|
|
6189
6804
|
return void 0;
|
|
6190
6805
|
}
|
|
6191
|
-
return
|
|
6806
|
+
return fs11.readFile(absolutePath, "utf8");
|
|
6192
6807
|
}
|
|
6193
6808
|
async function readExtractionArtifact(rootDir, manifest) {
|
|
6194
6809
|
if (!manifest.extractedMetadataPath) {
|
|
6195
6810
|
return void 0;
|
|
6196
6811
|
}
|
|
6197
|
-
const absolutePath =
|
|
6812
|
+
const absolutePath = path12.resolve(rootDir, manifest.extractedMetadataPath);
|
|
6198
6813
|
if (!await fileExists(absolutePath)) {
|
|
6199
6814
|
return void 0;
|
|
6200
6815
|
}
|
|
@@ -6202,20 +6817,20 @@ async function readExtractionArtifact(rootDir, manifest) {
|
|
|
6202
6817
|
}
|
|
6203
6818
|
|
|
6204
6819
|
// src/mcp.ts
|
|
6205
|
-
import
|
|
6206
|
-
import
|
|
6820
|
+
import fs19 from "fs/promises";
|
|
6821
|
+
import path23 from "path";
|
|
6207
6822
|
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
6208
6823
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
6209
6824
|
import { z as z8 } from "zod";
|
|
6210
6825
|
|
|
6211
6826
|
// src/schema.ts
|
|
6212
|
-
import
|
|
6213
|
-
import
|
|
6827
|
+
import fs12 from "fs/promises";
|
|
6828
|
+
import path13 from "path";
|
|
6214
6829
|
function normalizeSchemaContent(content) {
|
|
6215
6830
|
return content.trim() ? content.trim() : defaultVaultSchema().trim();
|
|
6216
6831
|
}
|
|
6217
6832
|
async function readSchemaFile(schemaPath, fallback = defaultVaultSchema()) {
|
|
6218
|
-
const content = await fileExists(schemaPath) ? await
|
|
6833
|
+
const content = await fileExists(schemaPath) ? await fs12.readFile(schemaPath, "utf8") : fallback;
|
|
6219
6834
|
const normalized = normalizeSchemaContent(content);
|
|
6220
6835
|
return {
|
|
6221
6836
|
path: schemaPath,
|
|
@@ -6224,7 +6839,7 @@ async function readSchemaFile(schemaPath, fallback = defaultVaultSchema()) {
|
|
|
6224
6839
|
};
|
|
6225
6840
|
}
|
|
6226
6841
|
function resolveProjectSchemaPath(rootDir, schemaPath) {
|
|
6227
|
-
return
|
|
6842
|
+
return path13.resolve(rootDir, schemaPath);
|
|
6228
6843
|
}
|
|
6229
6844
|
function composeVaultSchema(root, projectSchemas = []) {
|
|
6230
6845
|
if (!projectSchemas.length) {
|
|
@@ -6240,7 +6855,7 @@ function composeVaultSchema(root, projectSchemas = []) {
|
|
|
6240
6855
|
(schema) => [
|
|
6241
6856
|
`## Project Schema`,
|
|
6242
6857
|
"",
|
|
6243
|
-
`Path: ${toPosix(
|
|
6858
|
+
`Path: ${toPosix(path13.relative(path13.dirname(root.path), schema.path) || schema.path)}`,
|
|
6244
6859
|
"",
|
|
6245
6860
|
schema.content
|
|
6246
6861
|
].join("\n")
|
|
@@ -6316,13 +6931,13 @@ function buildSchemaPrompt(schema, instruction) {
|
|
|
6316
6931
|
}
|
|
6317
6932
|
|
|
6318
6933
|
// src/vault.ts
|
|
6319
|
-
import
|
|
6320
|
-
import
|
|
6934
|
+
import fs18 from "fs/promises";
|
|
6935
|
+
import path22 from "path";
|
|
6321
6936
|
import matter9 from "gray-matter";
|
|
6322
6937
|
import { z as z7 } from "zod";
|
|
6323
6938
|
|
|
6324
6939
|
// src/analysis.ts
|
|
6325
|
-
import
|
|
6940
|
+
import path14 from "path";
|
|
6326
6941
|
import { z as z2 } from "zod";
|
|
6327
6942
|
var ANALYSIS_FORMAT_VERSION = 6;
|
|
6328
6943
|
var sourceAnalysisSchema = z2.object({
|
|
@@ -6551,7 +7166,7 @@ function extractionWarningSummary(manifest, extraction) {
|
|
|
6551
7166
|
return `Imported ${manifest.sourceKind} source. Text extraction is not yet available for this source.`;
|
|
6552
7167
|
}
|
|
6553
7168
|
async function analyzeSource(manifest, extractedText, provider, paths, schema) {
|
|
6554
|
-
const cachePath =
|
|
7169
|
+
const cachePath = path14.join(paths.analysesDir, `${manifest.sourceId}.json`);
|
|
6555
7170
|
const cached = await readJsonFile(cachePath);
|
|
6556
7171
|
if (cached && cached.analysisVersion === ANALYSIS_FORMAT_VERSION && cached.sourceHash === manifest.contentHash && cached.extractionHash === manifest.extractionHash && cached.schemaHash === schema.hash) {
|
|
6557
7172
|
return cached;
|
|
@@ -6641,8 +7256,8 @@ function conflictConfidence(claimA, claimB) {
|
|
|
6641
7256
|
}
|
|
6642
7257
|
|
|
6643
7258
|
// src/deep-lint.ts
|
|
6644
|
-
import
|
|
6645
|
-
import
|
|
7259
|
+
import fs13 from "fs/promises";
|
|
7260
|
+
import path17 from "path";
|
|
6646
7261
|
import matter4 from "gray-matter";
|
|
6647
7262
|
import { z as z5 } from "zod";
|
|
6648
7263
|
|
|
@@ -6663,7 +7278,7 @@ function normalizeFindingSeverity(value) {
|
|
|
6663
7278
|
|
|
6664
7279
|
// src/orchestration.ts
|
|
6665
7280
|
import { spawn } from "child_process";
|
|
6666
|
-
import
|
|
7281
|
+
import path15 from "path";
|
|
6667
7282
|
import { z as z3 } from "zod";
|
|
6668
7283
|
var orchestrationRoleResultSchema = z3.object({
|
|
6669
7284
|
summary: z3.string().optional(),
|
|
@@ -6756,7 +7371,7 @@ async function runProviderRole(rootDir, role, roleConfig, input) {
|
|
|
6756
7371
|
}
|
|
6757
7372
|
async function runCommandRole(rootDir, role, executor, input) {
|
|
6758
7373
|
const [command, ...args] = executor.command;
|
|
6759
|
-
const cwd = executor.cwd ?
|
|
7374
|
+
const cwd = executor.cwd ? path15.resolve(rootDir, executor.cwd) : rootDir;
|
|
6760
7375
|
const child = spawn(command, args, {
|
|
6761
7376
|
cwd,
|
|
6762
7377
|
env: {
|
|
@@ -6850,7 +7465,7 @@ function summarizeRoleQuestions(results) {
|
|
|
6850
7465
|
}
|
|
6851
7466
|
|
|
6852
7467
|
// src/web-search/registry.ts
|
|
6853
|
-
import
|
|
7468
|
+
import path16 from "path";
|
|
6854
7469
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
6855
7470
|
import { z as z4 } from "zod";
|
|
6856
7471
|
|
|
@@ -6948,7 +7563,7 @@ async function createWebSearchAdapter(id, config, rootDir) {
|
|
|
6948
7563
|
if (!config.module) {
|
|
6949
7564
|
throw new Error(`Web search provider ${id} is type "custom" but no module path was configured.`);
|
|
6950
7565
|
}
|
|
6951
|
-
const resolvedModule =
|
|
7566
|
+
const resolvedModule = path16.isAbsolute(config.module) ? config.module : path16.resolve(rootDir, config.module);
|
|
6952
7567
|
const loaded = await import(pathToFileURL2(resolvedModule).href);
|
|
6953
7568
|
const parsed = customWebSearchModuleSchema.parse(loaded);
|
|
6954
7569
|
return parsed.createAdapter(id, config, rootDir);
|
|
@@ -7015,8 +7630,8 @@ async function loadContextPages(rootDir, graph) {
|
|
|
7015
7630
|
);
|
|
7016
7631
|
return Promise.all(
|
|
7017
7632
|
contextPages.slice(0, 18).map(async (page) => {
|
|
7018
|
-
const absolutePath =
|
|
7019
|
-
const raw = await
|
|
7633
|
+
const absolutePath = path17.join(paths.wikiDir, page.path);
|
|
7634
|
+
const raw = await fs13.readFile(absolutePath, "utf8").catch(() => "");
|
|
7020
7635
|
const parsed = matter4(raw);
|
|
7021
7636
|
return {
|
|
7022
7637
|
id: page.id,
|
|
@@ -7064,7 +7679,7 @@ function heuristicDeepFindings(contextPages, structuralFindings, graph) {
|
|
|
7064
7679
|
code: "missing_citation",
|
|
7065
7680
|
message: finding.message,
|
|
7066
7681
|
pagePath: finding.pagePath,
|
|
7067
|
-
suggestedQuery: finding.pagePath ? `Which sources support the claims in ${
|
|
7682
|
+
suggestedQuery: finding.pagePath ? `Which sources support the claims in ${path17.basename(finding.pagePath, ".md")}?` : void 0
|
|
7068
7683
|
});
|
|
7069
7684
|
}
|
|
7070
7685
|
for (const page of contextPages.filter((item) => item.kind === "source").slice(0, 3)) {
|
|
@@ -7243,8 +7858,8 @@ async function runDeepLint(rootDir, structuralFindings, options = {}) {
|
|
|
7243
7858
|
}
|
|
7244
7859
|
|
|
7245
7860
|
// src/embeddings.ts
|
|
7246
|
-
import
|
|
7247
|
-
import
|
|
7861
|
+
import fs14 from "fs/promises";
|
|
7862
|
+
import path18 from "path";
|
|
7248
7863
|
var MAX_EMBEDDING_BATCH = 32;
|
|
7249
7864
|
var MAX_SIMILARITY_NODES = 240;
|
|
7250
7865
|
function cosineSimilarity(left, right) {
|
|
@@ -7278,8 +7893,8 @@ async function loadPageContents(rootDir, graph) {
|
|
|
7278
7893
|
const contents = /* @__PURE__ */ new Map();
|
|
7279
7894
|
await Promise.all(
|
|
7280
7895
|
graph.pages.map(async (page) => {
|
|
7281
|
-
const absolutePath =
|
|
7282
|
-
const content = await
|
|
7896
|
+
const absolutePath = path18.join(paths.wikiDir, page.path);
|
|
7897
|
+
const content = await fs14.readFile(absolutePath, "utf8").catch(() => {
|
|
7283
7898
|
process.stderr.write(`[swarmvault] Warning: could not read page ${page.path} for embedding
|
|
7284
7899
|
`);
|
|
7285
7900
|
return "";
|
|
@@ -8781,15 +9396,15 @@ function sourceTypeForNode(node, pagesById) {
|
|
|
8781
9396
|
return pagesById.get(node.pageId)?.sourceType;
|
|
8782
9397
|
}
|
|
8783
9398
|
function supportingPathDetails(graph, edge) {
|
|
8784
|
-
const
|
|
9399
|
+
const path28 = shortestGraphPath(graph, edge.source, edge.target);
|
|
8785
9400
|
const edgesById = new Map(graph.edges.map((item) => [item.id, item]));
|
|
8786
|
-
const pathEdges =
|
|
9401
|
+
const pathEdges = path28.edgeIds.map((edgeId) => edgesById.get(edgeId)).filter((item) => Boolean(item));
|
|
8787
9402
|
return {
|
|
8788
|
-
pathNodeIds:
|
|
8789
|
-
pathEdgeIds:
|
|
9403
|
+
pathNodeIds: path28.nodeIds,
|
|
9404
|
+
pathEdgeIds: path28.edgeIds,
|
|
8790
9405
|
pathRelations: pathEdges.map((item) => item.relation),
|
|
8791
9406
|
pathEvidenceClasses: pathEdges.map((item) => item.evidenceClass),
|
|
8792
|
-
pathSummary:
|
|
9407
|
+
pathSummary: path28.summary
|
|
8793
9408
|
};
|
|
8794
9409
|
}
|
|
8795
9410
|
function surpriseScore(edge, graph, pagesById, hyperedgesByNodeId) {
|
|
@@ -8858,7 +9473,7 @@ function topSurprisingConnections(graph, pagesById) {
|
|
|
8858
9473
|
}).map((edge) => {
|
|
8859
9474
|
const source = nodesById.get(edge.source);
|
|
8860
9475
|
const target = nodesById.get(edge.target);
|
|
8861
|
-
const
|
|
9476
|
+
const path28 = supportingPathDetails(graph, edge);
|
|
8862
9477
|
const scored = surpriseScore(edge, graph, pagesById, hyperedgesByNodeId);
|
|
8863
9478
|
return {
|
|
8864
9479
|
id: edge.id,
|
|
@@ -8869,11 +9484,11 @@ function topSurprisingConnections(graph, pagesById) {
|
|
|
8869
9484
|
relation: edge.relation,
|
|
8870
9485
|
evidenceClass: edge.evidenceClass,
|
|
8871
9486
|
confidence: edge.confidence,
|
|
8872
|
-
pathNodeIds:
|
|
8873
|
-
pathEdgeIds:
|
|
8874
|
-
pathRelations:
|
|
8875
|
-
pathEvidenceClasses:
|
|
8876
|
-
pathSummary:
|
|
9487
|
+
pathNodeIds: path28.pathNodeIds,
|
|
9488
|
+
pathEdgeIds: path28.pathEdgeIds,
|
|
9489
|
+
pathRelations: path28.pathRelations,
|
|
9490
|
+
pathEvidenceClasses: path28.pathEvidenceClasses,
|
|
9491
|
+
pathSummary: path28.pathSummary,
|
|
8877
9492
|
why: scored.why,
|
|
8878
9493
|
explanation: scored.explanation,
|
|
8879
9494
|
surpriseScore: scored.score
|
|
@@ -8889,6 +9504,35 @@ function topGroupPatterns(graph) {
|
|
|
8889
9504
|
(left, right) => right.confidence - left.confidence || right.nodeIds.length - left.nodeIds.length || left.label.localeCompare(right.label)
|
|
8890
9505
|
).slice(0, 8);
|
|
8891
9506
|
}
|
|
9507
|
+
function fragmentedCommunityPresentation(graph, communityPages) {
|
|
9508
|
+
const thinCommunities = (graph.communities ?? []).filter((community) => community.nodeIds.length <= 2).sort((left, right) => right.nodeIds.length - left.nodeIds.length || left.label.localeCompare(right.label));
|
|
9509
|
+
const visibleCommunities = thinCommunities.slice(0, 6).map((community) => {
|
|
9510
|
+
const page = communityPages.find((candidate) => candidate.id === `graph:${community.id}`);
|
|
9511
|
+
return {
|
|
9512
|
+
id: community.id,
|
|
9513
|
+
label: community.label,
|
|
9514
|
+
nodeCount: community.nodeIds.length,
|
|
9515
|
+
pageId: page?.id,
|
|
9516
|
+
path: page?.path,
|
|
9517
|
+
title: page?.title
|
|
9518
|
+
};
|
|
9519
|
+
});
|
|
9520
|
+
const rolledUp = thinCommunities.slice(visibleCommunities.length);
|
|
9521
|
+
if (!rolledUp.length) {
|
|
9522
|
+
return {
|
|
9523
|
+
thinCommunities: visibleCommunities
|
|
9524
|
+
};
|
|
9525
|
+
}
|
|
9526
|
+
return {
|
|
9527
|
+
thinCommunities: visibleCommunities,
|
|
9528
|
+
fragmentedCommunityRollup: {
|
|
9529
|
+
totalCommunities: graph.communities?.length ?? 0,
|
|
9530
|
+
rolledUpCount: rolledUp.length,
|
|
9531
|
+
rolledUpNodes: rolledUp.reduce((sum, community) => sum + community.nodeIds.length, 0),
|
|
9532
|
+
exampleLabels: rolledUp.slice(0, 4).map((community) => community.label)
|
|
9533
|
+
}
|
|
9534
|
+
};
|
|
9535
|
+
}
|
|
8892
9536
|
function suggestedGraphQuestions(graph) {
|
|
8893
9537
|
const thinCommunities = (graph.communities ?? []).filter((community) => community.nodeIds.length <= 2);
|
|
8894
9538
|
const bridgeNodes = graph.nodes.filter((node) => (node.bridgeScore ?? 0) > 0).sort((left, right) => (right.bridgeScore ?? 0) - (left.bridgeScore ?? 0)).slice(0, 3);
|
|
@@ -8903,17 +9547,7 @@ function buildGraphReportArtifact(input) {
|
|
|
8903
9547
|
const pagesById = new Map(reportGraph.pages.map((page) => [page.id, page]));
|
|
8904
9548
|
const godNodes = reportGraph.nodes.filter((node) => node.isGodNode).sort((left, right) => (right.degree ?? 0) - (left.degree ?? 0)).slice(0, 8);
|
|
8905
9549
|
const bridgeNodes = reportGraph.nodes.filter((node) => (node.bridgeScore ?? 0) > 0).sort((left, right) => (right.bridgeScore ?? 0) - (left.bridgeScore ?? 0)).slice(0, 8);
|
|
8906
|
-
const
|
|
8907
|
-
const page = input.communityPages.find((candidate) => candidate.id === `graph:${community.id}`);
|
|
8908
|
-
return {
|
|
8909
|
-
id: community.id,
|
|
8910
|
-
label: community.label,
|
|
8911
|
-
nodeCount: community.nodeIds.length,
|
|
8912
|
-
pageId: page?.id,
|
|
8913
|
-
path: page?.path,
|
|
8914
|
-
title: page?.title
|
|
8915
|
-
};
|
|
8916
|
-
});
|
|
9550
|
+
const communityPresentation = fragmentedCommunityPresentation(reportGraph, input.communityPages);
|
|
8917
9551
|
const surprisingConnections = topSurprisingConnections(reportGraph, pagesById);
|
|
8918
9552
|
const groupPatterns = topGroupPatterns(reportGraph);
|
|
8919
9553
|
const breakdown = sourceClassBreakdown(input.graph);
|
|
@@ -8927,6 +9561,11 @@ function buildGraphReportArtifact(input) {
|
|
|
8927
9561
|
`Non-first-party material accounts for ${(nonFirstPartyNodes / Math.max(1, input.graph.nodes.length) * 100).toFixed(1)}% of graph nodes.`
|
|
8928
9562
|
);
|
|
8929
9563
|
}
|
|
9564
|
+
if (communityPresentation.fragmentedCommunityRollup) {
|
|
9565
|
+
warnings.push(
|
|
9566
|
+
`First-party report view is fragmented: ${communityPresentation.fragmentedCommunityRollup.rolledUpCount} tiny communities were rolled up for readability.`
|
|
9567
|
+
);
|
|
9568
|
+
}
|
|
8930
9569
|
return {
|
|
8931
9570
|
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8932
9571
|
graphHash: input.graphHash,
|
|
@@ -8964,10 +9603,11 @@ function buildGraphReportArtifact(input) {
|
|
|
8964
9603
|
degree: node.degree,
|
|
8965
9604
|
bridgeScore: node.bridgeScore
|
|
8966
9605
|
})),
|
|
8967
|
-
thinCommunities,
|
|
9606
|
+
thinCommunities: communityPresentation.thinCommunities,
|
|
9607
|
+
fragmentedCommunityRollup: communityPresentation.fragmentedCommunityRollup,
|
|
8968
9608
|
surprisingConnections,
|
|
8969
9609
|
groupPatterns,
|
|
8970
|
-
suggestedQuestions: suggestedGraphQuestions(
|
|
9610
|
+
suggestedQuestions: suggestedGraphQuestions(reportGraph),
|
|
8971
9611
|
communityPages: input.communityPages.map((page) => ({
|
|
8972
9612
|
id: page.id,
|
|
8973
9613
|
path: page.path,
|
|
@@ -9094,6 +9734,10 @@ function buildGraphReportPage(input) {
|
|
|
9094
9734
|
...input.report.thinCommunities.length ? input.report.thinCommunities.map(
|
|
9095
9735
|
(community) => community.path ? `- [[${community.path.replace(/\.md$/, "")}|${community.title ?? community.label}]] (${community.nodeCount} node(s))` : `- ${community.label} (${community.nodeCount} node(s))`
|
|
9096
9736
|
) : ["- No thin communities detected."],
|
|
9737
|
+
...input.report.fragmentedCommunityRollup ? [
|
|
9738
|
+
`- Rolled up ${input.report.fragmentedCommunityRollup.rolledUpCount} additional tiny communities covering ${input.report.fragmentedCommunityRollup.rolledUpNodes} node(s).`,
|
|
9739
|
+
`- Example rolled-up labels: ${input.report.fragmentedCommunityRollup.exampleLabels.join(", ")}`
|
|
9740
|
+
] : [],
|
|
9097
9741
|
"",
|
|
9098
9742
|
"## Surprising Connections",
|
|
9099
9743
|
"",
|
|
@@ -9771,13 +10415,13 @@ function buildOutputAssetManifest(input) {
|
|
|
9771
10415
|
}
|
|
9772
10416
|
|
|
9773
10417
|
// src/outputs.ts
|
|
9774
|
-
import
|
|
9775
|
-
import
|
|
10418
|
+
import fs16 from "fs/promises";
|
|
10419
|
+
import path20 from "path";
|
|
9776
10420
|
import matter7 from "gray-matter";
|
|
9777
10421
|
|
|
9778
10422
|
// src/pages.ts
|
|
9779
|
-
import
|
|
9780
|
-
import
|
|
10423
|
+
import fs15 from "fs/promises";
|
|
10424
|
+
import path19 from "path";
|
|
9781
10425
|
import matter6 from "gray-matter";
|
|
9782
10426
|
function normalizeStringArray(value) {
|
|
9783
10427
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
@@ -9855,7 +10499,7 @@ async function loadExistingManagedPageState(absolutePath, defaults = {}) {
|
|
|
9855
10499
|
updatedAt: updatedFallback
|
|
9856
10500
|
};
|
|
9857
10501
|
}
|
|
9858
|
-
const content = await
|
|
10502
|
+
const content = await fs15.readFile(absolutePath, "utf8");
|
|
9859
10503
|
const parsed = matter6(content);
|
|
9860
10504
|
return {
|
|
9861
10505
|
status: normalizePageStatus(parsed.data.status, defaults.status ?? "active"),
|
|
@@ -9894,7 +10538,7 @@ function parseStoredPage(relativePath, content, defaults = {}) {
|
|
|
9894
10538
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9895
10539
|
const fallbackCreatedAt = defaults.createdAt ?? now;
|
|
9896
10540
|
const fallbackUpdatedAt = defaults.updatedAt ?? fallbackCreatedAt;
|
|
9897
|
-
const title = typeof parsed.data.title === "string" ? parsed.data.title :
|
|
10541
|
+
const title = typeof parsed.data.title === "string" ? parsed.data.title : path19.basename(relativePath, ".md");
|
|
9898
10542
|
const kind = inferPageKind(relativePath, parsed.data.kind);
|
|
9899
10543
|
const sourceIds = normalizeStringArray(parsed.data.source_ids);
|
|
9900
10544
|
const projectIds = normalizeProjectIds(parsed.data.project_ids);
|
|
@@ -9935,18 +10579,18 @@ function parseStoredPage(relativePath, content, defaults = {}) {
|
|
|
9935
10579
|
};
|
|
9936
10580
|
}
|
|
9937
10581
|
async function loadInsightPages(wikiDir) {
|
|
9938
|
-
const insightsDir =
|
|
10582
|
+
const insightsDir = path19.join(wikiDir, "insights");
|
|
9939
10583
|
if (!await fileExists(insightsDir)) {
|
|
9940
10584
|
return [];
|
|
9941
10585
|
}
|
|
9942
|
-
const files = (await listFilesRecursive(insightsDir)).filter((filePath) => filePath.endsWith(".md")).filter((filePath) =>
|
|
10586
|
+
const files = (await listFilesRecursive(insightsDir)).filter((filePath) => filePath.endsWith(".md")).filter((filePath) => path19.basename(filePath) !== "index.md").sort((left, right) => left.localeCompare(right));
|
|
9943
10587
|
const insights = [];
|
|
9944
10588
|
for (const absolutePath of files) {
|
|
9945
|
-
const relativePath = toPosix(
|
|
9946
|
-
const content = await
|
|
10589
|
+
const relativePath = toPosix(path19.relative(wikiDir, absolutePath));
|
|
10590
|
+
const content = await fs15.readFile(absolutePath, "utf8");
|
|
9947
10591
|
const parsed = matter6(content);
|
|
9948
|
-
const stats = await
|
|
9949
|
-
const title = typeof parsed.data.title === "string" ? parsed.data.title :
|
|
10592
|
+
const stats = await fs15.stat(absolutePath);
|
|
10593
|
+
const title = typeof parsed.data.title === "string" ? parsed.data.title : path19.basename(absolutePath, ".md");
|
|
9950
10594
|
const sourceIds = normalizeStringArray(parsed.data.source_ids);
|
|
9951
10595
|
const projectIds = normalizeProjectIds(parsed.data.project_ids);
|
|
9952
10596
|
const nodeIds = normalizeStringArray(parsed.data.node_ids);
|
|
@@ -10009,79 +10653,94 @@ function relatedOutputsForPage(targetPage, outputPages) {
|
|
|
10009
10653
|
return outputPages.map((page) => ({ page, rank: relationRank(page, targetPage) })).filter((item) => item.rank > 0).sort((left, right) => right.rank - left.rank || left.page.title.localeCompare(right.page.title)).map((item) => item.page);
|
|
10010
10654
|
}
|
|
10011
10655
|
async function resolveUniqueOutputSlug(wikiDir, baseSlug) {
|
|
10012
|
-
const outputsDir =
|
|
10656
|
+
const outputsDir = path20.join(wikiDir, "outputs");
|
|
10013
10657
|
const root = baseSlug || "output";
|
|
10014
10658
|
let candidate = root;
|
|
10015
10659
|
let counter = 2;
|
|
10016
|
-
while (await fileExists(
|
|
10660
|
+
while (await fileExists(path20.join(outputsDir, `${candidate}.md`))) {
|
|
10017
10661
|
candidate = `${root}-${counter}`;
|
|
10018
10662
|
counter++;
|
|
10019
10663
|
}
|
|
10020
10664
|
return candidate;
|
|
10021
10665
|
}
|
|
10022
10666
|
async function loadSavedOutputPages(wikiDir) {
|
|
10023
|
-
const outputsDir =
|
|
10024
|
-
const entries = await
|
|
10667
|
+
const outputsDir = path20.join(wikiDir, "outputs");
|
|
10668
|
+
const entries = await fs16.readdir(outputsDir, { withFileTypes: true }).catch(() => []);
|
|
10025
10669
|
const outputs = [];
|
|
10026
|
-
|
|
10027
|
-
|
|
10670
|
+
const queue = [{ absoluteDir: outputsDir, relativeDir: "outputs" }];
|
|
10671
|
+
while (queue.length > 0) {
|
|
10672
|
+
const current = queue.shift();
|
|
10673
|
+
if (!current) {
|
|
10028
10674
|
continue;
|
|
10029
10675
|
}
|
|
10030
|
-
const
|
|
10031
|
-
const
|
|
10032
|
-
|
|
10033
|
-
|
|
10034
|
-
|
|
10035
|
-
|
|
10036
|
-
|
|
10037
|
-
|
|
10038
|
-
|
|
10039
|
-
|
|
10040
|
-
|
|
10041
|
-
|
|
10042
|
-
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
|
|
10046
|
-
|
|
10047
|
-
|
|
10048
|
-
|
|
10049
|
-
|
|
10050
|
-
|
|
10051
|
-
|
|
10052
|
-
|
|
10053
|
-
|
|
10054
|
-
|
|
10055
|
-
|
|
10056
|
-
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10064
|
-
|
|
10065
|
-
|
|
10066
|
-
|
|
10067
|
-
|
|
10068
|
-
|
|
10069
|
-
|
|
10070
|
-
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10676
|
+
const currentEntries = current.absoluteDir === outputsDir ? entries : await fs16.readdir(current.absoluteDir, { withFileTypes: true }).catch(() => []);
|
|
10677
|
+
for (const entry of currentEntries) {
|
|
10678
|
+
if (entry.isDirectory()) {
|
|
10679
|
+
queue.push({
|
|
10680
|
+
absoluteDir: path20.join(current.absoluteDir, entry.name),
|
|
10681
|
+
relativeDir: path20.posix.join(current.relativeDir, entry.name)
|
|
10682
|
+
});
|
|
10683
|
+
continue;
|
|
10684
|
+
}
|
|
10685
|
+
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
|
10686
|
+
continue;
|
|
10687
|
+
}
|
|
10688
|
+
const relativePath = path20.posix.join(current.relativeDir, entry.name);
|
|
10689
|
+
const absolutePath = path20.join(current.absoluteDir, entry.name);
|
|
10690
|
+
const content = await fs16.readFile(absolutePath, "utf8");
|
|
10691
|
+
const parsed = matter7(content);
|
|
10692
|
+
const slug = relativePath.replace(/^outputs\//, "").replace(/\.md$/, "");
|
|
10693
|
+
const title = typeof parsed.data.title === "string" ? parsed.data.title : path20.basename(slug);
|
|
10694
|
+
const pageId = typeof parsed.data.page_id === "string" ? parsed.data.page_id : `output:${slug}`;
|
|
10695
|
+
const sourceIds = normalizeStringArray(parsed.data.source_ids);
|
|
10696
|
+
const projectIds = normalizeProjectIds(parsed.data.project_ids);
|
|
10697
|
+
const nodeIds = normalizeStringArray(parsed.data.node_ids);
|
|
10698
|
+
const relatedPageIds = normalizeStringArray(parsed.data.related_page_ids);
|
|
10699
|
+
const relatedNodeIds = normalizeStringArray(parsed.data.related_node_ids);
|
|
10700
|
+
const relatedSourceIds = normalizeStringArray(parsed.data.related_source_ids);
|
|
10701
|
+
const backlinks = normalizeStringArray(parsed.data.backlinks);
|
|
10702
|
+
const compiledFrom = normalizeStringArray(parsed.data.compiled_from);
|
|
10703
|
+
const stats = await fs16.stat(absolutePath);
|
|
10704
|
+
const createdAt = typeof parsed.data.created_at === "string" ? parsed.data.created_at : stats.birthtimeMs > 0 ? stats.birthtime.toISOString() : stats.mtime.toISOString();
|
|
10705
|
+
const updatedAt = typeof parsed.data.updated_at === "string" ? parsed.data.updated_at : stats.mtime.toISOString();
|
|
10706
|
+
outputs.push({
|
|
10707
|
+
page: {
|
|
10708
|
+
id: pageId,
|
|
10709
|
+
path: relativePath,
|
|
10710
|
+
title,
|
|
10711
|
+
kind: "output",
|
|
10712
|
+
sourceIds,
|
|
10713
|
+
projectIds,
|
|
10714
|
+
nodeIds,
|
|
10715
|
+
freshness: parsed.data.freshness === "stale" ? "stale" : "fresh",
|
|
10716
|
+
status: normalizePageStatus(parsed.data.status, "active"),
|
|
10717
|
+
confidence: typeof parsed.data.confidence === "number" ? parsed.data.confidence : 0.74,
|
|
10718
|
+
backlinks,
|
|
10719
|
+
schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
|
|
10720
|
+
sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
|
|
10721
|
+
relatedPageIds,
|
|
10722
|
+
relatedNodeIds,
|
|
10723
|
+
relatedSourceIds,
|
|
10724
|
+
createdAt,
|
|
10725
|
+
updatedAt,
|
|
10726
|
+
compiledFrom: compiledFrom.length ? compiledFrom : relatedSourceIds,
|
|
10727
|
+
managedBy: normalizePageManager(parsed.data.managed_by, "system"),
|
|
10728
|
+
origin: typeof parsed.data.origin === "string" ? parsed.data.origin : void 0,
|
|
10729
|
+
question: typeof parsed.data.question === "string" ? parsed.data.question : void 0,
|
|
10730
|
+
outputFormat: parsed.data.output_format === "report" || parsed.data.output_format === "slides" || parsed.data.output_format === "chart" || parsed.data.output_format === "image" ? parsed.data.output_format : "markdown",
|
|
10731
|
+
outputAssets: normalizeOutputAssets(parsed.data.output_assets)
|
|
10732
|
+
},
|
|
10733
|
+
content,
|
|
10734
|
+
contentHash: sha256(content)
|
|
10735
|
+
});
|
|
10736
|
+
}
|
|
10078
10737
|
}
|
|
10079
10738
|
return outputs.sort((left, right) => left.page.title.localeCompare(right.page.title));
|
|
10080
10739
|
}
|
|
10081
10740
|
|
|
10082
10741
|
// src/search.ts
|
|
10083
|
-
import
|
|
10084
|
-
import
|
|
10742
|
+
import fs17 from "fs/promises";
|
|
10743
|
+
import path21 from "path";
|
|
10085
10744
|
import matter8 from "gray-matter";
|
|
10086
10745
|
function getDatabaseSync() {
|
|
10087
10746
|
const builtin = process.getBuiltinModule?.("node:sqlite");
|
|
@@ -10107,7 +10766,7 @@ function normalizeSourceClass2(value) {
|
|
|
10107
10766
|
return value === "first_party" || value === "third_party" || value === "resource" || value === "generated" ? value : void 0;
|
|
10108
10767
|
}
|
|
10109
10768
|
async function rebuildSearchIndex(dbPath, pages, wikiDir) {
|
|
10110
|
-
await ensureDir(
|
|
10769
|
+
await ensureDir(path21.dirname(dbPath));
|
|
10111
10770
|
const DatabaseSync = getDatabaseSync();
|
|
10112
10771
|
const db = new DatabaseSync(dbPath);
|
|
10113
10772
|
db.exec("PRAGMA journal_mode = WAL;");
|
|
@@ -10139,8 +10798,8 @@ async function rebuildSearchIndex(dbPath, pages, wikiDir) {
|
|
|
10139
10798
|
"INSERT INTO pages (id, path, title, body, kind, status, source_type, source_class, project_ids, project_key) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
10140
10799
|
);
|
|
10141
10800
|
for (const page of pages) {
|
|
10142
|
-
const absolutePath =
|
|
10143
|
-
const content = await
|
|
10801
|
+
const absolutePath = path21.join(wikiDir, page.path);
|
|
10802
|
+
const content = await fs17.readFile(absolutePath, "utf8");
|
|
10144
10803
|
const parsed = matter8(content);
|
|
10145
10804
|
insertPage.run(
|
|
10146
10805
|
page.id,
|
|
@@ -10236,9 +10895,41 @@ function searchPages(dbPath, query, limitOrOptions = 5) {
|
|
|
10236
10895
|
}
|
|
10237
10896
|
|
|
10238
10897
|
// src/vault.ts
|
|
10898
|
+
var COMPILE_PROGRESS_THRESHOLD = 120;
|
|
10899
|
+
var COMPILE_PROGRESS_UPDATE_INTERVAL = 50;
|
|
10239
10900
|
function uniqueStrings3(values) {
|
|
10240
10901
|
return uniqueBy(values.filter(Boolean), (value) => value);
|
|
10241
10902
|
}
|
|
10903
|
+
function createCompileProgressReporter(phase, totalItems) {
|
|
10904
|
+
if (totalItems < COMPILE_PROGRESS_THRESHOLD || !process.stderr?.isTTY) {
|
|
10905
|
+
return {
|
|
10906
|
+
tick: () => {
|
|
10907
|
+
},
|
|
10908
|
+
finish: () => {
|
|
10909
|
+
}
|
|
10910
|
+
};
|
|
10911
|
+
}
|
|
10912
|
+
let completed = 0;
|
|
10913
|
+
let nextUpdate = Math.min(COMPILE_PROGRESS_UPDATE_INTERVAL, totalItems);
|
|
10914
|
+
process.stderr.write(`[swarmvault compile] ${phase}: 0/${totalItems}
|
|
10915
|
+
`);
|
|
10916
|
+
return {
|
|
10917
|
+
tick: (label) => {
|
|
10918
|
+
completed += 1;
|
|
10919
|
+
if (completed >= nextUpdate || completed === totalItems) {
|
|
10920
|
+
process.stderr.write(`[swarmvault compile] ${phase}: ${completed}/${totalItems}${label ? ` (${label})` : ""}
|
|
10921
|
+
`);
|
|
10922
|
+
while (completed >= nextUpdate) {
|
|
10923
|
+
nextUpdate += COMPILE_PROGRESS_UPDATE_INTERVAL;
|
|
10924
|
+
}
|
|
10925
|
+
}
|
|
10926
|
+
},
|
|
10927
|
+
finish: (summary) => {
|
|
10928
|
+
process.stderr.write(`[swarmvault compile] ${phase}: ${totalItems}/${totalItems}${summary ? ` (${summary})` : ""}
|
|
10929
|
+
`);
|
|
10930
|
+
}
|
|
10931
|
+
};
|
|
10932
|
+
}
|
|
10242
10933
|
function normalizeOutputFormat2(format) {
|
|
10243
10934
|
return format === "report" || format === "slides" || format === "chart" || format === "image" ? format : "markdown";
|
|
10244
10935
|
}
|
|
@@ -10257,7 +10948,7 @@ function outputFormatInstruction(format) {
|
|
|
10257
10948
|
}
|
|
10258
10949
|
}
|
|
10259
10950
|
function outputAssetPath(slug, fileName) {
|
|
10260
|
-
return toPosix(
|
|
10951
|
+
return toPosix(path22.join("outputs", "assets", slug, fileName));
|
|
10261
10952
|
}
|
|
10262
10953
|
function outputAssetId(slug, role) {
|
|
10263
10954
|
return `output:${slug}:asset:${role}`;
|
|
@@ -10397,7 +11088,7 @@ async function resolveImageGenerationProvider(rootDir) {
|
|
|
10397
11088
|
if (!providerConfig) {
|
|
10398
11089
|
throw new Error(`No provider configured with id "${preferredProviderId}" for task "imageProvider".`);
|
|
10399
11090
|
}
|
|
10400
|
-
const { createProvider: createProvider2 } = await import("./registry-
|
|
11091
|
+
const { createProvider: createProvider2 } = await import("./registry-2REAPKPO.js");
|
|
10401
11092
|
return createProvider2(preferredProviderId, providerConfig, rootDir);
|
|
10402
11093
|
}
|
|
10403
11094
|
async function generateOutputArtifacts(rootDir, input) {
|
|
@@ -10595,7 +11286,7 @@ async function generateOutputArtifacts(rootDir, input) {
|
|
|
10595
11286
|
};
|
|
10596
11287
|
}
|
|
10597
11288
|
function normalizeProjectRoot(root) {
|
|
10598
|
-
const normalized = toPosix(
|
|
11289
|
+
const normalized = toPosix(path22.posix.normalize(root.replace(/\\/g, "/"))).replace(/^\.\/+/, "").replace(/\/+$/, "");
|
|
10599
11290
|
return normalized;
|
|
10600
11291
|
}
|
|
10601
11292
|
function projectEntries(config) {
|
|
@@ -10621,10 +11312,10 @@ function manifestPathForProject(rootDir, manifest) {
|
|
|
10621
11312
|
if (!rawPath) {
|
|
10622
11313
|
return toPosix(manifest.storedPath);
|
|
10623
11314
|
}
|
|
10624
|
-
if (!
|
|
11315
|
+
if (!path22.isAbsolute(rawPath)) {
|
|
10625
11316
|
return normalizeProjectRoot(rawPath);
|
|
10626
11317
|
}
|
|
10627
|
-
const relative = toPosix(
|
|
11318
|
+
const relative = toPosix(path22.relative(rootDir, rawPath));
|
|
10628
11319
|
return relative.startsWith("..") ? toPosix(rawPath) : normalizeProjectRoot(relative);
|
|
10629
11320
|
}
|
|
10630
11321
|
function prefixMatches(value, prefix) {
|
|
@@ -10798,7 +11489,7 @@ function pageHashes(pages) {
|
|
|
10798
11489
|
return Object.fromEntries(pages.map((page) => [page.page.id, page.contentHash]));
|
|
10799
11490
|
}
|
|
10800
11491
|
async function buildManagedGraphPage(absolutePath, defaults, build) {
|
|
10801
|
-
const existingContent = await fileExists(absolutePath) ? await
|
|
11492
|
+
const existingContent = await fileExists(absolutePath) ? await fs18.readFile(absolutePath, "utf8") : null;
|
|
10802
11493
|
let existing = await loadExistingManagedPageState(absolutePath, {
|
|
10803
11494
|
status: defaults.status ?? "active",
|
|
10804
11495
|
managedBy: defaults.managedBy
|
|
@@ -10836,7 +11527,7 @@ async function buildManagedGraphPage(absolutePath, defaults, build) {
|
|
|
10836
11527
|
return built;
|
|
10837
11528
|
}
|
|
10838
11529
|
async function buildManagedContent(absolutePath, defaults, build) {
|
|
10839
|
-
const existingContent = await fileExists(absolutePath) ? await
|
|
11530
|
+
const existingContent = await fileExists(absolutePath) ? await fs18.readFile(absolutePath, "utf8") : null;
|
|
10840
11531
|
let existing = await loadExistingManagedPageState(absolutePath, {
|
|
10841
11532
|
status: defaults.status ?? "active",
|
|
10842
11533
|
managedBy: defaults.managedBy
|
|
@@ -10959,7 +11650,7 @@ function resetGraphNodeMetrics(nodes) {
|
|
|
10959
11650
|
return nodes.map(({ communityId: _communityId, degree: _degree, bridgeScore: _bridgeScore, isGodNode: _isGodNode, ...node }) => node);
|
|
10960
11651
|
}
|
|
10961
11652
|
function manifestRepoPath(manifest) {
|
|
10962
|
-
return toPosix(manifest.repoRelativePath ??
|
|
11653
|
+
return toPosix(manifest.repoRelativePath ?? path22.basename(manifest.originalPath ?? manifest.storedPath));
|
|
10963
11654
|
}
|
|
10964
11655
|
function goPackageScopeKey(manifest, analysis) {
|
|
10965
11656
|
if (analysis.code?.language !== "go") {
|
|
@@ -10969,7 +11660,7 @@ function goPackageScopeKey(manifest, analysis) {
|
|
|
10969
11660
|
if (!packageName) {
|
|
10970
11661
|
return null;
|
|
10971
11662
|
}
|
|
10972
|
-
return `${packageName}:${
|
|
11663
|
+
return `${packageName}:${path22.posix.dirname(manifestRepoPath(manifest))}`;
|
|
10973
11664
|
}
|
|
10974
11665
|
function buildGoPackageSymbolLookups(analyses, manifestsById) {
|
|
10975
11666
|
const lookups = /* @__PURE__ */ new Map();
|
|
@@ -11438,7 +12129,7 @@ async function buildGraphOrientationPages(graph, paths, schemaHash, previousComp
|
|
|
11438
12129
|
const benchmark = await readJsonFile(paths.benchmarkPath);
|
|
11439
12130
|
const communityRecords = [];
|
|
11440
12131
|
for (const community of graph.communities ?? []) {
|
|
11441
|
-
const absolutePath =
|
|
12132
|
+
const absolutePath = path22.join(paths.wikiDir, "graph", "communities", `${community.id.replace(/^community:/, "")}.md`);
|
|
11442
12133
|
communityRecords.push(
|
|
11443
12134
|
await buildManagedGraphPage(
|
|
11444
12135
|
absolutePath,
|
|
@@ -11467,7 +12158,7 @@ async function buildGraphOrientationPages(graph, paths, schemaHash, previousComp
|
|
|
11467
12158
|
graphHash: graphHash(graph),
|
|
11468
12159
|
contradictions
|
|
11469
12160
|
});
|
|
11470
|
-
const reportAbsolutePath =
|
|
12161
|
+
const reportAbsolutePath = path22.join(paths.wikiDir, "graph", "report.md");
|
|
11471
12162
|
const reportRecord = await buildManagedGraphPage(
|
|
11472
12163
|
reportAbsolutePath,
|
|
11473
12164
|
{
|
|
@@ -11488,7 +12179,7 @@ async function buildGraphOrientationPages(graph, paths, schemaHash, previousComp
|
|
|
11488
12179
|
};
|
|
11489
12180
|
}
|
|
11490
12181
|
async function writePage(wikiDir, relativePath, content, changedPages) {
|
|
11491
|
-
const absolutePath =
|
|
12182
|
+
const absolutePath = path22.resolve(wikiDir, relativePath);
|
|
11492
12183
|
const changed = await writeFileIfChanged(absolutePath, content);
|
|
11493
12184
|
if (changed) {
|
|
11494
12185
|
changedPages.push(relativePath);
|
|
@@ -11551,29 +12242,29 @@ async function requiredCompileArtifactsExist(paths) {
|
|
|
11551
12242
|
paths.graphPath,
|
|
11552
12243
|
paths.codeIndexPath,
|
|
11553
12244
|
paths.searchDbPath,
|
|
11554
|
-
|
|
11555
|
-
|
|
11556
|
-
|
|
11557
|
-
|
|
11558
|
-
|
|
11559
|
-
|
|
11560
|
-
|
|
11561
|
-
|
|
12245
|
+
path22.join(paths.wikiDir, "index.md"),
|
|
12246
|
+
path22.join(paths.wikiDir, "sources", "index.md"),
|
|
12247
|
+
path22.join(paths.wikiDir, "code", "index.md"),
|
|
12248
|
+
path22.join(paths.wikiDir, "concepts", "index.md"),
|
|
12249
|
+
path22.join(paths.wikiDir, "entities", "index.md"),
|
|
12250
|
+
path22.join(paths.wikiDir, "outputs", "index.md"),
|
|
12251
|
+
path22.join(paths.wikiDir, "projects", "index.md"),
|
|
12252
|
+
path22.join(paths.wikiDir, "candidates", "index.md")
|
|
11562
12253
|
];
|
|
11563
12254
|
const checks = await Promise.all(requiredPaths.map((filePath) => fileExists(filePath)));
|
|
11564
12255
|
return checks.every(Boolean);
|
|
11565
12256
|
}
|
|
11566
12257
|
async function loadAvailableCachedAnalyses(paths, manifests) {
|
|
11567
12258
|
const analyses = await Promise.all(
|
|
11568
|
-
manifests.map(async (manifest) => readJsonFile(
|
|
12259
|
+
manifests.map(async (manifest) => readJsonFile(path22.join(paths.analysesDir, `${manifest.sourceId}.json`)))
|
|
11569
12260
|
);
|
|
11570
12261
|
return analyses.filter((analysis) => Boolean(analysis));
|
|
11571
12262
|
}
|
|
11572
12263
|
function approvalManifestPath(paths, approvalId) {
|
|
11573
|
-
return
|
|
12264
|
+
return path22.join(paths.approvalsDir, approvalId, "manifest.json");
|
|
11574
12265
|
}
|
|
11575
12266
|
function approvalGraphPath(paths, approvalId) {
|
|
11576
|
-
return
|
|
12267
|
+
return path22.join(paths.approvalsDir, approvalId, "state", "graph.json");
|
|
11577
12268
|
}
|
|
11578
12269
|
async function readApprovalManifest(paths, approvalId) {
|
|
11579
12270
|
const manifest = await readJsonFile(approvalManifestPath(paths, approvalId));
|
|
@@ -11583,7 +12274,7 @@ async function readApprovalManifest(paths, approvalId) {
|
|
|
11583
12274
|
return manifest;
|
|
11584
12275
|
}
|
|
11585
12276
|
async function writeApprovalManifest(paths, manifest) {
|
|
11586
|
-
await
|
|
12277
|
+
await fs18.writeFile(approvalManifestPath(paths, manifest.approvalId), `${JSON.stringify(manifest, null, 2)}
|
|
11587
12278
|
`, "utf8");
|
|
11588
12279
|
}
|
|
11589
12280
|
async function buildApprovalEntries(paths, changedFiles, deletedPaths, previousGraph, graph) {
|
|
@@ -11598,7 +12289,7 @@ async function buildApprovalEntries(paths, changedFiles, deletedPaths, previousG
|
|
|
11598
12289
|
continue;
|
|
11599
12290
|
}
|
|
11600
12291
|
const previousPage = previousPagesById.get(nextPage.id);
|
|
11601
|
-
const currentExists = await fileExists(
|
|
12292
|
+
const currentExists = await fileExists(path22.join(paths.wikiDir, file.relativePath));
|
|
11602
12293
|
if (previousPage && previousPage.path !== nextPage.path) {
|
|
11603
12294
|
entries.push({
|
|
11604
12295
|
pageId: nextPage.id,
|
|
@@ -11631,7 +12322,7 @@ async function buildApprovalEntries(paths, changedFiles, deletedPaths, previousG
|
|
|
11631
12322
|
const previousPage = previousPagesByPath.get(deletedPath);
|
|
11632
12323
|
entries.push({
|
|
11633
12324
|
pageId: previousPage?.id ?? `page:${slugify(deletedPath)}`,
|
|
11634
|
-
title: previousPage?.title ??
|
|
12325
|
+
title: previousPage?.title ?? path22.basename(deletedPath, ".md"),
|
|
11635
12326
|
kind: previousPage?.kind ?? "index",
|
|
11636
12327
|
changeType: "delete",
|
|
11637
12328
|
status: "pending",
|
|
@@ -11643,16 +12334,16 @@ async function buildApprovalEntries(paths, changedFiles, deletedPaths, previousG
|
|
|
11643
12334
|
}
|
|
11644
12335
|
async function stageApprovalBundle(paths, changedFiles, deletedPaths, previousGraph, graph) {
|
|
11645
12336
|
const approvalId = `compile-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`;
|
|
11646
|
-
const approvalDir =
|
|
12337
|
+
const approvalDir = path22.join(paths.approvalsDir, approvalId);
|
|
11647
12338
|
await ensureDir(approvalDir);
|
|
11648
|
-
await ensureDir(
|
|
11649
|
-
await ensureDir(
|
|
12339
|
+
await ensureDir(path22.join(approvalDir, "wiki"));
|
|
12340
|
+
await ensureDir(path22.join(approvalDir, "state"));
|
|
11650
12341
|
for (const file of changedFiles) {
|
|
11651
|
-
const targetPath =
|
|
11652
|
-
await ensureDir(
|
|
11653
|
-
await
|
|
12342
|
+
const targetPath = path22.join(approvalDir, "wiki", file.relativePath);
|
|
12343
|
+
await ensureDir(path22.dirname(targetPath));
|
|
12344
|
+
await fs18.writeFile(targetPath, file.content, "utf8");
|
|
11654
12345
|
}
|
|
11655
|
-
await
|
|
12346
|
+
await fs18.writeFile(path22.join(approvalDir, "state", "graph.json"), JSON.stringify(graph, null, 2), "utf8");
|
|
11656
12347
|
await writeApprovalManifest(paths, {
|
|
11657
12348
|
approvalId,
|
|
11658
12349
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -11714,7 +12405,7 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
11714
12405
|
confidence: 1
|
|
11715
12406
|
});
|
|
11716
12407
|
const sourceRecord = await buildManagedGraphPage(
|
|
11717
|
-
|
|
12408
|
+
path22.join(paths.wikiDir, preview.path),
|
|
11718
12409
|
{
|
|
11719
12410
|
managedBy: "system",
|
|
11720
12411
|
confidence: 1,
|
|
@@ -11760,7 +12451,7 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
11760
12451
|
);
|
|
11761
12452
|
records.push(
|
|
11762
12453
|
await buildManagedGraphPage(
|
|
11763
|
-
|
|
12454
|
+
path22.join(paths.wikiDir, modulePreview.path),
|
|
11764
12455
|
{
|
|
11765
12456
|
managedBy: "system",
|
|
11766
12457
|
confidence: 1,
|
|
@@ -11794,8 +12485,8 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
11794
12485
|
const relativePath = promoted ? activeAggregatePath(itemKind, slug) : candidatePagePathFor(itemKind, slug);
|
|
11795
12486
|
const aggregateSourceClass2 = aggregateManifestSourceClass(input.manifests, sourceIds);
|
|
11796
12487
|
const fallbackPaths = [
|
|
11797
|
-
|
|
11798
|
-
|
|
12488
|
+
path22.join(paths.wikiDir, activeAggregatePath(itemKind, slug)),
|
|
12489
|
+
path22.join(paths.wikiDir, candidatePagePathFor(itemKind, slug))
|
|
11799
12490
|
];
|
|
11800
12491
|
const confidence = nodeConfidence(aggregate.sourceAnalyses.length);
|
|
11801
12492
|
const preview = emptyGraphPage({
|
|
@@ -11813,7 +12504,7 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
11813
12504
|
status: promoted ? "active" : "candidate"
|
|
11814
12505
|
});
|
|
11815
12506
|
const pageRecord = await buildManagedGraphPage(
|
|
11816
|
-
|
|
12507
|
+
path22.join(paths.wikiDir, relativePath),
|
|
11817
12508
|
{
|
|
11818
12509
|
status: promoted ? "active" : "candidate",
|
|
11819
12510
|
managedBy: "system",
|
|
@@ -11930,7 +12621,7 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
11930
12621
|
confidence: 1
|
|
11931
12622
|
}),
|
|
11932
12623
|
content: await buildManagedContent(
|
|
11933
|
-
|
|
12624
|
+
path22.join(paths.wikiDir, "projects", "index.md"),
|
|
11934
12625
|
{
|
|
11935
12626
|
managedBy: "system",
|
|
11936
12627
|
compiledFrom: indexCompiledFrom(projectIndexRefs)
|
|
@@ -11954,7 +12645,7 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
11954
12645
|
records.push({
|
|
11955
12646
|
page: projectIndexRef,
|
|
11956
12647
|
content: await buildManagedContent(
|
|
11957
|
-
|
|
12648
|
+
path22.join(paths.wikiDir, projectIndexRef.path),
|
|
11958
12649
|
{
|
|
11959
12650
|
managedBy: "system",
|
|
11960
12651
|
compiledFrom: indexCompiledFrom(Object.values(sections).flat())
|
|
@@ -11982,7 +12673,7 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
11982
12673
|
confidence: 1
|
|
11983
12674
|
}),
|
|
11984
12675
|
content: await buildManagedContent(
|
|
11985
|
-
|
|
12676
|
+
path22.join(paths.wikiDir, "index.md"),
|
|
11986
12677
|
{
|
|
11987
12678
|
managedBy: "system",
|
|
11988
12679
|
compiledFrom: indexCompiledFrom(allPages)
|
|
@@ -12013,7 +12704,7 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
12013
12704
|
confidence: 1
|
|
12014
12705
|
}),
|
|
12015
12706
|
content: await buildManagedContent(
|
|
12016
|
-
|
|
12707
|
+
path22.join(paths.wikiDir, relativePath),
|
|
12017
12708
|
{
|
|
12018
12709
|
managedBy: "system",
|
|
12019
12710
|
compiledFrom: indexCompiledFrom(pages)
|
|
@@ -12024,12 +12715,12 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
12024
12715
|
}
|
|
12025
12716
|
const nextPagePaths = new Set(records.map((record) => record.page.path));
|
|
12026
12717
|
const obsoleteGraphPaths = (previousGraph?.pages ?? []).filter((page) => page.kind !== "output" && page.kind !== "insight").map((page) => page.path).filter((relativePath) => !nextPagePaths.has(relativePath));
|
|
12027
|
-
const existingProjectIndexPaths = (await listFilesRecursive(paths.projectsDir)).filter((absolutePath) => absolutePath.endsWith(".md")).map((absolutePath) => toPosix(
|
|
12718
|
+
const existingProjectIndexPaths = (await listFilesRecursive(paths.projectsDir)).filter((absolutePath) => absolutePath.endsWith(".md")).map((absolutePath) => toPosix(path22.relative(paths.wikiDir, absolutePath))).filter((relativePath) => !nextPagePaths.has(relativePath));
|
|
12028
12719
|
const obsoletePaths = uniqueStrings3([...obsoleteGraphPaths, ...existingProjectIndexPaths]);
|
|
12029
12720
|
const changedFiles = [];
|
|
12030
12721
|
for (const record of records) {
|
|
12031
|
-
const absolutePath =
|
|
12032
|
-
const current = await fileExists(absolutePath) ? await
|
|
12722
|
+
const absolutePath = path22.join(paths.wikiDir, record.page.path);
|
|
12723
|
+
const current = await fileExists(absolutePath) ? await fs18.readFile(absolutePath, "utf8") : null;
|
|
12033
12724
|
if (current !== record.content) {
|
|
12034
12725
|
changedPages.push(record.page.path);
|
|
12035
12726
|
changedFiles.push({ relativePath: record.page.path, content: record.content });
|
|
@@ -12054,10 +12745,10 @@ async function syncVaultArtifacts(rootDir, input) {
|
|
|
12054
12745
|
await writePage(paths.wikiDir, record.page.path, record.content, writeChanges);
|
|
12055
12746
|
}
|
|
12056
12747
|
for (const relativePath of obsoletePaths) {
|
|
12057
|
-
await
|
|
12748
|
+
await fs18.rm(path22.join(paths.wikiDir, relativePath), { force: true });
|
|
12058
12749
|
}
|
|
12059
12750
|
await writeJsonFile(paths.graphPath, graph);
|
|
12060
|
-
await writeJsonFile(
|
|
12751
|
+
await writeJsonFile(path22.join(paths.wikiDir, "graph", "report.json"), graphOrientation.report);
|
|
12061
12752
|
await writeJsonFile(paths.codeIndexPath, input.codeIndex);
|
|
12062
12753
|
await writeJsonFile(paths.compileStatePath, {
|
|
12063
12754
|
generatedAt: graph.generatedAt,
|
|
@@ -12128,17 +12819,17 @@ async function refreshIndexesAndSearch(rootDir, pages) {
|
|
|
12128
12819
|
})
|
|
12129
12820
|
);
|
|
12130
12821
|
await Promise.all([
|
|
12131
|
-
ensureDir(
|
|
12132
|
-
ensureDir(
|
|
12133
|
-
ensureDir(
|
|
12134
|
-
ensureDir(
|
|
12135
|
-
ensureDir(
|
|
12136
|
-
ensureDir(
|
|
12137
|
-
ensureDir(
|
|
12138
|
-
ensureDir(
|
|
12139
|
-
ensureDir(
|
|
12822
|
+
ensureDir(path22.join(paths.wikiDir, "sources")),
|
|
12823
|
+
ensureDir(path22.join(paths.wikiDir, "code")),
|
|
12824
|
+
ensureDir(path22.join(paths.wikiDir, "concepts")),
|
|
12825
|
+
ensureDir(path22.join(paths.wikiDir, "entities")),
|
|
12826
|
+
ensureDir(path22.join(paths.wikiDir, "outputs")),
|
|
12827
|
+
ensureDir(path22.join(paths.wikiDir, "graph")),
|
|
12828
|
+
ensureDir(path22.join(paths.wikiDir, "graph", "communities")),
|
|
12829
|
+
ensureDir(path22.join(paths.wikiDir, "projects")),
|
|
12830
|
+
ensureDir(path22.join(paths.wikiDir, "candidates"))
|
|
12140
12831
|
]);
|
|
12141
|
-
const projectsIndexPath =
|
|
12832
|
+
const projectsIndexPath = path22.join(paths.wikiDir, "projects", "index.md");
|
|
12142
12833
|
await writeFileIfChanged(
|
|
12143
12834
|
projectsIndexPath,
|
|
12144
12835
|
await buildManagedContent(
|
|
@@ -12159,7 +12850,7 @@ async function refreshIndexesAndSearch(rootDir, pages) {
|
|
|
12159
12850
|
outputs: pages.filter((page) => page.kind === "output" && page.projectIds.includes(project.id)),
|
|
12160
12851
|
candidates: pages.filter((page) => page.status === "candidate" && page.projectIds.includes(project.id))
|
|
12161
12852
|
};
|
|
12162
|
-
const absolutePath =
|
|
12853
|
+
const absolutePath = path22.join(paths.wikiDir, "projects", project.id, "index.md");
|
|
12163
12854
|
await writeFileIfChanged(
|
|
12164
12855
|
absolutePath,
|
|
12165
12856
|
await buildManagedContent(
|
|
@@ -12177,7 +12868,7 @@ async function refreshIndexesAndSearch(rootDir, pages) {
|
|
|
12177
12868
|
)
|
|
12178
12869
|
);
|
|
12179
12870
|
}
|
|
12180
|
-
const rootIndexPath =
|
|
12871
|
+
const rootIndexPath = path22.join(paths.wikiDir, "index.md");
|
|
12181
12872
|
await writeFileIfChanged(
|
|
12182
12873
|
rootIndexPath,
|
|
12183
12874
|
await buildManagedContent(
|
|
@@ -12198,7 +12889,7 @@ async function refreshIndexesAndSearch(rootDir, pages) {
|
|
|
12198
12889
|
["candidates/index.md", "candidates", pagesWithGraph.filter((page) => page.status === "candidate")],
|
|
12199
12890
|
["graph/index.md", "graph", pagesWithGraph.filter((page) => page.kind === "graph_report" || page.kind === "community_summary")]
|
|
12200
12891
|
]) {
|
|
12201
|
-
const absolutePath =
|
|
12892
|
+
const absolutePath = path22.join(paths.wikiDir, relativePath);
|
|
12202
12893
|
await writeFileIfChanged(
|
|
12203
12894
|
absolutePath,
|
|
12204
12895
|
await buildManagedContent(
|
|
@@ -12212,23 +12903,23 @@ async function refreshIndexesAndSearch(rootDir, pages) {
|
|
|
12212
12903
|
);
|
|
12213
12904
|
}
|
|
12214
12905
|
for (const record of graphOrientation.records) {
|
|
12215
|
-
await writeFileIfChanged(
|
|
12906
|
+
await writeFileIfChanged(path22.join(paths.wikiDir, record.page.path), record.content);
|
|
12216
12907
|
}
|
|
12217
12908
|
if (graphOrientation.report) {
|
|
12218
|
-
await writeJsonFile(
|
|
12909
|
+
await writeJsonFile(path22.join(paths.wikiDir, "graph", "report.json"), graphOrientation.report);
|
|
12219
12910
|
}
|
|
12220
|
-
const existingProjectIndexPaths = (await listFilesRecursive(paths.projectsDir)).filter((absolutePath) => absolutePath.endsWith(".md")).map((absolutePath) => toPosix(
|
|
12911
|
+
const existingProjectIndexPaths = (await listFilesRecursive(paths.projectsDir)).filter((absolutePath) => absolutePath.endsWith(".md")).map((absolutePath) => toPosix(path22.relative(paths.wikiDir, absolutePath)));
|
|
12221
12912
|
const allowedProjectIndexPaths = /* @__PURE__ */ new Set([
|
|
12222
12913
|
"projects/index.md",
|
|
12223
12914
|
...configuredProjects.map((project) => `projects/${project.id}/index.md`)
|
|
12224
12915
|
]);
|
|
12225
12916
|
await Promise.all(
|
|
12226
|
-
existingProjectIndexPaths.filter((relativePath) => !allowedProjectIndexPaths.has(relativePath)).map((relativePath) =>
|
|
12917
|
+
existingProjectIndexPaths.filter((relativePath) => !allowedProjectIndexPaths.has(relativePath)).map((relativePath) => fs18.rm(path22.join(paths.wikiDir, relativePath), { force: true }))
|
|
12227
12918
|
);
|
|
12228
|
-
const existingGraphPages = (await listFilesRecursive(
|
|
12919
|
+
const existingGraphPages = (await listFilesRecursive(path22.join(paths.wikiDir, "graph").replace(/\/$/, "")).catch(() => [])).filter((absolutePath) => absolutePath.endsWith(".md")).map((absolutePath) => toPosix(path22.relative(paths.wikiDir, absolutePath)));
|
|
12229
12920
|
const allowedGraphPages = /* @__PURE__ */ new Set(["graph/index.md", ...graphOrientation.records.map((record) => record.page.path)]);
|
|
12230
12921
|
await Promise.all(
|
|
12231
|
-
existingGraphPages.filter((relativePath) => !allowedGraphPages.has(relativePath)).map((relativePath) =>
|
|
12922
|
+
existingGraphPages.filter((relativePath) => !allowedGraphPages.has(relativePath)).map((relativePath) => fs18.rm(path22.join(paths.wikiDir, relativePath), { force: true }))
|
|
12232
12923
|
);
|
|
12233
12924
|
await rebuildSearchIndex(paths.searchDbPath, pagesWithGraph, paths.wikiDir);
|
|
12234
12925
|
}
|
|
@@ -12248,7 +12939,7 @@ async function prepareOutputPageSave(rootDir, input) {
|
|
|
12248
12939
|
confidence: 0.74
|
|
12249
12940
|
}
|
|
12250
12941
|
});
|
|
12251
|
-
const absolutePath =
|
|
12942
|
+
const absolutePath = path22.join(paths.wikiDir, output.page.path);
|
|
12252
12943
|
return {
|
|
12253
12944
|
page: output.page,
|
|
12254
12945
|
savedPath: absolutePath,
|
|
@@ -12260,15 +12951,15 @@ async function prepareOutputPageSave(rootDir, input) {
|
|
|
12260
12951
|
async function persistOutputPage(rootDir, input) {
|
|
12261
12952
|
const { paths } = await loadVaultConfig(rootDir);
|
|
12262
12953
|
const prepared = await prepareOutputPageSave(rootDir, input);
|
|
12263
|
-
await ensureDir(
|
|
12264
|
-
await
|
|
12954
|
+
await ensureDir(path22.dirname(prepared.savedPath));
|
|
12955
|
+
await fs18.writeFile(prepared.savedPath, prepared.content, "utf8");
|
|
12265
12956
|
for (const assetFile of prepared.assetFiles) {
|
|
12266
|
-
const assetPath =
|
|
12267
|
-
await ensureDir(
|
|
12957
|
+
const assetPath = path22.join(paths.wikiDir, assetFile.relativePath);
|
|
12958
|
+
await ensureDir(path22.dirname(assetPath));
|
|
12268
12959
|
if (typeof assetFile.content === "string") {
|
|
12269
|
-
await
|
|
12960
|
+
await fs18.writeFile(assetPath, assetFile.content, assetFile.encoding ?? "utf8");
|
|
12270
12961
|
} else {
|
|
12271
|
-
await
|
|
12962
|
+
await fs18.writeFile(assetPath, assetFile.content);
|
|
12272
12963
|
}
|
|
12273
12964
|
}
|
|
12274
12965
|
return { page: prepared.page, savedPath: prepared.savedPath, outputAssets: prepared.outputAssets };
|
|
@@ -12289,7 +12980,7 @@ async function prepareExploreHubSave(rootDir, input) {
|
|
|
12289
12980
|
confidence: 0.76
|
|
12290
12981
|
}
|
|
12291
12982
|
});
|
|
12292
|
-
const absolutePath =
|
|
12983
|
+
const absolutePath = path22.join(paths.wikiDir, hub.page.path);
|
|
12293
12984
|
return {
|
|
12294
12985
|
page: hub.page,
|
|
12295
12986
|
savedPath: absolutePath,
|
|
@@ -12301,15 +12992,15 @@ async function prepareExploreHubSave(rootDir, input) {
|
|
|
12301
12992
|
async function persistExploreHub(rootDir, input) {
|
|
12302
12993
|
const { paths } = await loadVaultConfig(rootDir);
|
|
12303
12994
|
const prepared = await prepareExploreHubSave(rootDir, input);
|
|
12304
|
-
await ensureDir(
|
|
12305
|
-
await
|
|
12995
|
+
await ensureDir(path22.dirname(prepared.savedPath));
|
|
12996
|
+
await fs18.writeFile(prepared.savedPath, prepared.content, "utf8");
|
|
12306
12997
|
for (const assetFile of prepared.assetFiles) {
|
|
12307
|
-
const assetPath =
|
|
12308
|
-
await ensureDir(
|
|
12998
|
+
const assetPath = path22.join(paths.wikiDir, assetFile.relativePath);
|
|
12999
|
+
await ensureDir(path22.dirname(assetPath));
|
|
12309
13000
|
if (typeof assetFile.content === "string") {
|
|
12310
|
-
await
|
|
13001
|
+
await fs18.writeFile(assetPath, assetFile.content, assetFile.encoding ?? "utf8");
|
|
12311
13002
|
} else {
|
|
12312
|
-
await
|
|
13003
|
+
await fs18.writeFile(assetPath, assetFile.content);
|
|
12313
13004
|
}
|
|
12314
13005
|
}
|
|
12315
13006
|
return { page: prepared.page, savedPath: prepared.savedPath, outputAssets: prepared.outputAssets };
|
|
@@ -12326,17 +13017,17 @@ async function stageOutputApprovalBundle(rootDir, stagedPages) {
|
|
|
12326
13017
|
}))
|
|
12327
13018
|
]);
|
|
12328
13019
|
const approvalId = `schedule-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`;
|
|
12329
|
-
const approvalDir =
|
|
13020
|
+
const approvalDir = path22.join(paths.approvalsDir, approvalId);
|
|
12330
13021
|
await ensureDir(approvalDir);
|
|
12331
|
-
await ensureDir(
|
|
12332
|
-
await ensureDir(
|
|
13022
|
+
await ensureDir(path22.join(approvalDir, "wiki"));
|
|
13023
|
+
await ensureDir(path22.join(approvalDir, "state"));
|
|
12333
13024
|
for (const file of changedFiles) {
|
|
12334
|
-
const targetPath =
|
|
12335
|
-
await ensureDir(
|
|
13025
|
+
const targetPath = path22.join(approvalDir, "wiki", file.relativePath);
|
|
13026
|
+
await ensureDir(path22.dirname(targetPath));
|
|
12336
13027
|
if ("binary" in file && file.binary) {
|
|
12337
|
-
await
|
|
13028
|
+
await fs18.writeFile(targetPath, Buffer.from(file.content, "base64"));
|
|
12338
13029
|
} else {
|
|
12339
|
-
await
|
|
13030
|
+
await fs18.writeFile(targetPath, file.content, "utf8");
|
|
12340
13031
|
}
|
|
12341
13032
|
}
|
|
12342
13033
|
const nextPages = sortGraphPages([
|
|
@@ -12351,7 +13042,7 @@ async function stageOutputApprovalBundle(rootDir, stagedPages) {
|
|
|
12351
13042
|
sources: previousGraph?.sources ?? [],
|
|
12352
13043
|
pages: nextPages
|
|
12353
13044
|
};
|
|
12354
|
-
await
|
|
13045
|
+
await fs18.writeFile(path22.join(approvalDir, "state", "graph.json"), JSON.stringify(graph, null, 2), "utf8");
|
|
12355
13046
|
await writeApprovalManifest(paths, {
|
|
12356
13047
|
approvalId,
|
|
12357
13048
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -12380,9 +13071,9 @@ async function executeQuery(rootDir, question, format) {
|
|
|
12380
13071
|
const searchResults = searchPages(paths.searchDbPath, question, 5);
|
|
12381
13072
|
const excerpts = await Promise.all(
|
|
12382
13073
|
searchResults.map(async (result) => {
|
|
12383
|
-
const absolutePath =
|
|
13074
|
+
const absolutePath = path22.join(paths.wikiDir, result.path);
|
|
12384
13075
|
try {
|
|
12385
|
-
const content = await
|
|
13076
|
+
const content = await fs18.readFile(absolutePath, "utf8");
|
|
12386
13077
|
const parsed = matter9(content);
|
|
12387
13078
|
return `# ${result.title}
|
|
12388
13079
|
${truncate(normalizeWhitespace(parsed.content), 1200)}`;
|
|
@@ -12616,7 +13307,7 @@ function computeChangeSummary(current, staged, changeType) {
|
|
|
12616
13307
|
async function listApprovals(rootDir) {
|
|
12617
13308
|
const { paths } = await loadVaultConfig(rootDir);
|
|
12618
13309
|
const manifests = await Promise.all(
|
|
12619
|
-
(await
|
|
13310
|
+
(await fs18.readdir(paths.approvalsDir, { withFileTypes: true }).catch(() => [])).filter((entry) => entry.isDirectory()).map(async (entry) => {
|
|
12620
13311
|
try {
|
|
12621
13312
|
return await readApprovalManifest(paths, entry.name);
|
|
12622
13313
|
} catch {
|
|
@@ -12632,8 +13323,8 @@ async function readApproval(rootDir, approvalId, options) {
|
|
|
12632
13323
|
const details = await Promise.all(
|
|
12633
13324
|
manifest.entries.map(async (entry) => {
|
|
12634
13325
|
const currentPath = entry.previousPath ?? entry.nextPath;
|
|
12635
|
-
const currentContent = currentPath ? await
|
|
12636
|
-
const stagedContent = entry.nextPath ? await
|
|
13326
|
+
const currentContent = currentPath ? await fs18.readFile(path22.join(paths.wikiDir, currentPath), "utf8").catch(() => void 0) : void 0;
|
|
13327
|
+
const stagedContent = entry.nextPath ? await fs18.readFile(path22.join(paths.approvalsDir, approvalId, "wiki", entry.nextPath), "utf8").catch(() => void 0) : void 0;
|
|
12637
13328
|
const detail = {
|
|
12638
13329
|
...entry,
|
|
12639
13330
|
currentContent,
|
|
@@ -12666,26 +13357,26 @@ async function acceptApproval(rootDir, approvalId, targets = []) {
|
|
|
12666
13357
|
if (!entry.nextPath) {
|
|
12667
13358
|
throw new Error(`Approval entry ${entry.pageId} is missing a staged path.`);
|
|
12668
13359
|
}
|
|
12669
|
-
const stagedAbsolutePath =
|
|
12670
|
-
const stagedContent = await
|
|
12671
|
-
const targetAbsolutePath =
|
|
12672
|
-
await ensureDir(
|
|
12673
|
-
await
|
|
13360
|
+
const stagedAbsolutePath = path22.join(paths.approvalsDir, approvalId, "wiki", entry.nextPath);
|
|
13361
|
+
const stagedContent = await fs18.readFile(stagedAbsolutePath, "utf8");
|
|
13362
|
+
const targetAbsolutePath = path22.join(paths.wikiDir, entry.nextPath);
|
|
13363
|
+
await ensureDir(path22.dirname(targetAbsolutePath));
|
|
13364
|
+
await fs18.writeFile(targetAbsolutePath, stagedContent, "utf8");
|
|
12674
13365
|
if (entry.changeType === "promote" && entry.previousPath) {
|
|
12675
|
-
await
|
|
13366
|
+
await fs18.rm(path22.join(paths.wikiDir, entry.previousPath), { force: true });
|
|
12676
13367
|
}
|
|
12677
13368
|
const nextPage = bundleGraph?.pages.find((page) => page.id === entry.pageId && page.path === entry.nextPath) ?? parseStoredPage(entry.nextPath, stagedContent);
|
|
12678
13369
|
if (nextPage.kind === "output" && nextPage.outputAssets?.length) {
|
|
12679
|
-
const outputAssetDir =
|
|
12680
|
-
await
|
|
13370
|
+
const outputAssetDir = path22.join(paths.wikiDir, "outputs", "assets", path22.basename(nextPage.path, ".md"));
|
|
13371
|
+
await fs18.rm(outputAssetDir, { recursive: true, force: true });
|
|
12681
13372
|
for (const asset of nextPage.outputAssets) {
|
|
12682
|
-
const stagedAssetPath =
|
|
13373
|
+
const stagedAssetPath = path22.join(paths.approvalsDir, approvalId, "wiki", asset.path);
|
|
12683
13374
|
if (!await fileExists(stagedAssetPath)) {
|
|
12684
13375
|
continue;
|
|
12685
13376
|
}
|
|
12686
|
-
const targetAssetPath =
|
|
12687
|
-
await ensureDir(
|
|
12688
|
-
await
|
|
13377
|
+
const targetAssetPath = path22.join(paths.wikiDir, asset.path);
|
|
13378
|
+
await ensureDir(path22.dirname(targetAssetPath));
|
|
13379
|
+
await fs18.copyFile(stagedAssetPath, targetAssetPath);
|
|
12689
13380
|
}
|
|
12690
13381
|
}
|
|
12691
13382
|
nextPages = nextPages.filter(
|
|
@@ -12696,10 +13387,10 @@ async function acceptApproval(rootDir, approvalId, targets = []) {
|
|
|
12696
13387
|
} else {
|
|
12697
13388
|
const deletedPage = nextPages.find((page) => page.id === entry.pageId || page.path === entry.previousPath) ?? bundleGraph?.pages.find((page) => page.id === entry.pageId || page.path === entry.previousPath) ?? null;
|
|
12698
13389
|
if (entry.previousPath) {
|
|
12699
|
-
await
|
|
13390
|
+
await fs18.rm(path22.join(paths.wikiDir, entry.previousPath), { force: true });
|
|
12700
13391
|
}
|
|
12701
13392
|
if (deletedPage?.kind === "output") {
|
|
12702
|
-
await
|
|
13393
|
+
await fs18.rm(path22.join(paths.wikiDir, "outputs", "assets", path22.basename(deletedPage.path, ".md")), {
|
|
12703
13394
|
recursive: true,
|
|
12704
13395
|
force: true
|
|
12705
13396
|
});
|
|
@@ -12790,7 +13481,7 @@ async function promoteCandidate(rootDir, target) {
|
|
|
12790
13481
|
const { paths } = await loadVaultConfig(rootDir);
|
|
12791
13482
|
const graph = await readJsonFile(paths.graphPath);
|
|
12792
13483
|
const candidate = resolveCandidateTarget(graph?.pages ?? [], target);
|
|
12793
|
-
const raw = await
|
|
13484
|
+
const raw = await fs18.readFile(path22.join(paths.wikiDir, candidate.path), "utf8");
|
|
12794
13485
|
const parsed = matter9(raw);
|
|
12795
13486
|
const nextUpdatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12796
13487
|
const nextContent = matter9.stringify(parsed.content, {
|
|
@@ -12802,10 +13493,10 @@ async function promoteCandidate(rootDir, target) {
|
|
|
12802
13493
|
)
|
|
12803
13494
|
});
|
|
12804
13495
|
const nextPath = candidateActivePath(candidate);
|
|
12805
|
-
const nextAbsolutePath =
|
|
12806
|
-
await ensureDir(
|
|
12807
|
-
await
|
|
12808
|
-
await
|
|
13496
|
+
const nextAbsolutePath = path22.join(paths.wikiDir, nextPath);
|
|
13497
|
+
await ensureDir(path22.dirname(nextAbsolutePath));
|
|
13498
|
+
await fs18.writeFile(nextAbsolutePath, nextContent, "utf8");
|
|
13499
|
+
await fs18.rm(path22.join(paths.wikiDir, candidate.path), { force: true });
|
|
12809
13500
|
const nextPage = parseStoredPage(nextPath, nextContent, { createdAt: candidate.createdAt, updatedAt: nextUpdatedAt });
|
|
12810
13501
|
const nextPages = sortGraphPages(
|
|
12811
13502
|
(graph?.pages ?? []).filter((page) => page.id !== candidate.id && page.path !== candidate.path).concat(nextPage)
|
|
@@ -12850,7 +13541,7 @@ async function archiveCandidate(rootDir, target) {
|
|
|
12850
13541
|
const { paths } = await loadVaultConfig(rootDir);
|
|
12851
13542
|
const graph = await readJsonFile(paths.graphPath);
|
|
12852
13543
|
const candidate = resolveCandidateTarget(graph?.pages ?? [], target);
|
|
12853
|
-
await
|
|
13544
|
+
await fs18.rm(path22.join(paths.wikiDir, candidate.path), { force: true });
|
|
12854
13545
|
const nextPages = sortGraphPages((graph?.pages ?? []).filter((page) => page.id !== candidate.id && page.path !== candidate.path));
|
|
12855
13546
|
const nextGraph = {
|
|
12856
13547
|
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -12889,18 +13580,18 @@ async function archiveCandidate(rootDir, target) {
|
|
|
12889
13580
|
}
|
|
12890
13581
|
async function ensureObsidianWorkspace(rootDir) {
|
|
12891
13582
|
const { config } = await loadVaultConfig(rootDir);
|
|
12892
|
-
const obsidianDir =
|
|
13583
|
+
const obsidianDir = path22.join(rootDir, ".obsidian");
|
|
12893
13584
|
const projectIds = projectEntries(config).map((project) => project.id);
|
|
12894
13585
|
await ensureDir(obsidianDir);
|
|
12895
13586
|
await Promise.all([
|
|
12896
|
-
writeJsonFile(
|
|
13587
|
+
writeJsonFile(path22.join(obsidianDir, "app.json"), {
|
|
12897
13588
|
alwaysUpdateLinks: true,
|
|
12898
13589
|
newFileLocation: "folder",
|
|
12899
13590
|
newFileFolderPath: "wiki/insights",
|
|
12900
13591
|
useMarkdownLinks: false,
|
|
12901
13592
|
attachmentFolderPath: "raw/assets"
|
|
12902
13593
|
}),
|
|
12903
|
-
writeJsonFile(
|
|
13594
|
+
writeJsonFile(path22.join(obsidianDir, "core-plugins.json"), [
|
|
12904
13595
|
"file-explorer",
|
|
12905
13596
|
"global-search",
|
|
12906
13597
|
"switcher",
|
|
@@ -12910,7 +13601,7 @@ async function ensureObsidianWorkspace(rootDir) {
|
|
|
12910
13601
|
"tag-pane",
|
|
12911
13602
|
"page-preview"
|
|
12912
13603
|
]),
|
|
12913
|
-
writeJsonFile(
|
|
13604
|
+
writeJsonFile(path22.join(obsidianDir, "graph.json"), {
|
|
12914
13605
|
"collapse-filter": false,
|
|
12915
13606
|
search: "",
|
|
12916
13607
|
showTags: true,
|
|
@@ -12922,7 +13613,7 @@ async function ensureObsidianWorkspace(rootDir) {
|
|
|
12922
13613
|
})),
|
|
12923
13614
|
localJumps: false
|
|
12924
13615
|
}),
|
|
12925
|
-
writeJsonFile(
|
|
13616
|
+
writeJsonFile(path22.join(obsidianDir, "workspace.json"), {
|
|
12926
13617
|
active: "root",
|
|
12927
13618
|
lastOpenFiles: ["wiki/index.md", "wiki/projects/index.md", "wiki/candidates/index.md", "wiki/insights/index.md"],
|
|
12928
13619
|
left: {
|
|
@@ -12937,7 +13628,7 @@ async function ensureObsidianWorkspace(rootDir) {
|
|
|
12937
13628
|
async function initVault(rootDir, options = {}) {
|
|
12938
13629
|
const { paths } = await initWorkspace(rootDir);
|
|
12939
13630
|
await installConfiguredAgents(rootDir);
|
|
12940
|
-
const insightsIndexPath =
|
|
13631
|
+
const insightsIndexPath = path22.join(paths.wikiDir, "insights", "index.md");
|
|
12941
13632
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12942
13633
|
await writeFileIfChanged(
|
|
12943
13634
|
insightsIndexPath,
|
|
@@ -12973,7 +13664,7 @@ async function initVault(rootDir, options = {}) {
|
|
|
12973
13664
|
)
|
|
12974
13665
|
);
|
|
12975
13666
|
await writeFileIfChanged(
|
|
12976
|
-
|
|
13667
|
+
path22.join(paths.wikiDir, "projects", "index.md"),
|
|
12977
13668
|
matter9.stringify(["# Projects", "", "- Run `swarmvault compile` to build project rollups.", ""].join("\n"), {
|
|
12978
13669
|
page_id: "projects:index",
|
|
12979
13670
|
kind: "index",
|
|
@@ -12995,7 +13686,7 @@ async function initVault(rootDir, options = {}) {
|
|
|
12995
13686
|
})
|
|
12996
13687
|
);
|
|
12997
13688
|
await writeFileIfChanged(
|
|
12998
|
-
|
|
13689
|
+
path22.join(paths.wikiDir, "candidates", "index.md"),
|
|
12999
13690
|
matter9.stringify(["# Candidates", "", "- Run `swarmvault compile` to stage candidate pages.", ""].join("\n"), {
|
|
13000
13691
|
page_id: "candidates:index",
|
|
13001
13692
|
kind: "index",
|
|
@@ -13117,34 +13808,41 @@ async function compileVault(rootDir, options = {}) {
|
|
|
13117
13808
|
candidatePageCount: (graph?.pages ?? []).filter((page) => page.status === "candidate").length
|
|
13118
13809
|
};
|
|
13119
13810
|
}
|
|
13811
|
+
const analysisProgress = createCompileProgressReporter("analyze", manifests.length);
|
|
13120
13812
|
const [dirtyAnalyses, cleanAnalyses] = await Promise.all([
|
|
13121
13813
|
Promise.all(
|
|
13122
|
-
dirty.map(
|
|
13123
|
-
|
|
13814
|
+
dirty.map(async (manifest) => {
|
|
13815
|
+
const analysis = await analyzeSource(
|
|
13124
13816
|
manifest,
|
|
13125
13817
|
await readExtractedText(rootDir, manifest),
|
|
13126
13818
|
provider,
|
|
13127
13819
|
paths,
|
|
13128
13820
|
getEffectiveSchema(schemas, sourceProjects[manifest.sourceId] ?? null)
|
|
13129
|
-
)
|
|
13130
|
-
|
|
13821
|
+
);
|
|
13822
|
+
analysisProgress.tick(manifest.title);
|
|
13823
|
+
return analysis;
|
|
13824
|
+
})
|
|
13131
13825
|
),
|
|
13132
13826
|
Promise.all(
|
|
13133
13827
|
clean.map(async (manifest) => {
|
|
13134
|
-
const cached = await readJsonFile(
|
|
13828
|
+
const cached = await readJsonFile(path22.join(paths.analysesDir, `${manifest.sourceId}.json`));
|
|
13135
13829
|
if (cached) {
|
|
13830
|
+
analysisProgress.tick(manifest.title);
|
|
13136
13831
|
return cached;
|
|
13137
13832
|
}
|
|
13138
|
-
|
|
13833
|
+
const analysis = await analyzeSource(
|
|
13139
13834
|
manifest,
|
|
13140
13835
|
await readExtractedText(rootDir, manifest),
|
|
13141
13836
|
provider,
|
|
13142
13837
|
paths,
|
|
13143
13838
|
getEffectiveSchema(schemas, sourceProjects[manifest.sourceId] ?? null)
|
|
13144
13839
|
);
|
|
13840
|
+
analysisProgress.tick(manifest.title);
|
|
13841
|
+
return analysis;
|
|
13145
13842
|
})
|
|
13146
13843
|
)
|
|
13147
13844
|
]);
|
|
13845
|
+
analysisProgress.finish(`dirty=${dirty.length}, clean=${clean.length}`);
|
|
13148
13846
|
const initialAnalyses = [...dirtyAnalyses, ...cleanAnalyses];
|
|
13149
13847
|
const codeIndex = await buildCodeIndex(rootDir, manifests, initialAnalyses);
|
|
13150
13848
|
const analyses = await Promise.all(
|
|
@@ -13155,22 +13853,22 @@ async function compileVault(rootDir, options = {}) {
|
|
|
13155
13853
|
}
|
|
13156
13854
|
const enriched = enrichResolvedCodeImports(manifest, analysis, codeIndex);
|
|
13157
13855
|
if (analysisSignature(enriched) !== analysisSignature(analysis)) {
|
|
13158
|
-
await writeJsonFile(
|
|
13856
|
+
await writeJsonFile(path22.join(paths.analysesDir, `${analysis.sourceId}.json`), enriched);
|
|
13159
13857
|
}
|
|
13160
13858
|
return enriched;
|
|
13161
13859
|
})
|
|
13162
13860
|
);
|
|
13163
13861
|
await Promise.all([
|
|
13164
|
-
ensureDir(
|
|
13165
|
-
ensureDir(
|
|
13166
|
-
ensureDir(
|
|
13167
|
-
ensureDir(
|
|
13168
|
-
ensureDir(
|
|
13169
|
-
ensureDir(
|
|
13170
|
-
ensureDir(
|
|
13171
|
-
ensureDir(
|
|
13172
|
-
ensureDir(
|
|
13173
|
-
ensureDir(
|
|
13862
|
+
ensureDir(path22.join(paths.wikiDir, "sources")),
|
|
13863
|
+
ensureDir(path22.join(paths.wikiDir, "code")),
|
|
13864
|
+
ensureDir(path22.join(paths.wikiDir, "concepts")),
|
|
13865
|
+
ensureDir(path22.join(paths.wikiDir, "entities")),
|
|
13866
|
+
ensureDir(path22.join(paths.wikiDir, "outputs")),
|
|
13867
|
+
ensureDir(path22.join(paths.wikiDir, "projects")),
|
|
13868
|
+
ensureDir(path22.join(paths.wikiDir, "insights")),
|
|
13869
|
+
ensureDir(path22.join(paths.wikiDir, "candidates")),
|
|
13870
|
+
ensureDir(path22.join(paths.wikiDir, "candidates", "concepts")),
|
|
13871
|
+
ensureDir(path22.join(paths.wikiDir, "candidates", "entities"))
|
|
13174
13872
|
]);
|
|
13175
13873
|
const sync = await syncVaultArtifacts(rootDir, {
|
|
13176
13874
|
schemas,
|
|
@@ -13317,7 +14015,7 @@ async function queryVault(rootDir, options) {
|
|
|
13317
14015
|
assetFiles: staged.assetFiles
|
|
13318
14016
|
}
|
|
13319
14017
|
]);
|
|
13320
|
-
stagedPath =
|
|
14018
|
+
stagedPath = path22.join(approval.approvalDir, "wiki", staged.page.path);
|
|
13321
14019
|
savedPageId = staged.page.id;
|
|
13322
14020
|
approvalId = approval.approvalId;
|
|
13323
14021
|
approvalDir = approval.approvalDir;
|
|
@@ -13573,9 +14271,9 @@ ${orchestrationNotes.join("\n")}
|
|
|
13573
14271
|
approvalId = approval.approvalId;
|
|
13574
14272
|
approvalDir = approval.approvalDir;
|
|
13575
14273
|
stepResults.forEach((result, index) => {
|
|
13576
|
-
result.stagedPath =
|
|
14274
|
+
result.stagedPath = path22.join(approval.approvalDir, "wiki", stagedStepPages[index]?.page.path ?? "");
|
|
13577
14275
|
});
|
|
13578
|
-
stagedHubPath =
|
|
14276
|
+
stagedHubPath = path22.join(approval.approvalDir, "wiki", hubPage.path);
|
|
13579
14277
|
} else {
|
|
13580
14278
|
await refreshVaultAfterOutputSave(rootDir);
|
|
13581
14279
|
}
|
|
@@ -13662,11 +14360,11 @@ async function benchmarkVault(rootDir, options = {}) {
|
|
|
13662
14360
|
}
|
|
13663
14361
|
}
|
|
13664
14362
|
for (const page of graph.pages) {
|
|
13665
|
-
const absolutePath =
|
|
14363
|
+
const absolutePath = path22.join(paths.wikiDir, page.path);
|
|
13666
14364
|
if (!await fileExists(absolutePath)) {
|
|
13667
14365
|
continue;
|
|
13668
14366
|
}
|
|
13669
|
-
const parsed = matter9(await
|
|
14367
|
+
const parsed = matter9(await fs18.readFile(absolutePath, "utf8"));
|
|
13670
14368
|
pageContentsById.set(page.id, parsed.content);
|
|
13671
14369
|
}
|
|
13672
14370
|
const configuredQuestions = (config.benchmark?.questions ?? []).map((question) => normalizeWhitespace(question)).filter(Boolean);
|
|
@@ -13711,7 +14409,7 @@ async function listGraphHyperedges(rootDir, target, limit = 25) {
|
|
|
13711
14409
|
}
|
|
13712
14410
|
async function readGraphReport(rootDir) {
|
|
13713
14411
|
const { paths } = await loadVaultConfig(rootDir);
|
|
13714
|
-
return readJsonFile(
|
|
14412
|
+
return readJsonFile(path22.join(paths.wikiDir, "graph", "report.json"));
|
|
13715
14413
|
}
|
|
13716
14414
|
async function listGodNodes(rootDir, limit = 10) {
|
|
13717
14415
|
const graph = await ensureCompiledGraph(rootDir);
|
|
@@ -13724,15 +14422,15 @@ async function listPages(rootDir) {
|
|
|
13724
14422
|
}
|
|
13725
14423
|
async function readPage(rootDir, relativePath) {
|
|
13726
14424
|
const { paths } = await loadVaultConfig(rootDir);
|
|
13727
|
-
const absolutePath =
|
|
14425
|
+
const absolutePath = path22.resolve(paths.wikiDir, relativePath);
|
|
13728
14426
|
if (!absolutePath.startsWith(paths.wikiDir) || !await fileExists(absolutePath)) {
|
|
13729
14427
|
return null;
|
|
13730
14428
|
}
|
|
13731
|
-
const raw = await
|
|
14429
|
+
const raw = await fs18.readFile(absolutePath, "utf8");
|
|
13732
14430
|
const parsed = matter9(raw);
|
|
13733
14431
|
return {
|
|
13734
14432
|
path: relativePath,
|
|
13735
|
-
title: typeof parsed.data.title === "string" ? parsed.data.title :
|
|
14433
|
+
title: typeof parsed.data.title === "string" ? parsed.data.title : path22.basename(relativePath, path22.extname(relativePath)),
|
|
13736
14434
|
frontmatter: parsed.data,
|
|
13737
14435
|
content: parsed.content
|
|
13738
14436
|
};
|
|
@@ -13768,7 +14466,7 @@ function structuralLintFindings(_rootDir, paths, graph, schemas, manifests, sour
|
|
|
13768
14466
|
severity: "warning",
|
|
13769
14467
|
code: "stale_page",
|
|
13770
14468
|
message: `Page ${page.title} is stale because the vault schema changed.`,
|
|
13771
|
-
pagePath:
|
|
14469
|
+
pagePath: path22.join(paths.wikiDir, page.path),
|
|
13772
14470
|
relatedPageIds: [page.id]
|
|
13773
14471
|
});
|
|
13774
14472
|
}
|
|
@@ -13779,7 +14477,7 @@ function structuralLintFindings(_rootDir, paths, graph, schemas, manifests, sour
|
|
|
13779
14477
|
severity: "warning",
|
|
13780
14478
|
code: "stale_page",
|
|
13781
14479
|
message: `Page ${page.title} is stale because source ${sourceId} changed.`,
|
|
13782
|
-
pagePath:
|
|
14480
|
+
pagePath: path22.join(paths.wikiDir, page.path),
|
|
13783
14481
|
relatedSourceIds: [sourceId],
|
|
13784
14482
|
relatedPageIds: [page.id]
|
|
13785
14483
|
});
|
|
@@ -13790,13 +14488,13 @@ function structuralLintFindings(_rootDir, paths, graph, schemas, manifests, sour
|
|
|
13790
14488
|
severity: "info",
|
|
13791
14489
|
code: "orphan_page",
|
|
13792
14490
|
message: `Page ${page.title} has no backlinks.`,
|
|
13793
|
-
pagePath:
|
|
14491
|
+
pagePath: path22.join(paths.wikiDir, page.path),
|
|
13794
14492
|
relatedPageIds: [page.id]
|
|
13795
14493
|
});
|
|
13796
14494
|
}
|
|
13797
|
-
const absolutePath =
|
|
14495
|
+
const absolutePath = path22.join(paths.wikiDir, page.path);
|
|
13798
14496
|
if (await fileExists(absolutePath)) {
|
|
13799
|
-
const content = await
|
|
14497
|
+
const content = await fs18.readFile(absolutePath, "utf8");
|
|
13800
14498
|
if (content.includes("## Claims")) {
|
|
13801
14499
|
const uncited = content.split("\n").filter((line) => line.startsWith("- ") && !line.includes("[source:"));
|
|
13802
14500
|
if (uncited.length) {
|
|
@@ -13913,7 +14611,7 @@ async function bootstrapDemo(rootDir, input) {
|
|
|
13913
14611
|
}
|
|
13914
14612
|
|
|
13915
14613
|
// src/mcp.ts
|
|
13916
|
-
var SERVER_VERSION = "0.
|
|
14614
|
+
var SERVER_VERSION = "0.2.0";
|
|
13917
14615
|
async function createMcpServer(rootDir) {
|
|
13918
14616
|
const server = new McpServer({
|
|
13919
14617
|
name: "swarmvault",
|
|
@@ -14184,7 +14882,7 @@ async function createMcpServer(rootDir) {
|
|
|
14184
14882
|
},
|
|
14185
14883
|
async () => {
|
|
14186
14884
|
const { paths } = await loadVaultConfig(rootDir);
|
|
14187
|
-
const files = (await listFilesRecursive(paths.sessionsDir)).filter((filePath) => filePath.endsWith(".md")).map((filePath) => toPosix(
|
|
14885
|
+
const files = (await listFilesRecursive(paths.sessionsDir)).filter((filePath) => filePath.endsWith(".md")).map((filePath) => toPosix(path23.relative(paths.sessionsDir, filePath))).sort();
|
|
14188
14886
|
return asTextResource("swarmvault://sessions", JSON.stringify(files, null, 2));
|
|
14189
14887
|
}
|
|
14190
14888
|
);
|
|
@@ -14217,8 +14915,8 @@ async function createMcpServer(rootDir) {
|
|
|
14217
14915
|
return asTextResource(`swarmvault://pages/${encodedPath}`, `Page not found: ${relativePath}`);
|
|
14218
14916
|
}
|
|
14219
14917
|
const { paths } = await loadVaultConfig(rootDir);
|
|
14220
|
-
const absolutePath =
|
|
14221
|
-
return asTextResource(`swarmvault://pages/${encodedPath}`, await
|
|
14918
|
+
const absolutePath = path23.resolve(paths.wikiDir, relativePath);
|
|
14919
|
+
return asTextResource(`swarmvault://pages/${encodedPath}`, await fs19.readFile(absolutePath, "utf8"));
|
|
14222
14920
|
}
|
|
14223
14921
|
);
|
|
14224
14922
|
server.registerResource(
|
|
@@ -14226,11 +14924,11 @@ async function createMcpServer(rootDir) {
|
|
|
14226
14924
|
new ResourceTemplate("swarmvault://sessions/{path}", {
|
|
14227
14925
|
list: async () => {
|
|
14228
14926
|
const { paths } = await loadVaultConfig(rootDir);
|
|
14229
|
-
const files = (await listFilesRecursive(paths.sessionsDir)).filter((filePath) => filePath.endsWith(".md")).map((filePath) => toPosix(
|
|
14927
|
+
const files = (await listFilesRecursive(paths.sessionsDir)).filter((filePath) => filePath.endsWith(".md")).map((filePath) => toPosix(path23.relative(paths.sessionsDir, filePath))).sort();
|
|
14230
14928
|
return {
|
|
14231
14929
|
resources: files.map((relativePath) => ({
|
|
14232
14930
|
uri: `swarmvault://sessions/${encodeURIComponent(relativePath)}`,
|
|
14233
|
-
name:
|
|
14931
|
+
name: path23.basename(relativePath, ".md"),
|
|
14234
14932
|
title: relativePath,
|
|
14235
14933
|
description: "SwarmVault session artifact",
|
|
14236
14934
|
mimeType: "text/markdown"
|
|
@@ -14247,11 +14945,11 @@ async function createMcpServer(rootDir) {
|
|
|
14247
14945
|
const { paths } = await loadVaultConfig(rootDir);
|
|
14248
14946
|
const encodedPath = typeof variables.path === "string" ? variables.path : "";
|
|
14249
14947
|
const relativePath = decodeURIComponent(encodedPath);
|
|
14250
|
-
const absolutePath =
|
|
14948
|
+
const absolutePath = path23.resolve(paths.sessionsDir, relativePath);
|
|
14251
14949
|
if (!absolutePath.startsWith(paths.sessionsDir) || !await fileExists(absolutePath)) {
|
|
14252
14950
|
return asTextResource(`swarmvault://sessions/${encodedPath}`, `Session not found: ${relativePath}`);
|
|
14253
14951
|
}
|
|
14254
|
-
return asTextResource(`swarmvault://sessions/${encodedPath}`, await
|
|
14952
|
+
return asTextResource(`swarmvault://sessions/${encodedPath}`, await fs19.readFile(absolutePath, "utf8"));
|
|
14255
14953
|
}
|
|
14256
14954
|
);
|
|
14257
14955
|
return server;
|
|
@@ -14299,13 +14997,13 @@ function asTextResource(uri, text) {
|
|
|
14299
14997
|
}
|
|
14300
14998
|
|
|
14301
14999
|
// src/schedule.ts
|
|
14302
|
-
import
|
|
14303
|
-
import
|
|
15000
|
+
import fs20 from "fs/promises";
|
|
15001
|
+
import path24 from "path";
|
|
14304
15002
|
function scheduleStatePath(schedulesDir, jobId) {
|
|
14305
|
-
return
|
|
15003
|
+
return path24.join(schedulesDir, `${encodeURIComponent(jobId)}.json`);
|
|
14306
15004
|
}
|
|
14307
15005
|
function scheduleLockPath(schedulesDir, jobId) {
|
|
14308
|
-
return
|
|
15006
|
+
return path24.join(schedulesDir, `${encodeURIComponent(jobId)}.lock`);
|
|
14309
15007
|
}
|
|
14310
15008
|
function parseEveryDuration(value) {
|
|
14311
15009
|
const match = value.trim().match(/^(\d+)(m|h|d)$/i);
|
|
@@ -14408,13 +15106,13 @@ async function acquireJobLease(rootDir, jobId) {
|
|
|
14408
15106
|
const { paths } = await loadVaultConfig(rootDir);
|
|
14409
15107
|
const leasePath = scheduleLockPath(paths.schedulesDir, jobId);
|
|
14410
15108
|
await ensureDir(paths.schedulesDir);
|
|
14411
|
-
const handle = await
|
|
15109
|
+
const handle = await fs20.open(leasePath, "wx");
|
|
14412
15110
|
await handle.writeFile(`${process.pid}
|
|
14413
15111
|
${(/* @__PURE__ */ new Date()).toISOString()}
|
|
14414
15112
|
`);
|
|
14415
15113
|
await handle.close();
|
|
14416
15114
|
return async () => {
|
|
14417
|
-
await
|
|
15115
|
+
await fs20.rm(leasePath, { force: true });
|
|
14418
15116
|
};
|
|
14419
15117
|
}
|
|
14420
15118
|
async function listSchedules(rootDir) {
|
|
@@ -14560,33 +15258,750 @@ async function serveSchedules(rootDir, pollMs = 3e4) {
|
|
|
14560
15258
|
};
|
|
14561
15259
|
}
|
|
14562
15260
|
|
|
15261
|
+
// src/sources.ts
|
|
15262
|
+
import { spawn as spawn2 } from "child_process";
|
|
15263
|
+
import fs21 from "fs/promises";
|
|
15264
|
+
import path25 from "path";
|
|
15265
|
+
import { JSDOM as JSDOM3 } from "jsdom";
|
|
15266
|
+
var DEFAULT_CRAWL_MAX_PAGES = 12;
|
|
15267
|
+
var DEFAULT_CRAWL_MAX_DEPTH = 2;
|
|
15268
|
+
var DOCS_HINT_SEGMENTS = /* @__PURE__ */ new Set([
|
|
15269
|
+
"docs",
|
|
15270
|
+
"documentation",
|
|
15271
|
+
"wiki",
|
|
15272
|
+
"help",
|
|
15273
|
+
"reference",
|
|
15274
|
+
"references",
|
|
15275
|
+
"guide",
|
|
15276
|
+
"guides",
|
|
15277
|
+
"tutorial",
|
|
15278
|
+
"tutorials",
|
|
15279
|
+
"manual",
|
|
15280
|
+
"api",
|
|
15281
|
+
"apis",
|
|
15282
|
+
"getting-started"
|
|
15283
|
+
]);
|
|
15284
|
+
function uniqueStrings4(values) {
|
|
15285
|
+
return uniqueBy(values.filter(Boolean), (value) => value);
|
|
15286
|
+
}
|
|
15287
|
+
function normalizeManagedStatus(value) {
|
|
15288
|
+
return value === "missing" || value === "error" ? value : "ready";
|
|
15289
|
+
}
|
|
15290
|
+
function withinRoot2(rootPath, targetPath) {
|
|
15291
|
+
const relative = path25.relative(rootPath, targetPath);
|
|
15292
|
+
return relative === "" || !relative.startsWith("..") && !path25.isAbsolute(relative);
|
|
15293
|
+
}
|
|
15294
|
+
async function findNearestGitRoot3(startPath) {
|
|
15295
|
+
let current = path25.resolve(startPath);
|
|
15296
|
+
try {
|
|
15297
|
+
const stat = await fs21.stat(current);
|
|
15298
|
+
if (!stat.isDirectory()) {
|
|
15299
|
+
current = path25.dirname(current);
|
|
15300
|
+
}
|
|
15301
|
+
} catch {
|
|
15302
|
+
current = path25.dirname(current);
|
|
15303
|
+
}
|
|
15304
|
+
while (true) {
|
|
15305
|
+
if (await fileExists(path25.join(current, ".git"))) {
|
|
15306
|
+
return current;
|
|
15307
|
+
}
|
|
15308
|
+
const parent = path25.dirname(current);
|
|
15309
|
+
if (parent === current) {
|
|
15310
|
+
return null;
|
|
15311
|
+
}
|
|
15312
|
+
current = parent;
|
|
15313
|
+
}
|
|
15314
|
+
}
|
|
15315
|
+
function normalizeUrlWithoutHash(input) {
|
|
15316
|
+
const url = new URL(input);
|
|
15317
|
+
url.hash = "";
|
|
15318
|
+
if (url.protocol === "http:" && url.port === "80" || url.protocol === "https:" && url.port === "443") {
|
|
15319
|
+
url.port = "";
|
|
15320
|
+
}
|
|
15321
|
+
return url.toString();
|
|
15322
|
+
}
|
|
15323
|
+
function normalizeGitHubRepoRootUrl(input) {
|
|
15324
|
+
let parsed;
|
|
15325
|
+
try {
|
|
15326
|
+
parsed = new URL(input);
|
|
15327
|
+
} catch {
|
|
15328
|
+
return null;
|
|
15329
|
+
}
|
|
15330
|
+
const host = parsed.hostname.toLowerCase();
|
|
15331
|
+
if (host !== "github.com" && host !== "www.github.com") {
|
|
15332
|
+
return null;
|
|
15333
|
+
}
|
|
15334
|
+
const segments = parsed.pathname.split("/").map((segment) => segment.trim()).filter(Boolean);
|
|
15335
|
+
if (segments.length !== 2) {
|
|
15336
|
+
return null;
|
|
15337
|
+
}
|
|
15338
|
+
const [owner, repoSegment] = segments;
|
|
15339
|
+
const repo = repoSegment.replace(/\.git$/i, "");
|
|
15340
|
+
if (!owner || !repo) {
|
|
15341
|
+
return null;
|
|
15342
|
+
}
|
|
15343
|
+
const url = `https://github.com/${owner}/${repo}`;
|
|
15344
|
+
return {
|
|
15345
|
+
url,
|
|
15346
|
+
cloneUrl: `${url}.git`,
|
|
15347
|
+
title: `${owner}/${repo}`
|
|
15348
|
+
};
|
|
15349
|
+
}
|
|
15350
|
+
function looksLikeDocsPathname(pathname) {
|
|
15351
|
+
const segments = pathname.split("/").map((segment) => segment.trim().toLowerCase()).filter(Boolean);
|
|
15352
|
+
return segments.some((segment) => DOCS_HINT_SEGMENTS.has(segment));
|
|
15353
|
+
}
|
|
15354
|
+
function isLikelyDocsStartUrl(url) {
|
|
15355
|
+
return looksLikeDocsPathname(url.pathname) || url.hostname.toLowerCase().startsWith("docs.");
|
|
15356
|
+
}
|
|
15357
|
+
function normalizeCrawlCandidate(href, baseUrl) {
|
|
15358
|
+
try {
|
|
15359
|
+
const url = new URL(href, baseUrl);
|
|
15360
|
+
if (!["http:", "https:"].includes(url.protocol)) {
|
|
15361
|
+
return null;
|
|
15362
|
+
}
|
|
15363
|
+
if (url.hash) {
|
|
15364
|
+
url.hash = "";
|
|
15365
|
+
}
|
|
15366
|
+
return url.toString();
|
|
15367
|
+
} catch {
|
|
15368
|
+
return null;
|
|
15369
|
+
}
|
|
15370
|
+
}
|
|
15371
|
+
function pathPrefix(pathname) {
|
|
15372
|
+
const segments = pathname.split("/").filter(Boolean);
|
|
15373
|
+
if (segments.length === 0) {
|
|
15374
|
+
return "/";
|
|
15375
|
+
}
|
|
15376
|
+
return `/${segments[0]}`;
|
|
15377
|
+
}
|
|
15378
|
+
function isAllowedDocsCandidate(candidate, startUrl) {
|
|
15379
|
+
if (candidate.origin !== startUrl.origin) {
|
|
15380
|
+
return false;
|
|
15381
|
+
}
|
|
15382
|
+
const extension = path25.extname(candidate.pathname).toLowerCase();
|
|
15383
|
+
if (extension && extension !== ".html" && extension !== ".htm" && extension !== ".md") {
|
|
15384
|
+
return false;
|
|
15385
|
+
}
|
|
15386
|
+
if (looksLikeDocsPathname(candidate.pathname)) {
|
|
15387
|
+
return true;
|
|
15388
|
+
}
|
|
15389
|
+
const startPrefix = pathPrefix(startUrl.pathname);
|
|
15390
|
+
const candidatePrefix = pathPrefix(candidate.pathname);
|
|
15391
|
+
return startPrefix !== "/" && candidatePrefix === startPrefix;
|
|
15392
|
+
}
|
|
15393
|
+
async function fetchHtml(url) {
|
|
15394
|
+
await validateUrlSafety(url);
|
|
15395
|
+
const response = await fetch(url);
|
|
15396
|
+
if (!response.ok) {
|
|
15397
|
+
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
|
|
15398
|
+
}
|
|
15399
|
+
const contentType = response.headers.get("content-type")?.split(";")[0]?.trim() ?? "text/html";
|
|
15400
|
+
if (!contentType.includes("html")) {
|
|
15401
|
+
throw new Error(`Unsupported docs crawl content type at ${url}: ${contentType}`);
|
|
15402
|
+
}
|
|
15403
|
+
const html = await response.text();
|
|
15404
|
+
const dom = new JSDOM3(html, { url });
|
|
15405
|
+
const document = dom.window.document;
|
|
15406
|
+
const title = document.title.trim() || url;
|
|
15407
|
+
const links = [...document.querySelectorAll("a[href]")].map((anchor) => normalizeCrawlCandidate(anchor.getAttribute("href") ?? "", url)).filter((value) => Boolean(value));
|
|
15408
|
+
return { title, links };
|
|
15409
|
+
}
|
|
15410
|
+
async function crawlDocsSource(url, maxPages, maxDepth) {
|
|
15411
|
+
const startUrl = new URL(normalizeUrlWithoutHash(url));
|
|
15412
|
+
const initial = await fetchHtml(startUrl.toString());
|
|
15413
|
+
const sameDomainDocsLinks = uniqueStrings4(
|
|
15414
|
+
initial.links.filter((candidate) => {
|
|
15415
|
+
const parsed = new URL(candidate);
|
|
15416
|
+
return isAllowedDocsCandidate(parsed, startUrl);
|
|
15417
|
+
})
|
|
15418
|
+
);
|
|
15419
|
+
if (!isLikelyDocsStartUrl(startUrl) && sameDomainDocsLinks.length < 3) {
|
|
15420
|
+
throw new Error(
|
|
15421
|
+
"This URL does not look like a docs hub. Use `swarmvault add` for single articles or `swarmvault ingest` for direct files."
|
|
15422
|
+
);
|
|
15423
|
+
}
|
|
15424
|
+
const visited = /* @__PURE__ */ new Set();
|
|
15425
|
+
const queued = /* @__PURE__ */ new Set();
|
|
15426
|
+
const pages = [];
|
|
15427
|
+
const queue = [{ url: startUrl.toString(), depth: 0 }];
|
|
15428
|
+
queued.add(startUrl.toString());
|
|
15429
|
+
while (queue.length > 0 && pages.length < maxPages) {
|
|
15430
|
+
const current = queue.shift();
|
|
15431
|
+
if (!current) {
|
|
15432
|
+
continue;
|
|
15433
|
+
}
|
|
15434
|
+
if (visited.has(current.url)) {
|
|
15435
|
+
continue;
|
|
15436
|
+
}
|
|
15437
|
+
visited.add(current.url);
|
|
15438
|
+
pages.push(current.url);
|
|
15439
|
+
if (current.depth >= maxDepth) {
|
|
15440
|
+
continue;
|
|
15441
|
+
}
|
|
15442
|
+
const { links } = await fetchHtml(current.url);
|
|
15443
|
+
for (const candidate of links) {
|
|
15444
|
+
if (pages.length + queue.length >= maxPages) {
|
|
15445
|
+
break;
|
|
15446
|
+
}
|
|
15447
|
+
if (queued.has(candidate) || visited.has(candidate)) {
|
|
15448
|
+
continue;
|
|
15449
|
+
}
|
|
15450
|
+
const parsed = new URL(candidate);
|
|
15451
|
+
if (!isAllowedDocsCandidate(parsed, startUrl)) {
|
|
15452
|
+
continue;
|
|
15453
|
+
}
|
|
15454
|
+
queued.add(candidate);
|
|
15455
|
+
queue.push({ url: candidate, depth: current.depth + 1 });
|
|
15456
|
+
}
|
|
15457
|
+
}
|
|
15458
|
+
return {
|
|
15459
|
+
title: initial.title,
|
|
15460
|
+
pages
|
|
15461
|
+
};
|
|
15462
|
+
}
|
|
15463
|
+
function stableManagedSourceId(kind, raw, fallbackTitle) {
|
|
15464
|
+
return `${kind}-${slugify(fallbackTitle)}-${sha256(raw).slice(0, 8)}`;
|
|
15465
|
+
}
|
|
15466
|
+
function matchesManagedSourceSpec(existing, input) {
|
|
15467
|
+
if (existing.kind !== input.kind) {
|
|
15468
|
+
return false;
|
|
15469
|
+
}
|
|
15470
|
+
if (input.kind === "directory") {
|
|
15471
|
+
return path25.resolve(existing.path ?? "") === path25.resolve(input.path);
|
|
15472
|
+
}
|
|
15473
|
+
return (existing.url ?? "") === input.url;
|
|
15474
|
+
}
|
|
15475
|
+
async function resolveManagedSourceInput(rootDir, input) {
|
|
15476
|
+
const absoluteInput = path25.resolve(rootDir, input);
|
|
15477
|
+
if (!(input.startsWith("http://") || input.startsWith("https://"))) {
|
|
15478
|
+
const stat = await fs21.stat(absoluteInput).catch(() => null);
|
|
15479
|
+
if (!stat) {
|
|
15480
|
+
throw new Error(`Source not found: ${input}`);
|
|
15481
|
+
}
|
|
15482
|
+
if (!stat.isDirectory()) {
|
|
15483
|
+
throw new Error(
|
|
15484
|
+
"`swarmvault source add` supports directories, public GitHub repo root URLs, and docs hubs. Use `swarmvault ingest` for single files."
|
|
15485
|
+
);
|
|
15486
|
+
}
|
|
15487
|
+
const detectedRepoRoot = await findNearestGitRoot3(absoluteInput);
|
|
15488
|
+
const repoRoot = detectedRepoRoot && !(withinRoot2(rootDir, absoluteInput) && !withinRoot2(rootDir, detectedRepoRoot)) ? detectedRepoRoot : absoluteInput;
|
|
15489
|
+
return {
|
|
15490
|
+
kind: "directory",
|
|
15491
|
+
path: absoluteInput,
|
|
15492
|
+
repoRoot,
|
|
15493
|
+
title: path25.basename(absoluteInput) || absoluteInput
|
|
15494
|
+
};
|
|
15495
|
+
}
|
|
15496
|
+
const github = normalizeGitHubRepoRootUrl(input);
|
|
15497
|
+
if (github) {
|
|
15498
|
+
return {
|
|
15499
|
+
kind: "github_repo",
|
|
15500
|
+
...github
|
|
15501
|
+
};
|
|
15502
|
+
}
|
|
15503
|
+
const parsed = new URL(input);
|
|
15504
|
+
if (parsed.hostname.toLowerCase().includes("github.com")) {
|
|
15505
|
+
throw new Error(
|
|
15506
|
+
"`swarmvault source add` only supports public GitHub repo root URLs. Use a repo root like https://github.com/owner/repo."
|
|
15507
|
+
);
|
|
15508
|
+
}
|
|
15509
|
+
return {
|
|
15510
|
+
kind: "crawl_url",
|
|
15511
|
+
url: normalizeUrlWithoutHash(input),
|
|
15512
|
+
title: parsed.hostname
|
|
15513
|
+
};
|
|
15514
|
+
}
|
|
15515
|
+
function directorySourceIdsFor(manifests, inputPath) {
|
|
15516
|
+
return manifests.filter((manifest) => manifest.originalPath && withinRoot2(path25.resolve(inputPath), path25.resolve(manifest.originalPath))).map((manifest) => manifest.sourceId).sort((left, right) => left.localeCompare(right));
|
|
15517
|
+
}
|
|
15518
|
+
async function syncDirectorySource(rootDir, inputPath, repoRoot) {
|
|
15519
|
+
const manifestsBefore = await listManifests(rootDir);
|
|
15520
|
+
const previousInScope = manifestsBefore.filter(
|
|
15521
|
+
(manifest) => manifest.originalPath && withinRoot2(path25.resolve(inputPath), path25.resolve(manifest.originalPath))
|
|
15522
|
+
);
|
|
15523
|
+
const result = await ingestDirectory(rootDir, inputPath, { repoRoot });
|
|
15524
|
+
const removed = [];
|
|
15525
|
+
for (const manifest of previousInScope) {
|
|
15526
|
+
if (!manifest.originalPath) {
|
|
15527
|
+
continue;
|
|
15528
|
+
}
|
|
15529
|
+
if (await fileExists(path25.resolve(manifest.originalPath))) {
|
|
15530
|
+
continue;
|
|
15531
|
+
}
|
|
15532
|
+
const removedManifest = await removeManifestBySourceId(rootDir, manifest.sourceId);
|
|
15533
|
+
if (removedManifest) {
|
|
15534
|
+
removed.push(removedManifest.sourceId);
|
|
15535
|
+
}
|
|
15536
|
+
}
|
|
15537
|
+
const manifestsAfter = await listManifests(rootDir);
|
|
15538
|
+
return {
|
|
15539
|
+
title: path25.basename(inputPath) || inputPath,
|
|
15540
|
+
sourceIds: directorySourceIdsFor(manifestsAfter, inputPath),
|
|
15541
|
+
counts: {
|
|
15542
|
+
scannedCount: result.scannedCount,
|
|
15543
|
+
importedCount: result.imported.length,
|
|
15544
|
+
updatedCount: result.updated.length,
|
|
15545
|
+
removedCount: removed.length,
|
|
15546
|
+
skippedCount: result.skipped.length
|
|
15547
|
+
},
|
|
15548
|
+
changed: result.imported.length + result.updated.length + removed.length > 0
|
|
15549
|
+
};
|
|
15550
|
+
}
|
|
15551
|
+
async function runGitCommand(cwd, args) {
|
|
15552
|
+
await new Promise((resolve, reject) => {
|
|
15553
|
+
const child = spawn2("git", args, {
|
|
15554
|
+
cwd,
|
|
15555
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
15556
|
+
});
|
|
15557
|
+
let stderr = "";
|
|
15558
|
+
child.stderr.on("data", (chunk) => {
|
|
15559
|
+
stderr += chunk.toString("utf8");
|
|
15560
|
+
});
|
|
15561
|
+
child.on("error", reject);
|
|
15562
|
+
child.on("close", (code) => {
|
|
15563
|
+
if (code === 0) {
|
|
15564
|
+
resolve();
|
|
15565
|
+
return;
|
|
15566
|
+
}
|
|
15567
|
+
reject(new Error(stderr.trim() || `git ${args.join(" ")} failed with exit code ${code ?? 1}`));
|
|
15568
|
+
});
|
|
15569
|
+
});
|
|
15570
|
+
}
|
|
15571
|
+
async function syncGitHubRepoSource(rootDir, entry) {
|
|
15572
|
+
const workingDir = await managedSourceWorkingDir(rootDir, entry.id);
|
|
15573
|
+
const checkoutDir = path25.join(workingDir, "checkout");
|
|
15574
|
+
await fs21.rm(checkoutDir, { recursive: true, force: true });
|
|
15575
|
+
await ensureDir(workingDir);
|
|
15576
|
+
if (!entry.url) {
|
|
15577
|
+
throw new Error(`Managed source ${entry.id} is missing its repository URL.`);
|
|
15578
|
+
}
|
|
15579
|
+
const github = normalizeGitHubRepoRootUrl(entry.url);
|
|
15580
|
+
if (!github) {
|
|
15581
|
+
throw new Error(`Managed source ${entry.id} has an invalid GitHub repo URL.`);
|
|
15582
|
+
}
|
|
15583
|
+
await runGitCommand(workingDir, ["clone", "--depth", "1", github.cloneUrl, "checkout"]);
|
|
15584
|
+
return await syncDirectorySource(rootDir, checkoutDir, checkoutDir);
|
|
15585
|
+
}
|
|
15586
|
+
async function syncCrawlSource(rootDir, entry, options) {
|
|
15587
|
+
if (!entry.url) {
|
|
15588
|
+
throw new Error(`Managed source ${entry.id} is missing its URL.`);
|
|
15589
|
+
}
|
|
15590
|
+
const crawl = await crawlDocsSource(entry.url, options.maxPages ?? DEFAULT_CRAWL_MAX_PAGES, options.maxDepth ?? DEFAULT_CRAWL_MAX_DEPTH);
|
|
15591
|
+
const previousSourceIds = [...entry.sourceIds];
|
|
15592
|
+
const currentSourceIds = [];
|
|
15593
|
+
let importedCount = 0;
|
|
15594
|
+
let updatedCount = 0;
|
|
15595
|
+
for (const pageUrl of crawl.pages) {
|
|
15596
|
+
const persisted = await ingestInputDetailed(rootDir, pageUrl);
|
|
15597
|
+
currentSourceIds.push(persisted.manifest.sourceId);
|
|
15598
|
+
if (persisted.isNew) {
|
|
15599
|
+
importedCount += 1;
|
|
15600
|
+
} else if (persisted.wasUpdated) {
|
|
15601
|
+
updatedCount += 1;
|
|
15602
|
+
}
|
|
15603
|
+
}
|
|
15604
|
+
let removedCount = 0;
|
|
15605
|
+
for (const sourceId of previousSourceIds) {
|
|
15606
|
+
if (currentSourceIds.includes(sourceId)) {
|
|
15607
|
+
continue;
|
|
15608
|
+
}
|
|
15609
|
+
if (await removeManifestBySourceId(rootDir, sourceId)) {
|
|
15610
|
+
removedCount += 1;
|
|
15611
|
+
}
|
|
15612
|
+
}
|
|
15613
|
+
return {
|
|
15614
|
+
title: crawl.title,
|
|
15615
|
+
sourceIds: uniqueStrings4(currentSourceIds).sort((left, right) => left.localeCompare(right)),
|
|
15616
|
+
counts: {
|
|
15617
|
+
scannedCount: crawl.pages.length,
|
|
15618
|
+
importedCount,
|
|
15619
|
+
updatedCount,
|
|
15620
|
+
removedCount,
|
|
15621
|
+
skippedCount: 0
|
|
15622
|
+
},
|
|
15623
|
+
changed: importedCount + updatedCount + removedCount > 0
|
|
15624
|
+
};
|
|
15625
|
+
}
|
|
15626
|
+
async function syncManagedSource(rootDir, entry, options) {
|
|
15627
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15628
|
+
try {
|
|
15629
|
+
let sync;
|
|
15630
|
+
if (entry.kind === "directory") {
|
|
15631
|
+
if (!entry.path || !entry.repoRoot) {
|
|
15632
|
+
throw new Error(`Managed source ${entry.id} is missing its directory path.`);
|
|
15633
|
+
}
|
|
15634
|
+
if (!await fileExists(entry.path)) {
|
|
15635
|
+
return {
|
|
15636
|
+
...entry,
|
|
15637
|
+
status: "missing",
|
|
15638
|
+
updatedAt: now,
|
|
15639
|
+
lastSyncAt: now,
|
|
15640
|
+
lastSyncStatus: "error",
|
|
15641
|
+
lastError: `Directory not found: ${entry.path}`,
|
|
15642
|
+
changed: false
|
|
15643
|
+
};
|
|
15644
|
+
}
|
|
15645
|
+
sync = await syncDirectorySource(rootDir, entry.path, entry.repoRoot);
|
|
15646
|
+
} else if (entry.kind === "github_repo") {
|
|
15647
|
+
sync = await syncGitHubRepoSource(rootDir, entry);
|
|
15648
|
+
} else {
|
|
15649
|
+
sync = await syncCrawlSource(rootDir, entry, options);
|
|
15650
|
+
}
|
|
15651
|
+
return {
|
|
15652
|
+
...entry,
|
|
15653
|
+
title: sync.title || entry.title,
|
|
15654
|
+
sourceIds: sync.sourceIds,
|
|
15655
|
+
status: "ready",
|
|
15656
|
+
updatedAt: now,
|
|
15657
|
+
lastSyncAt: now,
|
|
15658
|
+
lastSyncStatus: "success",
|
|
15659
|
+
lastSyncCounts: sync.counts,
|
|
15660
|
+
lastError: void 0,
|
|
15661
|
+
changed: sync.changed
|
|
15662
|
+
};
|
|
15663
|
+
} catch (error) {
|
|
15664
|
+
return {
|
|
15665
|
+
...entry,
|
|
15666
|
+
status: normalizeManagedStatus(entry.status),
|
|
15667
|
+
updatedAt: now,
|
|
15668
|
+
lastSyncAt: now,
|
|
15669
|
+
lastSyncStatus: "error",
|
|
15670
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
15671
|
+
changed: false
|
|
15672
|
+
};
|
|
15673
|
+
}
|
|
15674
|
+
}
|
|
15675
|
+
function scopedSourcePages(graph, sourceIds) {
|
|
15676
|
+
const scopedSet = new Set(sourceIds);
|
|
15677
|
+
return graph.pages.filter((page) => page.sourceIds.some((sourceId) => scopedSet.has(sourceId)));
|
|
15678
|
+
}
|
|
15679
|
+
function scopedNodeIds(graph, sourceIds) {
|
|
15680
|
+
const scopedSet = new Set(sourceIds);
|
|
15681
|
+
return graph.nodes.filter((node) => node.sourceIds.some((sourceId) => scopedSet.has(sourceId))).map((node) => node.id);
|
|
15682
|
+
}
|
|
15683
|
+
async function loadSourceAnalyses(rootDir, sourceIds) {
|
|
15684
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
15685
|
+
const analyses = await Promise.all(
|
|
15686
|
+
sourceIds.map(async (sourceId) => await readJsonFile(path25.join(paths.analysesDir, `${sourceId}.json`)))
|
|
15687
|
+
);
|
|
15688
|
+
return analyses.filter((analysis) => Boolean(analysis?.sourceId));
|
|
15689
|
+
}
|
|
15690
|
+
function renderDeterministicSourceBrief(input) {
|
|
15691
|
+
const modulePages = input.sourcePages.filter((page) => page.kind === "module").slice(0, 6);
|
|
15692
|
+
const sourcePages = input.sourcePages.filter((page) => page.kind === "source").slice(0, 6);
|
|
15693
|
+
const conceptPages = input.sourcePages.filter((page) => page.kind === "concept").slice(0, 6);
|
|
15694
|
+
const entityPages = input.sourcePages.filter((page) => page.kind === "entity").slice(0, 6);
|
|
15695
|
+
const questions = uniqueStrings4(input.analyses.flatMap((analysis) => analysis.questions)).slice(0, 5);
|
|
15696
|
+
const summary = truncate(
|
|
15697
|
+
normalizeWhitespace(
|
|
15698
|
+
uniqueStrings4(input.analyses.map((analysis) => analysis.summary).filter(Boolean)).join(" ") || `${input.source.title} has been compiled into a local source graph.`
|
|
15699
|
+
),
|
|
15700
|
+
320
|
|
15701
|
+
);
|
|
15702
|
+
const scopedNodeIdSet = new Set(scopedNodeIds(input.graph, input.source.sourceIds));
|
|
15703
|
+
const surprises = input.report?.surprisingConnections.filter((connection) => scopedNodeIdSet.has(connection.sourceNodeId) || scopedNodeIdSet.has(connection.targetNodeId)).slice(0, 4) ?? [];
|
|
15704
|
+
const contradictions = input.report?.contradictions.filter(
|
|
15705
|
+
(contradiction) => input.source.sourceIds.includes(contradiction.sourceIdA) || input.source.sourceIds.includes(contradiction.sourceIdB)
|
|
15706
|
+
) ?? [];
|
|
15707
|
+
return [
|
|
15708
|
+
`# Source Brief: ${input.source.title}`,
|
|
15709
|
+
"",
|
|
15710
|
+
"## What This Source Is",
|
|
15711
|
+
"",
|
|
15712
|
+
summary,
|
|
15713
|
+
"",
|
|
15714
|
+
"## Read First",
|
|
15715
|
+
"",
|
|
15716
|
+
...sourcePages.length ? sourcePages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No source page links are available yet."],
|
|
15717
|
+
"",
|
|
15718
|
+
"## Core Pages",
|
|
15719
|
+
"",
|
|
15720
|
+
...modulePages.length ? modulePages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`) : ["- No module pages are available yet."],
|
|
15721
|
+
...conceptPages.length ? ["", "Concept pages:", ...conceptPages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`)] : [],
|
|
15722
|
+
...entityPages.length ? ["", "Entity pages:", ...entityPages.map((page) => `- [[${page.path.replace(/\.md$/, "")}|${page.title}]]`)] : [],
|
|
15723
|
+
"",
|
|
15724
|
+
"## How The Important Parts Fit Together",
|
|
15725
|
+
"",
|
|
15726
|
+
`- Compiled source pages: ${sourcePages.length}`,
|
|
15727
|
+
`- Module pages: ${modulePages.length}`,
|
|
15728
|
+
`- Graph nodes touching this source: ${scopedNodeIdSet.size}`,
|
|
15729
|
+
`- Current tracked source ids: ${input.source.sourceIds.length}`,
|
|
15730
|
+
"",
|
|
15731
|
+
"## Surprises",
|
|
15732
|
+
"",
|
|
15733
|
+
...surprises.length ? surprises.map((surprise) => `- ${surprise.explanation}`) : ["- No surprising cross-source connections were highlighted for this source yet."],
|
|
15734
|
+
"",
|
|
15735
|
+
"## Contradictions",
|
|
15736
|
+
"",
|
|
15737
|
+
...contradictions.length ? contradictions.map(
|
|
15738
|
+
(contradiction) => `- ${contradiction.claimA} / ${contradiction.claimB} (sources: ${contradiction.sourceIdA}, ${contradiction.sourceIdB})`
|
|
15739
|
+
) : ["- No contradictions were detected for this source."],
|
|
15740
|
+
"",
|
|
15741
|
+
"## Open Questions",
|
|
15742
|
+
"",
|
|
15743
|
+
...questions.length ? questions.map((question) => `- ${question}`) : ["- No extracted open questions yet."],
|
|
15744
|
+
"",
|
|
15745
|
+
"## Suggested Next Questions",
|
|
15746
|
+
"",
|
|
15747
|
+
...(input.report?.suggestedQuestions ?? []).slice(0, 5).map((question) => `- ${question}`) || [
|
|
15748
|
+
"- Ask `swarmvault query` about the main modules or sections in this source."
|
|
15749
|
+
],
|
|
15750
|
+
""
|
|
15751
|
+
].join("\n");
|
|
15752
|
+
}
|
|
15753
|
+
async function generateSourceBriefMarkdown(rootDir, source) {
|
|
15754
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
15755
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
15756
|
+
if (!graph) {
|
|
15757
|
+
return null;
|
|
15758
|
+
}
|
|
15759
|
+
const sourcePages = scopedSourcePages(graph, source.sourceIds);
|
|
15760
|
+
const analyses = await loadSourceAnalyses(rootDir, source.sourceIds);
|
|
15761
|
+
const report = await readGraphReport(rootDir);
|
|
15762
|
+
const fallback = renderDeterministicSourceBrief({
|
|
15763
|
+
source,
|
|
15764
|
+
sourcePages,
|
|
15765
|
+
graph,
|
|
15766
|
+
analyses,
|
|
15767
|
+
report
|
|
15768
|
+
});
|
|
15769
|
+
const provider = await getProviderForTask(rootDir, "queryProvider");
|
|
15770
|
+
if (provider.type === "heuristic") {
|
|
15771
|
+
return fallback;
|
|
15772
|
+
}
|
|
15773
|
+
try {
|
|
15774
|
+
const schemas = await loadVaultSchemas(rootDir);
|
|
15775
|
+
const pageContext = sourcePages.slice(0, 10).map((page) => `- ${page.title} (${page.kind}) -> ${page.path}`).join("\n");
|
|
15776
|
+
const analysisContext = analyses.slice(0, 6).map(
|
|
15777
|
+
(analysis) => `# ${analysis.title}
|
|
15778
|
+
Summary: ${analysis.summary}
|
|
15779
|
+
Questions: ${analysis.questions.join(" | ") || "none"}
|
|
15780
|
+
Concepts: ${analysis.concepts.map((concept) => concept.name).join(", ") || "none"}
|
|
15781
|
+
Entities: ${analysis.entities.map((entity) => entity.name).join(", ") || "none"}`
|
|
15782
|
+
).join("\n\n---\n\n");
|
|
15783
|
+
const response = await provider.generateText({
|
|
15784
|
+
system: buildSchemaPrompt(
|
|
15785
|
+
schemas.effective.global,
|
|
15786
|
+
"Write a concise markdown source brief with sections: What This Source Is, Read First, Core Pages, How The Important Parts Fit Together, Surprises, Contradictions, Open Questions, Suggested Next Questions. Ground every claim in the provided context."
|
|
15787
|
+
),
|
|
15788
|
+
prompt: [
|
|
15789
|
+
`Source title: ${source.title}`,
|
|
15790
|
+
`Source kind: ${source.kind}`,
|
|
15791
|
+
`Tracked source ids: ${source.sourceIds.join(", ") || "none"}`,
|
|
15792
|
+
"",
|
|
15793
|
+
"Pages:",
|
|
15794
|
+
pageContext || "- none",
|
|
15795
|
+
"",
|
|
15796
|
+
"Analyses:",
|
|
15797
|
+
analysisContext || "No analysis context available.",
|
|
15798
|
+
"",
|
|
15799
|
+
"Deterministic fallback draft:",
|
|
15800
|
+
fallback
|
|
15801
|
+
].join("\n")
|
|
15802
|
+
});
|
|
15803
|
+
return response.text?.trim() ? response.text.trim() : fallback;
|
|
15804
|
+
} catch {
|
|
15805
|
+
return fallback;
|
|
15806
|
+
}
|
|
15807
|
+
}
|
|
15808
|
+
async function writeSourceBrief(rootDir, source) {
|
|
15809
|
+
if (!source.sourceIds.length) {
|
|
15810
|
+
return null;
|
|
15811
|
+
}
|
|
15812
|
+
const { paths } = await loadVaultConfig(rootDir);
|
|
15813
|
+
const markdown = await generateSourceBriefMarkdown(rootDir, source);
|
|
15814
|
+
if (!markdown) {
|
|
15815
|
+
return null;
|
|
15816
|
+
}
|
|
15817
|
+
const graph = await readJsonFile(paths.graphPath);
|
|
15818
|
+
const relatedPages = graph ? scopedSourcePages(graph, source.sourceIds) : [];
|
|
15819
|
+
const relatedPageIds = relatedPages.slice(0, 12).map((page) => page.id);
|
|
15820
|
+
const relatedNodeIds = graph ? scopedNodeIds(graph, source.sourceIds).slice(0, 20) : [];
|
|
15821
|
+
const projectIds = uniqueStrings4(relatedPages.flatMap((page) => page.projectIds));
|
|
15822
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15823
|
+
const output = buildOutputPage({
|
|
15824
|
+
title: `Source Brief: ${source.title}`,
|
|
15825
|
+
question: `Brief ${source.title}`,
|
|
15826
|
+
answer: markdown,
|
|
15827
|
+
citations: source.sourceIds,
|
|
15828
|
+
schemaHash: graph?.generatedAt ?? "",
|
|
15829
|
+
outputFormat: "report",
|
|
15830
|
+
relatedPageIds,
|
|
15831
|
+
relatedNodeIds,
|
|
15832
|
+
relatedSourceIds: source.sourceIds,
|
|
15833
|
+
projectIds,
|
|
15834
|
+
extraTags: ["source-brief"],
|
|
15835
|
+
origin: "query",
|
|
15836
|
+
slug: `source-briefs/${source.id}`,
|
|
15837
|
+
metadata: {
|
|
15838
|
+
status: "active",
|
|
15839
|
+
createdAt: now,
|
|
15840
|
+
updatedAt: now,
|
|
15841
|
+
compiledFrom: source.sourceIds,
|
|
15842
|
+
managedBy: "system",
|
|
15843
|
+
confidence: 0.82
|
|
15844
|
+
}
|
|
15845
|
+
});
|
|
15846
|
+
const absolutePath = path25.join(paths.wikiDir, output.page.path);
|
|
15847
|
+
await ensureDir(path25.dirname(absolutePath));
|
|
15848
|
+
await fs21.writeFile(absolutePath, output.content, "utf8");
|
|
15849
|
+
return absolutePath;
|
|
15850
|
+
}
|
|
15851
|
+
async function generateBriefsForSources(rootDir, sources) {
|
|
15852
|
+
const briefPaths = /* @__PURE__ */ new Map();
|
|
15853
|
+
for (const source of sources) {
|
|
15854
|
+
const briefPath = await writeSourceBrief(rootDir, source);
|
|
15855
|
+
if (briefPath) {
|
|
15856
|
+
briefPaths.set(source.id, briefPath);
|
|
15857
|
+
}
|
|
15858
|
+
}
|
|
15859
|
+
if (briefPaths.size > 0) {
|
|
15860
|
+
await refreshVaultAfterOutputSave(rootDir);
|
|
15861
|
+
}
|
|
15862
|
+
return briefPaths;
|
|
15863
|
+
}
|
|
15864
|
+
function shouldCompile(changedSources, graphExists, compileRequested) {
|
|
15865
|
+
return compileRequested && (!graphExists || changedSources.length > 0);
|
|
15866
|
+
}
|
|
15867
|
+
async function listManagedSourceRecords(rootDir) {
|
|
15868
|
+
await ensureManagedSourcesArtifact(rootDir);
|
|
15869
|
+
return await loadManagedSources(rootDir);
|
|
15870
|
+
}
|
|
15871
|
+
async function addManagedSource(rootDir, input, options = {}) {
|
|
15872
|
+
const compileRequested = options.compile ?? true;
|
|
15873
|
+
const briefRequested = options.brief ?? true;
|
|
15874
|
+
const sources = await loadManagedSources(rootDir);
|
|
15875
|
+
const resolved = await resolveManagedSourceInput(rootDir, input);
|
|
15876
|
+
const existing = sources.find((candidate) => matchesManagedSourceSpec(candidate, resolved));
|
|
15877
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15878
|
+
const source = existing ?? {
|
|
15879
|
+
id: resolved.kind === "directory" ? stableManagedSourceId("directory", path25.resolve(resolved.path), resolved.title) : stableManagedSourceId(resolved.kind, resolved.url, resolved.title),
|
|
15880
|
+
kind: resolved.kind,
|
|
15881
|
+
title: resolved.title,
|
|
15882
|
+
path: resolved.kind === "directory" ? resolved.path : void 0,
|
|
15883
|
+
repoRoot: resolved.kind === "directory" ? resolved.repoRoot : void 0,
|
|
15884
|
+
url: resolved.kind === "directory" ? void 0 : resolved.url,
|
|
15885
|
+
createdAt: now,
|
|
15886
|
+
updatedAt: now,
|
|
15887
|
+
status: "ready",
|
|
15888
|
+
sourceIds: []
|
|
15889
|
+
};
|
|
15890
|
+
const synced = await syncManagedSource(rootDir, source, options);
|
|
15891
|
+
if (synced.lastSyncStatus === "error") {
|
|
15892
|
+
throw new Error(synced.lastError ?? `Failed to add managed source ${synced.id}.`);
|
|
15893
|
+
}
|
|
15894
|
+
const graphExists = await loadVaultConfig(rootDir).then(({ paths }) => fileExists(paths.graphPath));
|
|
15895
|
+
let compile;
|
|
15896
|
+
if (shouldCompile([synced], graphExists, compileRequested)) {
|
|
15897
|
+
compile = await compileVault(rootDir, {});
|
|
15898
|
+
}
|
|
15899
|
+
let briefGenerated = false;
|
|
15900
|
+
let briefPath;
|
|
15901
|
+
if (compileRequested && briefRequested && synced.status === "ready") {
|
|
15902
|
+
const briefs = await generateBriefsForSources(rootDir, [synced]);
|
|
15903
|
+
briefPath = briefs.get(synced.id);
|
|
15904
|
+
briefGenerated = Boolean(briefPath);
|
|
15905
|
+
}
|
|
15906
|
+
const nextSource = {
|
|
15907
|
+
...synced,
|
|
15908
|
+
briefPath: briefPath ?? synced.briefPath,
|
|
15909
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15910
|
+
};
|
|
15911
|
+
const nextSources = existing ? sources.map((candidate) => candidate.id === nextSource.id ? nextSource : candidate) : [...sources, nextSource];
|
|
15912
|
+
await saveManagedSources(rootDir, nextSources);
|
|
15913
|
+
return {
|
|
15914
|
+
source: nextSource,
|
|
15915
|
+
compile,
|
|
15916
|
+
briefGenerated
|
|
15917
|
+
};
|
|
15918
|
+
}
|
|
15919
|
+
async function reloadManagedSources(rootDir, options = {}) {
|
|
15920
|
+
const compileRequested = options.compile ?? true;
|
|
15921
|
+
const briefRequested = options.brief ?? true;
|
|
15922
|
+
const sources = await loadManagedSources(rootDir);
|
|
15923
|
+
const selected = options.all || !options.id ? sources : sources.filter((source) => source.id === options.id);
|
|
15924
|
+
if (!selected.length) {
|
|
15925
|
+
throw new Error(options.id ? `Managed source not found: ${options.id}` : "No managed sources registered.");
|
|
15926
|
+
}
|
|
15927
|
+
const syncedSources = [];
|
|
15928
|
+
const changedSources = [];
|
|
15929
|
+
for (const source of selected) {
|
|
15930
|
+
const synced = await syncManagedSource(rootDir, source, options);
|
|
15931
|
+
syncedSources.push(synced);
|
|
15932
|
+
if (synced.changed) {
|
|
15933
|
+
changedSources.push(synced);
|
|
15934
|
+
}
|
|
15935
|
+
}
|
|
15936
|
+
const graphExists = await loadVaultConfig(rootDir).then(({ paths }) => fileExists(paths.graphPath));
|
|
15937
|
+
let compile;
|
|
15938
|
+
if (shouldCompile(changedSources, graphExists, compileRequested)) {
|
|
15939
|
+
compile = await compileVault(rootDir, {});
|
|
15940
|
+
}
|
|
15941
|
+
const briefPaths = compileRequested && briefRequested ? await generateBriefsForSources(
|
|
15942
|
+
rootDir,
|
|
15943
|
+
syncedSources.filter((source) => source.status === "ready")
|
|
15944
|
+
) : /* @__PURE__ */ new Map();
|
|
15945
|
+
const nextSources = sources.map((source) => {
|
|
15946
|
+
const synced = syncedSources.find((candidate) => candidate.id === source.id);
|
|
15947
|
+
if (!synced) {
|
|
15948
|
+
return source;
|
|
15949
|
+
}
|
|
15950
|
+
return {
|
|
15951
|
+
...synced,
|
|
15952
|
+
briefPath: briefPaths.get(synced.id) ?? synced.briefPath,
|
|
15953
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15954
|
+
};
|
|
15955
|
+
});
|
|
15956
|
+
await saveManagedSources(rootDir, nextSources);
|
|
15957
|
+
return {
|
|
15958
|
+
sources: nextSources.filter((source) => selected.some((candidate) => candidate.id === source.id)),
|
|
15959
|
+
compile,
|
|
15960
|
+
briefPaths: [...briefPaths.values()]
|
|
15961
|
+
};
|
|
15962
|
+
}
|
|
15963
|
+
async function deleteManagedSource(rootDir, id) {
|
|
15964
|
+
const sources = await loadManagedSources(rootDir);
|
|
15965
|
+
const target = sources.find((source) => source.id === id);
|
|
15966
|
+
if (!target) {
|
|
15967
|
+
throw new Error(`Managed source not found: ${id}`);
|
|
15968
|
+
}
|
|
15969
|
+
await saveManagedSources(
|
|
15970
|
+
rootDir,
|
|
15971
|
+
sources.filter((source) => source.id !== id)
|
|
15972
|
+
);
|
|
15973
|
+
const workingDir = await managedSourceWorkingDir(rootDir, id);
|
|
15974
|
+
await fs21.rm(workingDir, { recursive: true, force: true });
|
|
15975
|
+
return { removed: target };
|
|
15976
|
+
}
|
|
15977
|
+
|
|
14563
15978
|
// src/viewer.ts
|
|
14564
15979
|
import { execFile } from "child_process";
|
|
14565
|
-
import
|
|
15980
|
+
import fs22 from "fs/promises";
|
|
14566
15981
|
import http from "http";
|
|
14567
|
-
import
|
|
15982
|
+
import path27 from "path";
|
|
14568
15983
|
import { promisify } from "util";
|
|
14569
15984
|
import matter10 from "gray-matter";
|
|
14570
15985
|
import mime2 from "mime-types";
|
|
14571
15986
|
|
|
14572
15987
|
// src/watch.ts
|
|
14573
|
-
import
|
|
15988
|
+
import path26 from "path";
|
|
14574
15989
|
import process3 from "process";
|
|
14575
15990
|
import chokidar from "chokidar";
|
|
14576
15991
|
var MAX_BACKOFF_MS = 3e4;
|
|
14577
15992
|
var BACKOFF_THRESHOLD = 3;
|
|
14578
15993
|
var CRITICAL_THRESHOLD = 10;
|
|
14579
15994
|
var REPO_WATCH_IGNORES = /* @__PURE__ */ new Set([".git", ".venv"]);
|
|
14580
|
-
function
|
|
14581
|
-
const relative =
|
|
14582
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
15995
|
+
function withinRoot3(rootPath, targetPath) {
|
|
15996
|
+
const relative = path26.relative(rootPath, targetPath);
|
|
15997
|
+
return relative === "" || !relative.startsWith("..") && !path26.isAbsolute(relative);
|
|
14583
15998
|
}
|
|
14584
15999
|
function hasIgnoredRepoSegment(baseDir, targetPath) {
|
|
14585
|
-
const relativePath =
|
|
16000
|
+
const relativePath = path26.relative(baseDir, targetPath);
|
|
14586
16001
|
if (!relativePath || relativePath.startsWith("..")) {
|
|
14587
16002
|
return false;
|
|
14588
16003
|
}
|
|
14589
|
-
return relativePath.split(
|
|
16004
|
+
return relativePath.split(path26.sep).some((segment) => REPO_WATCH_IGNORES.has(segment));
|
|
14590
16005
|
}
|
|
14591
16006
|
function workspaceIgnoreRoots(rootDir, paths) {
|
|
14592
16007
|
return [
|
|
@@ -14595,16 +16010,16 @@ function workspaceIgnoreRoots(rootDir, paths) {
|
|
|
14595
16010
|
paths.stateDir,
|
|
14596
16011
|
paths.agentDir,
|
|
14597
16012
|
paths.inboxDir,
|
|
14598
|
-
|
|
14599
|
-
|
|
14600
|
-
|
|
14601
|
-
].map((candidate) =>
|
|
16013
|
+
path26.join(rootDir, ".claude"),
|
|
16014
|
+
path26.join(rootDir, ".cursor"),
|
|
16015
|
+
path26.join(rootDir, ".obsidian")
|
|
16016
|
+
].map((candidate) => path26.resolve(candidate));
|
|
14602
16017
|
}
|
|
14603
16018
|
async function resolveWatchTargets(rootDir, paths, options) {
|
|
14604
|
-
const targets = /* @__PURE__ */ new Set([
|
|
16019
|
+
const targets = /* @__PURE__ */ new Set([path26.resolve(paths.inboxDir)]);
|
|
14605
16020
|
if (options.repo) {
|
|
14606
16021
|
for (const repoRoot of await listTrackedRepoRoots(rootDir)) {
|
|
14607
|
-
targets.add(
|
|
16022
|
+
targets.add(path26.resolve(repoRoot));
|
|
14608
16023
|
}
|
|
14609
16024
|
}
|
|
14610
16025
|
return [...targets].sort((left, right) => left.localeCompare(right));
|
|
@@ -14734,7 +16149,7 @@ async function watchVault(rootDir, options = {}) {
|
|
|
14734
16149
|
const { paths } = await initWorkspace(rootDir);
|
|
14735
16150
|
const baseDebounceMs = options.debounceMs ?? 900;
|
|
14736
16151
|
const ignoredRoots = workspaceIgnoreRoots(rootDir, paths);
|
|
14737
|
-
const inboxWatchRoot =
|
|
16152
|
+
const inboxWatchRoot = path26.resolve(paths.inboxDir);
|
|
14738
16153
|
let watchTargets = await resolveWatchTargets(rootDir, paths, options);
|
|
14739
16154
|
let timer;
|
|
14740
16155
|
let running = false;
|
|
@@ -14749,12 +16164,12 @@ async function watchVault(rootDir, options = {}) {
|
|
|
14749
16164
|
usePolling: true,
|
|
14750
16165
|
interval: 100,
|
|
14751
16166
|
ignored: (targetPath) => {
|
|
14752
|
-
const absolutePath =
|
|
14753
|
-
const primaryTarget = watchTargets.filter((watchTarget) =>
|
|
16167
|
+
const absolutePath = path26.resolve(targetPath);
|
|
16168
|
+
const primaryTarget = watchTargets.filter((watchTarget) => withinRoot3(watchTarget, absolutePath)).sort((left, right) => right.length - left.length)[0] ?? null;
|
|
14754
16169
|
if (!primaryTarget) {
|
|
14755
16170
|
return false;
|
|
14756
16171
|
}
|
|
14757
|
-
if (primaryTarget !== inboxWatchRoot && ignoredRoots.some((ignoreRoot) =>
|
|
16172
|
+
if (primaryTarget !== inboxWatchRoot && ignoredRoots.some((ignoreRoot) => withinRoot3(ignoreRoot, absolutePath))) {
|
|
14758
16173
|
return true;
|
|
14759
16174
|
}
|
|
14760
16175
|
return hasIgnoredRepoSegment(primaryTarget, absolutePath);
|
|
@@ -14938,8 +16353,8 @@ async function watchVault(rootDir, options = {}) {
|
|
|
14938
16353
|
}
|
|
14939
16354
|
};
|
|
14940
16355
|
const reasonForPath = (targetPath) => {
|
|
14941
|
-
const baseDir = watchTargets.filter((watchTarget) =>
|
|
14942
|
-
return
|
|
16356
|
+
const baseDir = watchTargets.filter((watchTarget) => withinRoot3(watchTarget, path26.resolve(targetPath))).sort((left, right) => right.length - left.length)[0] ?? paths.inboxDir;
|
|
16357
|
+
return path26.relative(baseDir, targetPath) || ".";
|
|
14943
16358
|
};
|
|
14944
16359
|
watcher.on("add", (filePath) => schedule(`add:${reasonForPath(filePath)}`)).on("change", (filePath) => schedule(`change:${reasonForPath(filePath)}`)).on("unlink", (filePath) => schedule(`unlink:${reasonForPath(filePath)}`)).on("addDir", (dirPath) => schedule(`addDir:${reasonForPath(dirPath)}`)).on("unlinkDir", (dirPath) => schedule(`unlinkDir:${reasonForPath(dirPath)}`)).on("error", (caught) => schedule(`error:${caught instanceof Error ? caught.message : String(caught)}`));
|
|
14945
16360
|
await new Promise((resolve, reject) => {
|
|
@@ -14981,15 +16396,15 @@ async function getWatchStatus(rootDir) {
|
|
|
14981
16396
|
var execFileAsync = promisify(execFile);
|
|
14982
16397
|
async function readViewerPage(rootDir, relativePath) {
|
|
14983
16398
|
const { paths } = await loadVaultConfig(rootDir);
|
|
14984
|
-
const absolutePath =
|
|
16399
|
+
const absolutePath = path27.resolve(paths.wikiDir, relativePath);
|
|
14985
16400
|
if (!absolutePath.startsWith(paths.wikiDir) || !await fileExists(absolutePath)) {
|
|
14986
16401
|
return null;
|
|
14987
16402
|
}
|
|
14988
|
-
const raw = await
|
|
16403
|
+
const raw = await fs22.readFile(absolutePath, "utf8");
|
|
14989
16404
|
const parsed = matter10(raw);
|
|
14990
16405
|
return {
|
|
14991
16406
|
path: relativePath,
|
|
14992
|
-
title: typeof parsed.data.title === "string" ? parsed.data.title :
|
|
16407
|
+
title: typeof parsed.data.title === "string" ? parsed.data.title : path27.basename(relativePath, path27.extname(relativePath)),
|
|
14993
16408
|
frontmatter: parsed.data,
|
|
14994
16409
|
content: parsed.content,
|
|
14995
16410
|
assets: normalizeOutputAssets(parsed.data.output_assets)
|
|
@@ -14997,12 +16412,12 @@ async function readViewerPage(rootDir, relativePath) {
|
|
|
14997
16412
|
}
|
|
14998
16413
|
async function readViewerAsset(rootDir, relativePath) {
|
|
14999
16414
|
const { paths } = await loadVaultConfig(rootDir);
|
|
15000
|
-
const absolutePath =
|
|
16415
|
+
const absolutePath = path27.resolve(paths.wikiDir, relativePath);
|
|
15001
16416
|
if (!absolutePath.startsWith(paths.wikiDir) || !await fileExists(absolutePath)) {
|
|
15002
16417
|
return null;
|
|
15003
16418
|
}
|
|
15004
16419
|
return {
|
|
15005
|
-
buffer: await
|
|
16420
|
+
buffer: await fs22.readFile(absolutePath),
|
|
15006
16421
|
mimeType: mime2.lookup(absolutePath) || "application/octet-stream"
|
|
15007
16422
|
};
|
|
15008
16423
|
}
|
|
@@ -15025,12 +16440,12 @@ async function readJsonBody(request) {
|
|
|
15025
16440
|
return JSON.parse(raw);
|
|
15026
16441
|
}
|
|
15027
16442
|
async function ensureViewerDist(viewerDistDir) {
|
|
15028
|
-
const indexPath =
|
|
16443
|
+
const indexPath = path27.join(viewerDistDir, "index.html");
|
|
15029
16444
|
if (await fileExists(indexPath)) {
|
|
15030
16445
|
return;
|
|
15031
16446
|
}
|
|
15032
|
-
const viewerProjectDir =
|
|
15033
|
-
if (await fileExists(
|
|
16447
|
+
const viewerProjectDir = path27.dirname(viewerDistDir);
|
|
16448
|
+
if (await fileExists(path27.join(viewerProjectDir, "package.json"))) {
|
|
15034
16449
|
await execFileAsync("pnpm", ["build"], { cwd: viewerProjectDir });
|
|
15035
16450
|
}
|
|
15036
16451
|
}
|
|
@@ -15047,7 +16462,7 @@ async function startGraphServer(rootDir, port) {
|
|
|
15047
16462
|
return;
|
|
15048
16463
|
}
|
|
15049
16464
|
response.writeHead(200, { "content-type": "application/json" });
|
|
15050
|
-
response.end(await
|
|
16465
|
+
response.end(await fs22.readFile(paths.graphPath, "utf8"));
|
|
15051
16466
|
return;
|
|
15052
16467
|
}
|
|
15053
16468
|
if (url.pathname === "/api/graph/query") {
|
|
@@ -15104,14 +16519,14 @@ async function startGraphServer(rootDir, port) {
|
|
|
15104
16519
|
return;
|
|
15105
16520
|
}
|
|
15106
16521
|
if (url.pathname === "/api/graph-report") {
|
|
15107
|
-
const reportPath =
|
|
16522
|
+
const reportPath = path27.join(paths.wikiDir, "graph", "report.json");
|
|
15108
16523
|
if (!await fileExists(reportPath)) {
|
|
15109
16524
|
response.writeHead(404, { "content-type": "application/json" });
|
|
15110
16525
|
response.end(JSON.stringify({ error: "Graph report artifact not found. Run `swarmvault compile` first." }));
|
|
15111
16526
|
return;
|
|
15112
16527
|
}
|
|
15113
16528
|
response.writeHead(200, { "content-type": "application/json" });
|
|
15114
|
-
response.end(await
|
|
16529
|
+
response.end(await fs22.readFile(reportPath, "utf8"));
|
|
15115
16530
|
return;
|
|
15116
16531
|
}
|
|
15117
16532
|
if (url.pathname === "/api/watch-status") {
|
|
@@ -15194,8 +16609,8 @@ async function startGraphServer(rootDir, port) {
|
|
|
15194
16609
|
return;
|
|
15195
16610
|
}
|
|
15196
16611
|
const relativePath = url.pathname === "/" ? "index.html" : url.pathname.slice(1);
|
|
15197
|
-
const target =
|
|
15198
|
-
const fallback =
|
|
16612
|
+
const target = path27.join(paths.viewerDistDir, relativePath);
|
|
16613
|
+
const fallback = path27.join(paths.viewerDistDir, "index.html");
|
|
15199
16614
|
const filePath = await fileExists(target) ? target : fallback;
|
|
15200
16615
|
if (!await fileExists(filePath)) {
|
|
15201
16616
|
response.writeHead(503, { "content-type": "text/plain" });
|
|
@@ -15203,7 +16618,7 @@ async function startGraphServer(rootDir, port) {
|
|
|
15203
16618
|
return;
|
|
15204
16619
|
}
|
|
15205
16620
|
response.writeHead(200, { "content-type": mime2.lookup(filePath) || "text/plain" });
|
|
15206
|
-
response.end(await
|
|
16621
|
+
response.end(await fs22.readFile(filePath));
|
|
15207
16622
|
});
|
|
15208
16623
|
await new Promise((resolve) => {
|
|
15209
16624
|
server.listen(effectivePort, resolve);
|
|
@@ -15230,7 +16645,7 @@ async function exportGraphHtml(rootDir, outputPath) {
|
|
|
15230
16645
|
throw new Error("Graph artifact not found. Run `swarmvault compile` first.");
|
|
15231
16646
|
}
|
|
15232
16647
|
await ensureViewerDist(paths.viewerDistDir);
|
|
15233
|
-
const indexPath =
|
|
16648
|
+
const indexPath = path27.join(paths.viewerDistDir, "index.html");
|
|
15234
16649
|
if (!await fileExists(indexPath)) {
|
|
15235
16650
|
throw new Error("Viewer build not found. Run `pnpm build` first.");
|
|
15236
16651
|
}
|
|
@@ -15256,17 +16671,17 @@ async function exportGraphHtml(rootDir, outputPath) {
|
|
|
15256
16671
|
} : null;
|
|
15257
16672
|
})
|
|
15258
16673
|
);
|
|
15259
|
-
const rawHtml = await
|
|
16674
|
+
const rawHtml = await fs22.readFile(indexPath, "utf8");
|
|
15260
16675
|
const scriptMatch = rawHtml.match(/<script type="module" crossorigin src="([^"]+)"><\/script>/);
|
|
15261
16676
|
const styleMatch = rawHtml.match(/<link rel="stylesheet" crossorigin href="([^"]+)">/);
|
|
15262
|
-
const scriptPath = scriptMatch?.[1] ?
|
|
15263
|
-
const stylePath = styleMatch?.[1] ?
|
|
16677
|
+
const scriptPath = scriptMatch?.[1] ? path27.join(paths.viewerDistDir, scriptMatch[1].replace(/^\//, "")) : null;
|
|
16678
|
+
const stylePath = styleMatch?.[1] ? path27.join(paths.viewerDistDir, styleMatch[1].replace(/^\//, "")) : null;
|
|
15264
16679
|
if (!scriptPath || !await fileExists(scriptPath)) {
|
|
15265
16680
|
throw new Error("Viewer script bundle not found. Run `pnpm build` first.");
|
|
15266
16681
|
}
|
|
15267
|
-
const script = await
|
|
15268
|
-
const style = stylePath && await fileExists(stylePath) ? await
|
|
15269
|
-
const report = await readJsonFile(
|
|
16682
|
+
const script = await fs22.readFile(scriptPath, "utf8");
|
|
16683
|
+
const style = stylePath && await fileExists(stylePath) ? await fs22.readFile(stylePath, "utf8") : "";
|
|
16684
|
+
const report = await readJsonFile(path27.join(paths.wikiDir, "graph", "report.json"));
|
|
15270
16685
|
const embeddedData = JSON.stringify({ graph, pages: pages.filter(Boolean), report }, null, 2).replace(/</g, "\\u003c");
|
|
15271
16686
|
const html = [
|
|
15272
16687
|
"<!doctype html>",
|
|
@@ -15285,13 +16700,14 @@ async function exportGraphHtml(rootDir, outputPath) {
|
|
|
15285
16700
|
"</html>",
|
|
15286
16701
|
""
|
|
15287
16702
|
].filter(Boolean).join("\n");
|
|
15288
|
-
await
|
|
15289
|
-
await
|
|
15290
|
-
return
|
|
16703
|
+
await fs22.mkdir(path27.dirname(outputPath), { recursive: true });
|
|
16704
|
+
await fs22.writeFile(outputPath, html, "utf8");
|
|
16705
|
+
return path27.resolve(outputPath);
|
|
15291
16706
|
}
|
|
15292
16707
|
export {
|
|
15293
16708
|
acceptApproval,
|
|
15294
16709
|
addInput,
|
|
16710
|
+
addManagedSource,
|
|
15295
16711
|
archiveCandidate,
|
|
15296
16712
|
assertProviderCapability,
|
|
15297
16713
|
benchmarkVault,
|
|
@@ -15302,6 +16718,7 @@ export {
|
|
|
15302
16718
|
createWebSearchAdapter,
|
|
15303
16719
|
defaultVaultConfig,
|
|
15304
16720
|
defaultVaultSchema,
|
|
16721
|
+
deleteManagedSource,
|
|
15305
16722
|
explainGraphVault,
|
|
15306
16723
|
exploreVault,
|
|
15307
16724
|
exportGraphFormat,
|
|
@@ -15324,6 +16741,7 @@ export {
|
|
|
15324
16741
|
listCandidates,
|
|
15325
16742
|
listGodNodes,
|
|
15326
16743
|
listGraphHyperedges,
|
|
16744
|
+
listManagedSourceRecords,
|
|
15327
16745
|
listManifests,
|
|
15328
16746
|
listPages,
|
|
15329
16747
|
listSchedules,
|
|
@@ -15341,6 +16759,7 @@ export {
|
|
|
15341
16759
|
readGraphReport,
|
|
15342
16760
|
readPage,
|
|
15343
16761
|
rejectApproval,
|
|
16762
|
+
reloadManagedSources,
|
|
15344
16763
|
resolvePaths,
|
|
15345
16764
|
runSchedule,
|
|
15346
16765
|
runWatchCycle,
|