@hraness/wordcell 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -15
- package/dist/agent-context.js +1 -0
- package/dist/agent-guide-audit.js +1 -0
- package/dist/attachments.js +1 -0
- package/dist/authoring.js +1 -0
- package/dist/authority-4sp9d552.js +663 -0
- package/dist/benchmark.js +1 -0
- package/dist/capture.js +1 -0
- package/dist/cli.js +3781 -32
- package/dist/clip/acquire.js +1 -0
- package/dist/clip/args.js +1 -0
- package/dist/clip/bounded-byte-buffer.js +1 -0
- package/dist/clip/browser-profiles.js +1 -0
- package/dist/clip/bundle-reader.js +1 -0
- package/dist/clip/cli.js +1 -0
- package/dist/clip/cookies.js +1 -0
- package/dist/clip/doctor.js +1 -0
- package/dist/clip/jobs.js +1 -0
- package/dist/clip/network-proxy.js +1 -0
- package/dist/clip/network.js +1 -0
- package/dist/clip/persist.js +1 -0
- package/dist/clip/refresh.js +1 -0
- package/dist/clip/terminal.js +1 -0
- package/dist/evaluation-builder.js +13 -5
- package/dist/evaluation-kb.js +11 -4
- package/dist/evaluation.js +1 -0
- package/dist/git.js +1 -0
- package/dist/graph-authority.js +29 -0
- package/dist/graph-percolation.js +16 -0
- package/dist/graph.js +1 -0
- package/dist/index-0k2x4nn9.js +393 -0
- package/dist/{index-n05s3wsb.js → index-115b07ap.js} +156 -541
- package/dist/index-11621h23.js +160 -0
- package/dist/{index-tkpxp48t.js → index-51bap0mq.js} +2 -2
- package/dist/{index-8a08v6x8.js → index-59ssmqqm.js} +1 -1
- package/dist/index-bcfn9xah.js +472 -0
- package/dist/index-bgfzwt4h.js +51 -0
- package/dist/{index-bs0yf4pp.js → index-pbr6rv47.js} +2 -2
- package/dist/{index-m7et4rz4.js → index-vdw5yjfa.js} +61 -9
- package/dist/index-z1w83f81.js +4 -0
- package/dist/index.js +61 -34
- package/dist/navigation.js +1 -0
- package/dist/pdf.js +1 -0
- package/dist/percolate.js +1 -0
- package/dist/portfolio.js +11 -4
- package/dist/query.js +1 -0
- package/dist/repository-memory.js +1 -0
- package/dist/sdk.js +10 -3
- package/dist/search-rules.js +1 -0
- package/dist/search.js +1 -0
- package/dist/semantic.js +3 -1
- package/dist/source-inbox.js +1 -0
- package/dist/untrusted-content.js +1 -0
- package/dist/url-intelligence.js +1 -0
- package/dist/workflow.js +1 -0
- package/dist/workflows/decision-context.js +11 -4
- package/dist/workflows/explain-change.js +1 -0
- package/dist/workflows/index.js +11 -4
- package/dist/workflows/plan-radar.js +1 -0
- package/package.json +24 -6
- package/skills/wordcell/SKILL.md +2 -2
- package/skills/wordcell/references/percolate.md +4 -0
- package/skills/wordcell/references/query.md +6 -0
- package/src/cli-program.ts +45 -2
- package/src/graph-authority-model.ts +116 -0
- package/src/graph-authority.ts +219 -0
- package/src/graph-cache-filesystem.ts +4 -0
- package/src/graph-cli.ts +96 -0
- package/src/graph-facts.ts +206 -0
- package/src/graph-percolation.ts +39 -0
- package/src/graph-query.ts +122 -0
- package/src/index.ts +2 -0
- package/src/oh/authority.ts +193 -0
- package/src/oh/programs.ts +99 -0
- package/src/oh/schema.ts +40 -0
- package/src/oh/snapshot.ts +114 -0
- package/src/oh/validation.ts +61 -0
- package/src/sdk.ts +51 -6
- package/dist/index-sy0zety7.js +0 -3630
- package/dist/kb-alias.js +0 -55
- package/src/kb-alias.ts +0 -14
package/dist/clip/acquire.js
CHANGED
package/dist/clip/args.js
CHANGED
package/dist/clip/cli.js
CHANGED
package/dist/clip/cookies.js
CHANGED
package/dist/clip/doctor.js
CHANGED
package/dist/clip/jobs.js
CHANGED
package/dist/clip/network.js
CHANGED
package/dist/clip/persist.js
CHANGED
package/dist/clip/refresh.js
CHANGED
package/dist/clip/terminal.js
CHANGED
|
@@ -4,16 +4,22 @@ import {
|
|
|
4
4
|
knowledgeBaseEvaluationRetrieverIds,
|
|
5
5
|
openKnowledgeBaseEvaluation,
|
|
6
6
|
verifyFrozenEvaluationSnapshot
|
|
7
|
-
} from "./index-
|
|
8
|
-
import"./index-
|
|
7
|
+
} from "./index-pbr6rv47.js";
|
|
8
|
+
import"./index-vdw5yjfa.js";
|
|
9
|
+
import"./index-4j3tt0c3.js";
|
|
9
10
|
import"./index-adx6khj5.js";
|
|
10
11
|
import {
|
|
11
12
|
indexSemanticVault,
|
|
12
13
|
recommendedEmbeddingModel,
|
|
13
|
-
recommendedEmbeddingModelSha256
|
|
14
|
+
recommendedEmbeddingModelSha256
|
|
15
|
+
} from "./index-115b07ap.js";
|
|
16
|
+
import"./index-bgfzwt4h.js";
|
|
17
|
+
import"./index-bcfn9xah.js";
|
|
18
|
+
import"./index-11621h23.js";
|
|
19
|
+
import {
|
|
14
20
|
scanVault
|
|
15
|
-
} from "./index-
|
|
16
|
-
import"./index-
|
|
21
|
+
} from "./index-0k2x4nn9.js";
|
|
22
|
+
import"./index-nd6nynv2.js";
|
|
17
23
|
import {
|
|
18
24
|
runGitCommand
|
|
19
25
|
} from "./index-1gwbassd.js";
|
|
@@ -22,6 +28,7 @@ import {
|
|
|
22
28
|
MAX_EVALUATION_EVIDENCE_BYTES,
|
|
23
29
|
MAX_EVALUATION_RESULTS_PER_QUERY
|
|
24
30
|
} from "./index-b88v3vtm.js";
|
|
31
|
+
import"./index-3rm7cz6h.js";
|
|
25
32
|
import"./index-why54q5p.js";
|
|
26
33
|
import"./index-48pz4jpc.js";
|
|
27
34
|
import"./index-06c9ctr6.js";
|
|
@@ -29,6 +36,7 @@ import"./index-d13v9ckt.js";
|
|
|
29
36
|
import"./index-5vwpzb5a.js";
|
|
30
37
|
import"./index-ekpwvbra.js";
|
|
31
38
|
import"./index-1xxnjn0d.js";
|
|
39
|
+
import"./index-z1w83f81.js";
|
|
32
40
|
|
|
33
41
|
// src/evaluation-builder.ts
|
|
34
42
|
import { createHash as createHash9, randomBytes } from "crypto";
|
package/dist/evaluation-kb.js
CHANGED
|
@@ -4,13 +4,19 @@ import {
|
|
|
4
4
|
knowledgeBaseEvaluationRetrieverIds,
|
|
5
5
|
openKnowledgeBaseEvaluation,
|
|
6
6
|
verifyFrozenEvaluationSnapshot
|
|
7
|
-
} from "./index-
|
|
8
|
-
import"./index-
|
|
9
|
-
import"./index-adx6khj5.js";
|
|
10
|
-
import"./index-n05s3wsb.js";
|
|
7
|
+
} from "./index-pbr6rv47.js";
|
|
8
|
+
import"./index-vdw5yjfa.js";
|
|
11
9
|
import"./index-4j3tt0c3.js";
|
|
10
|
+
import"./index-adx6khj5.js";
|
|
11
|
+
import"./index-115b07ap.js";
|
|
12
|
+
import"./index-bgfzwt4h.js";
|
|
13
|
+
import"./index-bcfn9xah.js";
|
|
14
|
+
import"./index-11621h23.js";
|
|
15
|
+
import"./index-0k2x4nn9.js";
|
|
16
|
+
import"./index-nd6nynv2.js";
|
|
12
17
|
import"./index-1gwbassd.js";
|
|
13
18
|
import"./index-b88v3vtm.js";
|
|
19
|
+
import"./index-3rm7cz6h.js";
|
|
14
20
|
import"./index-why54q5p.js";
|
|
15
21
|
import"./index-48pz4jpc.js";
|
|
16
22
|
import"./index-06c9ctr6.js";
|
|
@@ -18,6 +24,7 @@ import"./index-d13v9ckt.js";
|
|
|
18
24
|
import"./index-5vwpzb5a.js";
|
|
19
25
|
import"./index-ekpwvbra.js";
|
|
20
26
|
import"./index-1xxnjn0d.js";
|
|
27
|
+
import"./index-z1w83f81.js";
|
|
21
28
|
export {
|
|
22
29
|
verifyFrozenEvaluationSnapshot,
|
|
23
30
|
openKnowledgeBaseEvaluation,
|
package/dist/evaluation.js
CHANGED
package/dist/git.js
CHANGED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
createGraphSnapshot,
|
|
4
|
+
openGraphAuthority,
|
|
5
|
+
queryGraph,
|
|
6
|
+
rebuildGraph,
|
|
7
|
+
verifyGraph
|
|
8
|
+
} from "./index-bcfn9xah.js";
|
|
9
|
+
import {
|
|
10
|
+
GRAPH_LIMITS,
|
|
11
|
+
GraphAuthorityError,
|
|
12
|
+
validateGraphQueryRequest
|
|
13
|
+
} from "./index-11621h23.js";
|
|
14
|
+
import"./index-0k2x4nn9.js";
|
|
15
|
+
import"./index-3rm7cz6h.js";
|
|
16
|
+
import"./index-06c9ctr6.js";
|
|
17
|
+
import"./index-5vwpzb5a.js";
|
|
18
|
+
import"./index-ekpwvbra.js";
|
|
19
|
+
import"./index-z1w83f81.js";
|
|
20
|
+
export {
|
|
21
|
+
verifyGraph,
|
|
22
|
+
validateGraphQueryRequest,
|
|
23
|
+
rebuildGraph,
|
|
24
|
+
queryGraph,
|
|
25
|
+
openGraphAuthority,
|
|
26
|
+
createGraphSnapshot,
|
|
27
|
+
GraphAuthorityError,
|
|
28
|
+
GRAPH_LIMITS
|
|
29
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
percolateWithGraph
|
|
4
|
+
} from "./index-bgfzwt4h.js";
|
|
5
|
+
import"./index-bcfn9xah.js";
|
|
6
|
+
import"./index-11621h23.js";
|
|
7
|
+
import"./index-0k2x4nn9.js";
|
|
8
|
+
import"./index-nd6nynv2.js";
|
|
9
|
+
import"./index-3rm7cz6h.js";
|
|
10
|
+
import"./index-06c9ctr6.js";
|
|
11
|
+
import"./index-5vwpzb5a.js";
|
|
12
|
+
import"./index-ekpwvbra.js";
|
|
13
|
+
import"./index-z1w83f81.js";
|
|
14
|
+
export {
|
|
15
|
+
percolateWithGraph
|
|
16
|
+
};
|
package/dist/graph.js
CHANGED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
MAX_ANALYZED_NOTES,
|
|
4
|
+
analyzeVault,
|
|
5
|
+
isCanonicalNoteId,
|
|
6
|
+
lookupNote,
|
|
7
|
+
normalizeVaultPath,
|
|
8
|
+
parseNote,
|
|
9
|
+
renderCatalog,
|
|
10
|
+
replaceCatalog
|
|
11
|
+
} from "./index-ekpwvbra.js";
|
|
12
|
+
|
|
13
|
+
// src/vault.ts
|
|
14
|
+
import { randomUUID } from "crypto";
|
|
15
|
+
import { constants } from "fs";
|
|
16
|
+
import {
|
|
17
|
+
lstat,
|
|
18
|
+
open,
|
|
19
|
+
readdir,
|
|
20
|
+
realpath,
|
|
21
|
+
rename,
|
|
22
|
+
rm
|
|
23
|
+
} from "fs/promises";
|
|
24
|
+
import { basename, dirname, join, relative, resolve, sep } from "path";
|
|
25
|
+
var MAX_SCANNED_NOTES = MAX_ANALYZED_NOTES;
|
|
26
|
+
var MAX_NOTE_UTF8_BYTES = 16 * 1024 * 1024;
|
|
27
|
+
var MAX_VAULT_UTF8_BYTES = 256 * 1024 * 1024;
|
|
28
|
+
|
|
29
|
+
class VaultScanBudgetError extends RangeError {
|
|
30
|
+
kind;
|
|
31
|
+
limit;
|
|
32
|
+
constructor(kind, limit, message) {
|
|
33
|
+
super(message);
|
|
34
|
+
this.name = "VaultScanBudgetError";
|
|
35
|
+
this.kind = kind;
|
|
36
|
+
this.limit = limit;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
var defaultIgnoredDirectories = new Set([
|
|
40
|
+
".git",
|
|
41
|
+
".next",
|
|
42
|
+
".turbo",
|
|
43
|
+
".vercel",
|
|
44
|
+
"coverage",
|
|
45
|
+
"dist",
|
|
46
|
+
"node_modules"
|
|
47
|
+
]);
|
|
48
|
+
async function markdownFiles(directory, ignoredDirectories = defaultIgnoredDirectories) {
|
|
49
|
+
const files = [];
|
|
50
|
+
const entries = await readdir(directory, { withFileTypes: true });
|
|
51
|
+
for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) {
|
|
52
|
+
if (entry.name.startsWith("."))
|
|
53
|
+
continue;
|
|
54
|
+
const entryPath = join(directory, entry.name);
|
|
55
|
+
if (entry.isDirectory()) {
|
|
56
|
+
if (ignoredDirectories.has(entry.name))
|
|
57
|
+
continue;
|
|
58
|
+
files.push(...await markdownFiles(entryPath, ignoredDirectories));
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (entry.isFile() && entry.name.endsWith(".md") && entry.name !== "AGENTS.md") {
|
|
62
|
+
files.push(entryPath);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return files;
|
|
66
|
+
}
|
|
67
|
+
function checkedScanLimit(value, hardMaximum, option) {
|
|
68
|
+
const limit = value ?? hardMaximum;
|
|
69
|
+
if (!Number.isSafeInteger(limit) || limit < 0 || limit > hardMaximum) {
|
|
70
|
+
throw new RangeError(`${option} must be a safe integer from 0 through ${hardMaximum}.`);
|
|
71
|
+
}
|
|
72
|
+
return limit;
|
|
73
|
+
}
|
|
74
|
+
function normalizedRawNoteId(rawId) {
|
|
75
|
+
return normalizeVaultPath(rawId).normalize("NFC");
|
|
76
|
+
}
|
|
77
|
+
function validateScannedNotePaths(root, paths) {
|
|
78
|
+
const files = paths.map((absolutePath) => {
|
|
79
|
+
const vaultPath = relative(root, absolutePath).split(sep).join("/");
|
|
80
|
+
return {
|
|
81
|
+
absolutePath,
|
|
82
|
+
vaultPath,
|
|
83
|
+
rawId: vaultPath.slice(0, -3)
|
|
84
|
+
};
|
|
85
|
+
});
|
|
86
|
+
const pathByNormalizedId = new Map;
|
|
87
|
+
for (const file of files) {
|
|
88
|
+
const normalizedId = normalizedRawNoteId(file.rawId);
|
|
89
|
+
const collision = pathByNormalizedId.get(normalizedId);
|
|
90
|
+
if (collision !== undefined && collision !== file.vaultPath) {
|
|
91
|
+
throw new Error(`Vault note paths ${JSON.stringify(collision)} and ` + `${JSON.stringify(file.vaultPath)} normalize to the same note ID ` + `${JSON.stringify(normalizedId)}.`);
|
|
92
|
+
}
|
|
93
|
+
pathByNormalizedId.set(normalizedId, file.vaultPath);
|
|
94
|
+
}
|
|
95
|
+
for (const file of files) {
|
|
96
|
+
if (isCanonicalNoteId(file.rawId))
|
|
97
|
+
continue;
|
|
98
|
+
if (file.rawId !== file.rawId.normalize("NFC")) {
|
|
99
|
+
throw new Error(`Vault note path ${JSON.stringify(file.vaultPath)} is not NFC; ` + `its extensionless note ID must be exactly ` + `${JSON.stringify(file.rawId.normalize("NFC"))}.`);
|
|
100
|
+
}
|
|
101
|
+
if (file.rawId.includes("\\")) {
|
|
102
|
+
throw new Error(`Vault note path ${JSON.stringify(file.vaultPath)} contains a backslash; ` + "note IDs must use exact vault-root directory separators.");
|
|
103
|
+
}
|
|
104
|
+
throw new Error(`Vault note path ${JSON.stringify(file.vaultPath)} must have an exact ` + "canonical extensionless vault-root note ID.");
|
|
105
|
+
}
|
|
106
|
+
return files;
|
|
107
|
+
}
|
|
108
|
+
function assertScannableNoteFile(vaultPath, metadata) {
|
|
109
|
+
if (metadata.isSymbolicLink()) {
|
|
110
|
+
throw new Error(`Vault note ${JSON.stringify(vaultPath)} must not be a symbolic link.`);
|
|
111
|
+
}
|
|
112
|
+
if (!metadata.isFile()) {
|
|
113
|
+
throw new Error(`Vault note ${JSON.stringify(vaultPath)} must be a regular file.`);
|
|
114
|
+
}
|
|
115
|
+
if (metadata.nlink !== 1n) {
|
|
116
|
+
throw new Error(`Vault note ${JSON.stringify(vaultPath)} must not be hard-linked.`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function noteBytesError(vaultPath, limit) {
|
|
120
|
+
return new VaultScanBudgetError("note-bytes", limit, `Vault note ${JSON.stringify(vaultPath)} exceeds the ${limit}-byte UTF-8 limit.`);
|
|
121
|
+
}
|
|
122
|
+
function totalBytesError(limit) {
|
|
123
|
+
return new VaultScanBudgetError("total-bytes", limit, `Vault scan exceeds the ${limit}-byte cumulative UTF-8 limit.`);
|
|
124
|
+
}
|
|
125
|
+
async function readBoundedNote(handle, vaultPath, maxNoteBytes, remainingTotalBytes, maxTotalBytes) {
|
|
126
|
+
const chunks = [];
|
|
127
|
+
let bytes = 0;
|
|
128
|
+
for (;; ) {
|
|
129
|
+
const remaining = Math.min(maxNoteBytes - bytes, remainingTotalBytes - bytes);
|
|
130
|
+
const buffer = new Uint8Array(Math.min(64 * 1024, Math.max(1, remaining + 1)));
|
|
131
|
+
const result = await handle.read(buffer, 0, buffer.byteLength, null);
|
|
132
|
+
if (result.bytesRead === 0)
|
|
133
|
+
break;
|
|
134
|
+
bytes += result.bytesRead;
|
|
135
|
+
if (bytes > maxNoteBytes)
|
|
136
|
+
throw noteBytesError(vaultPath, maxNoteBytes);
|
|
137
|
+
if (bytes > remainingTotalBytes) {
|
|
138
|
+
throw totalBytesError(maxTotalBytes);
|
|
139
|
+
}
|
|
140
|
+
chunks.push(buffer.slice(0, result.bytesRead));
|
|
141
|
+
}
|
|
142
|
+
const joined = new Uint8Array(bytes);
|
|
143
|
+
let offset = 0;
|
|
144
|
+
for (const chunk of chunks) {
|
|
145
|
+
joined.set(chunk, offset);
|
|
146
|
+
offset += chunk.byteLength;
|
|
147
|
+
}
|
|
148
|
+
try {
|
|
149
|
+
return {
|
|
150
|
+
content: new TextDecoder("utf-8", { fatal: true }).decode(joined),
|
|
151
|
+
bytes
|
|
152
|
+
};
|
|
153
|
+
} catch (error) {
|
|
154
|
+
throw new Error(`Vault note ${JSON.stringify(vaultPath)} is not valid UTF-8.`, { cause: error });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
async function readVaultNotes(root, ignoredDirectories = defaultIgnoredDirectories, limits = {}) {
|
|
158
|
+
const maxNotes = checkedScanLimit(limits.maxNotes, MAX_SCANNED_NOTES, "maxNotes");
|
|
159
|
+
const maxNoteBytes = checkedScanLimit(limits.maxNoteBytes, MAX_NOTE_UTF8_BYTES, "maxNoteBytes");
|
|
160
|
+
const maxTotalBytes = checkedScanLimit(limits.maxTotalBytes, MAX_VAULT_UTF8_BYTES, "maxTotalBytes");
|
|
161
|
+
const paths = await markdownFiles(root, ignoredDirectories);
|
|
162
|
+
if (paths.length > maxNotes) {
|
|
163
|
+
throw new VaultScanBudgetError("notes", maxNotes, `Vault scan exceeds the ${maxNotes} Markdown note limit.`);
|
|
164
|
+
}
|
|
165
|
+
const files = validateScannedNotePaths(root, paths);
|
|
166
|
+
let declaredTotal = 0n;
|
|
167
|
+
const preflight = [];
|
|
168
|
+
for (const file of files) {
|
|
169
|
+
const metadata = await lstat(file.absolutePath, { bigint: true });
|
|
170
|
+
assertScannableNoteFile(file.vaultPath, metadata);
|
|
171
|
+
if (metadata.size > BigInt(maxNoteBytes)) {
|
|
172
|
+
throw noteBytesError(file.vaultPath, maxNoteBytes);
|
|
173
|
+
}
|
|
174
|
+
declaredTotal += metadata.size;
|
|
175
|
+
if (declaredTotal > BigInt(maxTotalBytes)) {
|
|
176
|
+
throw totalBytesError(maxTotalBytes);
|
|
177
|
+
}
|
|
178
|
+
preflight.push({
|
|
179
|
+
...file,
|
|
180
|
+
device: metadata.dev,
|
|
181
|
+
inode: metadata.ino
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
const notes = [];
|
|
185
|
+
let observedTotal = 0;
|
|
186
|
+
for (const file of preflight) {
|
|
187
|
+
const handle = await open(file.absolutePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
188
|
+
try {
|
|
189
|
+
const beforeRead = await handle.stat({ bigint: true });
|
|
190
|
+
assertScannableNoteFile(file.vaultPath, beforeRead);
|
|
191
|
+
if (beforeRead.dev !== file.device || beforeRead.ino !== file.inode) {
|
|
192
|
+
throw new Error(`Vault note ${JSON.stringify(file.vaultPath)} changed during scan; retry.`);
|
|
193
|
+
}
|
|
194
|
+
if (beforeRead.size > BigInt(maxNoteBytes)) {
|
|
195
|
+
throw noteBytesError(file.vaultPath, maxNoteBytes);
|
|
196
|
+
}
|
|
197
|
+
if (BigInt(observedTotal) + beforeRead.size > BigInt(maxTotalBytes)) {
|
|
198
|
+
throw totalBytesError(maxTotalBytes);
|
|
199
|
+
}
|
|
200
|
+
const read = await readBoundedNote(handle, file.vaultPath, maxNoteBytes, maxTotalBytes - observedTotal, maxTotalBytes);
|
|
201
|
+
const afterRead = await handle.stat({ bigint: true });
|
|
202
|
+
if (afterRead.dev !== file.device || afterRead.ino !== file.inode || afterRead.size !== beforeRead.size || afterRead.size !== BigInt(read.bytes)) {
|
|
203
|
+
throw new Error(`Vault note ${JSON.stringify(file.vaultPath)} changed during scan; retry.`);
|
|
204
|
+
}
|
|
205
|
+
observedTotal += read.bytes;
|
|
206
|
+
notes.push(parseNote(file.vaultPath, read.content));
|
|
207
|
+
} finally {
|
|
208
|
+
await handle.close();
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return notes;
|
|
212
|
+
}
|
|
213
|
+
function confined(root, path) {
|
|
214
|
+
const fromRoot = relative(root, path);
|
|
215
|
+
return fromRoot !== "" && fromRoot !== ".." && !fromRoot.startsWith(`..${sep}`);
|
|
216
|
+
}
|
|
217
|
+
async function assertConfinedIndexParents(root, path) {
|
|
218
|
+
if (!confined(root, path))
|
|
219
|
+
throw new Error("The configured index must be a file inside the vault root.");
|
|
220
|
+
const parent = dirname(path);
|
|
221
|
+
const segments = relative(root, parent).split(sep).filter((segment) => segment !== "");
|
|
222
|
+
let current = root;
|
|
223
|
+
for (const segment of segments) {
|
|
224
|
+
current = join(current, segment);
|
|
225
|
+
const metadata = await lstat(current);
|
|
226
|
+
if (metadata.isSymbolicLink()) {
|
|
227
|
+
throw new Error("The configured index path must not traverse a symbolic link.");
|
|
228
|
+
}
|
|
229
|
+
if (!metadata.isDirectory()) {
|
|
230
|
+
throw new Error("Every configured index parent must be a directory.");
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
const canonicalParent = await realpath(parent);
|
|
234
|
+
if (!confined(root, join(canonicalParent, basename(path)))) {
|
|
235
|
+
throw new Error("The configured index parent resolves outside the vault root.");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
async function readIndexRevision(root, path, maxNoteBytes = MAX_NOTE_UTF8_BYTES) {
|
|
239
|
+
await assertConfinedIndexParents(root, path);
|
|
240
|
+
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
241
|
+
try {
|
|
242
|
+
const metadata = await handle.stat({ bigint: true });
|
|
243
|
+
if (!metadata.isFile())
|
|
244
|
+
throw new Error("The configured index must be a regular file.");
|
|
245
|
+
if (metadata.nlink !== 1n)
|
|
246
|
+
throw new Error("The configured index must not be hard-linked.");
|
|
247
|
+
const canonicalPath = await realpath(path);
|
|
248
|
+
if (!confined(root, canonicalPath)) {
|
|
249
|
+
throw new Error("The configured index resolves outside the vault root.");
|
|
250
|
+
}
|
|
251
|
+
const vaultPath = relative(root, path).split(sep).join("/");
|
|
252
|
+
if (metadata.size > BigInt(maxNoteBytes)) {
|
|
253
|
+
throw noteBytesError(vaultPath, maxNoteBytes);
|
|
254
|
+
}
|
|
255
|
+
const read = await readBoundedNote(handle, vaultPath, maxNoteBytes, maxNoteBytes, maxNoteBytes);
|
|
256
|
+
const afterRead = await handle.stat({ bigint: true });
|
|
257
|
+
if (afterRead.dev !== metadata.dev || afterRead.ino !== metadata.ino || afterRead.size !== metadata.size || afterRead.size !== BigInt(read.bytes)) {
|
|
258
|
+
throw new Error("The configured index changed during scan; retry.");
|
|
259
|
+
}
|
|
260
|
+
return {
|
|
261
|
+
content: read.content,
|
|
262
|
+
device: metadata.dev,
|
|
263
|
+
inode: metadata.ino,
|
|
264
|
+
mode: Number(metadata.mode & 0o777n)
|
|
265
|
+
};
|
|
266
|
+
} finally {
|
|
267
|
+
await handle.close();
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
function sameRevision(left, right) {
|
|
271
|
+
return left.device === right.device && left.inode === right.inode && left.content === right.content;
|
|
272
|
+
}
|
|
273
|
+
function parsedCatalogMode(value, source) {
|
|
274
|
+
if (value === undefined)
|
|
275
|
+
return;
|
|
276
|
+
if (value === "managed" || value === "authored")
|
|
277
|
+
return value;
|
|
278
|
+
throw new Error(`${source} must be exactly "managed" or "authored".`);
|
|
279
|
+
}
|
|
280
|
+
function declaredCatalogMode(indexNote) {
|
|
281
|
+
const declaration = Object.entries(indexNote.metadata).find(([name]) => name.toLocaleLowerCase("en-US") === "kb_catalog");
|
|
282
|
+
return parsedCatalogMode(declaration?.[1], `The configured index frontmatter property "kb_catalog"`);
|
|
283
|
+
}
|
|
284
|
+
async function atomicReplace(root, path, content, expected) {
|
|
285
|
+
const beforeWrite = await readIndexRevision(root, path);
|
|
286
|
+
if (!sameRevision(beforeWrite, expected)) {
|
|
287
|
+
throw new Error("The configured index changed during refresh; retry without overwriting the editor's changes.");
|
|
288
|
+
}
|
|
289
|
+
const directory = dirname(path);
|
|
290
|
+
const temporaryPath = join(directory, `.${basename(path)}.${process.pid}.${randomUUID()}.tmp`);
|
|
291
|
+
await assertConfinedIndexParents(root, path);
|
|
292
|
+
const handle = await open(temporaryPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, expected.mode);
|
|
293
|
+
let closed = false;
|
|
294
|
+
try {
|
|
295
|
+
await handle.writeFile(content, "utf8");
|
|
296
|
+
await handle.sync();
|
|
297
|
+
await handle.close();
|
|
298
|
+
closed = true;
|
|
299
|
+
const beforeRename = await readIndexRevision(root, path);
|
|
300
|
+
if (!sameRevision(beforeRename, expected)) {
|
|
301
|
+
throw new Error("The configured index changed during refresh; retry without overwriting the editor's changes.");
|
|
302
|
+
}
|
|
303
|
+
await assertConfinedIndexParents(root, path);
|
|
304
|
+
await rename(temporaryPath, path);
|
|
305
|
+
} catch (error) {
|
|
306
|
+
if (!closed)
|
|
307
|
+
await handle.close().catch(() => {
|
|
308
|
+
return;
|
|
309
|
+
});
|
|
310
|
+
await rm(temporaryPath, { force: true }).catch(() => {
|
|
311
|
+
return;
|
|
312
|
+
});
|
|
313
|
+
throw error;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
async function snapshot(rootInput, options, writeIndex) {
|
|
317
|
+
const requestedRoot = resolve(rootInput);
|
|
318
|
+
const root = await realpath(requestedRoot);
|
|
319
|
+
const rootMetadata = await lstat(root);
|
|
320
|
+
if (!rootMetadata.isDirectory())
|
|
321
|
+
throw new Error("The vault root must be a directory.");
|
|
322
|
+
const indexPath = resolve(root, options.index ?? "index.md");
|
|
323
|
+
const relativeIndex = relative(root, indexPath);
|
|
324
|
+
if (!confined(root, indexPath)) {
|
|
325
|
+
throw new Error("The configured index must be a file inside the vault root.");
|
|
326
|
+
}
|
|
327
|
+
if (!indexPath.toLowerCase().endsWith(".md")) {
|
|
328
|
+
throw new Error("The configured index must be a Markdown file.");
|
|
329
|
+
}
|
|
330
|
+
const vaultIndexPath = relativeIndex.split(sep).join("/");
|
|
331
|
+
const catalogNoteId = vaultIndexPath.toLowerCase().endsWith(".md") ? vaultIndexPath.slice(0, -3) : vaultIndexPath;
|
|
332
|
+
const notes = await readVaultNotes(root, options.ignoredDirectories, {
|
|
333
|
+
...options.maxNotes === undefined ? {} : { maxNotes: options.maxNotes },
|
|
334
|
+
...options.maxNoteBytes === undefined ? {} : { maxNoteBytes: options.maxNoteBytes },
|
|
335
|
+
...options.maxTotalBytes === undefined ? {} : { maxTotalBytes: options.maxTotalBytes }
|
|
336
|
+
});
|
|
337
|
+
const indexRevision = await readIndexRevision(root, indexPath, options.maxNoteBytes ?? MAX_NOTE_UTF8_BYTES);
|
|
338
|
+
const currentIndex = indexRevision.content;
|
|
339
|
+
const indexNote = parseNote(vaultIndexPath, currentIndex);
|
|
340
|
+
const catalogMode = parsedCatalogMode(options.catalogMode, "ScanVaultOptions.catalogMode") ?? declaredCatalogMode(indexNote) ?? "managed";
|
|
341
|
+
let index = "authored";
|
|
342
|
+
if (catalogMode === "managed") {
|
|
343
|
+
const expectedIndex = replaceCatalog(currentIndex, renderCatalog(notes, catalogNoteId));
|
|
344
|
+
const stale = currentIndex !== expectedIndex;
|
|
345
|
+
index = stale ? "stale" : "current";
|
|
346
|
+
if (writeIndex && stale) {
|
|
347
|
+
await atomicReplace(root, indexPath, expectedIndex, indexRevision);
|
|
348
|
+
index = "updated";
|
|
349
|
+
const parsed = parseNote(vaultIndexPath, expectedIndex);
|
|
350
|
+
const noteIndex = notes.findIndex((note) => note.path === vaultIndexPath);
|
|
351
|
+
if (noteIndex === -1)
|
|
352
|
+
notes.push(parsed);
|
|
353
|
+
else
|
|
354
|
+
notes[noteIndex] = parsed;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
const mentionScope = options.mentionScope;
|
|
358
|
+
const mentionIds = new Set;
|
|
359
|
+
if (typeof mentionScope === "string") {
|
|
360
|
+
const lookup = lookupNote(notes, mentionScope);
|
|
361
|
+
if (lookup.kind === "found")
|
|
362
|
+
mentionIds.add(lookup.note.id);
|
|
363
|
+
else if (lookup.kind === "ambiguous") {
|
|
364
|
+
for (const note of lookup.candidates)
|
|
365
|
+
mentionIds.add(note.id);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
const mentionScopePredicate = mentionScope === undefined ? undefined : (note) => mentionScope !== false && mentionIds.has(note.id);
|
|
369
|
+
return {
|
|
370
|
+
root,
|
|
371
|
+
indexPath,
|
|
372
|
+
catalogMode,
|
|
373
|
+
index,
|
|
374
|
+
notes,
|
|
375
|
+
analysis: analyzeVault(notes, {
|
|
376
|
+
catalogNoteId,
|
|
377
|
+
...options.includeInSuggestions === undefined ? {} : { includeInSuggestions: options.includeInSuggestions },
|
|
378
|
+
...mentionScopePredicate === undefined ? {} : { mentionScope: mentionScopePredicate },
|
|
379
|
+
...options.maxNotes === undefined ? {} : { maxNotes: options.maxNotes },
|
|
380
|
+
...options.maxConnectionObservations === undefined ? {} : { maxConnectionObservations: options.maxConnectionObservations },
|
|
381
|
+
...options.maxMentionPairs === undefined ? {} : { maxMentionPairs: options.maxMentionPairs },
|
|
382
|
+
...options.maxMentions === undefined ? {} : { maxMentions: options.maxMentions }
|
|
383
|
+
})
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
async function scanVault(root = ".", options = {}) {
|
|
387
|
+
return snapshot(root, options, false);
|
|
388
|
+
}
|
|
389
|
+
async function refreshVault(root = ".", options = {}) {
|
|
390
|
+
return snapshot(root, options, true);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
export { MAX_SCANNED_NOTES, MAX_NOTE_UTF8_BYTES, MAX_VAULT_UTF8_BYTES, VaultScanBudgetError, defaultIgnoredDirectories, markdownFiles, readVaultNotes, scanVault, refreshVault };
|