@gettymade/roux 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -48
- package/dist/cli/index.js +2058 -1226
- package/dist/cli/index.js.map +1 -1
- package/dist/index.d.ts +278 -54
- package/dist/index.js +1365 -575
- package/dist/index.js.map +1 -1
- package/package.json +6 -2
package/dist/index.js
CHANGED
|
@@ -88,7 +88,22 @@ function isNode(value) {
|
|
|
88
88
|
return false;
|
|
89
89
|
}
|
|
90
90
|
const obj = value;
|
|
91
|
-
|
|
91
|
+
if (typeof obj["id"] !== "string" || typeof obj["title"] !== "string" || typeof obj["content"] !== "string") {
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
if (!Array.isArray(obj["tags"]) || !obj["tags"].every((t) => typeof t === "string")) {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
if (!Array.isArray(obj["outgoingLinks"]) || !obj["outgoingLinks"].every((l) => typeof l === "string")) {
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
if (typeof obj["properties"] !== "object" || obj["properties"] === null || Array.isArray(obj["properties"])) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
if (obj["sourceRef"] !== void 0 && !isSourceRef(obj["sourceRef"])) {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
return true;
|
|
92
107
|
}
|
|
93
108
|
function isSourceRef(value) {
|
|
94
109
|
if (typeof value !== "object" || value === null) {
|
|
@@ -96,17 +111,68 @@ function isSourceRef(value) {
|
|
|
96
111
|
}
|
|
97
112
|
const obj = value;
|
|
98
113
|
const validTypes = ["file", "api", "manual"];
|
|
99
|
-
|
|
114
|
+
if (typeof obj["type"] !== "string" || !validTypes.includes(obj["type"])) {
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
if (obj["path"] !== void 0 && typeof obj["path"] !== "string") {
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
if (obj["lastModified"] !== void 0) {
|
|
121
|
+
if (!(obj["lastModified"] instanceof Date)) {
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
if (isNaN(obj["lastModified"].getTime())) {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return true;
|
|
100
129
|
}
|
|
101
130
|
|
|
102
131
|
// src/types/provider.ts
|
|
103
|
-
function
|
|
132
|
+
function isVectorIndex(value) {
|
|
104
133
|
if (value === null || typeof value !== "object") {
|
|
105
134
|
return false;
|
|
106
135
|
}
|
|
107
136
|
const obj = value;
|
|
108
137
|
return typeof obj.store === "function" && typeof obj.search === "function" && typeof obj.delete === "function" && typeof obj.getModel === "function" && typeof obj.hasEmbedding === "function";
|
|
109
138
|
}
|
|
139
|
+
function isStoreProvider(value) {
|
|
140
|
+
if (value === null || typeof value !== "object") {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
const obj = value;
|
|
144
|
+
return typeof obj.id === "string" && obj.id.trim().length > 0 && typeof obj.createNode === "function" && typeof obj.updateNode === "function" && typeof obj.deleteNode === "function" && typeof obj.getNode === "function" && typeof obj.getNodes === "function" && typeof obj.getNeighbors === "function" && typeof obj.findPath === "function" && typeof obj.getHubs === "function" && typeof obj.storeEmbedding === "function" && typeof obj.searchByVector === "function" && typeof obj.searchByTags === "function" && typeof obj.getRandomNode === "function" && typeof obj.resolveTitles === "function" && typeof obj.listNodes === "function" && typeof obj.resolveNodes === "function" && typeof obj.nodesExist === "function";
|
|
145
|
+
}
|
|
146
|
+
function isEmbeddingProvider(value) {
|
|
147
|
+
if (value === null || typeof value !== "object") {
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
const obj = value;
|
|
151
|
+
return typeof obj.id === "string" && obj.id.trim().length > 0 && typeof obj.embed === "function" && typeof obj.embedBatch === "function" && typeof obj.dimensions === "function" && typeof obj.modelId === "function";
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// src/types/guards.ts
|
|
155
|
+
function createGuard(schema) {
|
|
156
|
+
return (value) => {
|
|
157
|
+
if (value === null || typeof value !== "object") return false;
|
|
158
|
+
const obj = value;
|
|
159
|
+
for (const [key, spec] of Object.entries(schema)) {
|
|
160
|
+
const val = obj[key];
|
|
161
|
+
if (spec.optional && val === void 0) continue;
|
|
162
|
+
if (spec.type === "array") {
|
|
163
|
+
if (!Array.isArray(val)) return false;
|
|
164
|
+
} else if (spec.type === "object") {
|
|
165
|
+
if (typeof val !== "object" || val === null) return false;
|
|
166
|
+
} else {
|
|
167
|
+
if (typeof val !== spec.type) return false;
|
|
168
|
+
}
|
|
169
|
+
if (spec.nonEmpty && spec.type === "string" && val.length === 0) {
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return true;
|
|
174
|
+
};
|
|
175
|
+
}
|
|
110
176
|
|
|
111
177
|
// src/types/config.ts
|
|
112
178
|
var DEFAULT_CONFIG = {
|
|
@@ -129,15 +195,399 @@ var DEFAULT_CONFIG = {
|
|
|
129
195
|
};
|
|
130
196
|
|
|
131
197
|
// src/providers/docstore/index.ts
|
|
132
|
-
import {
|
|
133
|
-
import {
|
|
134
|
-
import {
|
|
198
|
+
import { writeFile, mkdir, rm, stat as stat2 } from "fs/promises";
|
|
199
|
+
import { mkdirSync as mkdirSync2 } from "fs";
|
|
200
|
+
import { join as join4, relative as relative2, dirname, extname as extname3 } from "path";
|
|
135
201
|
|
|
136
|
-
// src/
|
|
202
|
+
// src/graph/builder.ts
|
|
203
|
+
import { DirectedGraph } from "graphology";
|
|
204
|
+
function buildGraph(nodes) {
|
|
205
|
+
const graph = new DirectedGraph();
|
|
206
|
+
const nodeIds = /* @__PURE__ */ new Set();
|
|
207
|
+
for (const node of nodes) {
|
|
208
|
+
graph.addNode(node.id);
|
|
209
|
+
nodeIds.add(node.id);
|
|
210
|
+
}
|
|
211
|
+
for (const node of nodes) {
|
|
212
|
+
const seen = /* @__PURE__ */ new Set();
|
|
213
|
+
for (const target of node.outgoingLinks) {
|
|
214
|
+
if (!nodeIds.has(target) || seen.has(target)) {
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
seen.add(target);
|
|
218
|
+
graph.addDirectedEdge(node.id, target);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
return graph;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// src/graph/traversal.ts
|
|
225
|
+
import { bidirectional } from "graphology-shortest-path";
|
|
226
|
+
|
|
227
|
+
// src/utils/heap.ts
|
|
228
|
+
var MinHeap = class {
|
|
229
|
+
data = [];
|
|
230
|
+
compare;
|
|
231
|
+
constructor(comparator) {
|
|
232
|
+
this.compare = comparator;
|
|
233
|
+
}
|
|
234
|
+
size() {
|
|
235
|
+
return this.data.length;
|
|
236
|
+
}
|
|
237
|
+
peek() {
|
|
238
|
+
return this.data[0];
|
|
239
|
+
}
|
|
240
|
+
push(value) {
|
|
241
|
+
this.data.push(value);
|
|
242
|
+
this.bubbleUp(this.data.length - 1);
|
|
243
|
+
}
|
|
244
|
+
pop() {
|
|
245
|
+
if (this.data.length === 0) return void 0;
|
|
246
|
+
if (this.data.length === 1) return this.data.pop();
|
|
247
|
+
const min = this.data[0];
|
|
248
|
+
this.data[0] = this.data.pop();
|
|
249
|
+
this.bubbleDown(0);
|
|
250
|
+
return min;
|
|
251
|
+
}
|
|
252
|
+
toArray() {
|
|
253
|
+
return [...this.data];
|
|
254
|
+
}
|
|
255
|
+
bubbleUp(index) {
|
|
256
|
+
while (index > 0) {
|
|
257
|
+
const parentIndex = Math.floor((index - 1) / 2);
|
|
258
|
+
if (this.compare(this.data[index], this.data[parentIndex]) >= 0) {
|
|
259
|
+
break;
|
|
260
|
+
}
|
|
261
|
+
this.swap(index, parentIndex);
|
|
262
|
+
index = parentIndex;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
bubbleDown(index) {
|
|
266
|
+
const length = this.data.length;
|
|
267
|
+
while (true) {
|
|
268
|
+
const leftChild = 2 * index + 1;
|
|
269
|
+
const rightChild = 2 * index + 2;
|
|
270
|
+
let smallest = index;
|
|
271
|
+
if (leftChild < length && this.compare(this.data[leftChild], this.data[smallest]) < 0) {
|
|
272
|
+
smallest = leftChild;
|
|
273
|
+
}
|
|
274
|
+
if (rightChild < length && this.compare(this.data[rightChild], this.data[smallest]) < 0) {
|
|
275
|
+
smallest = rightChild;
|
|
276
|
+
}
|
|
277
|
+
if (smallest === index) break;
|
|
278
|
+
this.swap(index, smallest);
|
|
279
|
+
index = smallest;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
swap(i, j) {
|
|
283
|
+
const temp = this.data[i];
|
|
284
|
+
this.data[i] = this.data[j];
|
|
285
|
+
this.data[j] = temp;
|
|
286
|
+
}
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
// src/graph/traversal.ts
|
|
290
|
+
function getNeighborIds(graph, id, options) {
|
|
291
|
+
if (!graph.hasNode(id)) {
|
|
292
|
+
return [];
|
|
293
|
+
}
|
|
294
|
+
const limit = options.limit;
|
|
295
|
+
if (limit !== void 0 && limit <= 0) {
|
|
296
|
+
return [];
|
|
297
|
+
}
|
|
298
|
+
const maxCount = limit ?? Infinity;
|
|
299
|
+
const direction = options.direction;
|
|
300
|
+
if (direction === "both") {
|
|
301
|
+
const neighbors2 = [];
|
|
302
|
+
for (const entry of graph.neighborEntries(id)) {
|
|
303
|
+
if (neighbors2.length >= maxCount) break;
|
|
304
|
+
neighbors2.push(entry.neighbor);
|
|
305
|
+
}
|
|
306
|
+
return neighbors2;
|
|
307
|
+
}
|
|
308
|
+
const neighbors = [];
|
|
309
|
+
const iterator = direction === "in" ? graph.inNeighborEntries(id) : graph.outNeighborEntries(id);
|
|
310
|
+
for (const entry of iterator) {
|
|
311
|
+
if (neighbors.length >= maxCount) break;
|
|
312
|
+
neighbors.push(entry.neighbor);
|
|
313
|
+
}
|
|
314
|
+
return neighbors;
|
|
315
|
+
}
|
|
316
|
+
function findPath(graph, source, target) {
|
|
317
|
+
if (!graph.hasNode(source) || !graph.hasNode(target)) {
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
if (source === target) {
|
|
321
|
+
return [source];
|
|
322
|
+
}
|
|
323
|
+
const path = bidirectional(graph, source, target);
|
|
324
|
+
return path;
|
|
325
|
+
}
|
|
326
|
+
function getHubs(graph, metric, limit) {
|
|
327
|
+
if (limit <= 0) {
|
|
328
|
+
return [];
|
|
329
|
+
}
|
|
330
|
+
const heap = new MinHeap((a, b) => a[1] - b[1]);
|
|
331
|
+
graph.forEachNode((id) => {
|
|
332
|
+
const score = metric === "in_degree" ? graph.inDegree(id) : graph.outDegree(id);
|
|
333
|
+
if (heap.size() < limit) {
|
|
334
|
+
heap.push([id, score]);
|
|
335
|
+
} else if (score > heap.peek()[1]) {
|
|
336
|
+
heap.pop();
|
|
337
|
+
heap.push([id, score]);
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
return heap.toArray().sort((a, b) => {
|
|
341
|
+
const scoreDiff = b[1] - a[1];
|
|
342
|
+
if (scoreDiff !== 0) return scoreDiff;
|
|
343
|
+
return a[0].localeCompare(b[0]);
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// src/graph/analysis.ts
|
|
348
|
+
function computeCentrality(graph) {
|
|
349
|
+
const result = /* @__PURE__ */ new Map();
|
|
350
|
+
graph.forEachNode((id) => {
|
|
351
|
+
result.set(id, {
|
|
352
|
+
inDegree: graph.inDegree(id),
|
|
353
|
+
outDegree: graph.outDegree(id)
|
|
354
|
+
});
|
|
355
|
+
});
|
|
356
|
+
return result;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// src/graph/manager.ts
|
|
360
|
+
var GraphNotReadyError = class extends Error {
|
|
361
|
+
constructor() {
|
|
362
|
+
super("Graph not built. Call build() before querying.");
|
|
363
|
+
this.name = "GraphNotReadyError";
|
|
364
|
+
}
|
|
365
|
+
};
|
|
366
|
+
var GraphManager = class {
|
|
367
|
+
graph = null;
|
|
368
|
+
/** Build graph and return centrality metrics. Caller stores as needed. */
|
|
369
|
+
build(nodes) {
|
|
370
|
+
this.graph = buildGraph(nodes);
|
|
371
|
+
return computeCentrality(this.graph);
|
|
372
|
+
}
|
|
373
|
+
/** Throws GraphNotReadyError if not built. Returns graph for query use. */
|
|
374
|
+
assertReady() {
|
|
375
|
+
if (!this.graph) throw new GraphNotReadyError();
|
|
376
|
+
return this.graph;
|
|
377
|
+
}
|
|
378
|
+
isReady() {
|
|
379
|
+
return this.graph !== null;
|
|
380
|
+
}
|
|
381
|
+
getNeighborIds(id, options) {
|
|
382
|
+
return getNeighborIds(this.assertReady(), id, options);
|
|
383
|
+
}
|
|
384
|
+
findPath(source, target) {
|
|
385
|
+
return findPath(this.assertReady(), source, target);
|
|
386
|
+
}
|
|
387
|
+
getHubs(metric, limit) {
|
|
388
|
+
return getHubs(this.assertReady(), metric, limit);
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
// src/providers/store/resolve.ts
|
|
137
393
|
var import_string_similarity = __toESM(require_src(), 1);
|
|
394
|
+
function resolveNames(names, candidates, options) {
|
|
395
|
+
if (names.length === 0) return [];
|
|
396
|
+
const { strategy, threshold } = options;
|
|
397
|
+
if (candidates.length === 0) {
|
|
398
|
+
return names.map((query) => ({ query, match: null, score: 0 }));
|
|
399
|
+
}
|
|
400
|
+
const candidateTitles = candidates.map((c) => c.title.toLowerCase());
|
|
401
|
+
const titleToId = /* @__PURE__ */ new Map();
|
|
402
|
+
for (const c of candidates) {
|
|
403
|
+
titleToId.set(c.title.toLowerCase(), c.id);
|
|
404
|
+
}
|
|
405
|
+
return names.map((query) => {
|
|
406
|
+
const queryLower = query.toLowerCase();
|
|
407
|
+
if (strategy === "exact") {
|
|
408
|
+
const matchedId = titleToId.get(queryLower);
|
|
409
|
+
if (matchedId) {
|
|
410
|
+
return { query, match: matchedId, score: 1 };
|
|
411
|
+
}
|
|
412
|
+
return { query, match: null, score: 0 };
|
|
413
|
+
}
|
|
414
|
+
if (strategy === "fuzzy") {
|
|
415
|
+
const result = import_string_similarity.default.findBestMatch(queryLower, candidateTitles);
|
|
416
|
+
const bestMatch = result.bestMatch;
|
|
417
|
+
if (bestMatch.rating >= threshold) {
|
|
418
|
+
const matchedId = titleToId.get(bestMatch.target);
|
|
419
|
+
return { query, match: matchedId, score: bestMatch.rating };
|
|
420
|
+
}
|
|
421
|
+
return { query, match: null, score: 0 };
|
|
422
|
+
}
|
|
423
|
+
return { query, match: null, score: 0 };
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// src/providers/store/index.ts
|
|
428
|
+
var StoreProvider = class {
|
|
429
|
+
graphManager = new GraphManager();
|
|
430
|
+
vectorIndex;
|
|
431
|
+
constructor(options) {
|
|
432
|
+
this.vectorIndex = options?.vectorIndex ?? null;
|
|
433
|
+
}
|
|
434
|
+
// ── Graph operations (delegate to GraphManager) ────────────
|
|
435
|
+
async getNeighbors(id, options) {
|
|
436
|
+
if (!this.graphManager.isReady()) return [];
|
|
437
|
+
const neighborIds = this.graphManager.getNeighborIds(id, options);
|
|
438
|
+
return this.getNodesByIds(neighborIds);
|
|
439
|
+
}
|
|
440
|
+
async findPath(source, target) {
|
|
441
|
+
if (!this.graphManager.isReady()) return null;
|
|
442
|
+
return this.graphManager.findPath(source, target);
|
|
443
|
+
}
|
|
444
|
+
async getHubs(metric, limit) {
|
|
445
|
+
if (!this.graphManager.isReady()) return [];
|
|
446
|
+
return this.graphManager.getHubs(metric, limit);
|
|
447
|
+
}
|
|
448
|
+
// ── Vector operations (delegate to VectorIndex) ────────────
|
|
449
|
+
async storeEmbedding(id, vector, model) {
|
|
450
|
+
if (!this.vectorIndex) throw new Error("No VectorIndex configured");
|
|
451
|
+
return this.vectorIndex.store(id, vector, model);
|
|
452
|
+
}
|
|
453
|
+
async searchByVector(vector, limit) {
|
|
454
|
+
if (!this.vectorIndex) throw new Error("No VectorIndex configured");
|
|
455
|
+
return this.vectorIndex.search(vector, limit);
|
|
456
|
+
}
|
|
457
|
+
// ── Discovery ──────────────────────────────────────────────
|
|
458
|
+
async getRandomNode(tags) {
|
|
459
|
+
let candidates;
|
|
460
|
+
if (tags && tags.length > 0) {
|
|
461
|
+
candidates = await this.searchByTags(tags, "any");
|
|
462
|
+
} else {
|
|
463
|
+
candidates = await this.loadAllNodes();
|
|
464
|
+
}
|
|
465
|
+
if (candidates.length === 0) return null;
|
|
466
|
+
return candidates[Math.floor(Math.random() * candidates.length)];
|
|
467
|
+
}
|
|
468
|
+
// ── Default implementations (overridable) ──────────────────
|
|
469
|
+
async searchByTags(tags, mode, limit) {
|
|
470
|
+
const allNodes = await this.loadAllNodes();
|
|
471
|
+
const lowerTags = tags.map((t) => t.toLowerCase());
|
|
472
|
+
let results = allNodes.filter((node) => {
|
|
473
|
+
const nodeTags = node.tags.map((t) => t.toLowerCase());
|
|
474
|
+
return mode === "any" ? lowerTags.some((t) => nodeTags.includes(t)) : lowerTags.every((t) => nodeTags.includes(t));
|
|
475
|
+
});
|
|
476
|
+
if (limit !== void 0) results = results.slice(0, limit);
|
|
477
|
+
return results;
|
|
478
|
+
}
|
|
479
|
+
async listNodes(filter, options) {
|
|
480
|
+
let nodes = await this.loadAllNodes();
|
|
481
|
+
if (filter.tag) {
|
|
482
|
+
const lower = filter.tag.toLowerCase();
|
|
483
|
+
nodes = nodes.filter((n) => n.tags.some((t) => t.toLowerCase() === lower));
|
|
484
|
+
}
|
|
485
|
+
if (filter.path) {
|
|
486
|
+
const lowerPath = filter.path.toLowerCase();
|
|
487
|
+
nodes = nodes.filter((n) => n.id.startsWith(lowerPath));
|
|
488
|
+
}
|
|
489
|
+
const total = nodes.length;
|
|
490
|
+
const offset = options?.offset ?? 0;
|
|
491
|
+
const limit = Math.min(options?.limit ?? 100, 1e3);
|
|
492
|
+
const sliced = nodes.slice(offset, offset + limit);
|
|
493
|
+
return {
|
|
494
|
+
nodes: sliced.map((n) => ({ id: n.id, title: n.title })),
|
|
495
|
+
total
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
async nodesExist(ids) {
|
|
499
|
+
if (ids.length === 0) return /* @__PURE__ */ new Map();
|
|
500
|
+
const found = await this.getNodesByIds(ids);
|
|
501
|
+
const foundIds = new Set(found.map((n) => n.id));
|
|
502
|
+
const result = /* @__PURE__ */ new Map();
|
|
503
|
+
for (const id of ids) {
|
|
504
|
+
result.set(id, foundIds.has(id));
|
|
505
|
+
}
|
|
506
|
+
return result;
|
|
507
|
+
}
|
|
508
|
+
async resolveTitles(ids) {
|
|
509
|
+
if (ids.length === 0) return /* @__PURE__ */ new Map();
|
|
510
|
+
const nodes = await this.getNodesByIds(ids);
|
|
511
|
+
const result = /* @__PURE__ */ new Map();
|
|
512
|
+
for (const node of nodes) {
|
|
513
|
+
result.set(node.id, node.title);
|
|
514
|
+
}
|
|
515
|
+
return result;
|
|
516
|
+
}
|
|
517
|
+
async resolveNodes(names, options) {
|
|
518
|
+
const strategy = options?.strategy ?? "fuzzy";
|
|
519
|
+
if (strategy === "semantic") {
|
|
520
|
+
return names.map((query) => ({ query, match: null, score: 0 }));
|
|
521
|
+
}
|
|
522
|
+
const allNodes = await this.loadAllNodes();
|
|
523
|
+
let candidates = allNodes.map((n) => ({ id: n.id, title: n.title }));
|
|
524
|
+
if (options?.tag) {
|
|
525
|
+
const lower = options.tag.toLowerCase();
|
|
526
|
+
const filtered = allNodes.filter((n) => n.tags.some((t) => t.toLowerCase() === lower));
|
|
527
|
+
candidates = filtered.map((n) => ({ id: n.id, title: n.title }));
|
|
528
|
+
}
|
|
529
|
+
if (options?.path) {
|
|
530
|
+
const lowerPath = options.path.toLowerCase();
|
|
531
|
+
candidates = candidates.filter((c) => c.id.startsWith(lowerPath));
|
|
532
|
+
}
|
|
533
|
+
return resolveNames(names, candidates, {
|
|
534
|
+
strategy,
|
|
535
|
+
threshold: options?.threshold ?? 0.7
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
// ── Graph lifecycle ────────────────────────────────────────
|
|
539
|
+
async syncGraph() {
|
|
540
|
+
const nodes = await this.loadAllNodes();
|
|
541
|
+
const centrality = this.graphManager.build(nodes);
|
|
542
|
+
this.onCentralityComputed(centrality);
|
|
543
|
+
}
|
|
544
|
+
onCentralityComputed(_centrality) {
|
|
545
|
+
}
|
|
546
|
+
};
|
|
547
|
+
|
|
548
|
+
// src/providers/docstore/cache.ts
|
|
138
549
|
import Database from "better-sqlite3";
|
|
139
550
|
import { join } from "path";
|
|
140
551
|
import { mkdirSync } from "fs";
|
|
552
|
+
|
|
553
|
+
// src/providers/docstore/cache/centrality.ts
|
|
554
|
+
function initCentralitySchema(db) {
|
|
555
|
+
db.exec(`
|
|
556
|
+
CREATE TABLE IF NOT EXISTS centrality (
|
|
557
|
+
node_id TEXT PRIMARY KEY,
|
|
558
|
+
pagerank REAL,
|
|
559
|
+
in_degree INTEGER,
|
|
560
|
+
out_degree INTEGER,
|
|
561
|
+
computed_at INTEGER,
|
|
562
|
+
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
|
563
|
+
)
|
|
564
|
+
`);
|
|
565
|
+
}
|
|
566
|
+
function storeCentrality(db, nodeId, pagerank, inDegree, outDegree, computedAt) {
|
|
567
|
+
db.prepare(
|
|
568
|
+
`
|
|
569
|
+
INSERT INTO centrality (node_id, pagerank, in_degree, out_degree, computed_at)
|
|
570
|
+
VALUES (?, ?, ?, ?, ?)
|
|
571
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
572
|
+
pagerank = excluded.pagerank,
|
|
573
|
+
in_degree = excluded.in_degree,
|
|
574
|
+
out_degree = excluded.out_degree,
|
|
575
|
+
computed_at = excluded.computed_at
|
|
576
|
+
`
|
|
577
|
+
).run(nodeId, pagerank, inDegree, outDegree, computedAt);
|
|
578
|
+
}
|
|
579
|
+
function getCentrality(db, nodeId) {
|
|
580
|
+
const row = db.prepare("SELECT * FROM centrality WHERE node_id = ?").get(nodeId);
|
|
581
|
+
if (!row) return null;
|
|
582
|
+
return {
|
|
583
|
+
pagerank: row.pagerank,
|
|
584
|
+
inDegree: row.in_degree,
|
|
585
|
+
outDegree: row.out_degree,
|
|
586
|
+
computedAt: row.computed_at
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// src/providers/docstore/cache.ts
|
|
141
591
|
var Cache = class {
|
|
142
592
|
db;
|
|
143
593
|
constructor(cacheDir) {
|
|
@@ -161,24 +611,9 @@ var Cache = class {
|
|
|
161
611
|
source_modified INTEGER
|
|
162
612
|
);
|
|
163
613
|
|
|
164
|
-
CREATE TABLE IF NOT EXISTS embeddings (
|
|
165
|
-
node_id TEXT PRIMARY KEY,
|
|
166
|
-
model TEXT,
|
|
167
|
-
vector BLOB,
|
|
168
|
-
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
|
169
|
-
);
|
|
170
|
-
|
|
171
|
-
CREATE TABLE IF NOT EXISTS centrality (
|
|
172
|
-
node_id TEXT PRIMARY KEY,
|
|
173
|
-
pagerank REAL,
|
|
174
|
-
in_degree INTEGER,
|
|
175
|
-
out_degree INTEGER,
|
|
176
|
-
computed_at INTEGER,
|
|
177
|
-
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
|
178
|
-
);
|
|
179
|
-
|
|
180
614
|
CREATE INDEX IF NOT EXISTS idx_nodes_source_path ON nodes(source_path);
|
|
181
615
|
`);
|
|
616
|
+
initCentralitySchema(this.db);
|
|
182
617
|
this.db.pragma("foreign_keys = ON");
|
|
183
618
|
}
|
|
184
619
|
getTableNames() {
|
|
@@ -238,25 +673,37 @@ var Cache = class {
|
|
|
238
673
|
const rows = this.db.prepare("SELECT * FROM nodes").all();
|
|
239
674
|
return rows.map((row) => this.rowToNode(row));
|
|
240
675
|
}
|
|
241
|
-
searchByTags(tags, mode) {
|
|
676
|
+
searchByTags(tags, mode, limit) {
|
|
242
677
|
if (tags.length === 0) return [];
|
|
243
|
-
const allNodes = this.getAllNodes();
|
|
244
678
|
const lowerTags = tags.map((t) => t.toLowerCase());
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
}
|
|
252
|
-
|
|
679
|
+
let query;
|
|
680
|
+
const params = [];
|
|
681
|
+
if (mode === "any") {
|
|
682
|
+
const tagConditions = lowerTags.map(
|
|
683
|
+
() => "EXISTS (SELECT 1 FROM json_each(tags) WHERE LOWER(json_each.value) = ?)"
|
|
684
|
+
).join(" OR ");
|
|
685
|
+
query = `SELECT * FROM nodes WHERE ${tagConditions}`;
|
|
686
|
+
params.push(...lowerTags);
|
|
687
|
+
} else {
|
|
688
|
+
const tagConditions = lowerTags.map(
|
|
689
|
+
() => "EXISTS (SELECT 1 FROM json_each(tags) WHERE LOWER(json_each.value) = ?)"
|
|
690
|
+
).join(" AND ");
|
|
691
|
+
query = `SELECT * FROM nodes WHERE ${tagConditions}`;
|
|
692
|
+
params.push(...lowerTags);
|
|
693
|
+
}
|
|
694
|
+
if (limit !== void 0) {
|
|
695
|
+
query += " LIMIT ?";
|
|
696
|
+
params.push(limit);
|
|
697
|
+
}
|
|
698
|
+
const rows = this.db.prepare(query).all(...params);
|
|
699
|
+
return rows.map((row) => this.rowToNode(row));
|
|
253
700
|
}
|
|
254
701
|
getModifiedTime(sourcePath) {
|
|
255
702
|
const row = this.db.prepare("SELECT source_modified FROM nodes WHERE source_path = ?").get(sourcePath);
|
|
256
703
|
return row?.source_modified ?? null;
|
|
257
704
|
}
|
|
258
705
|
getNodeByPath(sourcePath) {
|
|
259
|
-
const row = this.db.prepare("SELECT * FROM nodes WHERE source_path = ?").get(sourcePath);
|
|
706
|
+
const row = this.db.prepare("SELECT * FROM nodes WHERE LOWER(source_path) = LOWER(?)").get(sourcePath);
|
|
260
707
|
if (!row) return null;
|
|
261
708
|
return this.rowToNode(row);
|
|
262
709
|
}
|
|
@@ -264,6 +711,13 @@ var Cache = class {
|
|
|
264
711
|
const rows = this.db.prepare("SELECT source_path FROM nodes").all();
|
|
265
712
|
return new Set(rows.map((r) => r.source_path));
|
|
266
713
|
}
|
|
714
|
+
updateSourcePath(id, newPath) {
|
|
715
|
+
this.db.prepare("UPDATE nodes SET source_path = ? WHERE id = ?").run(newPath, id);
|
|
716
|
+
}
|
|
717
|
+
getIdByPath(sourcePath) {
|
|
718
|
+
const row = this.db.prepare("SELECT id FROM nodes WHERE source_path = ?").get(sourcePath);
|
|
719
|
+
return row?.id ?? null;
|
|
720
|
+
}
|
|
267
721
|
resolveTitles(ids) {
|
|
268
722
|
if (ids.length === 0) return /* @__PURE__ */ new Map();
|
|
269
723
|
const placeholders = ids.map(() => "?").join(",");
|
|
@@ -295,7 +749,7 @@ var Cache = class {
|
|
|
295
749
|
params.push(filter.tag);
|
|
296
750
|
}
|
|
297
751
|
if (filter.path) {
|
|
298
|
-
conditions.push("
|
|
752
|
+
conditions.push("LOWER(source_path) LIKE '%' || LOWER(?) || '%'");
|
|
299
753
|
params.push(filter.path);
|
|
300
754
|
}
|
|
301
755
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
@@ -315,99 +769,27 @@ var Cache = class {
|
|
|
315
769
|
if (options?.tag) filter.tag = options.tag;
|
|
316
770
|
if (options?.path) filter.path = options.path;
|
|
317
771
|
const { nodes: candidates } = this.listNodes(filter, { limit: 1e3 });
|
|
318
|
-
|
|
319
|
-
return names.map((query) => ({ query, match: null, score: 0 }));
|
|
320
|
-
}
|
|
321
|
-
const candidateTitles = candidates.map((c) => c.title.toLowerCase());
|
|
322
|
-
const titleToId = /* @__PURE__ */ new Map();
|
|
323
|
-
for (const c of candidates) {
|
|
324
|
-
titleToId.set(c.title.toLowerCase(), c.id);
|
|
325
|
-
}
|
|
326
|
-
return names.map((query) => {
|
|
327
|
-
const queryLower = query.toLowerCase();
|
|
328
|
-
if (strategy === "exact") {
|
|
329
|
-
const matchedId = titleToId.get(queryLower);
|
|
330
|
-
if (matchedId) {
|
|
331
|
-
return { query, match: matchedId, score: 1 };
|
|
332
|
-
}
|
|
333
|
-
return { query, match: null, score: 0 };
|
|
334
|
-
}
|
|
335
|
-
if (strategy === "fuzzy") {
|
|
336
|
-
const result = import_string_similarity.default.findBestMatch(queryLower, candidateTitles);
|
|
337
|
-
const bestMatch = result.bestMatch;
|
|
338
|
-
if (bestMatch.rating >= threshold) {
|
|
339
|
-
const matchedId = titleToId.get(bestMatch.target);
|
|
340
|
-
return { query, match: matchedId, score: bestMatch.rating };
|
|
341
|
-
}
|
|
342
|
-
return { query, match: null, score: 0 };
|
|
343
|
-
}
|
|
344
|
-
return { query, match: null, score: 0 };
|
|
345
|
-
});
|
|
772
|
+
return resolveNames(names, candidates, { strategy, threshold });
|
|
346
773
|
}
|
|
347
774
|
updateOutgoingLinks(nodeId, links) {
|
|
348
775
|
this.db.prepare("UPDATE nodes SET outgoing_links = ? WHERE id = ?").run(JSON.stringify(links), nodeId);
|
|
349
776
|
}
|
|
350
|
-
storeEmbedding(nodeId, vector, model) {
|
|
351
|
-
const buffer = Buffer.from(new Float32Array(vector).buffer);
|
|
352
|
-
this.db.prepare(
|
|
353
|
-
`
|
|
354
|
-
INSERT INTO embeddings (node_id, model, vector)
|
|
355
|
-
VALUES (?, ?, ?)
|
|
356
|
-
ON CONFLICT(node_id) DO UPDATE SET
|
|
357
|
-
model = excluded.model,
|
|
358
|
-
vector = excluded.vector
|
|
359
|
-
`
|
|
360
|
-
).run(nodeId, model, buffer);
|
|
361
|
-
}
|
|
362
|
-
getEmbedding(nodeId) {
|
|
363
|
-
const row = this.db.prepare("SELECT model, vector FROM embeddings WHERE node_id = ?").get(nodeId);
|
|
364
|
-
if (!row) return null;
|
|
365
|
-
const float32 = new Float32Array(
|
|
366
|
-
row.vector.buffer,
|
|
367
|
-
row.vector.byteOffset,
|
|
368
|
-
row.vector.length / 4
|
|
369
|
-
);
|
|
370
|
-
return {
|
|
371
|
-
model: row.model,
|
|
372
|
-
vector: Array.from(float32)
|
|
373
|
-
};
|
|
374
|
-
}
|
|
375
777
|
storeCentrality(nodeId, pagerank, inDegree, outDegree, computedAt) {
|
|
376
|
-
this.db
|
|
377
|
-
`
|
|
378
|
-
INSERT INTO centrality (node_id, pagerank, in_degree, out_degree, computed_at)
|
|
379
|
-
VALUES (?, ?, ?, ?, ?)
|
|
380
|
-
ON CONFLICT(node_id) DO UPDATE SET
|
|
381
|
-
pagerank = excluded.pagerank,
|
|
382
|
-
in_degree = excluded.in_degree,
|
|
383
|
-
out_degree = excluded.out_degree,
|
|
384
|
-
computed_at = excluded.computed_at
|
|
385
|
-
`
|
|
386
|
-
).run(nodeId, pagerank, inDegree, outDegree, computedAt);
|
|
778
|
+
storeCentrality(this.db, nodeId, pagerank, inDegree, outDegree, computedAt);
|
|
387
779
|
}
|
|
388
780
|
getCentrality(nodeId) {
|
|
389
|
-
|
|
390
|
-
if (!row) return null;
|
|
391
|
-
return {
|
|
392
|
-
pagerank: row.pagerank,
|
|
393
|
-
inDegree: row.in_degree,
|
|
394
|
-
outDegree: row.out_degree,
|
|
395
|
-
computedAt: row.computed_at
|
|
396
|
-
};
|
|
781
|
+
return getCentrality(this.db, nodeId);
|
|
397
782
|
}
|
|
398
783
|
getStats() {
|
|
399
784
|
const nodeCount = this.db.prepare("SELECT COUNT(*) as count FROM nodes").get();
|
|
400
|
-
const embeddingCount = this.db.prepare("SELECT COUNT(*) as count FROM embeddings").get();
|
|
401
785
|
const edgeSum = this.db.prepare("SELECT SUM(in_degree) as total FROM centrality").get();
|
|
402
786
|
return {
|
|
403
787
|
nodeCount: nodeCount.count,
|
|
404
|
-
embeddingCount: embeddingCount.count,
|
|
405
788
|
edgeCount: edgeSum.total ?? 0
|
|
406
789
|
};
|
|
407
790
|
}
|
|
408
791
|
clear() {
|
|
409
792
|
this.db.exec("DELETE FROM centrality");
|
|
410
|
-
this.db.exec("DELETE FROM embeddings");
|
|
411
793
|
this.db.exec("DELETE FROM nodes");
|
|
412
794
|
}
|
|
413
795
|
close() {
|
|
@@ -434,9 +816,45 @@ var Cache = class {
|
|
|
434
816
|
// src/providers/vector/sqlite.ts
|
|
435
817
|
import Database2 from "better-sqlite3";
|
|
436
818
|
import { join as join2 } from "path";
|
|
437
|
-
|
|
819
|
+
|
|
820
|
+
// src/utils/math.ts
|
|
821
|
+
function cosineSimilarity(a, b) {
|
|
822
|
+
if (a.length === 0 || b.length === 0) {
|
|
823
|
+
throw new Error("Cannot compute similarity for empty vector");
|
|
824
|
+
}
|
|
825
|
+
if (a.length !== b.length) {
|
|
826
|
+
throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
|
|
827
|
+
}
|
|
828
|
+
let dotProduct = 0;
|
|
829
|
+
let normA = 0;
|
|
830
|
+
let normB = 0;
|
|
831
|
+
for (let i = 0; i < a.length; i++) {
|
|
832
|
+
dotProduct += a[i] * b[i];
|
|
833
|
+
normA += a[i] * a[i];
|
|
834
|
+
normB += b[i] * b[i];
|
|
835
|
+
}
|
|
836
|
+
if (normA === 0 || normB === 0) return 0;
|
|
837
|
+
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
838
|
+
}
|
|
839
|
+
function cosineDistance(a, b) {
|
|
840
|
+
const similarity = cosineSimilarity(a, b);
|
|
841
|
+
if (similarity === 0 && (isZeroVector(a) || isZeroVector(b))) {
|
|
842
|
+
return 1;
|
|
843
|
+
}
|
|
844
|
+
return 1 - similarity;
|
|
845
|
+
}
|
|
846
|
+
function isZeroVector(v) {
|
|
847
|
+
for (let i = 0; i < v.length; i++) {
|
|
848
|
+
if (v[i] !== 0) return false;
|
|
849
|
+
}
|
|
850
|
+
return true;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
// src/providers/vector/sqlite.ts
|
|
854
|
+
var SqliteVectorIndex = class {
|
|
438
855
|
db;
|
|
439
856
|
ownsDb;
|
|
857
|
+
modelMismatchWarned = false;
|
|
440
858
|
constructor(pathOrDb) {
|
|
441
859
|
if (typeof pathOrDb === "string") {
|
|
442
860
|
this.db = new Database2(join2(pathOrDb, "vectors.db"));
|
|
@@ -488,29 +906,45 @@ var SqliteVectorProvider = class {
|
|
|
488
906
|
if (limit <= 0) {
|
|
489
907
|
return [];
|
|
490
908
|
}
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
);
|
|
909
|
+
if (!this.modelMismatchWarned) {
|
|
910
|
+
const models = this.db.prepare("SELECT DISTINCT model FROM vectors").all();
|
|
911
|
+
if (models.length > 1) {
|
|
912
|
+
console.warn(
|
|
913
|
+
`Vector index contains embeddings from multiple models: ${models.map((m) => m.model).join(", ")}. Search results may be unreliable. Re-sync to re-embed all documents with current model.`
|
|
914
|
+
);
|
|
915
|
+
this.modelMismatchWarned = true;
|
|
916
|
+
}
|
|
500
917
|
}
|
|
501
918
|
const queryVec = new Float32Array(vector);
|
|
502
|
-
const
|
|
503
|
-
|
|
919
|
+
const stmt = this.db.prepare("SELECT id, vector FROM vectors");
|
|
920
|
+
const heap = new MinHeap(
|
|
921
|
+
(a, b) => b.distance - a.distance
|
|
922
|
+
);
|
|
923
|
+
let dimensionChecked = false;
|
|
924
|
+
for (const row of stmt.iterate()) {
|
|
925
|
+
if (!dimensionChecked) {
|
|
926
|
+
const storedDim = row.vector.byteLength / 4;
|
|
927
|
+
if (vector.length !== storedDim) {
|
|
928
|
+
throw new Error(
|
|
929
|
+
`Dimension mismatch: query has ${vector.length} dimensions, stored vectors have ${storedDim}`
|
|
930
|
+
);
|
|
931
|
+
}
|
|
932
|
+
dimensionChecked = true;
|
|
933
|
+
}
|
|
504
934
|
const storedVec = new Float32Array(
|
|
505
935
|
row.vector.buffer,
|
|
506
936
|
row.vector.byteOffset,
|
|
507
937
|
row.vector.byteLength / 4
|
|
508
938
|
);
|
|
509
939
|
const distance = cosineDistance(queryVec, storedVec);
|
|
510
|
-
|
|
940
|
+
if (heap.size() < limit) {
|
|
941
|
+
heap.push({ id: row.id, distance });
|
|
942
|
+
} else if (distance < heap.peek().distance) {
|
|
943
|
+
heap.pop();
|
|
944
|
+
heap.push({ id: row.id, distance });
|
|
945
|
+
}
|
|
511
946
|
}
|
|
512
|
-
|
|
513
|
-
return results.slice(0, limit);
|
|
947
|
+
return heap.toArray().sort((a, b) => a.distance - b.distance);
|
|
514
948
|
}
|
|
515
949
|
async delete(id) {
|
|
516
950
|
this.db.prepare("DELETE FROM vectors WHERE id = ?").run(id);
|
|
@@ -544,26 +978,30 @@ var SqliteVectorProvider = class {
|
|
|
544
978
|
}
|
|
545
979
|
}
|
|
546
980
|
};
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
981
|
+
|
|
982
|
+
// src/providers/docstore/parser.ts
|
|
983
|
+
import matter from "gray-matter";
|
|
984
|
+
|
|
985
|
+
// src/providers/docstore/normalize.ts
|
|
986
|
+
function hasFileExtension(path) {
|
|
987
|
+
const match = path.match(/\.([a-z0-9]{1,4})$/i);
|
|
988
|
+
if (!match?.[1]) return false;
|
|
989
|
+
return /[a-z]/i.test(match[1]);
|
|
990
|
+
}
|
|
991
|
+
function normalizePath(path) {
|
|
992
|
+
return path.toLowerCase().replace(/\\/g, "/");
|
|
993
|
+
}
|
|
994
|
+
function normalizeLinkTarget(target) {
|
|
995
|
+
let normalized = target.trim().toLowerCase().replace(/\\/g, "/");
|
|
996
|
+
if (!hasFileExtension(normalized)) {
|
|
997
|
+
normalized += ".md";
|
|
560
998
|
}
|
|
561
|
-
|
|
562
|
-
return 1 - similarity;
|
|
999
|
+
return normalized;
|
|
563
1000
|
}
|
|
564
1001
|
|
|
565
1002
|
// src/providers/docstore/parser.ts
|
|
566
|
-
|
|
1003
|
+
var RESERVED_FRONTMATTER_KEYS = ["id", "title", "tags"];
|
|
1004
|
+
var RESERVED_KEYS_SET = new Set(RESERVED_FRONTMATTER_KEYS);
|
|
567
1005
|
function parseMarkdown(raw) {
|
|
568
1006
|
let parsed;
|
|
569
1007
|
try {
|
|
@@ -573,10 +1011,12 @@ function parseMarkdown(raw) {
|
|
|
573
1011
|
title: void 0,
|
|
574
1012
|
tags: [],
|
|
575
1013
|
properties: {},
|
|
576
|
-
content: raw
|
|
1014
|
+
content: raw,
|
|
1015
|
+
rawLinks: extractWikiLinks(raw)
|
|
577
1016
|
};
|
|
578
1017
|
}
|
|
579
1018
|
const data = parsed.data;
|
|
1019
|
+
const id = typeof data["id"] === "string" ? data["id"] : void 0;
|
|
580
1020
|
const title = typeof data["title"] === "string" ? data["title"] : void 0;
|
|
581
1021
|
let tags = [];
|
|
582
1022
|
if (Array.isArray(data["tags"])) {
|
|
@@ -584,16 +1024,22 @@ function parseMarkdown(raw) {
|
|
|
584
1024
|
}
|
|
585
1025
|
const properties = {};
|
|
586
1026
|
for (const [key, value] of Object.entries(data)) {
|
|
587
|
-
if (key
|
|
1027
|
+
if (!RESERVED_KEYS_SET.has(key)) {
|
|
588
1028
|
properties[key] = value;
|
|
589
1029
|
}
|
|
590
1030
|
}
|
|
591
|
-
|
|
1031
|
+
const content = parsed.content.trim();
|
|
1032
|
+
const result = {
|
|
592
1033
|
title,
|
|
593
1034
|
tags,
|
|
594
1035
|
properties,
|
|
595
|
-
content
|
|
1036
|
+
content,
|
|
1037
|
+
rawLinks: extractWikiLinks(content)
|
|
596
1038
|
};
|
|
1039
|
+
if (id !== void 0) {
|
|
1040
|
+
result.id = id;
|
|
1041
|
+
}
|
|
1042
|
+
return result;
|
|
597
1043
|
}
|
|
598
1044
|
function extractWikiLinks(content) {
|
|
599
1045
|
const withoutCodeBlocks = content.replace(/```[\s\S]*?```/g, "");
|
|
@@ -611,9 +1057,7 @@ function extractWikiLinks(content) {
|
|
|
611
1057
|
}
|
|
612
1058
|
return links;
|
|
613
1059
|
}
|
|
614
|
-
|
|
615
|
-
return path.toLowerCase().replace(/\\/g, "/");
|
|
616
|
-
}
|
|
1060
|
+
var normalizeId = normalizePath;
|
|
617
1061
|
function titleFromPath(path) {
|
|
618
1062
|
const parts = path.split(/[/\\]/);
|
|
619
1063
|
const filename = parts.at(-1);
|
|
@@ -622,11 +1066,14 @@ function titleFromPath(path) {
|
|
|
622
1066
|
return spaced.split(" ").filter((w) => w.length > 0).map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
623
1067
|
}
|
|
624
1068
|
function serializeToMarkdown(parsed) {
|
|
625
|
-
const hasFrontmatter = parsed.title !== void 0 || parsed.tags.length > 0 || Object.keys(parsed.properties).length > 0;
|
|
1069
|
+
const hasFrontmatter = parsed.id !== void 0 || parsed.title !== void 0 || parsed.tags.length > 0 || Object.keys(parsed.properties).length > 0;
|
|
626
1070
|
if (!hasFrontmatter) {
|
|
627
1071
|
return parsed.content;
|
|
628
1072
|
}
|
|
629
1073
|
const frontmatter = {};
|
|
1074
|
+
if (parsed.id !== void 0) {
|
|
1075
|
+
frontmatter["id"] = parsed.id;
|
|
1076
|
+
}
|
|
630
1077
|
if (parsed.title !== void 0) {
|
|
631
1078
|
frontmatter["title"] = parsed.title;
|
|
632
1079
|
}
|
|
@@ -639,243 +1086,586 @@ function serializeToMarkdown(parsed) {
|
|
|
639
1086
|
return matter.stringify(parsed.content, frontmatter);
|
|
640
1087
|
}
|
|
641
1088
|
|
|
642
|
-
// src/
|
|
643
|
-
import {
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
1089
|
+
// src/providers/docstore/watcher.ts
|
|
1090
|
+
import { watch } from "chokidar";
|
|
1091
|
+
import { relative, extname } from "path";
|
|
1092
|
+
|
|
1093
|
+
// src/providers/docstore/constants.ts
|
|
1094
|
+
var EXCLUDED_DIRS = /* @__PURE__ */ new Set([
|
|
1095
|
+
".roux",
|
|
1096
|
+
"node_modules",
|
|
1097
|
+
".git",
|
|
1098
|
+
".obsidian"
|
|
1099
|
+
]);
|
|
1100
|
+
|
|
1101
|
+
// src/providers/docstore/watcher.ts
|
|
1102
|
+
var DEFAULT_DEBOUNCE_MS = 1e3;
|
|
1103
|
+
var FileWatcher = class {
|
|
1104
|
+
root;
|
|
1105
|
+
extensions;
|
|
1106
|
+
debounceMs;
|
|
1107
|
+
onBatch;
|
|
1108
|
+
watcher = null;
|
|
1109
|
+
debounceTimer = null;
|
|
1110
|
+
pendingChanges = /* @__PURE__ */ new Map();
|
|
1111
|
+
isPaused = false;
|
|
1112
|
+
constructor(options) {
|
|
1113
|
+
this.root = options.root;
|
|
1114
|
+
this.extensions = options.extensions;
|
|
1115
|
+
this.debounceMs = options.debounceMs ?? DEFAULT_DEBOUNCE_MS;
|
|
1116
|
+
this.onBatch = options.onBatch;
|
|
1117
|
+
}
|
|
1118
|
+
start() {
|
|
1119
|
+
if (this.watcher) {
|
|
1120
|
+
return Promise.reject(new Error("Already watching. Call stop() first."));
|
|
1121
|
+
}
|
|
1122
|
+
return new Promise((resolve2, reject) => {
|
|
1123
|
+
let isReady = false;
|
|
1124
|
+
this.watcher = watch(this.root, {
|
|
1125
|
+
ignoreInitial: true,
|
|
1126
|
+
ignored: [...EXCLUDED_DIRS].map((dir) => `**/${dir}/**`),
|
|
1127
|
+
awaitWriteFinish: {
|
|
1128
|
+
stabilityThreshold: 100
|
|
1129
|
+
},
|
|
1130
|
+
followSymlinks: false
|
|
1131
|
+
});
|
|
1132
|
+
this.watcher.on("ready", () => {
|
|
1133
|
+
isReady = true;
|
|
1134
|
+
resolve2();
|
|
1135
|
+
}).on("add", (path) => this.queueChange(path, "add")).on("change", (path) => this.queueChange(path, "change")).on("unlink", (path) => this.queueChange(path, "unlink")).on("error", (err) => {
|
|
1136
|
+
if (err.code === "EMFILE") {
|
|
1137
|
+
console.error(
|
|
1138
|
+
"File watcher hit file descriptor limit. Try: ulimit -n 65536 or reduce watched files."
|
|
1139
|
+
);
|
|
1140
|
+
}
|
|
1141
|
+
if (isReady) {
|
|
1142
|
+
console.error("FileWatcher error:", err);
|
|
1143
|
+
} else {
|
|
1144
|
+
reject(err);
|
|
1145
|
+
}
|
|
1146
|
+
});
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
stop() {
|
|
1150
|
+
if (this.debounceTimer) {
|
|
1151
|
+
clearTimeout(this.debounceTimer);
|
|
1152
|
+
this.debounceTimer = null;
|
|
1153
|
+
}
|
|
1154
|
+
this.pendingChanges.clear();
|
|
1155
|
+
if (this.watcher) {
|
|
1156
|
+
this.watcher.close();
|
|
1157
|
+
this.watcher = null;
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
isWatching() {
|
|
1161
|
+
return this.watcher !== null;
|
|
1162
|
+
}
|
|
1163
|
+
pause() {
|
|
1164
|
+
this.isPaused = true;
|
|
1165
|
+
}
|
|
1166
|
+
resume() {
|
|
1167
|
+
this.isPaused = false;
|
|
1168
|
+
}
|
|
1169
|
+
flush() {
|
|
1170
|
+
if (this.debounceTimer) {
|
|
1171
|
+
clearTimeout(this.debounceTimer);
|
|
1172
|
+
this.debounceTimer = null;
|
|
1173
|
+
}
|
|
1174
|
+
if (this.pendingChanges.size === 0) {
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
const batch = new Map(this.pendingChanges);
|
|
1178
|
+
this.pendingChanges.clear();
|
|
1179
|
+
try {
|
|
1180
|
+
const result = this.onBatch(batch);
|
|
1181
|
+
if (result && typeof result.catch === "function") {
|
|
1182
|
+
result.catch((err) => {
|
|
1183
|
+
console.error("FileWatcher onBatch callback threw an error:", err);
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
} catch (err) {
|
|
1187
|
+
console.error("FileWatcher onBatch callback threw an error:", err);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
queueChange(filePath, event) {
|
|
1191
|
+
if (this.isPaused) return;
|
|
1192
|
+
const relativePath = relative(this.root, filePath);
|
|
1193
|
+
const ext = extname(filePath).toLowerCase();
|
|
1194
|
+
if (!ext || !this.extensions.has(ext)) {
|
|
1195
|
+
return;
|
|
1196
|
+
}
|
|
1197
|
+
const pathParts = relativePath.split("/");
|
|
1198
|
+
for (const part of pathParts) {
|
|
1199
|
+
if (EXCLUDED_DIRS.has(part)) {
|
|
1200
|
+
return;
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
const id = relativePath.toLowerCase().replace(/\\/g, "/");
|
|
1204
|
+
const existing = this.pendingChanges.get(id);
|
|
1205
|
+
if (existing) {
|
|
1206
|
+
if (existing === "add" && event === "change") {
|
|
1207
|
+
return;
|
|
1208
|
+
} else if (existing === "add" && event === "unlink") {
|
|
1209
|
+
this.pendingChanges.delete(id);
|
|
1210
|
+
if (this.pendingChanges.size === 0) {
|
|
1211
|
+
if (this.debounceTimer) {
|
|
1212
|
+
clearTimeout(this.debounceTimer);
|
|
1213
|
+
this.debounceTimer = null;
|
|
1214
|
+
}
|
|
1215
|
+
return;
|
|
1216
|
+
}
|
|
1217
|
+
} else if (existing === "change" && event === "unlink") {
|
|
1218
|
+
this.pendingChanges.set(id, "unlink");
|
|
1219
|
+
} else if (existing === "change" && event === "add") {
|
|
1220
|
+
this.pendingChanges.set(id, "add");
|
|
1221
|
+
} else if (existing === "unlink" && event === "add") {
|
|
1222
|
+
this.pendingChanges.set(id, "add");
|
|
1223
|
+
} else if (existing === "unlink" && event === "change") {
|
|
1224
|
+
return;
|
|
1225
|
+
}
|
|
1226
|
+
} else {
|
|
1227
|
+
this.pendingChanges.set(id, event);
|
|
1228
|
+
}
|
|
1229
|
+
if (this.debounceTimer) {
|
|
1230
|
+
clearTimeout(this.debounceTimer);
|
|
1231
|
+
}
|
|
1232
|
+
this.debounceTimer = setTimeout(() => {
|
|
1233
|
+
this.flush();
|
|
1234
|
+
}, this.debounceMs);
|
|
650
1235
|
}
|
|
1236
|
+
};
|
|
1237
|
+
|
|
1238
|
+
// src/providers/docstore/links.ts
|
|
1239
|
+
var normalizeWikiLink = normalizeLinkTarget;
|
|
1240
|
+
function buildFilenameIndex(nodes) {
|
|
1241
|
+
const index = /* @__PURE__ */ new Map();
|
|
651
1242
|
for (const node of nodes) {
|
|
652
|
-
const
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
1243
|
+
const path = node.sourceRef?.path ?? "";
|
|
1244
|
+
const titleKey = node.title.toLowerCase();
|
|
1245
|
+
if (!titleKey && !path) {
|
|
1246
|
+
console.warn(
|
|
1247
|
+
`Node ${node.id} has no title or path \u2014 link resolution will fail`
|
|
1248
|
+
);
|
|
1249
|
+
}
|
|
1250
|
+
if (titleKey) {
|
|
1251
|
+
const existing = index.get(titleKey) ?? [];
|
|
1252
|
+
existing.push(node.id);
|
|
1253
|
+
index.set(titleKey, existing);
|
|
1254
|
+
}
|
|
1255
|
+
if (path) {
|
|
1256
|
+
const filename = path.split("/").pop()?.replace(/\.[^.]+$/, "").toLowerCase();
|
|
1257
|
+
if (filename && filename !== titleKey) {
|
|
1258
|
+
const existing = index.get(filename) ?? [];
|
|
1259
|
+
existing.push(node.id);
|
|
1260
|
+
index.set(filename, existing);
|
|
656
1261
|
}
|
|
657
|
-
seen.add(target);
|
|
658
|
-
graph.addDirectedEdge(node.id, target);
|
|
659
1262
|
}
|
|
660
1263
|
}
|
|
661
|
-
|
|
1264
|
+
for (const ids of index.values()) {
|
|
1265
|
+
ids.sort();
|
|
1266
|
+
}
|
|
1267
|
+
return index;
|
|
1268
|
+
}
|
|
1269
|
+
function resolveLinks(outgoingLinks, filenameIndex, validNodeIds) {
|
|
1270
|
+
return outgoingLinks.map((link) => {
|
|
1271
|
+
if (validNodeIds.has(link)) {
|
|
1272
|
+
return link;
|
|
1273
|
+
}
|
|
1274
|
+
if (link.includes("/")) {
|
|
1275
|
+
return link;
|
|
1276
|
+
}
|
|
1277
|
+
const lookupKey = link.replace(/\.md$/i, "").toLowerCase();
|
|
1278
|
+
const matches = filenameIndex.get(lookupKey);
|
|
1279
|
+
if (matches && matches.length > 0) {
|
|
1280
|
+
return matches[0];
|
|
1281
|
+
}
|
|
1282
|
+
const variant = spaceDashVariant(lookupKey);
|
|
1283
|
+
if (variant) {
|
|
1284
|
+
const variantMatches = filenameIndex.get(variant);
|
|
1285
|
+
if (variantMatches && variantMatches.length > 0) {
|
|
1286
|
+
return variantMatches[0];
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
return link;
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
function spaceDashVariant(filename) {
|
|
1293
|
+
const hasSpace = filename.includes(" ");
|
|
1294
|
+
const hasDash = filename.includes("-");
|
|
1295
|
+
if (hasSpace && !hasDash) {
|
|
1296
|
+
return filename.replace(/ /g, "-");
|
|
1297
|
+
}
|
|
1298
|
+
if (hasDash && !hasSpace) {
|
|
1299
|
+
return filename.replace(/-/g, " ");
|
|
1300
|
+
}
|
|
1301
|
+
return null;
|
|
662
1302
|
}
|
|
663
1303
|
|
|
664
|
-
// src/
|
|
665
|
-
import {
|
|
666
|
-
|
|
667
|
-
|
|
1304
|
+
// src/providers/docstore/file-operations.ts
|
|
1305
|
+
import { readFile, stat, readdir } from "fs/promises";
|
|
1306
|
+
import { join as join3, resolve, extname as extname2 } from "path";
|
|
1307
|
+
async function getFileMtime(filePath) {
|
|
1308
|
+
const stats = await stat(filePath);
|
|
1309
|
+
return stats.mtimeMs;
|
|
1310
|
+
}
|
|
1311
|
+
function validatePathWithinSource(sourceRoot, id) {
|
|
1312
|
+
const resolvedPath = resolve(sourceRoot, id);
|
|
1313
|
+
const resolvedRoot = resolve(sourceRoot);
|
|
1314
|
+
if (!resolvedPath.startsWith(resolvedRoot + "/")) {
|
|
1315
|
+
throw new Error(`Path traversal detected: ${id} resolves outside source root`);
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
async function collectFiles(dir, extensions) {
|
|
1319
|
+
if (extensions.size === 0) {
|
|
668
1320
|
return [];
|
|
669
1321
|
}
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
|
|
1322
|
+
const results = [];
|
|
1323
|
+
let entries;
|
|
1324
|
+
try {
|
|
1325
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
1326
|
+
} catch {
|
|
1327
|
+
return results;
|
|
1328
|
+
}
|
|
1329
|
+
for (const entry of entries) {
|
|
1330
|
+
const fullPath = join3(dir, entry.name);
|
|
1331
|
+
if (entry.isDirectory()) {
|
|
1332
|
+
if (EXCLUDED_DIRS.has(entry.name)) {
|
|
1333
|
+
continue;
|
|
1334
|
+
}
|
|
1335
|
+
const nested = await collectFiles(fullPath, extensions);
|
|
1336
|
+
results.push(...nested);
|
|
1337
|
+
} else if (entry.isFile()) {
|
|
1338
|
+
const ext = extname2(entry.name).toLowerCase();
|
|
1339
|
+
if (ext && extensions.has(ext)) {
|
|
1340
|
+
results.push(fullPath);
|
|
1341
|
+
}
|
|
688
1342
|
}
|
|
689
1343
|
}
|
|
690
|
-
return
|
|
1344
|
+
return results;
|
|
691
1345
|
}
|
|
692
|
-
function
|
|
693
|
-
|
|
694
|
-
|
|
1346
|
+
async function readFileContent(filePath) {
|
|
1347
|
+
return readFile(filePath, "utf-8");
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
// src/providers/docstore/id.ts
|
|
1351
|
+
import { nanoid } from "nanoid";
|
|
1352
|
+
var NANOID_PATTERN = /^[A-Za-z0-9_-]{12}$/;
|
|
1353
|
+
var isValidId = (id) => NANOID_PATTERN.test(id);
|
|
1354
|
+
var generateId = () => nanoid(12);
|
|
1355
|
+
|
|
1356
|
+
// src/providers/docstore/reader-registry.ts
|
|
1357
|
+
var ReaderRegistry = class {
|
|
1358
|
+
readers = /* @__PURE__ */ new Map();
|
|
1359
|
+
/**
|
|
1360
|
+
* Register a reader for its declared extensions.
|
|
1361
|
+
* Throws if any extension is already registered (atomic - no partial registration).
|
|
1362
|
+
*/
|
|
1363
|
+
register(reader) {
|
|
1364
|
+
for (const ext of reader.extensions) {
|
|
1365
|
+
const normalizedExt = ext.toLowerCase();
|
|
1366
|
+
if (this.readers.has(normalizedExt)) {
|
|
1367
|
+
throw new Error(`Extension already registered: ${ext}`);
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
for (const ext of reader.extensions) {
|
|
1371
|
+
const normalizedExt = ext.toLowerCase();
|
|
1372
|
+
this.readers.set(normalizedExt, reader);
|
|
1373
|
+
}
|
|
695
1374
|
}
|
|
696
|
-
|
|
697
|
-
|
|
1375
|
+
/**
|
|
1376
|
+
* Get reader for an extension, or null if none registered.
|
|
1377
|
+
* Case-insensitive.
|
|
1378
|
+
*/
|
|
1379
|
+
getReader(extension) {
|
|
1380
|
+
return this.readers.get(extension.toLowerCase()) ?? null;
|
|
698
1381
|
}
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
return [];
|
|
1382
|
+
/**
|
|
1383
|
+
* Get all registered extensions
|
|
1384
|
+
*/
|
|
1385
|
+
getExtensions() {
|
|
1386
|
+
return new Set(this.readers.keys());
|
|
705
1387
|
}
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
1388
|
+
/**
|
|
1389
|
+
* Check if an extension has a registered reader.
|
|
1390
|
+
* Case-insensitive.
|
|
1391
|
+
*/
|
|
1392
|
+
hasReader(extension) {
|
|
1393
|
+
return this.readers.has(extension.toLowerCase());
|
|
1394
|
+
}
|
|
1395
|
+
/**
|
|
1396
|
+
* Parse content using the appropriate reader for the file's extension.
|
|
1397
|
+
* Validates frontmatter ID and signals if writeback is needed.
|
|
1398
|
+
* Throws if no reader is registered for the extension.
|
|
1399
|
+
*
|
|
1400
|
+
* Note: Does NOT generate new IDs here - that happens in Phase 3's writeback.
|
|
1401
|
+
* Files without valid frontmatter IDs keep their path-based ID for now,
|
|
1402
|
+
* with needsIdWrite: true signaling that an ID should be generated and written.
|
|
1403
|
+
*/
|
|
1404
|
+
parse(content, context) {
|
|
1405
|
+
const reader = this.getReader(context.extension);
|
|
1406
|
+
if (!reader) {
|
|
1407
|
+
throw new Error(`No reader registered for extension: ${context.extension}`);
|
|
719
1408
|
}
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
1409
|
+
const node = reader.parse(content, context);
|
|
1410
|
+
const needsIdWrite = !isValidId(node.id);
|
|
1411
|
+
return { node, needsIdWrite };
|
|
1412
|
+
}
|
|
1413
|
+
};
|
|
1414
|
+
|
|
1415
|
+
// src/providers/docstore/readers/markdown.ts
|
|
1416
|
+
var MarkdownReader = class {
|
|
1417
|
+
extensions = [".md", ".markdown"];
|
|
1418
|
+
parse(content, context) {
|
|
1419
|
+
const parsed = parseMarkdown(content);
|
|
1420
|
+
const id = parsed.id ?? normalizeId(context.relativePath);
|
|
1421
|
+
const title = parsed.title ?? titleFromPath(id);
|
|
1422
|
+
const rawLinks = extractWikiLinks(parsed.content);
|
|
1423
|
+
const outgoingLinks = rawLinks.map((link) => normalizeWikiLink(link));
|
|
1424
|
+
return {
|
|
1425
|
+
id,
|
|
1426
|
+
title,
|
|
1427
|
+
content: parsed.content,
|
|
1428
|
+
tags: parsed.tags,
|
|
1429
|
+
outgoingLinks,
|
|
1430
|
+
properties: parsed.properties,
|
|
1431
|
+
sourceRef: {
|
|
1432
|
+
type: "file",
|
|
1433
|
+
path: context.absolutePath,
|
|
1434
|
+
lastModified: context.mtime
|
|
1435
|
+
}
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
};
|
|
735
1439
|
|
|
736
1440
|
// src/providers/docstore/index.ts
|
|
737
|
-
|
|
1441
|
+
function createDefaultRegistry() {
|
|
1442
|
+
const registry = new ReaderRegistry();
|
|
1443
|
+
registry.register(new MarkdownReader());
|
|
1444
|
+
return registry;
|
|
1445
|
+
}
|
|
1446
|
+
var DocStore = class extends StoreProvider {
|
|
1447
|
+
id;
|
|
738
1448
|
cache;
|
|
739
1449
|
sourceRoot;
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
watcher = null;
|
|
744
|
-
debounceTimer = null;
|
|
745
|
-
pendingChanges = /* @__PURE__ */ new Map();
|
|
1450
|
+
ownsVectorIndex;
|
|
1451
|
+
registry;
|
|
1452
|
+
fileWatcher = null;
|
|
746
1453
|
onChangeCallback;
|
|
747
|
-
constructor(
|
|
1454
|
+
constructor(options) {
|
|
1455
|
+
const {
|
|
1456
|
+
sourceRoot,
|
|
1457
|
+
cacheDir,
|
|
1458
|
+
id = "docstore",
|
|
1459
|
+
vectorIndex,
|
|
1460
|
+
registry,
|
|
1461
|
+
fileWatcher
|
|
1462
|
+
} = options;
|
|
1463
|
+
const ownsVector = !vectorIndex;
|
|
1464
|
+
if (!vectorIndex) mkdirSync2(cacheDir, { recursive: true });
|
|
1465
|
+
const vi = vectorIndex ?? new SqliteVectorIndex(cacheDir);
|
|
1466
|
+
super({ vectorIndex: vi });
|
|
1467
|
+
this.id = id;
|
|
748
1468
|
this.sourceRoot = sourceRoot;
|
|
749
1469
|
this.cache = new Cache(cacheDir);
|
|
750
|
-
this.
|
|
751
|
-
this.
|
|
1470
|
+
this.ownsVectorIndex = ownsVector;
|
|
1471
|
+
this.registry = registry ?? createDefaultRegistry();
|
|
1472
|
+
this.fileWatcher = fileWatcher ?? null;
|
|
752
1473
|
}
|
|
753
1474
|
async sync() {
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
1475
|
+
if (this.fileWatcher?.isWatching()) {
|
|
1476
|
+
this.fileWatcher.pause();
|
|
1477
|
+
}
|
|
1478
|
+
try {
|
|
1479
|
+
const extensions = this.registry.getExtensions();
|
|
1480
|
+
const currentPaths = await collectFiles(this.sourceRoot, extensions);
|
|
1481
|
+
const trackedPaths = this.cache.getAllTrackedPaths();
|
|
1482
|
+
const seenIds = /* @__PURE__ */ new Map();
|
|
1483
|
+
for (const filePath of currentPaths) {
|
|
1484
|
+
try {
|
|
1485
|
+
const mtime = await getFileMtime(filePath);
|
|
1486
|
+
const cachedMtime = this.cache.getModifiedTime(filePath);
|
|
1487
|
+
if (cachedMtime === null || mtime > cachedMtime) {
|
|
1488
|
+
const { node, needsIdWrite, newMtime } = await this.parseAndMaybeWriteId(filePath, mtime);
|
|
1489
|
+
const existingPath = seenIds.get(node.id);
|
|
1490
|
+
if (existingPath) {
|
|
1491
|
+
console.warn(
|
|
1492
|
+
`Duplicate ID ${node.id} found in ${filePath} (first seen in ${existingPath}):`,
|
|
1493
|
+
new Error("Skipping duplicate")
|
|
1494
|
+
);
|
|
1495
|
+
continue;
|
|
1496
|
+
}
|
|
1497
|
+
seenIds.set(node.id, filePath);
|
|
1498
|
+
const finalMtime = needsIdWrite ? newMtime ?? mtime : mtime;
|
|
1499
|
+
this.cache.upsertNode(node, "file", filePath, finalMtime);
|
|
1500
|
+
} else {
|
|
1501
|
+
const existingNode = this.cache.getNodeByPath(filePath);
|
|
1502
|
+
if (existingNode) {
|
|
1503
|
+
const existingPath = seenIds.get(existingNode.id);
|
|
1504
|
+
if (existingPath) {
|
|
1505
|
+
console.warn(
|
|
1506
|
+
`Duplicate ID ${existingNode.id} found in ${filePath} (first seen in ${existingPath}):`,
|
|
1507
|
+
new Error("Skipping duplicate")
|
|
1508
|
+
);
|
|
1509
|
+
this.cache.deleteNode(existingNode.id);
|
|
1510
|
+
} else {
|
|
1511
|
+
seenIds.set(existingNode.id, filePath);
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
} catch (err) {
|
|
1516
|
+
if (err.code === "ENOENT") {
|
|
1517
|
+
continue;
|
|
1518
|
+
}
|
|
1519
|
+
console.warn(`Failed to process file ${filePath}:`, err);
|
|
766
1520
|
continue;
|
|
767
1521
|
}
|
|
768
|
-
throw err;
|
|
769
1522
|
}
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
1523
|
+
const currentSet = new Set(currentPaths);
|
|
1524
|
+
for (const tracked of trackedPaths) {
|
|
1525
|
+
if (!currentSet.has(tracked)) {
|
|
1526
|
+
const node = this.cache.getNodeByPath(tracked);
|
|
1527
|
+
if (node) {
|
|
1528
|
+
this.cache.deleteNode(node.id);
|
|
1529
|
+
}
|
|
777
1530
|
}
|
|
778
1531
|
}
|
|
1532
|
+
this.resolveAllLinks();
|
|
1533
|
+
await this.syncGraph();
|
|
1534
|
+
} finally {
|
|
1535
|
+
if (this.fileWatcher?.isWatching()) {
|
|
1536
|
+
this.fileWatcher.resume();
|
|
1537
|
+
}
|
|
779
1538
|
}
|
|
780
|
-
const filenameIndex = this.buildFilenameIndex();
|
|
781
|
-
this.resolveOutgoingLinks(filenameIndex);
|
|
782
|
-
this.rebuildGraph();
|
|
783
1539
|
}
|
|
784
1540
|
async createNode(node) {
|
|
785
|
-
const
|
|
786
|
-
this.
|
|
787
|
-
const
|
|
788
|
-
if (
|
|
789
|
-
throw new Error(`Node already exists: ${
|
|
1541
|
+
const normalizedPath = normalizeId(node.id);
|
|
1542
|
+
validatePathWithinSource(this.sourceRoot, normalizedPath);
|
|
1543
|
+
const existingByPath = this.cache.getNodeByPath(join4(this.sourceRoot, normalizedPath));
|
|
1544
|
+
if (existingByPath) {
|
|
1545
|
+
throw new Error(`Node already exists: ${normalizedPath}`);
|
|
790
1546
|
}
|
|
791
|
-
const filePath =
|
|
1547
|
+
const filePath = join4(this.sourceRoot, normalizedPath);
|
|
792
1548
|
const dir = dirname(filePath);
|
|
793
1549
|
await mkdir(dir, { recursive: true });
|
|
1550
|
+
const stableId = generateId();
|
|
1551
|
+
const rawLinks = extractWikiLinks(node.content);
|
|
794
1552
|
const parsed = {
|
|
1553
|
+
id: stableId,
|
|
795
1554
|
title: node.title,
|
|
796
1555
|
tags: node.tags,
|
|
797
1556
|
properties: node.properties,
|
|
798
|
-
content: node.content
|
|
1557
|
+
content: node.content,
|
|
1558
|
+
rawLinks
|
|
799
1559
|
};
|
|
800
1560
|
const markdown = serializeToMarkdown(parsed);
|
|
801
1561
|
await writeFile(filePath, markdown, "utf-8");
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
1562
|
+
let outgoingLinks = node.outgoingLinks;
|
|
1563
|
+
if (node.content && (!outgoingLinks || outgoingLinks.length === 0)) {
|
|
1564
|
+
outgoingLinks = rawLinks.map((link) => normalizeWikiLink(link));
|
|
1565
|
+
}
|
|
1566
|
+
const mtime = await getFileMtime(filePath);
|
|
1567
|
+
const createdNode = {
|
|
1568
|
+
...node,
|
|
1569
|
+
id: stableId,
|
|
1570
|
+
outgoingLinks,
|
|
1571
|
+
sourceRef: {
|
|
1572
|
+
type: "file",
|
|
1573
|
+
path: filePath,
|
|
1574
|
+
lastModified: new Date(mtime)
|
|
1575
|
+
}
|
|
1576
|
+
};
|
|
1577
|
+
this.cache.upsertNode(createdNode, "file", filePath, mtime);
|
|
1578
|
+
this.resolveAllLinks();
|
|
1579
|
+
await this.syncGraph();
|
|
806
1580
|
}
|
|
807
1581
|
async updateNode(id, updates) {
|
|
808
|
-
|
|
809
|
-
const existing = this.cache.getNode(normalizedId);
|
|
1582
|
+
let existing = this.cache.getNode(id);
|
|
810
1583
|
if (!existing) {
|
|
811
|
-
|
|
1584
|
+
const normalizedId = normalizeId(id);
|
|
1585
|
+
existing = this.cache.getNode(normalizedId);
|
|
812
1586
|
}
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
1587
|
+
if (!existing && (id.includes(".") || id.includes("/"))) {
|
|
1588
|
+
const fullPath = join4(this.sourceRoot, normalizeId(id));
|
|
1589
|
+
existing = this.cache.getNodeByPath(fullPath);
|
|
1590
|
+
}
|
|
1591
|
+
if (!existing) {
|
|
1592
|
+
throw new Error(`Node not found: ${id}`);
|
|
817
1593
|
}
|
|
1594
|
+
const { ...safeUpdates } = updates;
|
|
1595
|
+
const contentForLinks = safeUpdates.content ?? existing.content;
|
|
1596
|
+
const rawLinks = extractWikiLinks(contentForLinks);
|
|
1597
|
+
const outgoingLinks = rawLinks.map((link) => normalizeWikiLink(link));
|
|
818
1598
|
const updated = {
|
|
819
1599
|
...existing,
|
|
820
|
-
...
|
|
821
|
-
outgoingLinks
|
|
1600
|
+
...safeUpdates,
|
|
1601
|
+
outgoingLinks,
|
|
822
1602
|
id: existing.id
|
|
823
|
-
//
|
|
1603
|
+
// Preserve original ID
|
|
824
1604
|
};
|
|
825
|
-
const filePath =
|
|
1605
|
+
const filePath = existing.sourceRef?.path ?? join4(this.sourceRoot, existing.id);
|
|
826
1606
|
const parsed = {
|
|
1607
|
+
id: existing.id,
|
|
1608
|
+
// Write the stable ID back to frontmatter
|
|
827
1609
|
title: updated.title,
|
|
828
1610
|
tags: updated.tags,
|
|
829
1611
|
properties: updated.properties,
|
|
830
|
-
content: updated.content
|
|
1612
|
+
content: updated.content,
|
|
1613
|
+
rawLinks
|
|
831
1614
|
};
|
|
832
1615
|
const markdown = serializeToMarkdown(parsed);
|
|
833
1616
|
await writeFile(filePath, markdown, "utf-8");
|
|
834
|
-
const mtime = await
|
|
1617
|
+
const mtime = await getFileMtime(filePath);
|
|
835
1618
|
this.cache.upsertNode(updated, "file", filePath, mtime);
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
}
|
|
1619
|
+
this.resolveAllLinks();
|
|
1620
|
+
await this.syncGraph();
|
|
839
1621
|
}
|
|
840
1622
|
async deleteNode(id) {
|
|
841
|
-
|
|
842
|
-
|
|
1623
|
+
let existing = this.cache.getNode(id);
|
|
1624
|
+
if (!existing) {
|
|
1625
|
+
const normalizedId = normalizeId(id);
|
|
1626
|
+
existing = this.cache.getNode(normalizedId);
|
|
1627
|
+
}
|
|
1628
|
+
if (!existing && (id.includes(".") || id.includes("/"))) {
|
|
1629
|
+
const fullPath = join4(this.sourceRoot, normalizeId(id));
|
|
1630
|
+
existing = this.cache.getNodeByPath(fullPath);
|
|
1631
|
+
}
|
|
843
1632
|
if (!existing) {
|
|
844
1633
|
throw new Error(`Node not found: ${id}`);
|
|
845
1634
|
}
|
|
846
|
-
const filePath =
|
|
1635
|
+
const filePath = existing.sourceRef?.path ?? join4(this.sourceRoot, existing.id);
|
|
847
1636
|
await rm(filePath);
|
|
848
1637
|
this.cache.deleteNode(existing.id);
|
|
849
|
-
await this.
|
|
850
|
-
this.
|
|
1638
|
+
if (this.vectorIndex) await this.vectorIndex.delete(existing.id);
|
|
1639
|
+
await this.syncGraph();
|
|
851
1640
|
}
|
|
852
1641
|
async getNode(id) {
|
|
1642
|
+
let node = this.cache.getNode(id);
|
|
1643
|
+
if (node) return node;
|
|
853
1644
|
const normalizedId = normalizeId(id);
|
|
854
|
-
|
|
1645
|
+
if (normalizedId !== id) {
|
|
1646
|
+
node = this.cache.getNode(normalizedId);
|
|
1647
|
+
if (node) return node;
|
|
1648
|
+
}
|
|
1649
|
+
if (id.includes(".") || id.includes("/")) {
|
|
1650
|
+
const fullPath = join4(this.sourceRoot, normalizedId);
|
|
1651
|
+
node = this.cache.getNodeByPath(fullPath);
|
|
1652
|
+
}
|
|
1653
|
+
return node;
|
|
855
1654
|
}
|
|
856
1655
|
async getNodes(ids) {
|
|
857
|
-
const
|
|
858
|
-
|
|
1656
|
+
const results = [];
|
|
1657
|
+
for (const id of ids) {
|
|
1658
|
+
const node = await this.getNode(id);
|
|
1659
|
+
if (node) results.push(node);
|
|
1660
|
+
}
|
|
1661
|
+
return results;
|
|
859
1662
|
}
|
|
860
1663
|
async getAllNodeIds() {
|
|
861
1664
|
const nodes = this.cache.getAllNodes();
|
|
862
1665
|
return nodes.map((n) => n.id);
|
|
863
1666
|
}
|
|
864
|
-
async searchByTags(tags, mode) {
|
|
865
|
-
return this.cache.searchByTags(tags, mode);
|
|
866
|
-
}
|
|
867
|
-
async getRandomNode(tags) {
|
|
868
|
-
let candidates;
|
|
869
|
-
if (tags && tags.length > 0) {
|
|
870
|
-
candidates = await this.searchByTags(tags, "any");
|
|
871
|
-
} else {
|
|
872
|
-
candidates = this.cache.getAllNodes();
|
|
873
|
-
}
|
|
874
|
-
if (candidates.length === 0) {
|
|
875
|
-
return null;
|
|
876
|
-
}
|
|
877
|
-
const randomIndex = Math.floor(Math.random() * candidates.length);
|
|
878
|
-
return candidates[randomIndex];
|
|
1667
|
+
async searchByTags(tags, mode, limit) {
|
|
1668
|
+
return this.cache.searchByTags(tags, mode, limit);
|
|
879
1669
|
}
|
|
880
1670
|
async resolveTitles(ids) {
|
|
881
1671
|
return this.cache.resolveTitles(ids);
|
|
@@ -891,267 +1681,209 @@ var DocStore = class _DocStore {
|
|
|
891
1681
|
return names.map((query) => ({ query, match: null, score: 0 }));
|
|
892
1682
|
}
|
|
893
1683
|
async nodesExist(ids) {
|
|
894
|
-
const
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
return this.cache.getNodes(neighborIds);
|
|
901
|
-
}
|
|
902
|
-
async findPath(source, target) {
|
|
903
|
-
this.ensureGraph();
|
|
904
|
-
return findPath(this.graph, source, target);
|
|
905
|
-
}
|
|
906
|
-
async getHubs(metric, limit) {
|
|
907
|
-
this.ensureGraph();
|
|
908
|
-
return getHubs(this.graph, metric, limit);
|
|
909
|
-
}
|
|
910
|
-
async storeEmbedding(id, vector, model) {
|
|
911
|
-
return this.vectorProvider.store(id, vector, model);
|
|
912
|
-
}
|
|
913
|
-
async searchByVector(vector, limit) {
|
|
914
|
-
return this.vectorProvider.search(vector, limit);
|
|
1684
|
+
const result = /* @__PURE__ */ new Map();
|
|
1685
|
+
for (const id of ids) {
|
|
1686
|
+
const node = await this.getNode(id);
|
|
1687
|
+
result.set(normalizeId(id), node !== null);
|
|
1688
|
+
}
|
|
1689
|
+
return result;
|
|
915
1690
|
}
|
|
916
1691
|
hasEmbedding(id) {
|
|
917
|
-
|
|
1692
|
+
if (!this.vectorIndex) return false;
|
|
1693
|
+
return this.vectorIndex.hasEmbedding(id);
|
|
918
1694
|
}
|
|
919
1695
|
close() {
|
|
920
1696
|
this.stopWatching();
|
|
921
1697
|
this.cache.close();
|
|
922
|
-
if (this.
|
|
923
|
-
this.
|
|
1698
|
+
if (this.ownsVectorIndex && this.vectorIndex && "close" in this.vectorIndex) {
|
|
1699
|
+
this.vectorIndex.close();
|
|
924
1700
|
}
|
|
925
1701
|
}
|
|
1702
|
+
// Lifecycle hooks
|
|
1703
|
+
async onRegister() {
|
|
1704
|
+
await this.sync();
|
|
1705
|
+
}
|
|
1706
|
+
async onUnregister() {
|
|
1707
|
+
this.close();
|
|
1708
|
+
}
|
|
926
1709
|
startWatching(onChange) {
|
|
927
|
-
if (this.
|
|
1710
|
+
if (this.fileWatcher?.isWatching()) {
|
|
928
1711
|
throw new Error("Already watching. Call stopWatching() first.");
|
|
929
1712
|
}
|
|
930
1713
|
this.onChangeCallback = onChange;
|
|
931
|
-
|
|
932
|
-
this.
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
stabilityThreshold: 100
|
|
937
|
-
},
|
|
938
|
-
followSymlinks: false
|
|
939
|
-
});
|
|
940
|
-
this.watcher.on("ready", () => resolve2()).on("add", (path) => this.queueChange(path, "add")).on("change", (path) => this.queueChange(path, "change")).on("unlink", (path) => this.queueChange(path, "unlink")).on("error", (err) => {
|
|
941
|
-
if (err.code === "EMFILE") {
|
|
942
|
-
console.error(
|
|
943
|
-
"File watcher hit file descriptor limit. Try: ulimit -n 65536 or reduce watched files."
|
|
944
|
-
);
|
|
945
|
-
}
|
|
946
|
-
reject(err);
|
|
1714
|
+
if (!this.fileWatcher) {
|
|
1715
|
+
this.fileWatcher = new FileWatcher({
|
|
1716
|
+
root: this.sourceRoot,
|
|
1717
|
+
extensions: this.registry.getExtensions(),
|
|
1718
|
+
onBatch: (events) => this.handleWatcherBatch(events)
|
|
947
1719
|
});
|
|
948
|
-
}
|
|
1720
|
+
}
|
|
1721
|
+
return this.fileWatcher.start();
|
|
949
1722
|
}
|
|
950
1723
|
stopWatching() {
|
|
951
|
-
if (this.
|
|
952
|
-
|
|
953
|
-
this.debounceTimer = null;
|
|
954
|
-
}
|
|
955
|
-
this.pendingChanges.clear();
|
|
956
|
-
if (this.watcher) {
|
|
957
|
-
this.watcher.close();
|
|
958
|
-
this.watcher = null;
|
|
1724
|
+
if (this.fileWatcher) {
|
|
1725
|
+
this.fileWatcher.stop();
|
|
959
1726
|
}
|
|
960
1727
|
}
|
|
961
1728
|
isWatching() {
|
|
962
|
-
return this.
|
|
963
|
-
}
|
|
964
|
-
queueChange(filePath, event) {
|
|
965
|
-
const relativePath = relative(this.sourceRoot, filePath);
|
|
966
|
-
const id = normalizeId(relativePath);
|
|
967
|
-
if (!filePath.endsWith(".md")) {
|
|
968
|
-
return;
|
|
969
|
-
}
|
|
970
|
-
const pathParts = relativePath.split("/");
|
|
971
|
-
for (const part of pathParts) {
|
|
972
|
-
if (_DocStore.EXCLUDED_DIRS.has(part)) {
|
|
973
|
-
return;
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
const existing = this.pendingChanges.get(id);
|
|
977
|
-
if (existing) {
|
|
978
|
-
if (existing === "add" && event === "change") {
|
|
979
|
-
return;
|
|
980
|
-
} else if (existing === "add" && event === "unlink") {
|
|
981
|
-
this.pendingChanges.delete(id);
|
|
982
|
-
} else if (existing === "change" && event === "unlink") {
|
|
983
|
-
this.pendingChanges.set(id, "unlink");
|
|
984
|
-
}
|
|
985
|
-
} else {
|
|
986
|
-
this.pendingChanges.set(id, event);
|
|
987
|
-
}
|
|
988
|
-
if (this.debounceTimer) {
|
|
989
|
-
clearTimeout(this.debounceTimer);
|
|
990
|
-
}
|
|
991
|
-
this.debounceTimer = setTimeout(() => {
|
|
992
|
-
this.processQueue();
|
|
993
|
-
}, 1e3);
|
|
1729
|
+
return this.fileWatcher?.isWatching() ?? false;
|
|
994
1730
|
}
|
|
995
|
-
async
|
|
996
|
-
|
|
997
|
-
this.pendingChanges.clear();
|
|
998
|
-
this.debounceTimer = null;
|
|
1731
|
+
async handleWatcherBatch(events) {
|
|
1732
|
+
this.fileWatcher?.pause();
|
|
999
1733
|
const processedIds = [];
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
if (
|
|
1005
|
-
this.cache.
|
|
1006
|
-
|
|
1007
|
-
|
|
1734
|
+
try {
|
|
1735
|
+
for (const [pathId, event] of events) {
|
|
1736
|
+
const filePath = join4(this.sourceRoot, pathId);
|
|
1737
|
+
try {
|
|
1738
|
+
if (event === "unlink") {
|
|
1739
|
+
const existing = this.cache.getNodeByPath(filePath);
|
|
1740
|
+
if (existing) {
|
|
1741
|
+
this.cache.deleteNode(existing.id);
|
|
1742
|
+
if (this.vectorIndex) {
|
|
1743
|
+
try {
|
|
1744
|
+
await this.vectorIndex.delete(existing.id);
|
|
1745
|
+
} catch (vectorErr) {
|
|
1746
|
+
console.warn(`Vector delete failed for ${pathId}:`, vectorErr);
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
processedIds.push(existing.id);
|
|
1750
|
+
}
|
|
1751
|
+
} else {
|
|
1752
|
+
const mtime = await getFileMtime(filePath);
|
|
1753
|
+
const { node, newMtime } = await this.parseAndMaybeWriteId(filePath, mtime);
|
|
1754
|
+
const finalMtime = newMtime ?? mtime;
|
|
1755
|
+
const existingByPath = this.cache.getNodeByPath(filePath);
|
|
1756
|
+
if (existingByPath && existingByPath.id !== node.id) {
|
|
1757
|
+
this.cache.deleteNode(existingByPath.id);
|
|
1758
|
+
if (this.vectorIndex) {
|
|
1759
|
+
try {
|
|
1760
|
+
await this.vectorIndex.delete(existingByPath.id);
|
|
1761
|
+
} catch {
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
this.cache.upsertNode(node, "file", filePath, finalMtime);
|
|
1766
|
+
processedIds.push(node.id);
|
|
1008
1767
|
}
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
const node = await this.fileToNode(filePath);
|
|
1012
|
-
const mtime = await this.getFileMtime(filePath);
|
|
1013
|
-
this.cache.upsertNode(node, "file", filePath, mtime);
|
|
1014
|
-
processedIds.push(id);
|
|
1768
|
+
} catch (err) {
|
|
1769
|
+
console.warn(`Failed to process file change for ${pathId}:`, err);
|
|
1015
1770
|
}
|
|
1016
|
-
} catch (err) {
|
|
1017
|
-
console.warn(`Failed to process file change for ${id}:`, err);
|
|
1018
1771
|
}
|
|
1772
|
+
if (processedIds.length > 0) {
|
|
1773
|
+
this.resolveAllLinks();
|
|
1774
|
+
await this.syncGraph();
|
|
1775
|
+
}
|
|
1776
|
+
if (this.onChangeCallback && processedIds.length > 0) {
|
|
1777
|
+
this.onChangeCallback(processedIds);
|
|
1778
|
+
}
|
|
1779
|
+
} finally {
|
|
1780
|
+
this.fileWatcher?.resume();
|
|
1019
1781
|
}
|
|
1020
|
-
if (processedIds.length > 0) {
|
|
1021
|
-
const filenameIndex = this.buildFilenameIndex();
|
|
1022
|
-
this.resolveOutgoingLinks(filenameIndex);
|
|
1023
|
-
this.rebuildGraph();
|
|
1024
|
-
}
|
|
1025
|
-
if (this.onChangeCallback && processedIds.length > 0) {
|
|
1026
|
-
this.onChangeCallback(processedIds);
|
|
1027
|
-
}
|
|
1028
|
-
}
|
|
1029
|
-
buildFilenameIndex() {
|
|
1030
|
-
const index = /* @__PURE__ */ new Map();
|
|
1031
|
-
for (const node of this.cache.getAllNodes()) {
|
|
1032
|
-
const basename = node.id.split("/").pop();
|
|
1033
|
-
const existing = index.get(basename) ?? [];
|
|
1034
|
-
existing.push(node.id);
|
|
1035
|
-
index.set(basename, existing);
|
|
1036
|
-
}
|
|
1037
|
-
for (const paths of index.values()) {
|
|
1038
|
-
paths.sort();
|
|
1039
|
-
}
|
|
1040
|
-
return index;
|
|
1041
1782
|
}
|
|
1042
|
-
|
|
1043
|
-
const
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1783
|
+
resolveAllLinks() {
|
|
1784
|
+
const nodes = this.cache.getAllNodes();
|
|
1785
|
+
const filenameIndex = buildFilenameIndex(nodes);
|
|
1786
|
+
const validNodeIds = new Set(nodes.map((n) => n.id));
|
|
1787
|
+
const pathToId = /* @__PURE__ */ new Map();
|
|
1788
|
+
for (const node of nodes) {
|
|
1789
|
+
if (node.sourceRef?.path) {
|
|
1790
|
+
const relativePath = relative2(this.sourceRoot, node.sourceRef.path);
|
|
1791
|
+
const normalizedPath = normalizeId(relativePath);
|
|
1792
|
+
pathToId.set(normalizedPath, node.id);
|
|
1047
1793
|
}
|
|
1048
1794
|
}
|
|
1049
|
-
for (const node of
|
|
1050
|
-
const
|
|
1795
|
+
for (const node of nodes) {
|
|
1796
|
+
const resolvedIds = resolveLinks(
|
|
1797
|
+
node.outgoingLinks,
|
|
1798
|
+
filenameIndex,
|
|
1799
|
+
validNodeIds
|
|
1800
|
+
);
|
|
1801
|
+
const finalIds = resolvedIds.map((link) => {
|
|
1051
1802
|
if (validNodeIds.has(link)) {
|
|
1052
1803
|
return link;
|
|
1053
1804
|
}
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
}
|
|
1057
|
-
const matches = filenameIndex.get(link);
|
|
1058
|
-
if (matches && matches.length > 0) {
|
|
1059
|
-
return matches[0];
|
|
1060
|
-
}
|
|
1061
|
-
return link;
|
|
1805
|
+
const stableId = pathToId.get(link);
|
|
1806
|
+
return stableId ?? link;
|
|
1062
1807
|
});
|
|
1063
|
-
if (
|
|
1064
|
-
this.cache.updateOutgoingLinks(node.id,
|
|
1808
|
+
if (finalIds.some((r, i) => r !== node.outgoingLinks[i])) {
|
|
1809
|
+
this.cache.updateOutgoingLinks(node.id, finalIds);
|
|
1065
1810
|
}
|
|
1066
1811
|
}
|
|
1067
1812
|
}
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1813
|
+
// ── Graph operations (override for path-based lookup) ─────
|
|
1814
|
+
async getNeighbors(id, options) {
|
|
1815
|
+
const node = await this.getNode(id);
|
|
1816
|
+
if (!node) return [];
|
|
1817
|
+
return super.getNeighbors(node.id, options);
|
|
1072
1818
|
}
|
|
1073
|
-
|
|
1074
|
-
const
|
|
1075
|
-
|
|
1076
|
-
|
|
1819
|
+
async findPath(source, target) {
|
|
1820
|
+
const sourceNode = await this.getNode(source);
|
|
1821
|
+
const targetNode = await this.getNode(target);
|
|
1822
|
+
if (!sourceNode || !targetNode) return null;
|
|
1823
|
+
return super.findPath(sourceNode.id, targetNode.id);
|
|
1824
|
+
}
|
|
1825
|
+
async getHubs(metric, limit) {
|
|
1826
|
+
return super.getHubs(metric, limit);
|
|
1827
|
+
}
|
|
1828
|
+
// ── StoreProvider abstract method implementations ─────────
|
|
1829
|
+
async loadAllNodes() {
|
|
1830
|
+
return this.cache.getAllNodes();
|
|
1831
|
+
}
|
|
1832
|
+
async getNodesByIds(ids) {
|
|
1833
|
+
return this.cache.getNodes(ids);
|
|
1834
|
+
}
|
|
1835
|
+
onCentralityComputed(centrality) {
|
|
1077
1836
|
const now = Date.now();
|
|
1078
1837
|
for (const [id, metrics] of centrality) {
|
|
1079
1838
|
this.cache.storeCentrality(id, 0, metrics.inDegree, metrics.outDegree, now);
|
|
1080
1839
|
}
|
|
1081
1840
|
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
results.push(fullPath);
|
|
1101
|
-
}
|
|
1841
|
+
/**
|
|
1842
|
+
* Parse a file and optionally write a generated ID back if missing.
|
|
1843
|
+
* Returns the node (with stable ID) and whether a write occurred.
|
|
1844
|
+
*/
|
|
1845
|
+
async parseAndMaybeWriteId(filePath, originalMtime) {
|
|
1846
|
+
const content = await readFileContent(filePath);
|
|
1847
|
+
const relativePath = relative2(this.sourceRoot, filePath);
|
|
1848
|
+
const ext = extname3(filePath).toLowerCase();
|
|
1849
|
+
const actualMtime = new Date(originalMtime);
|
|
1850
|
+
const context = {
|
|
1851
|
+
absolutePath: filePath,
|
|
1852
|
+
relativePath,
|
|
1853
|
+
extension: ext,
|
|
1854
|
+
mtime: actualMtime
|
|
1855
|
+
};
|
|
1856
|
+
const { node, needsIdWrite } = this.registry.parse(content, context);
|
|
1857
|
+
if (!needsIdWrite) {
|
|
1858
|
+
return { node, needsIdWrite: false };
|
|
1102
1859
|
}
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
const relativePath = relative(this.sourceRoot, filePath);
|
|
1113
|
-
const id = normalizeId(relativePath);
|
|
1114
|
-
const title = parsed.title ?? titleFromPath(id);
|
|
1115
|
-
const rawLinks = extractWikiLinks(parsed.content);
|
|
1116
|
-
const outgoingLinks = rawLinks.map((link) => this.normalizeWikiLink(link));
|
|
1117
|
-
return {
|
|
1118
|
-
id,
|
|
1119
|
-
title,
|
|
1120
|
-
content: parsed.content,
|
|
1121
|
-
tags: parsed.tags,
|
|
1122
|
-
outgoingLinks,
|
|
1123
|
-
properties: parsed.properties,
|
|
1124
|
-
sourceRef: {
|
|
1125
|
-
type: "file",
|
|
1126
|
-
path: filePath,
|
|
1127
|
-
lastModified: new Date(await this.getFileMtime(filePath))
|
|
1128
|
-
}
|
|
1860
|
+
const newId = generateId();
|
|
1861
|
+
const writebackSuccess = await this.writeIdBack(filePath, newId, originalMtime, content);
|
|
1862
|
+
if (!writebackSuccess) {
|
|
1863
|
+
console.warn(`File modified during sync, skipping ID writeback: ${filePath}`);
|
|
1864
|
+
return { node, needsIdWrite: true };
|
|
1865
|
+
}
|
|
1866
|
+
const updatedNode = {
|
|
1867
|
+
...node,
|
|
1868
|
+
id: newId
|
|
1129
1869
|
};
|
|
1870
|
+
const newMtime = await getFileMtime(filePath);
|
|
1871
|
+
return { node: updatedNode, needsIdWrite: true, newMtime };
|
|
1130
1872
|
}
|
|
1131
1873
|
/**
|
|
1132
|
-
*
|
|
1133
|
-
*
|
|
1134
|
-
* - If no extension, add .md
|
|
1135
|
-
* - Lowercase, forward slashes
|
|
1874
|
+
* Write a generated ID back to file's frontmatter.
|
|
1875
|
+
* Returns false if file was modified since originalMtime (race condition).
|
|
1136
1876
|
*/
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
if (
|
|
1140
|
-
|
|
1141
|
-
}
|
|
1142
|
-
return normalized;
|
|
1143
|
-
}
|
|
1144
|
-
hasFileExtension(path) {
|
|
1145
|
-
const match = path.match(/\.([a-z0-9]{1,4})$/i);
|
|
1146
|
-
if (!match?.[1]) return false;
|
|
1147
|
-
return /[a-z]/i.test(match[1]);
|
|
1148
|
-
}
|
|
1149
|
-
validatePathWithinSource(id) {
|
|
1150
|
-
const resolvedPath = resolve(this.sourceRoot, id);
|
|
1151
|
-
const resolvedRoot = resolve(this.sourceRoot);
|
|
1152
|
-
if (!resolvedPath.startsWith(resolvedRoot + "/")) {
|
|
1153
|
-
throw new Error(`Path traversal detected: ${id} resolves outside source root`);
|
|
1877
|
+
async writeIdBack(filePath, nodeId, originalMtime, originalContent) {
|
|
1878
|
+
const currentStat = await stat2(filePath);
|
|
1879
|
+
if (currentStat.mtimeMs !== originalMtime) {
|
|
1880
|
+
return false;
|
|
1154
1881
|
}
|
|
1882
|
+
const parsed = parseMarkdown(originalContent);
|
|
1883
|
+
parsed.id = nodeId;
|
|
1884
|
+
const newContent = serializeToMarkdown(parsed);
|
|
1885
|
+
await writeFile(filePath, newContent, "utf-8");
|
|
1886
|
+
return true;
|
|
1155
1887
|
}
|
|
1156
1888
|
};
|
|
1157
1889
|
|
|
@@ -1159,11 +1891,18 @@ var DocStore = class _DocStore {
|
|
|
1159
1891
|
import { pipeline } from "@xenova/transformers";
|
|
1160
1892
|
var DEFAULT_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
1161
1893
|
var DEFAULT_DIMENSIONS = 384;
|
|
1162
|
-
var
|
|
1894
|
+
var TransformersEmbedding = class {
|
|
1895
|
+
id;
|
|
1163
1896
|
model;
|
|
1164
1897
|
dims;
|
|
1165
1898
|
pipe = null;
|
|
1166
|
-
constructor(
|
|
1899
|
+
constructor(options = {}) {
|
|
1900
|
+
const {
|
|
1901
|
+
model = DEFAULT_MODEL,
|
|
1902
|
+
dimensions = DEFAULT_DIMENSIONS,
|
|
1903
|
+
id = "transformers-embedding"
|
|
1904
|
+
} = options;
|
|
1905
|
+
this.id = id;
|
|
1167
1906
|
this.model = model;
|
|
1168
1907
|
this.dims = dimensions;
|
|
1169
1908
|
}
|
|
@@ -1190,33 +1929,93 @@ var TransformersEmbeddingProvider = class {
|
|
|
1190
1929
|
modelId() {
|
|
1191
1930
|
return this.model;
|
|
1192
1931
|
}
|
|
1932
|
+
// Lifecycle hooks
|
|
1933
|
+
async onRegister() {
|
|
1934
|
+
}
|
|
1935
|
+
async onUnregister() {
|
|
1936
|
+
this.pipe = null;
|
|
1937
|
+
}
|
|
1193
1938
|
};
|
|
1194
1939
|
|
|
1195
1940
|
// src/core/graphcore.ts
|
|
1196
1941
|
var GraphCoreImpl = class _GraphCoreImpl {
|
|
1197
1942
|
store = null;
|
|
1198
1943
|
embedding = null;
|
|
1199
|
-
registerStore(provider) {
|
|
1944
|
+
async registerStore(provider) {
|
|
1200
1945
|
if (!provider) {
|
|
1201
1946
|
throw new Error("Store provider is required");
|
|
1202
1947
|
}
|
|
1948
|
+
if (!isStoreProvider(provider)) {
|
|
1949
|
+
throw new Error("Invalid Store provider: missing required methods or id");
|
|
1950
|
+
}
|
|
1951
|
+
if (this.store?.onUnregister) {
|
|
1952
|
+
try {
|
|
1953
|
+
await this.store.onUnregister();
|
|
1954
|
+
} catch (err) {
|
|
1955
|
+
console.warn("Error during store onUnregister:", err);
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1203
1958
|
this.store = provider;
|
|
1959
|
+
if (provider.onRegister) {
|
|
1960
|
+
try {
|
|
1961
|
+
await provider.onRegister();
|
|
1962
|
+
} catch (err) {
|
|
1963
|
+
this.store = null;
|
|
1964
|
+
throw err;
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1204
1967
|
}
|
|
1205
|
-
registerEmbedding(provider) {
|
|
1968
|
+
async registerEmbedding(provider) {
|
|
1206
1969
|
if (!provider) {
|
|
1207
1970
|
throw new Error("Embedding provider is required");
|
|
1208
1971
|
}
|
|
1972
|
+
if (!isEmbeddingProvider(provider)) {
|
|
1973
|
+
throw new Error("Invalid Embedding provider: missing required methods or id");
|
|
1974
|
+
}
|
|
1975
|
+
if (this.embedding?.onUnregister) {
|
|
1976
|
+
try {
|
|
1977
|
+
await this.embedding.onUnregister();
|
|
1978
|
+
} catch (err) {
|
|
1979
|
+
console.warn("Error during embedding onUnregister:", err);
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1209
1982
|
this.embedding = provider;
|
|
1983
|
+
if (provider.onRegister) {
|
|
1984
|
+
try {
|
|
1985
|
+
await provider.onRegister();
|
|
1986
|
+
} catch (err) {
|
|
1987
|
+
this.embedding = null;
|
|
1988
|
+
throw err;
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
async destroy() {
|
|
1993
|
+
if (this.embedding?.onUnregister) {
|
|
1994
|
+
try {
|
|
1995
|
+
await this.embedding.onUnregister();
|
|
1996
|
+
} catch (err) {
|
|
1997
|
+
console.warn("Error during embedding onUnregister in destroy:", err);
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
if (this.store?.onUnregister) {
|
|
2001
|
+
try {
|
|
2002
|
+
await this.store.onUnregister();
|
|
2003
|
+
} catch (err) {
|
|
2004
|
+
console.warn("Error during store onUnregister in destroy:", err);
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
this.embedding = null;
|
|
2008
|
+
this.store = null;
|
|
1210
2009
|
}
|
|
1211
2010
|
requireStore() {
|
|
1212
2011
|
if (!this.store) {
|
|
1213
|
-
throw new Error("
|
|
2012
|
+
throw new Error("Store not registered");
|
|
1214
2013
|
}
|
|
1215
2014
|
return this.store;
|
|
1216
2015
|
}
|
|
1217
2016
|
requireEmbedding() {
|
|
1218
2017
|
if (!this.embedding) {
|
|
1219
|
-
throw new Error("
|
|
2018
|
+
throw new Error("Embedding not registered");
|
|
1220
2019
|
}
|
|
1221
2020
|
return this.embedding;
|
|
1222
2021
|
}
|
|
@@ -1309,11 +2108,7 @@ var GraphCoreImpl = class _GraphCoreImpl {
|
|
|
1309
2108
|
}
|
|
1310
2109
|
async searchByTags(tags, mode, limit) {
|
|
1311
2110
|
const store = this.requireStore();
|
|
1312
|
-
|
|
1313
|
-
if (limit !== void 0) {
|
|
1314
|
-
return results.slice(0, limit);
|
|
1315
|
-
}
|
|
1316
|
-
return results;
|
|
2111
|
+
return store.searchByTags(tags, mode, limit);
|
|
1317
2112
|
}
|
|
1318
2113
|
async getRandomNode(tags) {
|
|
1319
2114
|
const store = this.requireStore();
|
|
@@ -1327,7 +2122,7 @@ var GraphCoreImpl = class _GraphCoreImpl {
|
|
|
1327
2122
|
const strategy = options?.strategy ?? "fuzzy";
|
|
1328
2123
|
if (strategy === "semantic") {
|
|
1329
2124
|
if (!this.embedding) {
|
|
1330
|
-
throw new Error("Semantic resolution requires
|
|
2125
|
+
throw new Error("Semantic resolution requires Embedding");
|
|
1331
2126
|
}
|
|
1332
2127
|
const filter = {};
|
|
1333
2128
|
if (options?.tag) filter.tag = options.tag;
|
|
@@ -1354,7 +2149,7 @@ var GraphCoreImpl = class _GraphCoreImpl {
|
|
|
1354
2149
|
let bestScore = 0;
|
|
1355
2150
|
let bestMatch = null;
|
|
1356
2151
|
for (let cIdx = 0; cIdx < candidates.length; cIdx++) {
|
|
1357
|
-
const similarity =
|
|
2152
|
+
const similarity = cosineSimilarity(queryVector, candidateVectors[cIdx]);
|
|
1358
2153
|
if (similarity > bestScore) {
|
|
1359
2154
|
bestScore = similarity;
|
|
1360
2155
|
bestMatch = candidates[cIdx].id;
|
|
@@ -1368,58 +2163,53 @@ var GraphCoreImpl = class _GraphCoreImpl {
|
|
|
1368
2163
|
}
|
|
1369
2164
|
return store.resolveNodes(names, options);
|
|
1370
2165
|
}
|
|
1371
|
-
|
|
1372
|
-
let dotProduct = 0;
|
|
1373
|
-
let normA = 0;
|
|
1374
|
-
let normB = 0;
|
|
1375
|
-
for (let i = 0; i < a.length; i++) {
|
|
1376
|
-
dotProduct += a[i] * b[i];
|
|
1377
|
-
normA += a[i] * a[i];
|
|
1378
|
-
normB += b[i] * b[i];
|
|
1379
|
-
}
|
|
1380
|
-
if (normA === 0 || normB === 0) return 0;
|
|
1381
|
-
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
1382
|
-
}
|
|
1383
|
-
static fromConfig(config) {
|
|
2166
|
+
static async fromConfig(config) {
|
|
1384
2167
|
if (!config.providers?.store) {
|
|
1385
|
-
throw new Error("
|
|
2168
|
+
throw new Error("Store configuration is required");
|
|
1386
2169
|
}
|
|
1387
2170
|
const core = new _GraphCoreImpl();
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
2171
|
+
try {
|
|
2172
|
+
if (config.providers.store.type === "docstore") {
|
|
2173
|
+
const sourcePath = config.source?.path ?? ".";
|
|
2174
|
+
const cachePath = config.cache?.path ?? ".roux";
|
|
2175
|
+
const store = new DocStore({ sourceRoot: sourcePath, cacheDir: cachePath });
|
|
2176
|
+
await core.registerStore(store);
|
|
2177
|
+
} else {
|
|
2178
|
+
throw new Error(
|
|
2179
|
+
`Unsupported store provider type: ${config.providers.store.type}. Supported: docstore`
|
|
2180
|
+
);
|
|
2181
|
+
}
|
|
2182
|
+
const embeddingConfig = config.providers.embedding;
|
|
2183
|
+
if (!embeddingConfig || embeddingConfig.type === "local") {
|
|
2184
|
+
const model = embeddingConfig?.model;
|
|
2185
|
+
const embedding = new TransformersEmbedding(model ? { model } : {});
|
|
2186
|
+
await core.registerEmbedding(embedding);
|
|
2187
|
+
} else {
|
|
2188
|
+
throw new Error(
|
|
2189
|
+
`Unsupported embedding provider type: ${embeddingConfig.type}. Supported: local`
|
|
2190
|
+
);
|
|
2191
|
+
}
|
|
2192
|
+
return core;
|
|
2193
|
+
} catch (err) {
|
|
2194
|
+
await core.destroy();
|
|
2195
|
+
throw err;
|
|
1407
2196
|
}
|
|
1408
|
-
return core;
|
|
1409
2197
|
}
|
|
1410
2198
|
};
|
|
1411
2199
|
|
|
1412
2200
|
// src/index.ts
|
|
1413
|
-
var VERSION = "0.1.
|
|
2201
|
+
var VERSION = "0.1.3";
|
|
1414
2202
|
export {
|
|
1415
2203
|
DEFAULT_CONFIG,
|
|
1416
2204
|
DocStore,
|
|
1417
2205
|
GraphCoreImpl,
|
|
1418
|
-
|
|
1419
|
-
|
|
2206
|
+
SqliteVectorIndex,
|
|
2207
|
+
StoreProvider,
|
|
2208
|
+
TransformersEmbedding,
|
|
1420
2209
|
VERSION,
|
|
2210
|
+
createGuard,
|
|
1421
2211
|
isNode,
|
|
1422
2212
|
isSourceRef,
|
|
1423
|
-
|
|
2213
|
+
isVectorIndex
|
|
1424
2214
|
};
|
|
1425
2215
|
//# sourceMappingURL=index.js.map
|