@zabaca/lattice 1.0.11 → 1.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/graph-sync.md +27 -69
- package/commands/research.md +5 -22
- package/dist/main.js +2240 -1608
- package/package.json +5 -4
- package/commands/entity-extract.md +0 -177
package/dist/main.js
CHANGED
|
@@ -23,17 +23,350 @@ import { CommandFactory } from "nest-commander";
|
|
|
23
23
|
import { Module as Module5 } from "@nestjs/common";
|
|
24
24
|
import { ConfigModule as ConfigModule2 } from "@nestjs/config";
|
|
25
25
|
|
|
26
|
+
// src/commands/extract.command.ts
|
|
27
|
+
import { existsSync, readFileSync } from "fs";
|
|
28
|
+
import { resolve } from "path";
|
|
29
|
+
import { query as query2 } from "@anthropic-ai/claude-agent-sdk";
|
|
30
|
+
import { Injectable as Injectable2 } from "@nestjs/common";
|
|
31
|
+
import { Command, CommandRunner, Option } from "nest-commander";
|
|
32
|
+
|
|
33
|
+
// src/sync/entity-extractor.service.ts
|
|
34
|
+
import { readFile } from "fs/promises";
|
|
35
|
+
import {
|
|
36
|
+
createSdkMcpServer,
|
|
37
|
+
query,
|
|
38
|
+
tool
|
|
39
|
+
} from "@anthropic-ai/claude-agent-sdk";
|
|
40
|
+
import { Injectable, Logger } from "@nestjs/common";
|
|
41
|
+
import { z } from "zod";
|
|
42
|
+
function validateExtraction(input, _filePath) {
|
|
43
|
+
const errors = [];
|
|
44
|
+
const { entities, relationships } = input ?? {};
|
|
45
|
+
if (!entities || !Array.isArray(entities)) {
|
|
46
|
+
return ["Entities array is missing or invalid"];
|
|
47
|
+
}
|
|
48
|
+
if (!relationships || !Array.isArray(relationships)) {
|
|
49
|
+
return ["Relationships array is missing or invalid"];
|
|
50
|
+
}
|
|
51
|
+
const entityNames = new Set(entities.map((e) => e.name));
|
|
52
|
+
for (const rel of relationships) {
|
|
53
|
+
if (rel.source !== "this" && !entityNames.has(rel.source)) {
|
|
54
|
+
errors.push(`Relationship source "${rel.source}" not found in extracted entities`);
|
|
55
|
+
}
|
|
56
|
+
if (!entityNames.has(rel.target)) {
|
|
57
|
+
errors.push(`Relationship target "${rel.target}" not found in extracted entities`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return errors;
|
|
61
|
+
}
|
|
62
|
+
function createValidationServer(filePath) {
|
|
63
|
+
return createSdkMcpServer({
|
|
64
|
+
name: "entity-validator",
|
|
65
|
+
version: "1.0.0",
|
|
66
|
+
tools: [
|
|
67
|
+
tool("validate_extraction", "Validate your extracted entities and relationships. Call this to check your work before finishing.", {
|
|
68
|
+
entities: z.array(z.object({
|
|
69
|
+
name: z.string().min(1),
|
|
70
|
+
type: z.enum([
|
|
71
|
+
"Topic",
|
|
72
|
+
"Technology",
|
|
73
|
+
"Concept",
|
|
74
|
+
"Tool",
|
|
75
|
+
"Process",
|
|
76
|
+
"Person",
|
|
77
|
+
"Organization",
|
|
78
|
+
"Document"
|
|
79
|
+
]),
|
|
80
|
+
description: z.string().min(1)
|
|
81
|
+
})),
|
|
82
|
+
relationships: z.array(z.object({
|
|
83
|
+
source: z.string().min(1),
|
|
84
|
+
relation: z.enum(["REFERENCES"]),
|
|
85
|
+
target: z.string().min(1)
|
|
86
|
+
})),
|
|
87
|
+
summary: z.string().min(10)
|
|
88
|
+
}, async (args) => {
|
|
89
|
+
const errors = validateExtraction(args, filePath);
|
|
90
|
+
if (errors.length === 0) {
|
|
91
|
+
return {
|
|
92
|
+
content: [
|
|
93
|
+
{
|
|
94
|
+
type: "text",
|
|
95
|
+
text: "\u2713 Validation passed. Your extraction is correct."
|
|
96
|
+
}
|
|
97
|
+
]
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
content: [
|
|
102
|
+
{
|
|
103
|
+
type: "text",
|
|
104
|
+
text: `\u2717 Validation failed:
|
|
105
|
+
${errors.map((e) => `- ${e}`).join(`
|
|
106
|
+
`)}
|
|
107
|
+
|
|
108
|
+
Please fix these errors and call validate_extraction again.`
|
|
109
|
+
}
|
|
110
|
+
]
|
|
111
|
+
};
|
|
112
|
+
})
|
|
113
|
+
]
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
class EntityExtractorService {
|
|
118
|
+
logger = new Logger(EntityExtractorService.name);
|
|
119
|
+
lastExtractionTime = 0;
|
|
120
|
+
minIntervalMs = 500;
|
|
121
|
+
async extractFromDocument(filePath) {
|
|
122
|
+
await this.rateLimit();
|
|
123
|
+
try {
|
|
124
|
+
const content = await readFile(filePath, "utf-8");
|
|
125
|
+
const promptText = this.buildExtractionPrompt(filePath, content);
|
|
126
|
+
const validationServer = createValidationServer(filePath);
|
|
127
|
+
let lastValidExtraction = null;
|
|
128
|
+
for await (const message of query({
|
|
129
|
+
prompt: promptText,
|
|
130
|
+
options: {
|
|
131
|
+
maxTurns: 3,
|
|
132
|
+
model: "claude-haiku-4-5-20251001",
|
|
133
|
+
mcpServers: {
|
|
134
|
+
"entity-validator": validationServer
|
|
135
|
+
},
|
|
136
|
+
allowedTools: ["mcp__entity-validator__validate_extraction"],
|
|
137
|
+
permissionMode: "default"
|
|
138
|
+
}
|
|
139
|
+
})) {
|
|
140
|
+
if (message.type === "assistant") {
|
|
141
|
+
for (const block of message.message?.content ?? []) {
|
|
142
|
+
if (block.type === "tool_use" && block.name === "mcp__entity-validator__validate_extraction") {
|
|
143
|
+
const input = block.input;
|
|
144
|
+
const validationErrors = validateExtraction(input, filePath);
|
|
145
|
+
if (validationErrors.length === 0) {
|
|
146
|
+
lastValidExtraction = input;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
} else if (message.type === "result") {
|
|
151
|
+
if (lastValidExtraction) {
|
|
152
|
+
this.logger.debug(`Extraction for ${filePath} completed in ${message.duration_ms}ms`);
|
|
153
|
+
return this.buildSuccessResult(lastValidExtraction, filePath);
|
|
154
|
+
}
|
|
155
|
+
const errorReason = message.subtype === "error_max_turns" ? "Max turns reached without valid extraction" : message.subtype === "error_during_execution" ? "Error during execution" : `Extraction failed: ${message.subtype}`;
|
|
156
|
+
return {
|
|
157
|
+
entities: [],
|
|
158
|
+
relationships: [],
|
|
159
|
+
summary: "",
|
|
160
|
+
success: false,
|
|
161
|
+
error: errorReason
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
throw new Error("No result received from SDK");
|
|
166
|
+
} catch (error) {
|
|
167
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
168
|
+
this.logger.error(`Entity extraction failed for ${filePath}: ${errorMsg}`);
|
|
169
|
+
return {
|
|
170
|
+
entities: [],
|
|
171
|
+
relationships: [],
|
|
172
|
+
summary: "",
|
|
173
|
+
success: false,
|
|
174
|
+
error: errorMsg
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
async extractFromDocuments(filePaths, onProgress) {
|
|
179
|
+
const results = new Map;
|
|
180
|
+
for (let i = 0;i < filePaths.length; i++) {
|
|
181
|
+
const path = filePaths[i];
|
|
182
|
+
onProgress?.(i, filePaths.length, path);
|
|
183
|
+
const result = await this.extractFromDocument(path);
|
|
184
|
+
results.set(path, result);
|
|
185
|
+
}
|
|
186
|
+
return results;
|
|
187
|
+
}
|
|
188
|
+
async rateLimit() {
|
|
189
|
+
const now = Date.now();
|
|
190
|
+
const elapsed = now - this.lastExtractionTime;
|
|
191
|
+
if (elapsed < this.minIntervalMs) {
|
|
192
|
+
const waitTime = this.minIntervalMs - elapsed;
|
|
193
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
|
194
|
+
}
|
|
195
|
+
this.lastExtractionTime = Date.now();
|
|
196
|
+
}
|
|
197
|
+
buildExtractionPrompt(filePath, content) {
|
|
198
|
+
return `Analyze this markdown document and extract entities, relationships, and a summary.
|
|
199
|
+
|
|
200
|
+
File: ${filePath}
|
|
201
|
+
|
|
202
|
+
<document>
|
|
203
|
+
${content}
|
|
204
|
+
</document>
|
|
205
|
+
|
|
206
|
+
## Instructions
|
|
207
|
+
|
|
208
|
+
Extract the following and call the validation tool with EXACTLY this schema:
|
|
209
|
+
|
|
210
|
+
### 1. Entities (array of 3-10 objects)
|
|
211
|
+
Each entity must have:
|
|
212
|
+
- "name": string (entity name)
|
|
213
|
+
- "type": one of "Topic", "Technology", "Concept", "Tool", "Process", "Person", "Organization", "Document"
|
|
214
|
+
- "description": string (brief description)
|
|
215
|
+
|
|
216
|
+
### 2. Relationships (array of objects)
|
|
217
|
+
Each relationship must have:
|
|
218
|
+
- "source": "this" (for document-to-entity) or an entity name
|
|
219
|
+
- "relation": "REFERENCES" (IMPORTANT: use "relation", not "type")
|
|
220
|
+
- "target": an entity name from your entities list
|
|
221
|
+
|
|
222
|
+
### 3. Summary
|
|
223
|
+
A 50-100 word summary of the document's main purpose and key concepts.
|
|
224
|
+
|
|
225
|
+
## IMPORTANT: Validation Required
|
|
226
|
+
|
|
227
|
+
You MUST call mcp__entity-validator__validate_extraction tool.
|
|
228
|
+
Pass the three fields DIRECTLY as top-level arguments (NOT wrapped in an "extraction" object):
|
|
229
|
+
- entities: your entities array
|
|
230
|
+
- relationships: your relationships array
|
|
231
|
+
- summary: your summary string
|
|
232
|
+
|
|
233
|
+
Example tool call structure:
|
|
234
|
+
{
|
|
235
|
+
"entities": [...],
|
|
236
|
+
"relationships": [...],
|
|
237
|
+
"summary": "..."
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
If validation fails, fix the errors and call the tool again.
|
|
241
|
+
Only finish after validation passes.`;
|
|
242
|
+
}
|
|
243
|
+
buildSuccessResult(extraction, filePath) {
|
|
244
|
+
const relationships = extraction.relationships.map((rel) => ({
|
|
245
|
+
...rel,
|
|
246
|
+
source: rel.source === "this" ? filePath : rel.source
|
|
247
|
+
}));
|
|
248
|
+
return {
|
|
249
|
+
entities: extraction.entities,
|
|
250
|
+
relationships,
|
|
251
|
+
summary: extraction.summary,
|
|
252
|
+
success: true
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
EntityExtractorService = __legacyDecorateClassTS([
|
|
257
|
+
Injectable()
|
|
258
|
+
], EntityExtractorService);
|
|
259
|
+
|
|
260
|
+
// src/commands/extract.command.ts
|
|
261
|
+
class ExtractCommand extends CommandRunner {
|
|
262
|
+
entityExtractor;
|
|
263
|
+
constructor(entityExtractor) {
|
|
264
|
+
super();
|
|
265
|
+
this.entityExtractor = entityExtractor;
|
|
266
|
+
}
|
|
267
|
+
async run(inputs, options) {
|
|
268
|
+
const [filePath] = inputs;
|
|
269
|
+
if (!filePath) {
|
|
270
|
+
console.error("Error: File path is required");
|
|
271
|
+
console.error("Usage: lattice extract <file>");
|
|
272
|
+
process.exit(1);
|
|
273
|
+
}
|
|
274
|
+
const absolutePath = resolve(process.cwd(), filePath);
|
|
275
|
+
if (!existsSync(absolutePath)) {
|
|
276
|
+
console.error(`Error: File not found: ${absolutePath}`);
|
|
277
|
+
process.exit(1);
|
|
278
|
+
}
|
|
279
|
+
try {
|
|
280
|
+
console.error(`Extracting entities from: ${absolutePath}
|
|
281
|
+
`);
|
|
282
|
+
if (options.raw) {
|
|
283
|
+
await this.extractRaw(absolutePath);
|
|
284
|
+
} else {
|
|
285
|
+
const result = await this.entityExtractor.extractFromDocument(absolutePath);
|
|
286
|
+
const output = options.pretty ? JSON.stringify(result, null, 2) : JSON.stringify(result);
|
|
287
|
+
console.log(output);
|
|
288
|
+
process.exit(result.success ? 0 : 1);
|
|
289
|
+
}
|
|
290
|
+
} catch (error) {
|
|
291
|
+
console.error("Error:", error instanceof Error ? error.message : String(error));
|
|
292
|
+
process.exit(1);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
async extractRaw(filePath) {
|
|
296
|
+
const content = readFileSync(filePath, "utf-8");
|
|
297
|
+
const prompt = this.entityExtractor.buildExtractionPrompt(filePath, content);
|
|
298
|
+
console.error("--- Prompt ---");
|
|
299
|
+
console.error(`${prompt.substring(0, 500)}...
|
|
300
|
+
`);
|
|
301
|
+
console.error("--- Raw Response ---");
|
|
302
|
+
let rawResponse = "";
|
|
303
|
+
for await (const message of query2({
|
|
304
|
+
prompt,
|
|
305
|
+
options: {
|
|
306
|
+
maxTurns: 5,
|
|
307
|
+
model: "claude-haiku-4-5-20251001",
|
|
308
|
+
allowedTools: [],
|
|
309
|
+
permissionMode: "default"
|
|
310
|
+
}
|
|
311
|
+
})) {
|
|
312
|
+
if (message.type === "assistant" && message.message?.content) {
|
|
313
|
+
for (const block of message.message.content) {
|
|
314
|
+
if ("text" in block) {
|
|
315
|
+
rawResponse += block.text;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
console.log(rawResponse);
|
|
321
|
+
process.exit(0);
|
|
322
|
+
}
|
|
323
|
+
parsePretty() {
|
|
324
|
+
return true;
|
|
325
|
+
}
|
|
326
|
+
parseRaw() {
|
|
327
|
+
return true;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
__legacyDecorateClassTS([
|
|
331
|
+
Option({
|
|
332
|
+
flags: "-p, --pretty",
|
|
333
|
+
description: "Pretty-print JSON output"
|
|
334
|
+
}),
|
|
335
|
+
__legacyMetadataTS("design:type", Function),
|
|
336
|
+
__legacyMetadataTS("design:paramtypes", []),
|
|
337
|
+
__legacyMetadataTS("design:returntype", Boolean)
|
|
338
|
+
], ExtractCommand.prototype, "parsePretty", null);
|
|
339
|
+
__legacyDecorateClassTS([
|
|
340
|
+
Option({
|
|
341
|
+
flags: "-r, --raw",
|
|
342
|
+
description: "Show raw Claude response (for debugging parse errors)"
|
|
343
|
+
}),
|
|
344
|
+
__legacyMetadataTS("design:type", Function),
|
|
345
|
+
__legacyMetadataTS("design:paramtypes", []),
|
|
346
|
+
__legacyMetadataTS("design:returntype", Boolean)
|
|
347
|
+
], ExtractCommand.prototype, "parseRaw", null);
|
|
348
|
+
ExtractCommand = __legacyDecorateClassTS([
|
|
349
|
+
Injectable2(),
|
|
350
|
+
Command({
|
|
351
|
+
name: "extract",
|
|
352
|
+
description: "Extract entities from a document (debug tool)",
|
|
353
|
+
arguments: "<file>"
|
|
354
|
+
}),
|
|
355
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
356
|
+
typeof EntityExtractorService === "undefined" ? Object : EntityExtractorService
|
|
357
|
+
])
|
|
358
|
+
], ExtractCommand);
|
|
26
359
|
// src/commands/init.command.ts
|
|
27
|
-
import { existsSync as
|
|
360
|
+
import { existsSync as existsSync3, writeFileSync } from "fs";
|
|
28
361
|
import * as fs from "fs/promises";
|
|
29
362
|
import { homedir as homedir2 } from "os";
|
|
30
363
|
import * as path from "path";
|
|
31
364
|
import { fileURLToPath } from "url";
|
|
32
|
-
import { Injectable } from "@nestjs/common";
|
|
33
|
-
import { Command, CommandRunner } from "nest-commander";
|
|
365
|
+
import { Injectable as Injectable3 } from "@nestjs/common";
|
|
366
|
+
import { Command as Command2, CommandRunner as CommandRunner2 } from "nest-commander";
|
|
34
367
|
|
|
35
368
|
// src/utils/paths.ts
|
|
36
|
-
import { existsSync, mkdirSync } from "fs";
|
|
369
|
+
import { existsSync as existsSync2, mkdirSync } from "fs";
|
|
37
370
|
import { homedir } from "os";
|
|
38
371
|
import { join } from "path";
|
|
39
372
|
var latticeHomeOverride = null;
|
|
@@ -60,11 +393,11 @@ function getEnvPath() {
|
|
|
60
393
|
}
|
|
61
394
|
function ensureLatticeHome() {
|
|
62
395
|
const home = getLatticeHomeInternal();
|
|
63
|
-
if (!
|
|
396
|
+
if (!existsSync2(home)) {
|
|
64
397
|
mkdirSync(home, { recursive: true });
|
|
65
398
|
}
|
|
66
399
|
const docsPath = getDocsPath();
|
|
67
|
-
if (!
|
|
400
|
+
if (!existsSync2(docsPath)) {
|
|
68
401
|
mkdirSync(docsPath, { recursive: true });
|
|
69
402
|
}
|
|
70
403
|
}
|
|
@@ -74,12 +407,12 @@ var __filename2 = fileURLToPath(import.meta.url);
|
|
|
74
407
|
var __dirname2 = path.dirname(__filename2);
|
|
75
408
|
var COMMANDS = ["research.md", "graph-sync.md", "entity-extract.md"];
|
|
76
409
|
|
|
77
|
-
class InitCommand extends
|
|
410
|
+
class InitCommand extends CommandRunner2 {
|
|
78
411
|
async run(_inputs, _options) {
|
|
79
412
|
try {
|
|
80
413
|
ensureLatticeHome();
|
|
81
414
|
const envPath = getEnvPath();
|
|
82
|
-
if (!
|
|
415
|
+
if (!existsSync3(envPath)) {
|
|
83
416
|
writeFileSync(envPath, `# Lattice Configuration
|
|
84
417
|
# Get your API key from: https://www.voyageai.com/
|
|
85
418
|
|
|
@@ -158,1504 +491,1224 @@ VOYAGE_API_KEY=
|
|
|
158
491
|
}
|
|
159
492
|
}
|
|
160
493
|
InitCommand = __legacyDecorateClassTS([
|
|
161
|
-
|
|
162
|
-
|
|
494
|
+
Injectable3(),
|
|
495
|
+
Command2({
|
|
163
496
|
name: "init",
|
|
164
497
|
description: "Install Claude Code slash commands for Lattice"
|
|
165
498
|
})
|
|
166
499
|
], InitCommand);
|
|
167
|
-
// src/commands/
|
|
168
|
-
import { Injectable as
|
|
169
|
-
import { Command as
|
|
170
|
-
|
|
171
|
-
// src/sync/ontology.service.ts
|
|
172
|
-
import { Injectable as Injectable3 } from "@nestjs/common";
|
|
173
|
-
|
|
174
|
-
// src/sync/document-parser.service.ts
|
|
175
|
-
import { createHash } from "crypto";
|
|
176
|
-
import { readFile as readFile2 } from "fs/promises";
|
|
177
|
-
import { Injectable as Injectable2, Logger } from "@nestjs/common";
|
|
178
|
-
import { glob } from "glob";
|
|
500
|
+
// src/commands/migrate.command.ts
|
|
501
|
+
import { Injectable as Injectable12, Logger as Logger8 } from "@nestjs/common";
|
|
502
|
+
import { Command as Command3, CommandRunner as CommandRunner3, Option as Option2 } from "nest-commander";
|
|
179
503
|
|
|
180
|
-
// src/
|
|
181
|
-
import
|
|
182
|
-
import {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
source: z.string().min(1),
|
|
201
|
-
relation: RelationTypeSchema,
|
|
202
|
-
target: z.string().min(1)
|
|
203
|
-
});
|
|
204
|
-
var GraphMetadataSchema = z.object({
|
|
205
|
-
importance: z.enum(["high", "medium", "low"]).optional(),
|
|
206
|
-
domain: z.string().optional()
|
|
207
|
-
});
|
|
208
|
-
var validateDateFormat = (dateStr) => {
|
|
209
|
-
const match = dateStr.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
|
210
|
-
if (!match)
|
|
211
|
-
return false;
|
|
212
|
-
const [, yearStr, monthStr, dayStr] = match;
|
|
213
|
-
const year = parseInt(yearStr, 10);
|
|
214
|
-
const month = parseInt(monthStr, 10);
|
|
215
|
-
const day = parseInt(dayStr, 10);
|
|
216
|
-
if (month < 1 || month > 12)
|
|
217
|
-
return false;
|
|
218
|
-
if (day < 1 || day > 31)
|
|
219
|
-
return false;
|
|
220
|
-
const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
221
|
-
if (year % 4 === 0 && year % 100 !== 0 || year % 400 === 0) {
|
|
222
|
-
daysInMonth[1] = 29;
|
|
504
|
+
// src/graph/graph.service.ts
|
|
505
|
+
import { DuckDBInstance } from "@duckdb/node-api";
|
|
506
|
+
import { Injectable as Injectable4, Logger as Logger2 } from "@nestjs/common";
|
|
507
|
+
import { ConfigService } from "@nestjs/config";
|
|
508
|
+
class GraphService {
|
|
509
|
+
configService;
|
|
510
|
+
logger = new Logger2(GraphService.name);
|
|
511
|
+
instance = null;
|
|
512
|
+
connection = null;
|
|
513
|
+
dbPath;
|
|
514
|
+
connecting = null;
|
|
515
|
+
vectorIndexes = new Set;
|
|
516
|
+
embeddingDimensions;
|
|
517
|
+
signalHandlersRegistered = false;
|
|
518
|
+
constructor(configService) {
|
|
519
|
+
this.configService = configService;
|
|
520
|
+
ensureLatticeHome();
|
|
521
|
+
this.dbPath = getDatabasePath();
|
|
522
|
+
this.embeddingDimensions = this.configService.get("EMBEDDING_DIMENSIONS") || 512;
|
|
523
|
+
this.registerSignalHandlers();
|
|
223
524
|
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
function parseFrontmatter(content) {
|
|
238
|
-
try {
|
|
239
|
-
const { data, content: markdown } = matter(content);
|
|
240
|
-
if (Object.keys(data).length === 0) {
|
|
241
|
-
return {
|
|
242
|
-
frontmatter: null,
|
|
243
|
-
content: markdown.trim(),
|
|
244
|
-
raw: content
|
|
245
|
-
};
|
|
246
|
-
}
|
|
247
|
-
const normalizedData = normalizeData(data);
|
|
248
|
-
const validated = FrontmatterSchema.safeParse(normalizedData);
|
|
249
|
-
return {
|
|
250
|
-
frontmatter: validated.success ? validated.data : normalizedData,
|
|
251
|
-
content: markdown.trim(),
|
|
252
|
-
raw: content
|
|
525
|
+
registerSignalHandlers() {
|
|
526
|
+
if (this.signalHandlersRegistered)
|
|
527
|
+
return;
|
|
528
|
+
this.signalHandlersRegistered = true;
|
|
529
|
+
const gracefulShutdown = async (signal) => {
|
|
530
|
+
this.logger.log(`Received ${signal}, checkpointing before exit...`);
|
|
531
|
+
try {
|
|
532
|
+
await this.checkpoint();
|
|
533
|
+
await this.disconnect();
|
|
534
|
+
} catch (error) {
|
|
535
|
+
this.logger.error(`Error during graceful shutdown: ${error instanceof Error ? error.message : String(error)}`);
|
|
536
|
+
}
|
|
537
|
+
process.exit(0);
|
|
253
538
|
};
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
return data.toISOString().split("T")[0];
|
|
539
|
+
process.on("SIGINT", () => gracefulShutdown("SIGINT"));
|
|
540
|
+
process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
|
|
541
|
+
process.on("beforeExit", async () => {
|
|
542
|
+
if (this.connection) {
|
|
543
|
+
await this.checkpoint();
|
|
544
|
+
}
|
|
545
|
+
});
|
|
262
546
|
}
|
|
263
|
-
|
|
264
|
-
|
|
547
|
+
async onModuleDestroy() {
|
|
548
|
+
await this.disconnect();
|
|
265
549
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
normalized[key] = normalizeData(value);
|
|
550
|
+
async ensureConnected() {
|
|
551
|
+
if (this.connection) {
|
|
552
|
+
return this.connection;
|
|
270
553
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
return this.docsPath;
|
|
554
|
+
if (this.connecting) {
|
|
555
|
+
await this.connecting;
|
|
556
|
+
if (!this.connection) {
|
|
557
|
+
throw new Error("Connection failed to establish");
|
|
558
|
+
}
|
|
559
|
+
return this.connection;
|
|
560
|
+
}
|
|
561
|
+
this.connecting = this.connect();
|
|
562
|
+
await this.connecting;
|
|
563
|
+
this.connecting = null;
|
|
564
|
+
if (!this.connection) {
|
|
565
|
+
throw new Error("Connection failed to establish");
|
|
566
|
+
}
|
|
567
|
+
return this.connection;
|
|
286
568
|
}
|
|
287
|
-
async
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const content = await readFile2(filePath, "utf-8");
|
|
296
|
-
const parsed = parseFrontmatter(content);
|
|
297
|
-
const title = this.extractTitle(content, filePath);
|
|
298
|
-
const contentHash = this.computeHash(content);
|
|
299
|
-
const frontmatterHash = this.computeHash(JSON.stringify(parsed.frontmatter || {}));
|
|
300
|
-
const entities = this.extractEntities(parsed.frontmatter, filePath);
|
|
301
|
-
const relationships = this.extractRelationships(parsed.frontmatter, filePath);
|
|
302
|
-
const graphMetadata = this.extractGraphMetadata(parsed.frontmatter);
|
|
303
|
-
return {
|
|
304
|
-
path: filePath,
|
|
305
|
-
title,
|
|
306
|
-
content: parsed.content,
|
|
307
|
-
contentHash,
|
|
308
|
-
frontmatterHash,
|
|
309
|
-
summary: parsed.frontmatter?.summary,
|
|
310
|
-
topic: parsed.frontmatter?.topic,
|
|
311
|
-
entities,
|
|
312
|
-
relationships,
|
|
313
|
-
graphMetadata,
|
|
314
|
-
tags: parsed.frontmatter?.tags || [],
|
|
315
|
-
created: parsed.frontmatter?.created,
|
|
316
|
-
updated: parsed.frontmatter?.updated,
|
|
317
|
-
status: parsed.frontmatter?.status
|
|
318
|
-
};
|
|
319
|
-
}
|
|
320
|
-
async parseAllDocuments() {
|
|
321
|
-
const { docs } = await this.parseAllDocumentsWithErrors();
|
|
322
|
-
return docs;
|
|
323
|
-
}
|
|
324
|
-
async parseAllDocumentsWithErrors() {
|
|
325
|
-
const files = await this.discoverDocuments();
|
|
326
|
-
const docs = [];
|
|
327
|
-
const errors = [];
|
|
328
|
-
for (const file of files) {
|
|
569
|
+
async connect() {
|
|
570
|
+
try {
|
|
571
|
+
this.instance = await DuckDBInstance.create(":memory:", {
|
|
572
|
+
allow_unsigned_extensions: "true"
|
|
573
|
+
});
|
|
574
|
+
this.connection = await this.instance.connect();
|
|
575
|
+
await this.connection.run("INSTALL vss; LOAD vss;");
|
|
576
|
+
await this.connection.run("SET hnsw_enable_experimental_persistence = true;");
|
|
329
577
|
try {
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
this.logger.warn(`
|
|
578
|
+
await this.connection.run("SET custom_extension_repository = 'http://duckpgq.s3.eu-north-1.amazonaws.com';");
|
|
579
|
+
await this.connection.run("FORCE INSTALL 'duckpgq';");
|
|
580
|
+
await this.connection.run("LOAD 'duckpgq';");
|
|
581
|
+
this.logger.log("DuckPGQ extension loaded successfully");
|
|
582
|
+
} catch (e) {
|
|
583
|
+
this.logger.warn(`DuckPGQ extension not available: ${e instanceof Error ? e.message : String(e)}`);
|
|
336
584
|
}
|
|
585
|
+
await this.connection.run(`ATTACH '${this.dbPath}' AS lattice (READ_WRITE);`);
|
|
586
|
+
await this.connection.run("USE lattice;");
|
|
587
|
+
await this.initializeSchema();
|
|
588
|
+
await this.connection.run("FORCE CHECKPOINT lattice;");
|
|
589
|
+
this.logger.log(`Connected to DuckDB (in-memory + ATTACH) at ${this.dbPath}`);
|
|
590
|
+
} catch (error) {
|
|
591
|
+
this.connection = null;
|
|
592
|
+
this.instance = null;
|
|
593
|
+
this.logger.error(`Failed to connect to DuckDB: ${error instanceof Error ? error.message : String(error)}`);
|
|
594
|
+
throw error;
|
|
337
595
|
}
|
|
338
|
-
return { docs, errors };
|
|
339
|
-
}
|
|
340
|
-
extractTitle(content, filePath) {
|
|
341
|
-
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
342
|
-
if (h1Match) {
|
|
343
|
-
return h1Match[1];
|
|
344
|
-
}
|
|
345
|
-
const parts = filePath.split("/");
|
|
346
|
-
return parts[parts.length - 1].replace(".md", "");
|
|
347
596
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
const errors = [];
|
|
355
|
-
for (let i = 0;i < fm.entities.length; i++) {
|
|
356
|
-
const e = fm.entities[i];
|
|
357
|
-
const result = EntitySchema.safeParse(e);
|
|
358
|
-
if (result.success) {
|
|
359
|
-
validEntities.push(result.data);
|
|
360
|
-
} else {
|
|
361
|
-
const entityPreview = typeof e === "string" ? `"${e}"` : JSON.stringify(e);
|
|
362
|
-
const zodErrors = result.error.issues.map((issue) => {
|
|
363
|
-
const path2 = issue.path.length > 0 ? issue.path.join(".") : "root";
|
|
364
|
-
return `${path2}: ${issue.message}`;
|
|
365
|
-
}).join("; ");
|
|
366
|
-
errors.push(`Entity[${i}]: ${entityPreview} - ${zodErrors}`);
|
|
367
|
-
}
|
|
597
|
+
async disconnect() {
|
|
598
|
+
if (this.connection) {
|
|
599
|
+
await this.checkpoint();
|
|
600
|
+
this.connection.closeSync();
|
|
601
|
+
this.connection = null;
|
|
602
|
+
this.logger.log("Disconnected from DuckDB");
|
|
368
603
|
}
|
|
369
|
-
if (
|
|
370
|
-
|
|
371
|
-
${errors.join(`
|
|
372
|
-
`)}`;
|
|
373
|
-
throw new Error(errorMsg);
|
|
604
|
+
if (this.instance) {
|
|
605
|
+
this.instance = null;
|
|
374
606
|
}
|
|
375
|
-
return validEntities;
|
|
376
607
|
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
return [];
|
|
381
|
-
}
|
|
382
|
-
const validRelationships = [];
|
|
383
|
-
const errors = [];
|
|
384
|
-
const validRelationTypes = RelationTypeSchema.options;
|
|
385
|
-
for (let i = 0;i < fm.relationships.length; i++) {
|
|
386
|
-
const r = fm.relationships[i];
|
|
387
|
-
const result = RelationshipSchema.safeParse(r);
|
|
388
|
-
if (result.success) {
|
|
389
|
-
const rel = result.data;
|
|
390
|
-
if (rel.source === "this") {
|
|
391
|
-
rel.source = docPath;
|
|
392
|
-
}
|
|
393
|
-
if (rel.target === "this") {
|
|
394
|
-
rel.target = docPath;
|
|
395
|
-
}
|
|
396
|
-
validRelationships.push(rel);
|
|
397
|
-
} else {
|
|
398
|
-
if (typeof r === "string") {
|
|
399
|
-
errors.push(`Relationship[${i}]: "${r}" - Expected object with {source, relation, target}, got string`);
|
|
400
|
-
} else if (typeof r === "object" && r !== null) {
|
|
401
|
-
const issues = [];
|
|
402
|
-
if (!r.source)
|
|
403
|
-
issues.push("missing source");
|
|
404
|
-
if (!r.target)
|
|
405
|
-
issues.push("missing target");
|
|
406
|
-
if (!r.relation) {
|
|
407
|
-
issues.push("missing relation");
|
|
408
|
-
} else if (!validRelationTypes.includes(r.relation)) {
|
|
409
|
-
issues.push(`invalid relation "${r.relation}" (allowed: ${validRelationTypes.join(", ")})`);
|
|
410
|
-
}
|
|
411
|
-
errors.push(`Relationship[${i}]: ${issues.join(", ")}`);
|
|
412
|
-
} else {
|
|
413
|
-
errors.push(`Relationship[${i}]: Expected object, got ${typeof r}`);
|
|
414
|
-
}
|
|
415
|
-
}
|
|
608
|
+
async checkpoint() {
|
|
609
|
+
if (!this.connection) {
|
|
610
|
+
return;
|
|
416
611
|
}
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
612
|
+
try {
|
|
613
|
+
await this.connection.run("FORCE CHECKPOINT lattice;");
|
|
614
|
+
this.logger.debug("Checkpoint completed");
|
|
615
|
+
} catch (error) {
|
|
616
|
+
this.logger.warn(`Checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
422
617
|
}
|
|
423
|
-
return validRelationships;
|
|
424
618
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
return;
|
|
619
|
+
async initializeSchema() {
|
|
620
|
+
if (!this.connection) {
|
|
621
|
+
throw new Error("Cannot initialize schema: not connected");
|
|
429
622
|
}
|
|
430
|
-
const
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
623
|
+
const conn = this.connection;
|
|
624
|
+
await conn.run(`
|
|
625
|
+
CREATE TABLE IF NOT EXISTS nodes (
|
|
626
|
+
label VARCHAR NOT NULL,
|
|
627
|
+
name VARCHAR NOT NULL,
|
|
628
|
+
properties JSON,
|
|
629
|
+
embedding FLOAT[${this.embeddingDimensions}],
|
|
630
|
+
created_at TIMESTAMP DEFAULT NOW(),
|
|
631
|
+
updated_at TIMESTAMP DEFAULT NOW(),
|
|
632
|
+
PRIMARY KEY(label, name)
|
|
633
|
+
)
|
|
634
|
+
`);
|
|
635
|
+
await conn.run(`
|
|
636
|
+
CREATE TABLE IF NOT EXISTS relationships (
|
|
637
|
+
source_label VARCHAR NOT NULL,
|
|
638
|
+
source_name VARCHAR NOT NULL,
|
|
639
|
+
relation_type VARCHAR NOT NULL,
|
|
640
|
+
target_label VARCHAR NOT NULL,
|
|
641
|
+
target_name VARCHAR NOT NULL,
|
|
642
|
+
properties JSON,
|
|
643
|
+
created_at TIMESTAMP DEFAULT NOW(),
|
|
644
|
+
PRIMARY KEY(source_label, source_name, relation_type, target_label, target_name)
|
|
645
|
+
)
|
|
646
|
+
`);
|
|
647
|
+
await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label)");
|
|
648
|
+
await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label_name ON nodes(label, name)");
|
|
649
|
+
await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_source ON relationships(source_label, source_name)");
|
|
650
|
+
await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_target ON relationships(target_label, target_name)");
|
|
651
|
+
await this.applyV2SchemaMigration(conn);
|
|
435
652
|
}
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
653
|
+
async applyV2SchemaMigration(conn) {
|
|
654
|
+
try {
|
|
655
|
+
await conn.run("ALTER TABLE nodes ADD COLUMN IF NOT EXISTS content_hash VARCHAR");
|
|
656
|
+
} catch {}
|
|
657
|
+
try {
|
|
658
|
+
await conn.run("ALTER TABLE nodes ADD COLUMN IF NOT EXISTS embedding_source_hash VARCHAR");
|
|
659
|
+
} catch {}
|
|
660
|
+
try {
|
|
661
|
+
await conn.run("ALTER TABLE nodes ADD COLUMN IF NOT EXISTS extraction_method VARCHAR DEFAULT 'frontmatter'");
|
|
662
|
+
} catch {}
|
|
663
|
+
try {
|
|
664
|
+
await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_content_hash ON nodes(content_hash) WHERE label = 'Document'");
|
|
665
|
+
} catch {}
|
|
447
666
|
}
|
|
448
|
-
async
|
|
449
|
-
const
|
|
450
|
-
|
|
667
|
+
async runV2Migration() {
|
|
668
|
+
const conn = await this.ensureConnected();
|
|
669
|
+
await this.applyV2SchemaMigration(conn);
|
|
670
|
+
this.logger.log("V2 schema migration completed");
|
|
451
671
|
}
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
} else {
|
|
465
|
-
documentsWithoutEntities++;
|
|
466
|
-
}
|
|
467
|
-
for (const entity of doc.entities) {
|
|
468
|
-
entityTypeSet.add(entity.type);
|
|
469
|
-
entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
|
|
470
|
-
if (!entityExamples[entity.name]) {
|
|
471
|
-
entityExamples[entity.name] = { type: entity.type, documents: [] };
|
|
472
|
-
}
|
|
473
|
-
if (!entityExamples[entity.name].documents.includes(doc.path)) {
|
|
474
|
-
entityExamples[entity.name].documents.push(doc.path);
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
for (const rel of doc.relationships) {
|
|
478
|
-
relationshipTypeSet.add(rel.relation);
|
|
479
|
-
relationshipCounts[rel.relation] = (relationshipCounts[rel.relation] || 0) + 1;
|
|
480
|
-
totalRelationships++;
|
|
481
|
-
}
|
|
672
|
+
async query(sql, _params) {
|
|
673
|
+
try {
|
|
674
|
+
const conn = await this.ensureConnected();
|
|
675
|
+
const reader = await conn.runAndReadAll(sql);
|
|
676
|
+
const rows = reader.getRows();
|
|
677
|
+
return {
|
|
678
|
+
resultSet: rows,
|
|
679
|
+
stats: undefined
|
|
680
|
+
};
|
|
681
|
+
} catch (error) {
|
|
682
|
+
this.logger.error(`Query failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
683
|
+
throw error;
|
|
482
684
|
}
|
|
483
|
-
return {
|
|
484
|
-
entityTypes: Array.from(entityTypeSet).sort(),
|
|
485
|
-
relationshipTypes: Array.from(relationshipTypeSet).sort(),
|
|
486
|
-
entityCounts,
|
|
487
|
-
relationshipCounts,
|
|
488
|
-
totalEntities: Object.keys(entityExamples).length,
|
|
489
|
-
totalRelationships,
|
|
490
|
-
documentsWithEntities,
|
|
491
|
-
documentsWithoutEntities,
|
|
492
|
-
entityExamples
|
|
493
|
-
};
|
|
494
685
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
Top Entities (by document count):`);
|
|
514
|
-
const sorted = Object.entries(ontology.entityExamples).sort((a, b) => b[1].documents.length - a[1].documents.length).slice(0, 10);
|
|
515
|
-
for (const [name, info] of sorted) {
|
|
516
|
-
console.log(` ${name} (${info.type}): ${info.documents.length} docs`);
|
|
686
|
+
async upsertNode(label, properties) {
|
|
687
|
+
try {
|
|
688
|
+
const { name, ...otherProps } = properties;
|
|
689
|
+
if (!name) {
|
|
690
|
+
throw new Error("Node must have a 'name' property");
|
|
691
|
+
}
|
|
692
|
+
const conn = await this.ensureConnected();
|
|
693
|
+
const propsJson = JSON.stringify(otherProps);
|
|
694
|
+
await conn.run(`
|
|
695
|
+
INSERT INTO nodes (label, name, properties)
|
|
696
|
+
VALUES ('${this.escape(String(label))}', '${this.escape(String(name))}', '${this.escape(propsJson)}')
|
|
697
|
+
ON CONFLICT (label, name) DO UPDATE SET
|
|
698
|
+
properties = EXCLUDED.properties,
|
|
699
|
+
updated_at = NOW()
|
|
700
|
+
`);
|
|
701
|
+
} catch (error) {
|
|
702
|
+
this.logger.error(`Failed to upsert node: ${error instanceof Error ? error.message : String(error)}`);
|
|
703
|
+
throw error;
|
|
517
704
|
}
|
|
518
705
|
}
|
|
519
|
-
|
|
520
|
-
OntologyService = __legacyDecorateClassTS([
|
|
521
|
-
Injectable3(),
|
|
522
|
-
__legacyMetadataTS("design:paramtypes", [
|
|
523
|
-
typeof DocumentParserService === "undefined" ? Object : DocumentParserService
|
|
524
|
-
])
|
|
525
|
-
], OntologyService);
|
|
526
|
-
|
|
527
|
-
// src/commands/ontology.command.ts
|
|
528
|
-
class OntologyCommand extends CommandRunner2 {
|
|
529
|
-
ontologyService;
|
|
530
|
-
constructor(ontologyService) {
|
|
531
|
-
super();
|
|
532
|
-
this.ontologyService = ontologyService;
|
|
533
|
-
}
|
|
534
|
-
async run() {
|
|
706
|
+
async upsertRelationship(sourceLabel, sourceName, relation, targetLabel, targetName, properties) {
|
|
535
707
|
try {
|
|
536
|
-
const
|
|
537
|
-
|
|
538
|
-
|
|
708
|
+
const conn = await this.ensureConnected();
|
|
709
|
+
await conn.run(`
|
|
710
|
+
INSERT INTO nodes (label, name, properties)
|
|
711
|
+
VALUES ('${this.escape(sourceLabel)}', '${this.escape(sourceName)}', '{}')
|
|
712
|
+
ON CONFLICT (label, name) DO NOTHING
|
|
713
|
+
`);
|
|
714
|
+
await conn.run(`
|
|
715
|
+
INSERT INTO nodes (label, name, properties)
|
|
716
|
+
VALUES ('${this.escape(targetLabel)}', '${this.escape(targetName)}', '{}')
|
|
717
|
+
ON CONFLICT (label, name) DO NOTHING
|
|
718
|
+
`);
|
|
719
|
+
const propsJson = properties ? JSON.stringify(properties) : "{}";
|
|
720
|
+
await conn.run(`
|
|
721
|
+
INSERT INTO relationships (source_label, source_name, relation_type, target_label, target_name, properties)
|
|
722
|
+
VALUES (
|
|
723
|
+
'${this.escape(sourceLabel)}',
|
|
724
|
+
'${this.escape(sourceName)}',
|
|
725
|
+
'${this.escape(relation)}',
|
|
726
|
+
'${this.escape(targetLabel)}',
|
|
727
|
+
'${this.escape(targetName)}',
|
|
728
|
+
'${this.escape(propsJson)}'
|
|
729
|
+
)
|
|
730
|
+
ON CONFLICT (source_label, source_name, relation_type, target_label, target_name) DO UPDATE SET
|
|
731
|
+
properties = EXCLUDED.properties
|
|
732
|
+
`);
|
|
539
733
|
} catch (error) {
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
process.exit(1);
|
|
734
|
+
this.logger.error(`Failed to upsert relationship: ${error instanceof Error ? error.message : String(error)}`);
|
|
735
|
+
throw error;
|
|
543
736
|
}
|
|
544
737
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
// src/embedding/embedding.service.ts
|
|
561
|
-
import { Injectable as Injectable5, Logger as Logger2 } from "@nestjs/common";
|
|
562
|
-
import { ConfigService } from "@nestjs/config";
|
|
563
|
-
|
|
564
|
-
// src/schemas/config.schemas.ts
|
|
565
|
-
import { z as z2 } from "zod";
|
|
566
|
-
var DuckDBConfigSchema = z2.object({
|
|
567
|
-
embeddingDimensions: z2.coerce.number().int().positive().default(512)
|
|
568
|
-
});
|
|
569
|
-
var EmbeddingConfigSchema = z2.object({
|
|
570
|
-
provider: z2.enum(["openai", "voyage", "nomic", "mock"]).default("voyage"),
|
|
571
|
-
apiKey: z2.string().optional(),
|
|
572
|
-
model: z2.string().min(1).default("voyage-3.5-lite"),
|
|
573
|
-
dimensions: z2.coerce.number().int().positive().default(512)
|
|
574
|
-
});
|
|
575
|
-
|
|
576
|
-
// src/embedding/embedding.types.ts
|
|
577
|
-
var DEFAULT_EMBEDDING_CONFIG = {
|
|
578
|
-
provider: "voyage",
|
|
579
|
-
model: "voyage-3.5-lite",
|
|
580
|
-
dimensions: 512
|
|
581
|
-
};
|
|
582
|
-
|
|
583
|
-
// src/embedding/providers/mock.provider.ts
|
|
584
|
-
class MockEmbeddingProvider {
|
|
585
|
-
name = "mock";
|
|
586
|
-
dimensions;
|
|
587
|
-
constructor(dimensions = 1536) {
|
|
588
|
-
this.dimensions = dimensions;
|
|
589
|
-
}
|
|
590
|
-
async generateEmbedding(text) {
|
|
591
|
-
return this.generateDeterministicEmbedding(text);
|
|
592
|
-
}
|
|
593
|
-
async generateEmbeddings(texts) {
|
|
594
|
-
return Promise.all(texts.map((text) => this.generateEmbedding(text)));
|
|
595
|
-
}
|
|
596
|
-
generateDeterministicEmbedding(text) {
|
|
597
|
-
const embedding = new Array(this.dimensions);
|
|
598
|
-
let hash = 0;
|
|
599
|
-
for (let i = 0;i < text.length; i++) {
|
|
600
|
-
hash = (hash << 5) - hash + text.charCodeAt(i);
|
|
601
|
-
hash = hash & hash;
|
|
602
|
-
}
|
|
603
|
-
for (let i = 0;i < this.dimensions; i++) {
|
|
604
|
-
const seed = hash * (i + 1) ^ i * 31;
|
|
605
|
-
embedding[i] = (seed % 2000 - 1000) / 1000;
|
|
738
|
+
async deleteNode(label, name) {
|
|
739
|
+
try {
|
|
740
|
+
const conn = await this.ensureConnected();
|
|
741
|
+
await conn.run(`
|
|
742
|
+
DELETE FROM relationships
|
|
743
|
+
WHERE (source_label = '${this.escape(label)}' AND source_name = '${this.escape(name)}')
|
|
744
|
+
OR (target_label = '${this.escape(label)}' AND target_name = '${this.escape(name)}')
|
|
745
|
+
`);
|
|
746
|
+
await conn.run(`
|
|
747
|
+
DELETE FROM nodes
|
|
748
|
+
WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
|
|
749
|
+
`);
|
|
750
|
+
} catch (error) {
|
|
751
|
+
this.logger.error(`Failed to delete node: ${error instanceof Error ? error.message : String(error)}`);
|
|
752
|
+
throw error;
|
|
606
753
|
}
|
|
607
|
-
const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
|
|
608
|
-
return embedding.map((val) => val / magnitude);
|
|
609
754
|
}
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
const apiKey = config?.apiKey || process.env.OPENAI_API_KEY;
|
|
621
|
-
if (!apiKey) {
|
|
622
|
-
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass apiKey in config.");
|
|
755
|
+
async deleteDocumentRelationships(documentPath) {
|
|
756
|
+
try {
|
|
757
|
+
const conn = await this.ensureConnected();
|
|
758
|
+
await conn.run(`
|
|
759
|
+
DELETE FROM relationships
|
|
760
|
+
WHERE properties->>'documentPath' = '${this.escape(documentPath)}'
|
|
761
|
+
`);
|
|
762
|
+
} catch (error) {
|
|
763
|
+
this.logger.error(`Failed to delete document relationships: ${error instanceof Error ? error.message : String(error)}`);
|
|
764
|
+
throw error;
|
|
623
765
|
}
|
|
624
|
-
this.apiKey = apiKey;
|
|
625
|
-
this.model = config?.model || "text-embedding-3-small";
|
|
626
|
-
this.dimensions = config?.dimensions || 1536;
|
|
627
|
-
}
|
|
628
|
-
async generateEmbedding(text) {
|
|
629
|
-
const embeddings = await this.generateEmbeddings([text]);
|
|
630
|
-
return embeddings[0];
|
|
631
766
|
}
|
|
632
|
-
async
|
|
633
|
-
|
|
767
|
+
async findNodesByLabel(label, limit) {
|
|
768
|
+
try {
|
|
769
|
+
const conn = await this.ensureConnected();
|
|
770
|
+
const limitClause = limit ? ` LIMIT ${limit}` : "";
|
|
771
|
+
const reader = await conn.runAndReadAll(`
|
|
772
|
+
SELECT name, properties
|
|
773
|
+
FROM nodes
|
|
774
|
+
WHERE label = '${this.escape(label)}'${limitClause}
|
|
775
|
+
`);
|
|
776
|
+
return reader.getRows().map((row) => {
|
|
777
|
+
const [name, properties] = row;
|
|
778
|
+
const props = properties ? JSON.parse(properties) : {};
|
|
779
|
+
return { name, ...props };
|
|
780
|
+
});
|
|
781
|
+
} catch (error) {
|
|
782
|
+
this.logger.error(`Failed to find nodes by label: ${error instanceof Error ? error.message : String(error)}`);
|
|
634
783
|
return [];
|
|
635
784
|
}
|
|
785
|
+
}
|
|
786
|
+
async findRelationships(nodeName) {
|
|
636
787
|
try {
|
|
637
|
-
const
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
})
|
|
788
|
+
const conn = await this.ensureConnected();
|
|
789
|
+
const reader = await conn.runAndReadAll(`
|
|
790
|
+
SELECT relation_type, target_name, source_name
|
|
791
|
+
FROM relationships
|
|
792
|
+
WHERE source_name = '${this.escape(nodeName)}'
|
|
793
|
+
OR target_name = '${this.escape(nodeName)}'
|
|
794
|
+
`);
|
|
795
|
+
return reader.getRows().map((row) => {
|
|
796
|
+
const [relType, targetName, sourceName] = row;
|
|
797
|
+
return [relType, sourceName === nodeName ? targetName : sourceName];
|
|
648
798
|
});
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
799
|
+
} catch (error) {
|
|
800
|
+
this.logger.error(`Failed to find relationships: ${error instanceof Error ? error.message : String(error)}`);
|
|
801
|
+
return [];
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
async createVectorIndex(label, property, dimensions) {
|
|
805
|
+
try {
|
|
806
|
+
const indexKey = `${label}_${property}`;
|
|
807
|
+
if (this.vectorIndexes.has(indexKey)) {
|
|
808
|
+
return;
|
|
652
809
|
}
|
|
653
|
-
const
|
|
654
|
-
|
|
655
|
-
|
|
810
|
+
const conn = await this.ensureConnected();
|
|
811
|
+
try {
|
|
812
|
+
await conn.run(`
|
|
813
|
+
CREATE INDEX idx_embedding_${this.escape(label)}
|
|
814
|
+
ON nodes USING HNSW (embedding)
|
|
815
|
+
WITH (metric = 'cosine')
|
|
816
|
+
`);
|
|
817
|
+
} catch {
|
|
818
|
+
this.logger.debug(`Vector index on ${label}.${property} already exists`);
|
|
819
|
+
}
|
|
820
|
+
this.vectorIndexes.add(indexKey);
|
|
821
|
+
this.logger.log(`Created vector index on ${label}.${property} with ${dimensions} dimensions`);
|
|
656
822
|
} catch (error) {
|
|
657
|
-
|
|
658
|
-
|
|
823
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
824
|
+
if (!errorMessage.includes("already exists")) {
|
|
825
|
+
this.logger.error(`Failed to create vector index: ${errorMessage}`);
|
|
826
|
+
throw error;
|
|
659
827
|
}
|
|
660
|
-
throw error;
|
|
661
828
|
}
|
|
662
829
|
}
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
usage: z3.object({
|
|
676
|
-
total_tokens: z3.number().int().nonnegative()
|
|
677
|
-
})
|
|
678
|
-
});
|
|
679
|
-
|
|
680
|
-
// src/embedding/providers/voyage.provider.ts
|
|
681
|
-
class VoyageEmbeddingProvider {
|
|
682
|
-
name = "voyage";
|
|
683
|
-
dimensions;
|
|
684
|
-
model;
|
|
685
|
-
apiKey;
|
|
686
|
-
inputType;
|
|
687
|
-
baseUrl = "https://api.voyageai.com/v1";
|
|
688
|
-
constructor(config) {
|
|
689
|
-
const apiKey = config?.apiKey || process.env.VOYAGE_API_KEY;
|
|
690
|
-
if (!apiKey) {
|
|
691
|
-
throw new Error("Voyage API key is required. Set VOYAGE_API_KEY environment variable or pass apiKey in config.");
|
|
830
|
+
async updateNodeEmbedding(label, name, embedding) {
|
|
831
|
+
try {
|
|
832
|
+
const conn = await this.ensureConnected();
|
|
833
|
+
const vectorStr = `[${embedding.join(", ")}]`;
|
|
834
|
+
await conn.run(`
|
|
835
|
+
UPDATE nodes
|
|
836
|
+
SET embedding = ${vectorStr}::FLOAT[${this.embeddingDimensions}]
|
|
837
|
+
WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
|
|
838
|
+
`);
|
|
839
|
+
} catch (error) {
|
|
840
|
+
this.logger.error(`Failed to update node embedding: ${error instanceof Error ? error.message : String(error)}`);
|
|
841
|
+
throw error;
|
|
692
842
|
}
|
|
693
|
-
this.apiKey = apiKey;
|
|
694
|
-
this.model = config?.model || "voyage-3.5-lite";
|
|
695
|
-
this.dimensions = config?.dimensions || 512;
|
|
696
|
-
this.inputType = config?.inputType || "document";
|
|
697
843
|
}
|
|
698
|
-
async
|
|
699
|
-
|
|
700
|
-
|
|
844
|
+
async vectorSearch(label, queryVector, k = 10) {
|
|
845
|
+
try {
|
|
846
|
+
const conn = await this.ensureConnected();
|
|
847
|
+
const vectorStr = `[${queryVector.join(", ")}]`;
|
|
848
|
+
const reader = await conn.runAndReadAll(`
|
|
849
|
+
SELECT
|
|
850
|
+
name,
|
|
851
|
+
properties->>'title' as title,
|
|
852
|
+
array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
|
|
853
|
+
FROM nodes
|
|
854
|
+
WHERE label = '${this.escape(label)}'
|
|
855
|
+
AND embedding IS NOT NULL
|
|
856
|
+
ORDER BY similarity DESC
|
|
857
|
+
LIMIT ${k}
|
|
858
|
+
`);
|
|
859
|
+
return reader.getRows().map((row) => {
|
|
860
|
+
const [name, title, similarity] = row;
|
|
861
|
+
return {
|
|
862
|
+
name,
|
|
863
|
+
title: title || undefined,
|
|
864
|
+
score: similarity
|
|
865
|
+
};
|
|
866
|
+
});
|
|
867
|
+
} catch (error) {
|
|
868
|
+
this.logger.error(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
869
|
+
throw error;
|
|
870
|
+
}
|
|
701
871
|
}
|
|
702
|
-
async
|
|
703
|
-
const
|
|
704
|
-
|
|
872
|
+
async vectorSearchAll(queryVector, k = 10) {
|
|
873
|
+
const allResults = [];
|
|
874
|
+
const conn = await this.ensureConnected();
|
|
875
|
+
const vectorStr = `[${queryVector.join(", ")}]`;
|
|
876
|
+
try {
|
|
877
|
+
const reader = await conn.runAndReadAll(`
|
|
878
|
+
SELECT
|
|
879
|
+
name,
|
|
880
|
+
label,
|
|
881
|
+
properties->>'title' as title,
|
|
882
|
+
properties->>'description' as description,
|
|
883
|
+
array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
|
|
884
|
+
FROM nodes
|
|
885
|
+
WHERE embedding IS NOT NULL
|
|
886
|
+
ORDER BY similarity DESC
|
|
887
|
+
LIMIT ${k}
|
|
888
|
+
`);
|
|
889
|
+
for (const row of reader.getRows()) {
|
|
890
|
+
const [name, label, title, description, similarity] = row;
|
|
891
|
+
allResults.push({
|
|
892
|
+
name,
|
|
893
|
+
label,
|
|
894
|
+
title: title || undefined,
|
|
895
|
+
description: description || undefined,
|
|
896
|
+
score: similarity
|
|
897
|
+
});
|
|
898
|
+
}
|
|
899
|
+
} catch (error) {
|
|
900
|
+
this.logger.debug(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
901
|
+
}
|
|
902
|
+
return allResults.sort((a, b) => b.score - a.score).slice(0, k);
|
|
705
903
|
}
|
|
706
|
-
|
|
707
|
-
return
|
|
904
|
+
escape(value) {
|
|
905
|
+
return value.replace(/'/g, "''");
|
|
708
906
|
}
|
|
709
|
-
async
|
|
710
|
-
|
|
711
|
-
|
|
907
|
+
async loadAllDocumentHashes() {
|
|
908
|
+
try {
|
|
909
|
+
const conn = await this.ensureConnected();
|
|
910
|
+
const reader = await conn.runAndReadAll(`
|
|
911
|
+
SELECT name, content_hash, embedding_source_hash
|
|
912
|
+
FROM nodes
|
|
913
|
+
WHERE label = 'Document'
|
|
914
|
+
`);
|
|
915
|
+
const hashMap = new Map;
|
|
916
|
+
for (const row of reader.getRows()) {
|
|
917
|
+
const [name, contentHash, embeddingSourceHash] = row;
|
|
918
|
+
hashMap.set(name, { contentHash, embeddingSourceHash });
|
|
919
|
+
}
|
|
920
|
+
return hashMap;
|
|
921
|
+
} catch (error) {
|
|
922
|
+
this.logger.error(`Failed to load document hashes: ${error instanceof Error ? error.message : String(error)}`);
|
|
923
|
+
return new Map;
|
|
712
924
|
}
|
|
925
|
+
}
|
|
926
|
+
async updateDocumentHashes(path2, contentHash, embeddingSourceHash) {
|
|
713
927
|
try {
|
|
714
|
-
const
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
928
|
+
const conn = await this.ensureConnected();
|
|
929
|
+
if (embeddingSourceHash) {
|
|
930
|
+
await conn.run(`
|
|
931
|
+
UPDATE nodes
|
|
932
|
+
SET content_hash = '${this.escape(contentHash)}',
|
|
933
|
+
embedding_source_hash = '${this.escape(embeddingSourceHash)}',
|
|
934
|
+
updated_at = NOW()
|
|
935
|
+
WHERE label = 'Document' AND name = '${this.escape(path2)}'
|
|
936
|
+
`);
|
|
937
|
+
} else {
|
|
938
|
+
await conn.run(`
|
|
939
|
+
UPDATE nodes
|
|
940
|
+
SET content_hash = '${this.escape(contentHash)}',
|
|
941
|
+
updated_at = NOW()
|
|
942
|
+
WHERE label = 'Document' AND name = '${this.escape(path2)}'
|
|
943
|
+
`);
|
|
730
944
|
}
|
|
731
|
-
const data = VoyageEmbeddingResponseSchema.parse(await response.json());
|
|
732
|
-
const sortedData = data.data.sort((a, b) => a.index - b.index);
|
|
733
|
-
return sortedData.map((item) => item.embedding);
|
|
734
945
|
} catch (error) {
|
|
735
|
-
|
|
736
|
-
|
|
946
|
+
this.logger.error(`Failed to update document hashes: ${error instanceof Error ? error.message : String(error)}`);
|
|
947
|
+
throw error;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
async batchUpdateDocumentHashes(updates) {
|
|
951
|
+
if (updates.length === 0)
|
|
952
|
+
return;
|
|
953
|
+
try {
|
|
954
|
+
const conn = await this.ensureConnected();
|
|
955
|
+
await conn.run("BEGIN TRANSACTION");
|
|
956
|
+
for (const { path: path2, contentHash, embeddingSourceHash } of updates) {
|
|
957
|
+
if (embeddingSourceHash) {
|
|
958
|
+
await conn.run(`
|
|
959
|
+
UPDATE nodes
|
|
960
|
+
SET content_hash = '${this.escape(contentHash)}',
|
|
961
|
+
embedding_source_hash = '${this.escape(embeddingSourceHash)}',
|
|
962
|
+
updated_at = NOW()
|
|
963
|
+
WHERE label = 'Document' AND name = '${this.escape(path2)}'
|
|
964
|
+
`);
|
|
965
|
+
} else {
|
|
966
|
+
await conn.run(`
|
|
967
|
+
UPDATE nodes
|
|
968
|
+
SET content_hash = '${this.escape(contentHash)}',
|
|
969
|
+
updated_at = NOW()
|
|
970
|
+
WHERE label = 'Document' AND name = '${this.escape(path2)}'
|
|
971
|
+
`);
|
|
972
|
+
}
|
|
737
973
|
}
|
|
974
|
+
await conn.run("COMMIT");
|
|
975
|
+
} catch (error) {
|
|
976
|
+
this.logger.error(`Failed to batch update document hashes: ${error instanceof Error ? error.message : String(error)}`);
|
|
738
977
|
throw error;
|
|
739
978
|
}
|
|
740
979
|
}
|
|
980
|
+
async getTrackedDocumentPaths() {
|
|
981
|
+
try {
|
|
982
|
+
const conn = await this.ensureConnected();
|
|
983
|
+
const reader = await conn.runAndReadAll(`
|
|
984
|
+
SELECT name FROM nodes WHERE label = 'Document'
|
|
985
|
+
`);
|
|
986
|
+
return reader.getRows().map((row) => row[0]);
|
|
987
|
+
} catch (error) {
|
|
988
|
+
this.logger.error(`Failed to get tracked document paths: ${error instanceof Error ? error.message : String(error)}`);
|
|
989
|
+
return [];
|
|
990
|
+
}
|
|
991
|
+
}
|
|
741
992
|
}
|
|
993
|
+
GraphService = __legacyDecorateClassTS([
|
|
994
|
+
Injectable4(),
|
|
995
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
996
|
+
typeof ConfigService === "undefined" ? Object : ConfigService
|
|
997
|
+
])
|
|
998
|
+
], GraphService);
|
|
742
999
|
|
|
743
|
-
// src/
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
1000
|
+
// src/sync/manifest.service.ts
|
|
1001
|
+
import { createHash } from "crypto";
|
|
1002
|
+
import { existsSync as existsSync4 } from "fs";
|
|
1003
|
+
import { readFile as readFile3, writeFile } from "fs/promises";
|
|
1004
|
+
import { Injectable as Injectable5 } from "@nestjs/common";
|
|
1005
|
+
|
|
1006
|
+
// src/schemas/manifest.schemas.ts
|
|
1007
|
+
import { z as z2 } from "zod";
|
|
1008
|
+
var ManifestEntrySchema = z2.object({
|
|
1009
|
+
contentHash: z2.string(),
|
|
1010
|
+
frontmatterHash: z2.string(),
|
|
1011
|
+
lastSynced: z2.string(),
|
|
1012
|
+
entityCount: z2.number().int().nonnegative(),
|
|
1013
|
+
relationshipCount: z2.number().int().nonnegative()
|
|
1014
|
+
});
|
|
1015
|
+
var SyncManifestSchema = z2.object({
|
|
1016
|
+
version: z2.string(),
|
|
1017
|
+
lastSync: z2.string(),
|
|
1018
|
+
documents: z2.record(z2.string(), ManifestEntrySchema)
|
|
1019
|
+
});
|
|
1020
|
+
|
|
1021
|
+
// src/sync/manifest.service.ts
|
|
1022
|
+
class ManifestService {
|
|
1023
|
+
manifestPath;
|
|
1024
|
+
manifest = null;
|
|
1025
|
+
constructor() {
|
|
1026
|
+
this.manifestPath = getManifestPath();
|
|
754
1027
|
}
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
1028
|
+
async load() {
|
|
1029
|
+
try {
|
|
1030
|
+
if (existsSync4(this.manifestPath)) {
|
|
1031
|
+
const content = await readFile3(this.manifestPath, "utf-8");
|
|
1032
|
+
this.manifest = SyncManifestSchema.parse(JSON.parse(content));
|
|
1033
|
+
} else {
|
|
1034
|
+
this.manifest = this.createEmptyManifest();
|
|
1035
|
+
}
|
|
1036
|
+
} catch (_error) {
|
|
1037
|
+
this.manifest = this.createEmptyManifest();
|
|
763
1038
|
}
|
|
764
|
-
return
|
|
765
|
-
provider: providerEnv,
|
|
766
|
-
apiKey,
|
|
767
|
-
model: this.configService.get("EMBEDDING_MODEL"),
|
|
768
|
-
dimensions: this.configService.get("EMBEDDING_DIMENSIONS")
|
|
769
|
-
});
|
|
1039
|
+
return this.manifest;
|
|
770
1040
|
}
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
if (!this.config.apiKey) {
|
|
775
|
-
throw new Error("OPENAI_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
|
|
776
|
-
}
|
|
777
|
-
return new OpenAIEmbeddingProvider({
|
|
778
|
-
apiKey: this.config.apiKey,
|
|
779
|
-
model: this.config.model,
|
|
780
|
-
dimensions: this.config.dimensions
|
|
781
|
-
});
|
|
782
|
-
case "mock":
|
|
783
|
-
return new MockEmbeddingProvider(this.config.dimensions);
|
|
784
|
-
case "voyage":
|
|
785
|
-
if (!this.config.apiKey) {
|
|
786
|
-
throw new Error("VOYAGE_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
|
|
787
|
-
}
|
|
788
|
-
return new VoyageEmbeddingProvider({
|
|
789
|
-
apiKey: this.config.apiKey,
|
|
790
|
-
model: this.config.model,
|
|
791
|
-
dimensions: this.config.dimensions
|
|
792
|
-
});
|
|
793
|
-
case "nomic":
|
|
794
|
-
throw new Error(`Provider ${this.config.provider} not yet implemented. Use 'voyage', 'openai', or 'mock'.`);
|
|
795
|
-
default:
|
|
796
|
-
throw new Error(`Unknown embedding provider: ${this.config.provider}. Use 'voyage', 'openai', or 'mock'.`);
|
|
1041
|
+
async save() {
|
|
1042
|
+
if (!this.manifest) {
|
|
1043
|
+
throw new Error("Manifest not loaded. Call load() first.");
|
|
797
1044
|
}
|
|
1045
|
+
this.manifest.lastSync = new Date().toISOString();
|
|
1046
|
+
const content = JSON.stringify(this.manifest, null, 2);
|
|
1047
|
+
await writeFile(this.manifestPath, content, "utf-8");
|
|
798
1048
|
}
|
|
799
|
-
|
|
800
|
-
return
|
|
801
|
-
}
|
|
802
|
-
getDimensions() {
|
|
803
|
-
return this.provider.dimensions;
|
|
1049
|
+
getContentHash(content) {
|
|
1050
|
+
return createHash("sha256").update(content).digest("hex");
|
|
804
1051
|
}
|
|
805
|
-
|
|
806
|
-
if (!
|
|
807
|
-
throw new Error("
|
|
1052
|
+
detectChange(path2, contentHash, frontmatterHash) {
|
|
1053
|
+
if (!this.manifest) {
|
|
1054
|
+
throw new Error("Manifest not loaded. Call load() first.");
|
|
808
1055
|
}
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
if (!text || text.trim().length === 0) {
|
|
813
|
-
throw new Error("Cannot generate embedding for empty text");
|
|
1056
|
+
const existing = this.manifest.documents[path2];
|
|
1057
|
+
if (!existing) {
|
|
1058
|
+
return "new";
|
|
814
1059
|
}
|
|
815
|
-
if (
|
|
816
|
-
return
|
|
1060
|
+
if (existing.contentHash === contentHash && existing.frontmatterHash === frontmatterHash) {
|
|
1061
|
+
return "unchanged";
|
|
817
1062
|
}
|
|
818
|
-
return
|
|
1063
|
+
return "updated";
|
|
819
1064
|
}
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
return [];
|
|
1065
|
+
updateEntry(path2, contentHash, frontmatterHash, entityCount, relationshipCount) {
|
|
1066
|
+
if (!this.manifest) {
|
|
1067
|
+
throw new Error("Manifest not loaded. Call load() first.");
|
|
824
1068
|
}
|
|
825
|
-
|
|
1069
|
+
this.manifest.documents[path2] = {
|
|
1070
|
+
contentHash,
|
|
1071
|
+
frontmatterHash,
|
|
1072
|
+
lastSynced: new Date().toISOString(),
|
|
1073
|
+
entityCount,
|
|
1074
|
+
relationshipCount
|
|
1075
|
+
};
|
|
826
1076
|
}
|
|
827
|
-
|
|
828
|
-
|
|
1077
|
+
removeEntry(path2) {
|
|
1078
|
+
if (!this.manifest) {
|
|
1079
|
+
throw new Error("Manifest not loaded. Call load() first.");
|
|
1080
|
+
}
|
|
1081
|
+
delete this.manifest.documents[path2];
|
|
1082
|
+
}
|
|
1083
|
+
getTrackedPaths() {
|
|
1084
|
+
if (!this.manifest) {
|
|
1085
|
+
throw new Error("Manifest not loaded. Call load() first.");
|
|
1086
|
+
}
|
|
1087
|
+
return Object.keys(this.manifest.documents);
|
|
1088
|
+
}
|
|
1089
|
+
createEmptyManifest() {
|
|
1090
|
+
return {
|
|
1091
|
+
version: "1.0",
|
|
1092
|
+
lastSync: new Date().toISOString(),
|
|
1093
|
+
documents: {}
|
|
1094
|
+
};
|
|
829
1095
|
}
|
|
830
1096
|
}
|
|
831
|
-
|
|
1097
|
+
ManifestService = __legacyDecorateClassTS([
|
|
832
1098
|
Injectable5(),
|
|
833
|
-
__legacyMetadataTS("design:paramtypes", [
|
|
834
|
-
|
|
835
|
-
])
|
|
836
|
-
], EmbeddingService);
|
|
1099
|
+
__legacyMetadataTS("design:paramtypes", [])
|
|
1100
|
+
], ManifestService);
|
|
837
1101
|
|
|
838
|
-
// src/
|
|
839
|
-
import {
|
|
1102
|
+
// src/sync/sync.service.ts
|
|
1103
|
+
import { Injectable as Injectable11, Logger as Logger7 } from "@nestjs/common";
|
|
1104
|
+
|
|
1105
|
+
// src/embedding/embedding.service.ts
|
|
840
1106
|
import { Injectable as Injectable6, Logger as Logger3 } from "@nestjs/common";
|
|
841
1107
|
import { ConfigService as ConfigService2 } from "@nestjs/config";
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
1108
|
+
|
|
1109
|
+
// src/schemas/config.schemas.ts
|
|
1110
|
+
import { z as z3 } from "zod";
|
|
1111
|
+
var DuckDBConfigSchema = z3.object({
|
|
1112
|
+
embeddingDimensions: z3.coerce.number().int().positive().default(512)
|
|
1113
|
+
});
|
|
1114
|
+
var EmbeddingConfigSchema = z3.object({
|
|
1115
|
+
provider: z3.enum(["openai", "voyage", "nomic", "mock"]).default("voyage"),
|
|
1116
|
+
apiKey: z3.string().optional(),
|
|
1117
|
+
model: z3.string().min(1).default("voyage-3.5-lite"),
|
|
1118
|
+
dimensions: z3.coerce.number().int().positive().default(512)
|
|
1119
|
+
});
|
|
1120
|
+
|
|
1121
|
+
// src/embedding/embedding.types.ts
|
|
1122
|
+
var DEFAULT_EMBEDDING_CONFIG = {
|
|
1123
|
+
provider: "voyage",
|
|
1124
|
+
model: "voyage-3.5-lite",
|
|
1125
|
+
dimensions: 512
|
|
1126
|
+
};
|
|
1127
|
+
|
|
1128
|
+
// src/embedding/providers/mock.provider.ts
|
|
1129
|
+
class MockEmbeddingProvider {
|
|
1130
|
+
name = "mock";
|
|
1131
|
+
dimensions;
|
|
1132
|
+
constructor(dimensions = 1536) {
|
|
1133
|
+
this.dimensions = dimensions;
|
|
858
1134
|
}
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
return;
|
|
862
|
-
this.signalHandlersRegistered = true;
|
|
863
|
-
const gracefulShutdown = async (signal) => {
|
|
864
|
-
this.logger.log(`Received ${signal}, checkpointing before exit...`);
|
|
865
|
-
try {
|
|
866
|
-
await this.checkpoint();
|
|
867
|
-
await this.disconnect();
|
|
868
|
-
} catch (error) {
|
|
869
|
-
this.logger.error(`Error during graceful shutdown: ${error instanceof Error ? error.message : String(error)}`);
|
|
870
|
-
}
|
|
871
|
-
process.exit(0);
|
|
872
|
-
};
|
|
873
|
-
process.on("SIGINT", () => gracefulShutdown("SIGINT"));
|
|
874
|
-
process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
|
|
875
|
-
process.on("beforeExit", async () => {
|
|
876
|
-
if (this.connection) {
|
|
877
|
-
await this.checkpoint();
|
|
878
|
-
}
|
|
879
|
-
});
|
|
1135
|
+
async generateEmbedding(text) {
|
|
1136
|
+
return this.generateDeterministicEmbedding(text);
|
|
880
1137
|
}
|
|
881
|
-
async
|
|
882
|
-
|
|
1138
|
+
async generateEmbeddings(texts) {
|
|
1139
|
+
return Promise.all(texts.map((text) => this.generateEmbedding(text)));
|
|
883
1140
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
1141
|
+
generateDeterministicEmbedding(text) {
|
|
1142
|
+
const embedding = new Array(this.dimensions);
|
|
1143
|
+
let hash = 0;
|
|
1144
|
+
for (let i = 0;i < text.length; i++) {
|
|
1145
|
+
hash = (hash << 5) - hash + text.charCodeAt(i);
|
|
1146
|
+
hash = hash & hash;
|
|
887
1147
|
}
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
throw new Error("Connection failed to establish");
|
|
892
|
-
}
|
|
893
|
-
return this.connection;
|
|
1148
|
+
for (let i = 0;i < this.dimensions; i++) {
|
|
1149
|
+
const seed = hash * (i + 1) ^ i * 31;
|
|
1150
|
+
embedding[i] = (seed % 2000 - 1000) / 1000;
|
|
894
1151
|
}
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
1152
|
+
const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
|
|
1153
|
+
return embedding.map((val) => val / magnitude);
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
// src/embedding/providers/openai.provider.ts
|
|
1158
|
+
class OpenAIEmbeddingProvider {
|
|
1159
|
+
name = "openai";
|
|
1160
|
+
dimensions;
|
|
1161
|
+
model;
|
|
1162
|
+
apiKey;
|
|
1163
|
+
baseUrl = "https://api.openai.com/v1";
|
|
1164
|
+
constructor(config) {
|
|
1165
|
+
const apiKey = config?.apiKey || process.env.OPENAI_API_KEY;
|
|
1166
|
+
if (!apiKey) {
|
|
1167
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass apiKey in config.");
|
|
900
1168
|
}
|
|
901
|
-
|
|
1169
|
+
this.apiKey = apiKey;
|
|
1170
|
+
this.model = config?.model || "text-embedding-3-small";
|
|
1171
|
+
this.dimensions = config?.dimensions || 1536;
|
|
902
1172
|
}
|
|
903
|
-
async
|
|
1173
|
+
async generateEmbedding(text) {
|
|
1174
|
+
const embeddings = await this.generateEmbeddings([text]);
|
|
1175
|
+
return embeddings[0];
|
|
1176
|
+
}
|
|
1177
|
+
async generateEmbeddings(texts) {
|
|
1178
|
+
if (!texts || texts.length === 0) {
|
|
1179
|
+
return [];
|
|
1180
|
+
}
|
|
904
1181
|
try {
|
|
905
|
-
|
|
906
|
-
|
|
1182
|
+
const response = await fetch(`${this.baseUrl}/embeddings`, {
|
|
1183
|
+
method: "POST",
|
|
1184
|
+
headers: {
|
|
1185
|
+
"Content-Type": "application/json",
|
|
1186
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
1187
|
+
},
|
|
1188
|
+
body: JSON.stringify({
|
|
1189
|
+
model: this.model,
|
|
1190
|
+
input: texts,
|
|
1191
|
+
dimensions: this.dimensions
|
|
1192
|
+
})
|
|
907
1193
|
});
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
try {
|
|
912
|
-
await this.connection.run("SET custom_extension_repository = 'http://duckpgq.s3.eu-north-1.amazonaws.com';");
|
|
913
|
-
await this.connection.run("FORCE INSTALL 'duckpgq';");
|
|
914
|
-
await this.connection.run("LOAD 'duckpgq';");
|
|
915
|
-
this.logger.log("DuckPGQ extension loaded successfully");
|
|
916
|
-
} catch (e) {
|
|
917
|
-
this.logger.warn(`DuckPGQ extension not available: ${e instanceof Error ? e.message : String(e)}`);
|
|
1194
|
+
if (!response.ok) {
|
|
1195
|
+
const error = await response.json().catch(() => ({}));
|
|
1196
|
+
throw new Error(`OpenAI API error: ${response.status} ${JSON.stringify(error)}`);
|
|
918
1197
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
this.logger.log(`Connected to DuckDB (in-memory + ATTACH) at ${this.dbPath}`);
|
|
1198
|
+
const data = await response.json();
|
|
1199
|
+
const sortedData = data.data.sort((a, b) => a.index - b.index);
|
|
1200
|
+
return sortedData.map((item) => item.embedding);
|
|
923
1201
|
} catch (error) {
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1202
|
+
if (error instanceof Error) {
|
|
1203
|
+
throw new Error(`Failed to generate embeddings: ${error.message}`);
|
|
1204
|
+
}
|
|
927
1205
|
throw error;
|
|
928
1206
|
}
|
|
929
1207
|
}
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
// src/schemas/embedding.schemas.ts
|
|
1211
|
+
import { z as z4 } from "zod";
|
|
1212
|
+
var VoyageEmbeddingResponseSchema = z4.object({
|
|
1213
|
+
object: z4.string(),
|
|
1214
|
+
data: z4.array(z4.object({
|
|
1215
|
+
object: z4.string(),
|
|
1216
|
+
embedding: z4.array(z4.number()),
|
|
1217
|
+
index: z4.number().int().nonnegative()
|
|
1218
|
+
})),
|
|
1219
|
+
model: z4.string(),
|
|
1220
|
+
usage: z4.object({
|
|
1221
|
+
total_tokens: z4.number().int().nonnegative()
|
|
1222
|
+
})
|
|
1223
|
+
});
|
|
1224
|
+
|
|
1225
|
+
// src/embedding/providers/voyage.provider.ts
|
|
1226
|
+
class VoyageEmbeddingProvider {
|
|
1227
|
+
name = "voyage";
|
|
1228
|
+
dimensions;
|
|
1229
|
+
model;
|
|
1230
|
+
apiKey;
|
|
1231
|
+
inputType;
|
|
1232
|
+
baseUrl = "https://api.voyageai.com/v1";
|
|
1233
|
+
constructor(config) {
|
|
1234
|
+
const apiKey = config?.apiKey || process.env.VOYAGE_API_KEY;
|
|
1235
|
+
if (!apiKey) {
|
|
1236
|
+
throw new Error("Voyage API key is required. Set VOYAGE_API_KEY environment variable or pass apiKey in config.");
|
|
939
1237
|
}
|
|
1238
|
+
this.apiKey = apiKey;
|
|
1239
|
+
this.model = config?.model || "voyage-3.5-lite";
|
|
1240
|
+
this.dimensions = config?.dimensions || 512;
|
|
1241
|
+
this.inputType = config?.inputType || "document";
|
|
940
1242
|
}
|
|
941
|
-
async
|
|
942
|
-
|
|
943
|
-
|
|
1243
|
+
async generateEmbedding(text) {
|
|
1244
|
+
const embeddings = await this.generateEmbeddings([text]);
|
|
1245
|
+
return embeddings[0];
|
|
1246
|
+
}
|
|
1247
|
+
async generateQueryEmbedding(text) {
|
|
1248
|
+
const embeddings = await this.generateEmbeddingsWithType([text], "query");
|
|
1249
|
+
return embeddings[0];
|
|
1250
|
+
}
|
|
1251
|
+
async generateEmbeddings(texts) {
|
|
1252
|
+
return this.generateEmbeddingsWithType(texts, this.inputType);
|
|
1253
|
+
}
|
|
1254
|
+
async generateEmbeddingsWithType(texts, inputType) {
|
|
1255
|
+
if (!texts || texts.length === 0) {
|
|
1256
|
+
return [];
|
|
944
1257
|
}
|
|
945
1258
|
try {
|
|
946
|
-
await this.
|
|
947
|
-
|
|
1259
|
+
const response = await fetch(`${this.baseUrl}/embeddings`, {
|
|
1260
|
+
method: "POST",
|
|
1261
|
+
headers: {
|
|
1262
|
+
"Content-Type": "application/json",
|
|
1263
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
1264
|
+
},
|
|
1265
|
+
body: JSON.stringify({
|
|
1266
|
+
model: this.model,
|
|
1267
|
+
input: texts,
|
|
1268
|
+
output_dimension: this.dimensions,
|
|
1269
|
+
input_type: inputType
|
|
1270
|
+
})
|
|
1271
|
+
});
|
|
1272
|
+
if (!response.ok) {
|
|
1273
|
+
const error = await response.json().catch(() => ({}));
|
|
1274
|
+
throw new Error(`Voyage API error: ${response.status} ${JSON.stringify(error)}`);
|
|
1275
|
+
}
|
|
1276
|
+
const data = VoyageEmbeddingResponseSchema.parse(await response.json());
|
|
1277
|
+
const sortedData = data.data.sort((a, b) => a.index - b.index);
|
|
1278
|
+
return sortedData.map((item) => item.embedding);
|
|
948
1279
|
} catch (error) {
|
|
949
|
-
|
|
1280
|
+
if (error instanceof Error) {
|
|
1281
|
+
throw new Error(`Failed to generate embeddings: ${error.message}`);
|
|
1282
|
+
}
|
|
1283
|
+
throw error;
|
|
950
1284
|
}
|
|
951
1285
|
}
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
PRIMARY KEY(label, name)
|
|
966
|
-
)
|
|
967
|
-
`);
|
|
968
|
-
await conn.run(`
|
|
969
|
-
CREATE TABLE IF NOT EXISTS relationships (
|
|
970
|
-
source_label VARCHAR NOT NULL,
|
|
971
|
-
source_name VARCHAR NOT NULL,
|
|
972
|
-
relation_type VARCHAR NOT NULL,
|
|
973
|
-
target_label VARCHAR NOT NULL,
|
|
974
|
-
target_name VARCHAR NOT NULL,
|
|
975
|
-
properties JSON,
|
|
976
|
-
created_at TIMESTAMP DEFAULT NOW(),
|
|
977
|
-
PRIMARY KEY(source_label, source_name, relation_type, target_label, target_name)
|
|
978
|
-
)
|
|
979
|
-
`);
|
|
980
|
-
await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label)");
|
|
981
|
-
await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label_name ON nodes(label, name)");
|
|
982
|
-
await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_source ON relationships(source_label, source_name)");
|
|
983
|
-
await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_target ON relationships(target_label, target_name)");
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
// src/embedding/embedding.service.ts
|
|
1289
|
+
class EmbeddingService {
|
|
1290
|
+
configService;
|
|
1291
|
+
logger = new Logger3(EmbeddingService.name);
|
|
1292
|
+
provider;
|
|
1293
|
+
config;
|
|
1294
|
+
constructor(configService) {
|
|
1295
|
+
this.configService = configService;
|
|
1296
|
+
this.config = this.loadConfig();
|
|
1297
|
+
this.provider = this.createProvider();
|
|
1298
|
+
this.logger.log(`Initialized embedding service with provider: ${this.provider.name}`);
|
|
984
1299
|
}
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
};
|
|
994
|
-
} catch (error) {
|
|
995
|
-
this.logger.error(`Query failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
996
|
-
throw error;
|
|
1300
|
+
loadConfig() {
|
|
1301
|
+
const providerEnv = this.configService.get("EMBEDDING_PROVIDER");
|
|
1302
|
+
const provider = providerEnv ?? DEFAULT_EMBEDDING_CONFIG.provider;
|
|
1303
|
+
let apiKey;
|
|
1304
|
+
if (provider === "voyage") {
|
|
1305
|
+
apiKey = this.configService.get("VOYAGE_API_KEY");
|
|
1306
|
+
} else if (provider === "openai") {
|
|
1307
|
+
apiKey = this.configService.get("OPENAI_API_KEY");
|
|
997
1308
|
}
|
|
1309
|
+
return EmbeddingConfigSchema.parse({
|
|
1310
|
+
provider: providerEnv,
|
|
1311
|
+
apiKey,
|
|
1312
|
+
model: this.configService.get("EMBEDDING_MODEL"),
|
|
1313
|
+
dimensions: this.configService.get("EMBEDDING_DIMENSIONS")
|
|
1314
|
+
});
|
|
998
1315
|
}
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1316
|
+
createProvider() {
|
|
1317
|
+
switch (this.config.provider) {
|
|
1318
|
+
case "openai":
|
|
1319
|
+
if (!this.config.apiKey) {
|
|
1320
|
+
throw new Error("OPENAI_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
|
|
1321
|
+
}
|
|
1322
|
+
return new OpenAIEmbeddingProvider({
|
|
1323
|
+
apiKey: this.config.apiKey,
|
|
1324
|
+
model: this.config.model,
|
|
1325
|
+
dimensions: this.config.dimensions
|
|
1326
|
+
});
|
|
1327
|
+
case "mock":
|
|
1328
|
+
return new MockEmbeddingProvider(this.config.dimensions);
|
|
1329
|
+
case "voyage":
|
|
1330
|
+
if (!this.config.apiKey) {
|
|
1331
|
+
throw new Error("VOYAGE_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
|
|
1332
|
+
}
|
|
1333
|
+
return new VoyageEmbeddingProvider({
|
|
1334
|
+
apiKey: this.config.apiKey,
|
|
1335
|
+
model: this.config.model,
|
|
1336
|
+
dimensions: this.config.dimensions
|
|
1337
|
+
});
|
|
1338
|
+
case "nomic":
|
|
1339
|
+
throw new Error(`Provider ${this.config.provider} not yet implemented. Use 'voyage', 'openai', or 'mock'.`);
|
|
1340
|
+
default:
|
|
1341
|
+
throw new Error(`Unknown embedding provider: ${this.config.provider}. Use 'voyage', 'openai', or 'mock'.`);
|
|
1017
1342
|
}
|
|
1018
1343
|
}
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
const conn = await this.ensureConnected();
|
|
1022
|
-
await conn.run(`
|
|
1023
|
-
INSERT INTO nodes (label, name, properties)
|
|
1024
|
-
VALUES ('${this.escape(sourceLabel)}', '${this.escape(sourceName)}', '{}')
|
|
1025
|
-
ON CONFLICT (label, name) DO NOTHING
|
|
1026
|
-
`);
|
|
1027
|
-
await conn.run(`
|
|
1028
|
-
INSERT INTO nodes (label, name, properties)
|
|
1029
|
-
VALUES ('${this.escape(targetLabel)}', '${this.escape(targetName)}', '{}')
|
|
1030
|
-
ON CONFLICT (label, name) DO NOTHING
|
|
1031
|
-
`);
|
|
1032
|
-
const propsJson = properties ? JSON.stringify(properties) : "{}";
|
|
1033
|
-
await conn.run(`
|
|
1034
|
-
INSERT INTO relationships (source_label, source_name, relation_type, target_label, target_name, properties)
|
|
1035
|
-
VALUES (
|
|
1036
|
-
'${this.escape(sourceLabel)}',
|
|
1037
|
-
'${this.escape(sourceName)}',
|
|
1038
|
-
'${this.escape(relation)}',
|
|
1039
|
-
'${this.escape(targetLabel)}',
|
|
1040
|
-
'${this.escape(targetName)}',
|
|
1041
|
-
'${this.escape(propsJson)}'
|
|
1042
|
-
)
|
|
1043
|
-
ON CONFLICT (source_label, source_name, relation_type, target_label, target_name) DO UPDATE SET
|
|
1044
|
-
properties = EXCLUDED.properties
|
|
1045
|
-
`);
|
|
1046
|
-
} catch (error) {
|
|
1047
|
-
this.logger.error(`Failed to upsert relationship: ${error instanceof Error ? error.message : String(error)}`);
|
|
1048
|
-
throw error;
|
|
1049
|
-
}
|
|
1344
|
+
getProviderName() {
|
|
1345
|
+
return this.provider.name;
|
|
1050
1346
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
const conn = await this.ensureConnected();
|
|
1054
|
-
await conn.run(`
|
|
1055
|
-
DELETE FROM relationships
|
|
1056
|
-
WHERE (source_label = '${this.escape(label)}' AND source_name = '${this.escape(name)}')
|
|
1057
|
-
OR (target_label = '${this.escape(label)}' AND target_name = '${this.escape(name)}')
|
|
1058
|
-
`);
|
|
1059
|
-
await conn.run(`
|
|
1060
|
-
DELETE FROM nodes
|
|
1061
|
-
WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
|
|
1062
|
-
`);
|
|
1063
|
-
} catch (error) {
|
|
1064
|
-
this.logger.error(`Failed to delete node: ${error instanceof Error ? error.message : String(error)}`);
|
|
1065
|
-
throw error;
|
|
1066
|
-
}
|
|
1347
|
+
getDimensions() {
|
|
1348
|
+
return this.provider.dimensions;
|
|
1067
1349
|
}
|
|
1068
|
-
async
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
await conn.run(`
|
|
1072
|
-
DELETE FROM relationships
|
|
1073
|
-
WHERE properties->>'documentPath' = '${this.escape(documentPath)}'
|
|
1074
|
-
`);
|
|
1075
|
-
} catch (error) {
|
|
1076
|
-
this.logger.error(`Failed to delete document relationships: ${error instanceof Error ? error.message : String(error)}`);
|
|
1077
|
-
throw error;
|
|
1350
|
+
async generateEmbedding(text) {
|
|
1351
|
+
if (!text || text.trim().length === 0) {
|
|
1352
|
+
throw new Error("Cannot generate embedding for empty text");
|
|
1078
1353
|
}
|
|
1354
|
+
return this.provider.generateEmbedding(text);
|
|
1079
1355
|
}
|
|
1080
|
-
async
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
const limitClause = limit ? ` LIMIT ${limit}` : "";
|
|
1084
|
-
const reader = await conn.runAndReadAll(`
|
|
1085
|
-
SELECT name, properties
|
|
1086
|
-
FROM nodes
|
|
1087
|
-
WHERE label = '${this.escape(label)}'${limitClause}
|
|
1088
|
-
`);
|
|
1089
|
-
return reader.getRows().map((row) => {
|
|
1090
|
-
const [name, properties] = row;
|
|
1091
|
-
const props = properties ? JSON.parse(properties) : {};
|
|
1092
|
-
return { name, ...props };
|
|
1093
|
-
});
|
|
1094
|
-
} catch (error) {
|
|
1095
|
-
this.logger.error(`Failed to find nodes by label: ${error instanceof Error ? error.message : String(error)}`);
|
|
1096
|
-
return [];
|
|
1356
|
+
async generateQueryEmbedding(text) {
|
|
1357
|
+
if (!text || text.trim().length === 0) {
|
|
1358
|
+
throw new Error("Cannot generate embedding for empty text");
|
|
1097
1359
|
}
|
|
1360
|
+
if (this.provider.generateQueryEmbedding) {
|
|
1361
|
+
return this.provider.generateQueryEmbedding(text);
|
|
1362
|
+
}
|
|
1363
|
+
return this.provider.generateEmbedding(text);
|
|
1098
1364
|
}
|
|
1099
|
-
async
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
const reader = await conn.runAndReadAll(`
|
|
1103
|
-
SELECT relation_type, target_name, source_name
|
|
1104
|
-
FROM relationships
|
|
1105
|
-
WHERE source_name = '${this.escape(nodeName)}'
|
|
1106
|
-
OR target_name = '${this.escape(nodeName)}'
|
|
1107
|
-
`);
|
|
1108
|
-
return reader.getRows().map((row) => {
|
|
1109
|
-
const [relType, targetName, sourceName] = row;
|
|
1110
|
-
return [relType, sourceName === nodeName ? targetName : sourceName];
|
|
1111
|
-
});
|
|
1112
|
-
} catch (error) {
|
|
1113
|
-
this.logger.error(`Failed to find relationships: ${error instanceof Error ? error.message : String(error)}`);
|
|
1365
|
+
async generateEmbeddings(texts) {
|
|
1366
|
+
const validTexts = texts.filter((t) => t && t.trim().length > 0);
|
|
1367
|
+
if (validTexts.length === 0) {
|
|
1114
1368
|
return [];
|
|
1115
1369
|
}
|
|
1370
|
+
return this.provider.generateEmbeddings(validTexts);
|
|
1116
1371
|
}
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
const indexKey = `${label}_${property}`;
|
|
1120
|
-
if (this.vectorIndexes.has(indexKey)) {
|
|
1121
|
-
return;
|
|
1122
|
-
}
|
|
1123
|
-
const conn = await this.ensureConnected();
|
|
1124
|
-
try {
|
|
1125
|
-
await conn.run(`
|
|
1126
|
-
CREATE INDEX idx_embedding_${this.escape(label)}
|
|
1127
|
-
ON nodes USING HNSW (embedding)
|
|
1128
|
-
WITH (metric = 'cosine')
|
|
1129
|
-
`);
|
|
1130
|
-
} catch {
|
|
1131
|
-
this.logger.debug(`Vector index on ${label}.${property} already exists`);
|
|
1132
|
-
}
|
|
1133
|
-
this.vectorIndexes.add(indexKey);
|
|
1134
|
-
this.logger.log(`Created vector index on ${label}.${property} with ${dimensions} dimensions`);
|
|
1135
|
-
} catch (error) {
|
|
1136
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1137
|
-
if (!errorMessage.includes("already exists")) {
|
|
1138
|
-
this.logger.error(`Failed to create vector index: ${errorMessage}`);
|
|
1139
|
-
throw error;
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
}
|
|
1143
|
-
async updateNodeEmbedding(label, name, embedding) {
|
|
1144
|
-
try {
|
|
1145
|
-
const conn = await this.ensureConnected();
|
|
1146
|
-
const vectorStr = `[${embedding.join(", ")}]`;
|
|
1147
|
-
await conn.run(`
|
|
1148
|
-
UPDATE nodes
|
|
1149
|
-
SET embedding = ${vectorStr}::FLOAT[${this.embeddingDimensions}]
|
|
1150
|
-
WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
|
|
1151
|
-
`);
|
|
1152
|
-
} catch (error) {
|
|
1153
|
-
this.logger.error(`Failed to update node embedding: ${error instanceof Error ? error.message : String(error)}`);
|
|
1154
|
-
throw error;
|
|
1155
|
-
}
|
|
1156
|
-
}
|
|
1157
|
-
async vectorSearch(label, queryVector, k = 10) {
|
|
1158
|
-
try {
|
|
1159
|
-
const conn = await this.ensureConnected();
|
|
1160
|
-
const vectorStr = `[${queryVector.join(", ")}]`;
|
|
1161
|
-
const reader = await conn.runAndReadAll(`
|
|
1162
|
-
SELECT
|
|
1163
|
-
name,
|
|
1164
|
-
properties->>'title' as title,
|
|
1165
|
-
array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
|
|
1166
|
-
FROM nodes
|
|
1167
|
-
WHERE label = '${this.escape(label)}'
|
|
1168
|
-
AND embedding IS NOT NULL
|
|
1169
|
-
ORDER BY similarity DESC
|
|
1170
|
-
LIMIT ${k}
|
|
1171
|
-
`);
|
|
1172
|
-
return reader.getRows().map((row) => {
|
|
1173
|
-
const [name, title, similarity] = row;
|
|
1174
|
-
return {
|
|
1175
|
-
name,
|
|
1176
|
-
title: title || undefined,
|
|
1177
|
-
score: similarity
|
|
1178
|
-
};
|
|
1179
|
-
});
|
|
1180
|
-
} catch (error) {
|
|
1181
|
-
this.logger.error(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1182
|
-
throw error;
|
|
1183
|
-
}
|
|
1184
|
-
}
|
|
1185
|
-
async vectorSearchAll(queryVector, k = 10) {
|
|
1186
|
-
const allResults = [];
|
|
1187
|
-
const conn = await this.ensureConnected();
|
|
1188
|
-
const vectorStr = `[${queryVector.join(", ")}]`;
|
|
1189
|
-
try {
|
|
1190
|
-
const reader = await conn.runAndReadAll(`
|
|
1191
|
-
SELECT
|
|
1192
|
-
name,
|
|
1193
|
-
label,
|
|
1194
|
-
properties->>'title' as title,
|
|
1195
|
-
properties->>'description' as description,
|
|
1196
|
-
array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
|
|
1197
|
-
FROM nodes
|
|
1198
|
-
WHERE embedding IS NOT NULL
|
|
1199
|
-
ORDER BY similarity DESC
|
|
1200
|
-
LIMIT ${k}
|
|
1201
|
-
`);
|
|
1202
|
-
for (const row of reader.getRows()) {
|
|
1203
|
-
const [name, label, title, description, similarity] = row;
|
|
1204
|
-
allResults.push({
|
|
1205
|
-
name,
|
|
1206
|
-
label,
|
|
1207
|
-
title: title || undefined,
|
|
1208
|
-
description: description || undefined,
|
|
1209
|
-
score: similarity
|
|
1210
|
-
});
|
|
1211
|
-
}
|
|
1212
|
-
} catch (error) {
|
|
1213
|
-
this.logger.debug(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1214
|
-
}
|
|
1215
|
-
return allResults.sort((a, b) => b.score - a.score).slice(0, k);
|
|
1216
|
-
}
|
|
1217
|
-
escape(value) {
|
|
1218
|
-
return value.replace(/'/g, "''");
|
|
1372
|
+
isRealProvider() {
|
|
1373
|
+
return this.provider.name !== "mock";
|
|
1219
1374
|
}
|
|
1220
1375
|
}
|
|
1221
|
-
|
|
1376
|
+
EmbeddingService = __legacyDecorateClassTS([
|
|
1222
1377
|
Injectable6(),
|
|
1223
1378
|
__legacyMetadataTS("design:paramtypes", [
|
|
1224
1379
|
typeof ConfigService2 === "undefined" ? Object : ConfigService2
|
|
1225
1380
|
])
|
|
1226
|
-
],
|
|
1381
|
+
], EmbeddingService);
|
|
1227
1382
|
|
|
1228
|
-
// src/
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
super();
|
|
1234
|
-
this.graphService = graphService;
|
|
1235
|
-
this.embeddingService = embeddingService;
|
|
1383
|
+
// src/pure/embedding-text.ts
|
|
1384
|
+
function composeDocumentEmbeddingText(doc) {
|
|
1385
|
+
const parts = [];
|
|
1386
|
+
if (doc.title) {
|
|
1387
|
+
parts.push(`Title: ${doc.title}`);
|
|
1236
1388
|
}
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1389
|
+
if (doc.topic) {
|
|
1390
|
+
parts.push(`Topic: ${doc.topic}`);
|
|
1391
|
+
}
|
|
1392
|
+
if (doc.tags && doc.tags.length > 0) {
|
|
1393
|
+
parts.push(`Tags: ${doc.tags.join(", ")}`);
|
|
1394
|
+
}
|
|
1395
|
+
if (doc.entities && doc.entities.length > 0) {
|
|
1396
|
+
const entityNames = doc.entities.map((e) => e.name).join(", ");
|
|
1397
|
+
parts.push(`Entities: ${entityNames}`);
|
|
1398
|
+
}
|
|
1399
|
+
if (doc.summary) {
|
|
1400
|
+
parts.push(doc.summary);
|
|
1401
|
+
} else {
|
|
1402
|
+
parts.push(doc.content.slice(0, 500));
|
|
1403
|
+
}
|
|
1404
|
+
return parts.join(" | ");
|
|
1405
|
+
}
|
|
1406
|
+
function composeEntityEmbeddingText(entity) {
|
|
1407
|
+
const parts = [`${entity.type}: ${entity.name}`];
|
|
1408
|
+
if (entity.description) {
|
|
1409
|
+
parts.push(entity.description);
|
|
1410
|
+
}
|
|
1411
|
+
return parts.join(". ");
|
|
1412
|
+
}
|
|
1413
|
+
function collectUniqueEntities(docs) {
|
|
1414
|
+
const entities = new Map;
|
|
1415
|
+
for (const doc of docs) {
|
|
1416
|
+
for (const entity of doc.entities) {
|
|
1417
|
+
const key = `${entity.type}:${entity.name}`;
|
|
1418
|
+
const existing = entities.get(key);
|
|
1419
|
+
if (!existing) {
|
|
1420
|
+
entities.set(key, {
|
|
1421
|
+
type: entity.type,
|
|
1422
|
+
name: entity.name,
|
|
1423
|
+
description: entity.description,
|
|
1424
|
+
documentPaths: [doc.path]
|
|
1425
|
+
});
|
|
1251
1426
|
} else {
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
console.log(`
|
|
1256
|
-
=== Semantic Search Results for "${query}"${labelSuffix} ===
|
|
1257
|
-
`);
|
|
1258
|
-
if (results.length === 0) {
|
|
1259
|
-
console.log(`No results found.
|
|
1260
|
-
`);
|
|
1261
|
-
if (options.label) {
|
|
1262
|
-
console.log(`Tip: Try without --label to search all entity types.
|
|
1263
|
-
`);
|
|
1427
|
+
existing.documentPaths.push(doc.path);
|
|
1428
|
+
if (entity.description && (!existing.description || entity.description.length > existing.description.length)) {
|
|
1429
|
+
existing.description = entity.description;
|
|
1264
1430
|
}
|
|
1265
|
-
process.exit(0);
|
|
1266
1431
|
}
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
return entities;
|
|
1435
|
+
}
|
|
1436
|
+
// src/pure/validation.ts
|
|
1437
|
+
function validateDocuments(docs) {
|
|
1438
|
+
const errors = [];
|
|
1439
|
+
for (const doc of docs) {
|
|
1440
|
+
if (!doc.title || doc.title.trim() === "") {
|
|
1441
|
+
errors.push({
|
|
1442
|
+
path: doc.path,
|
|
1443
|
+
error: "Missing required field: title"
|
|
1277
1444
|
});
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1445
|
+
}
|
|
1446
|
+
if (!doc.summary || doc.summary.trim() === "") {
|
|
1447
|
+
errors.push({
|
|
1448
|
+
path: doc.path,
|
|
1449
|
+
error: "Missing required field: summary"
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
if (!doc.created) {
|
|
1453
|
+
errors.push({
|
|
1454
|
+
path: doc.path,
|
|
1455
|
+
error: "Missing required field: created"
|
|
1456
|
+
});
|
|
1457
|
+
}
|
|
1458
|
+
if (!doc.updated) {
|
|
1459
|
+
errors.push({
|
|
1460
|
+
path: doc.path,
|
|
1461
|
+
error: "Missing required field: updated"
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
if (!doc.status) {
|
|
1465
|
+
errors.push({
|
|
1466
|
+
path: doc.path,
|
|
1467
|
+
error: "Missing required field: status"
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
const entityIndex = new Map;
|
|
1472
|
+
for (const doc of docs) {
|
|
1473
|
+
for (const entity of doc.entities) {
|
|
1474
|
+
let docSet = entityIndex.get(entity.name);
|
|
1475
|
+
if (!docSet) {
|
|
1476
|
+
docSet = new Set;
|
|
1477
|
+
entityIndex.set(entity.name, docSet);
|
|
1288
1478
|
}
|
|
1289
|
-
|
|
1479
|
+
docSet.add(doc.path);
|
|
1290
1480
|
}
|
|
1291
1481
|
}
|
|
1292
|
-
|
|
1293
|
-
|
|
1482
|
+
for (const doc of docs) {
|
|
1483
|
+
for (const rel of doc.relationships) {
|
|
1484
|
+
if (rel.source !== doc.path && !entityIndex.has(rel.source)) {
|
|
1485
|
+
errors.push({
|
|
1486
|
+
path: doc.path,
|
|
1487
|
+
error: `Relationship source "${rel.source}" not found in any document`
|
|
1488
|
+
});
|
|
1489
|
+
}
|
|
1490
|
+
const isDocPath = rel.target.endsWith(".md");
|
|
1491
|
+
const isKnownEntity = entityIndex.has(rel.target);
|
|
1492
|
+
const isSelfReference = rel.target === doc.path;
|
|
1493
|
+
if (!isDocPath && !isKnownEntity && !isSelfReference) {
|
|
1494
|
+
errors.push({
|
|
1495
|
+
path: doc.path,
|
|
1496
|
+
error: `Relationship target "${rel.target}" not found as entity`
|
|
1497
|
+
});
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1294
1500
|
}
|
|
1295
|
-
|
|
1296
|
-
|
|
1501
|
+
return errors;
|
|
1502
|
+
}
|
|
1503
|
+
function getChangeReason(changeType) {
|
|
1504
|
+
switch (changeType) {
|
|
1505
|
+
case "new":
|
|
1506
|
+
return "New document";
|
|
1507
|
+
case "updated":
|
|
1508
|
+
return "Content or frontmatter changed";
|
|
1509
|
+
case "deleted":
|
|
1510
|
+
return "File no longer exists";
|
|
1511
|
+
case "unchanged":
|
|
1512
|
+
return "No changes detected";
|
|
1297
1513
|
}
|
|
1298
1514
|
}
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
flags: "-l, --label <label>",
|
|
1302
|
-
description: "Filter by entity label (e.g., Technology, Concept, Document)"
|
|
1303
|
-
}),
|
|
1304
|
-
__legacyMetadataTS("design:type", Function),
|
|
1305
|
-
__legacyMetadataTS("design:paramtypes", [
|
|
1306
|
-
String
|
|
1307
|
-
]),
|
|
1308
|
-
__legacyMetadataTS("design:returntype", String)
|
|
1309
|
-
], SearchCommand.prototype, "parseLabel", null);
|
|
1310
|
-
__legacyDecorateClassTS([
|
|
1311
|
-
Option({
|
|
1312
|
-
flags: "--limit <n>",
|
|
1313
|
-
description: "Limit results",
|
|
1314
|
-
defaultValue: "20"
|
|
1315
|
-
}),
|
|
1316
|
-
__legacyMetadataTS("design:type", Function),
|
|
1317
|
-
__legacyMetadataTS("design:paramtypes", [
|
|
1318
|
-
String
|
|
1319
|
-
]),
|
|
1320
|
-
__legacyMetadataTS("design:returntype", String)
|
|
1321
|
-
], SearchCommand.prototype, "parseLimit", null);
|
|
1322
|
-
SearchCommand = __legacyDecorateClassTS([
|
|
1323
|
-
Injectable7(),
|
|
1324
|
-
Command3({
|
|
1325
|
-
name: "search",
|
|
1326
|
-
arguments: "<query>",
|
|
1327
|
-
description: "Semantic search across the knowledge graph"
|
|
1328
|
-
}),
|
|
1329
|
-
__legacyMetadataTS("design:paramtypes", [
|
|
1330
|
-
typeof GraphService === "undefined" ? Object : GraphService,
|
|
1331
|
-
typeof EmbeddingService === "undefined" ? Object : EmbeddingService
|
|
1332
|
-
])
|
|
1333
|
-
], SearchCommand);
|
|
1515
|
+
// src/sync/cascade.service.ts
|
|
1516
|
+
import { Injectable as Injectable8, Logger as Logger5 } from "@nestjs/common";
|
|
1334
1517
|
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1518
|
+
// src/sync/document-parser.service.ts
|
|
1519
|
+
import { createHash as createHash2 } from "crypto";
|
|
1520
|
+
import { readFile as readFile4 } from "fs/promises";
|
|
1521
|
+
import { Injectable as Injectable7, Logger as Logger4 } from "@nestjs/common";
|
|
1522
|
+
import { glob } from "glob";
|
|
1523
|
+
|
|
1524
|
+
// src/utils/frontmatter.ts
|
|
1525
|
+
import matter from "gray-matter";
|
|
1526
|
+
import { z as z5 } from "zod";
|
|
1527
|
+
var EntityTypeSchema = z5.enum([
|
|
1528
|
+
"Topic",
|
|
1529
|
+
"Technology",
|
|
1530
|
+
"Concept",
|
|
1531
|
+
"Tool",
|
|
1532
|
+
"Process",
|
|
1533
|
+
"Person",
|
|
1534
|
+
"Organization",
|
|
1535
|
+
"Document"
|
|
1536
|
+
]);
|
|
1537
|
+
var RelationTypeSchema = z5.enum(["REFERENCES"]);
|
|
1538
|
+
var EntitySchema = z5.object({
|
|
1539
|
+
name: z5.string().min(1),
|
|
1540
|
+
type: EntityTypeSchema,
|
|
1541
|
+
description: z5.string().min(1)
|
|
1542
|
+
});
|
|
1543
|
+
var RelationshipSchema = z5.object({
|
|
1544
|
+
source: z5.string().min(1),
|
|
1545
|
+
relation: RelationTypeSchema,
|
|
1546
|
+
target: z5.string().min(1)
|
|
1547
|
+
});
|
|
1548
|
+
var GraphMetadataSchema = z5.object({
|
|
1549
|
+
importance: z5.enum(["high", "medium", "low"]).optional(),
|
|
1550
|
+
domain: z5.string().optional()
|
|
1551
|
+
});
|
|
1552
|
+
var validateDateFormat = (dateStr) => {
|
|
1553
|
+
const match = dateStr.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
|
1554
|
+
if (!match)
|
|
1555
|
+
return false;
|
|
1556
|
+
const [, yearStr, monthStr, dayStr] = match;
|
|
1557
|
+
const year = parseInt(yearStr, 10);
|
|
1558
|
+
const month = parseInt(monthStr, 10);
|
|
1559
|
+
const day = parseInt(dayStr, 10);
|
|
1560
|
+
if (month < 1 || month > 12)
|
|
1561
|
+
return false;
|
|
1562
|
+
if (day < 1 || day > 31)
|
|
1563
|
+
return false;
|
|
1564
|
+
const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
1565
|
+
if (year % 4 === 0 && year % 100 !== 0 || year % 400 === 0) {
|
|
1566
|
+
daysInMonth[1] = 29;
|
|
1340
1567
|
}
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1568
|
+
return day <= daysInMonth[month - 1];
|
|
1569
|
+
};
|
|
1570
|
+
var FrontmatterSchema = z5.object({
|
|
1571
|
+
created: z5.string().refine(validateDateFormat, "Date must be in YYYY-MM-DD format"),
|
|
1572
|
+
updated: z5.string().refine(validateDateFormat, "Date must be in YYYY-MM-DD format"),
|
|
1573
|
+
status: z5.enum(["draft", "ongoing", "complete"]).optional(),
|
|
1574
|
+
topic: z5.string().optional(),
|
|
1575
|
+
tags: z5.array(z5.string()).optional(),
|
|
1576
|
+
summary: z5.string().optional(),
|
|
1577
|
+
entities: z5.array(EntitySchema).optional(),
|
|
1578
|
+
relationships: z5.array(RelationshipSchema).optional(),
|
|
1579
|
+
graph: GraphMetadataSchema.optional()
|
|
1580
|
+
}).passthrough();
|
|
1581
|
+
function parseFrontmatter(content) {
|
|
1582
|
+
try {
|
|
1583
|
+
const { data, content: markdown } = matter(content);
|
|
1584
|
+
if (Object.keys(data).length === 0) {
|
|
1585
|
+
return {
|
|
1586
|
+
frontmatter: null,
|
|
1587
|
+
content: markdown.trim(),
|
|
1588
|
+
raw: content
|
|
1589
|
+
};
|
|
1363
1590
|
}
|
|
1591
|
+
const normalizedData = normalizeData(data);
|
|
1592
|
+
const validated = FrontmatterSchema.safeParse(normalizedData);
|
|
1593
|
+
return {
|
|
1594
|
+
frontmatter: validated.success ? validated.data : normalizedData,
|
|
1595
|
+
content: markdown.trim(),
|
|
1596
|
+
raw: content
|
|
1597
|
+
};
|
|
1598
|
+
} catch (error) {
|
|
1599
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1600
|
+
throw new Error(`YAML parsing error: ${errorMessage}`);
|
|
1364
1601
|
}
|
|
1365
1602
|
}
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
name: "rels",
|
|
1370
|
-
arguments: "<name>",
|
|
1371
|
-
description: "Show relationships for a node"
|
|
1372
|
-
}),
|
|
1373
|
-
__legacyMetadataTS("design:paramtypes", [
|
|
1374
|
-
typeof GraphService === "undefined" ? Object : GraphService
|
|
1375
|
-
])
|
|
1376
|
-
], RelsCommand);
|
|
1377
|
-
|
|
1378
|
-
class SqlCommand extends CommandRunner3 {
|
|
1379
|
-
graphService;
|
|
1380
|
-
constructor(graphService) {
|
|
1381
|
-
super();
|
|
1382
|
-
this.graphService = graphService;
|
|
1603
|
+
function normalizeData(data) {
|
|
1604
|
+
if (data instanceof Date) {
|
|
1605
|
+
return data.toISOString().split("T")[0];
|
|
1383
1606
|
}
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
const replacer = (_key, value) => typeof value === "bigint" ? Number(value) : value;
|
|
1392
|
-
console.log(JSON.stringify(result, replacer, 2));
|
|
1393
|
-
console.log();
|
|
1394
|
-
process.exit(0);
|
|
1395
|
-
} catch (error) {
|
|
1396
|
-
console.error("Error:", error instanceof Error ? error.message : String(error));
|
|
1397
|
-
process.exit(1);
|
|
1607
|
+
if (Array.isArray(data)) {
|
|
1608
|
+
return data.map(normalizeData);
|
|
1609
|
+
}
|
|
1610
|
+
if (data !== null && typeof data === "object") {
|
|
1611
|
+
const normalized = {};
|
|
1612
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1613
|
+
normalized[key] = normalizeData(value);
|
|
1398
1614
|
}
|
|
1615
|
+
return normalized;
|
|
1399
1616
|
}
|
|
1617
|
+
return data;
|
|
1400
1618
|
}
|
|
1401
|
-
SqlCommand = __legacyDecorateClassTS([
|
|
1402
|
-
Injectable7(),
|
|
1403
|
-
Command3({
|
|
1404
|
-
name: "sql",
|
|
1405
|
-
arguments: "<query>",
|
|
1406
|
-
description: "Execute raw SQL query against DuckDB"
|
|
1407
|
-
}),
|
|
1408
|
-
__legacyMetadataTS("design:paramtypes", [
|
|
1409
|
-
typeof GraphService === "undefined" ? Object : GraphService
|
|
1410
|
-
])
|
|
1411
|
-
], SqlCommand);
|
|
1412
|
-
// src/commands/status.command.ts
|
|
1413
|
-
import { Injectable as Injectable12 } from "@nestjs/common";
|
|
1414
|
-
import { Command as Command4, CommandRunner as CommandRunner4, Option as Option2 } from "nest-commander";
|
|
1415
|
-
|
|
1416
|
-
// src/sync/manifest.service.ts
|
|
1417
|
-
import { createHash as createHash2 } from "crypto";
|
|
1418
|
-
import { existsSync as existsSync3 } from "fs";
|
|
1419
|
-
import { readFile as readFile3, writeFile } from "fs/promises";
|
|
1420
|
-
import { Injectable as Injectable8 } from "@nestjs/common";
|
|
1421
1619
|
|
|
1422
|
-
// src/
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
frontmatterHash: z4.string(),
|
|
1427
|
-
lastSynced: z4.string(),
|
|
1428
|
-
entityCount: z4.number().int().nonnegative(),
|
|
1429
|
-
relationshipCount: z4.number().int().nonnegative()
|
|
1430
|
-
});
|
|
1431
|
-
var SyncManifestSchema = z4.object({
|
|
1432
|
-
version: z4.string(),
|
|
1433
|
-
lastSync: z4.string(),
|
|
1434
|
-
documents: z4.record(z4.string(), ManifestEntrySchema)
|
|
1435
|
-
});
|
|
1436
|
-
|
|
1437
|
-
// src/sync/manifest.service.ts
|
|
1438
|
-
class ManifestService {
|
|
1439
|
-
manifestPath;
|
|
1440
|
-
manifest = null;
|
|
1620
|
+
// src/sync/document-parser.service.ts
|
|
1621
|
+
class DocumentParserService {
|
|
1622
|
+
logger = new Logger4(DocumentParserService.name);
|
|
1623
|
+
docsPath;
|
|
1441
1624
|
constructor() {
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
async load() {
|
|
1445
|
-
try {
|
|
1446
|
-
if (existsSync3(this.manifestPath)) {
|
|
1447
|
-
const content = await readFile3(this.manifestPath, "utf-8");
|
|
1448
|
-
this.manifest = SyncManifestSchema.parse(JSON.parse(content));
|
|
1449
|
-
} else {
|
|
1450
|
-
this.manifest = this.createEmptyManifest();
|
|
1451
|
-
}
|
|
1452
|
-
} catch (_error) {
|
|
1453
|
-
this.manifest = this.createEmptyManifest();
|
|
1454
|
-
}
|
|
1455
|
-
return this.manifest;
|
|
1456
|
-
}
|
|
1457
|
-
async save() {
|
|
1458
|
-
if (!this.manifest) {
|
|
1459
|
-
throw new Error("Manifest not loaded. Call load() first.");
|
|
1460
|
-
}
|
|
1461
|
-
this.manifest.lastSync = new Date().toISOString();
|
|
1462
|
-
const content = JSON.stringify(this.manifest, null, 2);
|
|
1463
|
-
await writeFile(this.manifestPath, content, "utf-8");
|
|
1625
|
+
ensureLatticeHome();
|
|
1626
|
+
this.docsPath = getDocsPath();
|
|
1464
1627
|
}
|
|
1465
|
-
|
|
1466
|
-
return
|
|
1628
|
+
getDocsPath() {
|
|
1629
|
+
return this.docsPath;
|
|
1467
1630
|
}
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
return "new";
|
|
1475
|
-
}
|
|
1476
|
-
if (existing.contentHash === contentHash && existing.frontmatterHash === frontmatterHash) {
|
|
1477
|
-
return "unchanged";
|
|
1478
|
-
}
|
|
1479
|
-
return "updated";
|
|
1631
|
+
async discoverDocuments() {
|
|
1632
|
+
const pattern = `${this.docsPath}/**/*.md`;
|
|
1633
|
+
const files = await glob(pattern, {
|
|
1634
|
+
ignore: ["**/node_modules/**", "**/.git/**"]
|
|
1635
|
+
});
|
|
1636
|
+
return files.sort();
|
|
1480
1637
|
}
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1638
|
+
async parseDocument(filePath) {
|
|
1639
|
+
const content = await readFile4(filePath, "utf-8");
|
|
1640
|
+
const parsed = parseFrontmatter(content);
|
|
1641
|
+
const title = this.extractTitle(content, filePath);
|
|
1642
|
+
const contentHash = this.computeHash(content);
|
|
1643
|
+
const frontmatterHash = this.computeHash(JSON.stringify(parsed.frontmatter || {}));
|
|
1644
|
+
const graphMetadata = this.extractGraphMetadata(parsed.frontmatter);
|
|
1645
|
+
return {
|
|
1646
|
+
path: filePath,
|
|
1647
|
+
title,
|
|
1648
|
+
content: parsed.content,
|
|
1486
1649
|
contentHash,
|
|
1487
1650
|
frontmatterHash,
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
delete this.manifest.documents[path2];
|
|
1498
|
-
}
|
|
1499
|
-
getTrackedPaths() {
|
|
1500
|
-
if (!this.manifest) {
|
|
1501
|
-
throw new Error("Manifest not loaded. Call load() first.");
|
|
1502
|
-
}
|
|
1503
|
-
return Object.keys(this.manifest.documents);
|
|
1504
|
-
}
|
|
1505
|
-
createEmptyManifest() {
|
|
1506
|
-
return {
|
|
1507
|
-
version: "1.0",
|
|
1508
|
-
lastSync: new Date().toISOString(),
|
|
1509
|
-
documents: {}
|
|
1651
|
+
summary: parsed.frontmatter?.summary,
|
|
1652
|
+
topic: parsed.frontmatter?.topic,
|
|
1653
|
+
entities: [],
|
|
1654
|
+
relationships: [],
|
|
1655
|
+
graphMetadata,
|
|
1656
|
+
tags: parsed.frontmatter?.tags || [],
|
|
1657
|
+
created: parsed.frontmatter?.created,
|
|
1658
|
+
updated: parsed.frontmatter?.updated,
|
|
1659
|
+
status: parsed.frontmatter?.status
|
|
1510
1660
|
};
|
|
1511
1661
|
}
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
__legacyMetadataTS("design:paramtypes", [])
|
|
1516
|
-
], ManifestService);
|
|
1517
|
-
|
|
1518
|
-
// src/sync/sync.service.ts
|
|
1519
|
-
import { Injectable as Injectable11, Logger as Logger5 } from "@nestjs/common";
|
|
1520
|
-
|
|
1521
|
-
// src/pure/embedding-text.ts
|
|
1522
|
-
function composeDocumentEmbeddingText(doc) {
|
|
1523
|
-
const parts = [];
|
|
1524
|
-
if (doc.title) {
|
|
1525
|
-
parts.push(`Title: ${doc.title}`);
|
|
1526
|
-
}
|
|
1527
|
-
if (doc.topic) {
|
|
1528
|
-
parts.push(`Topic: ${doc.topic}`);
|
|
1529
|
-
}
|
|
1530
|
-
if (doc.tags && doc.tags.length > 0) {
|
|
1531
|
-
parts.push(`Tags: ${doc.tags.join(", ")}`);
|
|
1532
|
-
}
|
|
1533
|
-
if (doc.entities && doc.entities.length > 0) {
|
|
1534
|
-
const entityNames = doc.entities.map((e) => e.name).join(", ");
|
|
1535
|
-
parts.push(`Entities: ${entityNames}`);
|
|
1536
|
-
}
|
|
1537
|
-
if (doc.summary) {
|
|
1538
|
-
parts.push(doc.summary);
|
|
1539
|
-
} else {
|
|
1540
|
-
parts.push(doc.content.slice(0, 500));
|
|
1541
|
-
}
|
|
1542
|
-
return parts.join(" | ");
|
|
1543
|
-
}
|
|
1544
|
-
function composeEntityEmbeddingText(entity) {
|
|
1545
|
-
const parts = [`${entity.type}: ${entity.name}`];
|
|
1546
|
-
if (entity.description) {
|
|
1547
|
-
parts.push(entity.description);
|
|
1662
|
+
async parseAllDocuments() {
|
|
1663
|
+
const { docs } = await this.parseAllDocumentsWithErrors();
|
|
1664
|
+
return docs;
|
|
1548
1665
|
}
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
description: entity.description,
|
|
1562
|
-
documentPaths: [doc.path]
|
|
1563
|
-
});
|
|
1564
|
-
} else {
|
|
1565
|
-
existing.documentPaths.push(doc.path);
|
|
1566
|
-
if (entity.description && (!existing.description || entity.description.length > existing.description.length)) {
|
|
1567
|
-
existing.description = entity.description;
|
|
1568
|
-
}
|
|
1666
|
+
async parseAllDocumentsWithErrors() {
|
|
1667
|
+
const files = await this.discoverDocuments();
|
|
1668
|
+
const docs = [];
|
|
1669
|
+
const errors = [];
|
|
1670
|
+
for (const file of files) {
|
|
1671
|
+
try {
|
|
1672
|
+
const parsed = await this.parseDocument(file);
|
|
1673
|
+
docs.push(parsed);
|
|
1674
|
+
} catch (error) {
|
|
1675
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1676
|
+
errors.push({ path: file, error: errorMsg });
|
|
1677
|
+
this.logger.warn(`Failed to parse ${file}: ${error}`);
|
|
1569
1678
|
}
|
|
1570
1679
|
}
|
|
1680
|
+
return { docs, errors };
|
|
1571
1681
|
}
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
const errors = [];
|
|
1577
|
-
for (const doc of docs) {
|
|
1578
|
-
if (!doc.title || doc.title.trim() === "") {
|
|
1579
|
-
errors.push({
|
|
1580
|
-
path: doc.path,
|
|
1581
|
-
error: "Missing required field: title"
|
|
1582
|
-
});
|
|
1583
|
-
}
|
|
1584
|
-
if (!doc.summary || doc.summary.trim() === "") {
|
|
1585
|
-
errors.push({
|
|
1586
|
-
path: doc.path,
|
|
1587
|
-
error: "Missing required field: summary"
|
|
1588
|
-
});
|
|
1589
|
-
}
|
|
1590
|
-
if (!doc.created) {
|
|
1591
|
-
errors.push({
|
|
1592
|
-
path: doc.path,
|
|
1593
|
-
error: "Missing required field: created"
|
|
1594
|
-
});
|
|
1595
|
-
}
|
|
1596
|
-
if (!doc.updated) {
|
|
1597
|
-
errors.push({
|
|
1598
|
-
path: doc.path,
|
|
1599
|
-
error: "Missing required field: updated"
|
|
1600
|
-
});
|
|
1601
|
-
}
|
|
1602
|
-
if (!doc.status) {
|
|
1603
|
-
errors.push({
|
|
1604
|
-
path: doc.path,
|
|
1605
|
-
error: "Missing required field: status"
|
|
1606
|
-
});
|
|
1607
|
-
}
|
|
1608
|
-
}
|
|
1609
|
-
const entityIndex = new Map;
|
|
1610
|
-
for (const doc of docs) {
|
|
1611
|
-
for (const entity of doc.entities) {
|
|
1612
|
-
let docSet = entityIndex.get(entity.name);
|
|
1613
|
-
if (!docSet) {
|
|
1614
|
-
docSet = new Set;
|
|
1615
|
-
entityIndex.set(entity.name, docSet);
|
|
1616
|
-
}
|
|
1617
|
-
docSet.add(doc.path);
|
|
1682
|
+
extractTitle(content, filePath) {
|
|
1683
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
1684
|
+
if (h1Match) {
|
|
1685
|
+
return h1Match[1];
|
|
1618
1686
|
}
|
|
1687
|
+
const parts = filePath.split("/");
|
|
1688
|
+
return parts[parts.length - 1].replace(".md", "");
|
|
1619
1689
|
}
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
path: doc.path,
|
|
1625
|
-
error: `Relationship source "${rel.source}" not found in any document`
|
|
1626
|
-
});
|
|
1627
|
-
}
|
|
1628
|
-
const isDocPath = rel.target.endsWith(".md");
|
|
1629
|
-
const isKnownEntity = entityIndex.has(rel.target);
|
|
1630
|
-
const isSelfReference = rel.target === doc.path;
|
|
1631
|
-
if (!isDocPath && !isKnownEntity && !isSelfReference) {
|
|
1632
|
-
errors.push({
|
|
1633
|
-
path: doc.path,
|
|
1634
|
-
error: `Relationship target "${rel.target}" not found as entity`
|
|
1635
|
-
});
|
|
1636
|
-
}
|
|
1690
|
+
extractGraphMetadata(frontmatter) {
|
|
1691
|
+
const fm = frontmatter;
|
|
1692
|
+
if (!fm?.graph) {
|
|
1693
|
+
return;
|
|
1637
1694
|
}
|
|
1695
|
+
const result = GraphMetadataSchema.safeParse(fm.graph);
|
|
1696
|
+
return result.success ? result.data : undefined;
|
|
1638
1697
|
}
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
function getChangeReason(changeType) {
|
|
1642
|
-
switch (changeType) {
|
|
1643
|
-
case "new":
|
|
1644
|
-
return "New document";
|
|
1645
|
-
case "updated":
|
|
1646
|
-
return "Content or frontmatter changed";
|
|
1647
|
-
case "deleted":
|
|
1648
|
-
return "File no longer exists";
|
|
1649
|
-
case "unchanged":
|
|
1650
|
-
return "No changes detected";
|
|
1698
|
+
computeHash(content) {
|
|
1699
|
+
return createHash2("sha256").update(content).digest("hex");
|
|
1651
1700
|
}
|
|
1652
1701
|
}
|
|
1702
|
+
DocumentParserService = __legacyDecorateClassTS([
|
|
1703
|
+
Injectable7(),
|
|
1704
|
+
__legacyMetadataTS("design:paramtypes", [])
|
|
1705
|
+
], DocumentParserService);
|
|
1706
|
+
|
|
1653
1707
|
// src/sync/cascade.service.ts
|
|
1654
|
-
import { Injectable as Injectable9, Logger as Logger4 } from "@nestjs/common";
|
|
1655
1708
|
class CascadeService {
|
|
1656
1709
|
graph;
|
|
1657
1710
|
_parser;
|
|
1658
|
-
logger = new
|
|
1711
|
+
logger = new Logger5(CascadeService.name);
|
|
1659
1712
|
constructor(graph, _parser) {
|
|
1660
1713
|
this.graph = graph;
|
|
1661
1714
|
this._parser = _parser;
|
|
@@ -1764,14 +1817,14 @@ class CascadeService {
|
|
|
1764
1817
|
async findAffectedByRename(entityName, _newName) {
|
|
1765
1818
|
try {
|
|
1766
1819
|
const escapedName = this.escapeForSql(entityName);
|
|
1767
|
-
const
|
|
1820
|
+
const query3 = `
|
|
1768
1821
|
SELECT DISTINCT n.name, n.properties->>'title' as title
|
|
1769
1822
|
FROM nodes n
|
|
1770
1823
|
INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
|
|
1771
1824
|
WHERE r.source_name = '${escapedName}'
|
|
1772
1825
|
AND n.label = 'Document'
|
|
1773
1826
|
`.trim();
|
|
1774
|
-
const result = await this.graph.query(
|
|
1827
|
+
const result = await this.graph.query(query3);
|
|
1775
1828
|
return (result.resultSet || []).map((row) => ({
|
|
1776
1829
|
path: row[0],
|
|
1777
1830
|
reason: `References "${entityName}" in entities`,
|
|
@@ -1787,14 +1840,14 @@ class CascadeService {
|
|
|
1787
1840
|
async findAffectedByDeletion(entityName) {
|
|
1788
1841
|
try {
|
|
1789
1842
|
const escapedName = this.escapeForSql(entityName);
|
|
1790
|
-
const
|
|
1843
|
+
const query3 = `
|
|
1791
1844
|
SELECT DISTINCT n.name, n.properties->>'title' as title
|
|
1792
1845
|
FROM nodes n
|
|
1793
1846
|
INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
|
|
1794
1847
|
WHERE r.source_name = '${escapedName}'
|
|
1795
1848
|
AND n.label = 'Document'
|
|
1796
1849
|
`.trim();
|
|
1797
|
-
const result = await this.graph.query(
|
|
1850
|
+
const result = await this.graph.query(query3);
|
|
1798
1851
|
return (result.resultSet || []).map((row) => ({
|
|
1799
1852
|
path: row[0],
|
|
1800
1853
|
reason: `References deleted entity "${entityName}"`,
|
|
@@ -1810,14 +1863,14 @@ class CascadeService {
|
|
|
1810
1863
|
async findAffectedByTypeChange(entityName, oldType, newType) {
|
|
1811
1864
|
try {
|
|
1812
1865
|
const escapedName = this.escapeForSql(entityName);
|
|
1813
|
-
const
|
|
1866
|
+
const query3 = `
|
|
1814
1867
|
SELECT DISTINCT n.name, n.properties->>'title' as title
|
|
1815
1868
|
FROM nodes n
|
|
1816
1869
|
INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
|
|
1817
1870
|
WHERE r.source_name = '${escapedName}'
|
|
1818
1871
|
AND n.label = 'Document'
|
|
1819
1872
|
`.trim();
|
|
1820
|
-
const result = await this.graph.query(
|
|
1873
|
+
const result = await this.graph.query(query3);
|
|
1821
1874
|
return (result.resultSet || []).map((row) => ({
|
|
1822
1875
|
path: row[0],
|
|
1823
1876
|
reason: `References "${entityName}" with type "${oldType}" (now "${newType}")`,
|
|
@@ -1833,14 +1886,14 @@ class CascadeService {
|
|
|
1833
1886
|
async findAffectedByRelationshipChange(entityName) {
|
|
1834
1887
|
try {
|
|
1835
1888
|
const escapedName = this.escapeForSql(entityName);
|
|
1836
|
-
const
|
|
1889
|
+
const query3 = `
|
|
1837
1890
|
SELECT DISTINCT n.name, n.properties->>'title' as title, r.relation_type
|
|
1838
1891
|
FROM nodes n
|
|
1839
1892
|
INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
|
|
1840
1893
|
WHERE r.source_name = '${escapedName}'
|
|
1841
1894
|
AND n.label = 'Document'
|
|
1842
1895
|
`.trim();
|
|
1843
|
-
const result = await this.graph.query(
|
|
1896
|
+
const result = await this.graph.query(query3);
|
|
1844
1897
|
return (result.resultSet || []).map((row) => ({
|
|
1845
1898
|
path: row[0],
|
|
1846
1899
|
reason: `Has relationship with "${entityName}"`,
|
|
@@ -1856,14 +1909,14 @@ class CascadeService {
|
|
|
1856
1909
|
async findAffectedByDocumentDeletion(documentPath) {
|
|
1857
1910
|
try {
|
|
1858
1911
|
const escapedPath = this.escapeForSql(documentPath);
|
|
1859
|
-
const
|
|
1912
|
+
const query3 = `
|
|
1860
1913
|
SELECT DISTINCT n.name, r.relation_type
|
|
1861
1914
|
FROM nodes n
|
|
1862
1915
|
INNER JOIN relationships r ON r.source_label = n.label AND r.source_name = n.name
|
|
1863
1916
|
WHERE r.target_name = '${escapedPath}'
|
|
1864
1917
|
AND n.label = 'Document'
|
|
1865
1918
|
`.trim();
|
|
1866
|
-
const result = await this.graph.query(
|
|
1919
|
+
const result = await this.graph.query(query3);
|
|
1867
1920
|
return (result.resultSet || []).map((row) => ({
|
|
1868
1921
|
path: row[0],
|
|
1869
1922
|
reason: `Links to deleted document "${documentPath}"`,
|
|
@@ -1953,16 +2006,118 @@ class CascadeService {
|
|
|
1953
2006
|
}
|
|
1954
2007
|
}
|
|
1955
2008
|
CascadeService = __legacyDecorateClassTS([
|
|
1956
|
-
|
|
2009
|
+
Injectable8(),
|
|
1957
2010
|
__legacyMetadataTS("design:paramtypes", [
|
|
1958
2011
|
typeof GraphService === "undefined" ? Object : GraphService,
|
|
1959
2012
|
typeof DocumentParserService === "undefined" ? Object : DocumentParserService
|
|
1960
2013
|
])
|
|
1961
2014
|
], CascadeService);
|
|
1962
2015
|
|
|
2016
|
+
// src/sync/database-change-detector.service.ts
|
|
2017
|
+
import { createHash as createHash3 } from "crypto";
|
|
2018
|
+
import { Injectable as Injectable9, Logger as Logger6 } from "@nestjs/common";
|
|
2019
|
+
class DatabaseChangeDetectorService {
|
|
2020
|
+
graph;
|
|
2021
|
+
logger = new Logger6(DatabaseChangeDetectorService.name);
|
|
2022
|
+
hashCache = new Map;
|
|
2023
|
+
loaded = false;
|
|
2024
|
+
constructor(graph) {
|
|
2025
|
+
this.graph = graph;
|
|
2026
|
+
}
|
|
2027
|
+
async loadHashes() {
|
|
2028
|
+
this.hashCache = await this.graph.loadAllDocumentHashes();
|
|
2029
|
+
this.loaded = true;
|
|
2030
|
+
this.logger.debug(`Loaded ${this.hashCache.size} document hashes from DB`);
|
|
2031
|
+
}
|
|
2032
|
+
isLoaded() {
|
|
2033
|
+
return this.loaded;
|
|
2034
|
+
}
|
|
2035
|
+
reset() {
|
|
2036
|
+
this.hashCache.clear();
|
|
2037
|
+
this.loaded = false;
|
|
2038
|
+
}
|
|
2039
|
+
getContentHash(content) {
|
|
2040
|
+
return createHash3("sha256").update(content).digest("hex");
|
|
2041
|
+
}
|
|
2042
|
+
detectChange(path2, currentContentHash) {
|
|
2043
|
+
if (!this.loaded) {
|
|
2044
|
+
throw new Error("Hashes not loaded. Call loadHashes() before detectChange().");
|
|
2045
|
+
}
|
|
2046
|
+
const cached = this.hashCache.get(path2);
|
|
2047
|
+
if (!cached) {
|
|
2048
|
+
return "new";
|
|
2049
|
+
}
|
|
2050
|
+
if (!cached.contentHash) {
|
|
2051
|
+
return "updated";
|
|
2052
|
+
}
|
|
2053
|
+
return cached.contentHash === currentContentHash ? "unchanged" : "updated";
|
|
2054
|
+
}
|
|
2055
|
+
detectChangeWithReason(path2, currentContentHash) {
|
|
2056
|
+
const changeType = this.detectChange(path2, currentContentHash);
|
|
2057
|
+
let reason;
|
|
2058
|
+
switch (changeType) {
|
|
2059
|
+
case "new":
|
|
2060
|
+
reason = "New document not in database";
|
|
2061
|
+
break;
|
|
2062
|
+
case "updated":
|
|
2063
|
+
reason = "Content hash changed";
|
|
2064
|
+
break;
|
|
2065
|
+
case "unchanged":
|
|
2066
|
+
reason = "Content unchanged";
|
|
2067
|
+
break;
|
|
2068
|
+
default:
|
|
2069
|
+
reason = "Unknown";
|
|
2070
|
+
}
|
|
2071
|
+
return { path: path2, changeType, reason };
|
|
2072
|
+
}
|
|
2073
|
+
getTrackedPaths() {
|
|
2074
|
+
if (!this.loaded) {
|
|
2075
|
+
throw new Error("Hashes not loaded. Call loadHashes() before getTrackedPaths().");
|
|
2076
|
+
}
|
|
2077
|
+
return Array.from(this.hashCache.keys());
|
|
2078
|
+
}
|
|
2079
|
+
isEmbeddingStale(path2, currentSourceHash) {
|
|
2080
|
+
if (!this.loaded) {
|
|
2081
|
+
throw new Error("Hashes not loaded. Call loadHashes() before isEmbeddingStale().");
|
|
2082
|
+
}
|
|
2083
|
+
const cached = this.hashCache.get(path2);
|
|
2084
|
+
if (!cached?.embeddingSourceHash) {
|
|
2085
|
+
return true;
|
|
2086
|
+
}
|
|
2087
|
+
return cached.embeddingSourceHash !== currentSourceHash;
|
|
2088
|
+
}
|
|
2089
|
+
getCachedEntry(path2) {
|
|
2090
|
+
if (!this.loaded) {
|
|
2091
|
+
throw new Error("Hashes not loaded. Call loadHashes() before getCachedEntry().");
|
|
2092
|
+
}
|
|
2093
|
+
return this.hashCache.get(path2);
|
|
2094
|
+
}
|
|
2095
|
+
getCacheSize() {
|
|
2096
|
+
return this.hashCache.size;
|
|
2097
|
+
}
|
|
2098
|
+
findDocumentsNeedingEmbeddings() {
|
|
2099
|
+
if (!this.loaded) {
|
|
2100
|
+
throw new Error("Hashes not loaded. Call loadHashes() before findDocumentsNeedingEmbeddings().");
|
|
2101
|
+
}
|
|
2102
|
+
const needsEmbedding = [];
|
|
2103
|
+
for (const [path2, entry] of this.hashCache) {
|
|
2104
|
+
if (!entry.embeddingSourceHash) {
|
|
2105
|
+
needsEmbedding.push(path2);
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
return needsEmbedding;
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
2111
|
+
DatabaseChangeDetectorService = __legacyDecorateClassTS([
|
|
2112
|
+
Injectable9(),
|
|
2113
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
2114
|
+
typeof GraphService === "undefined" ? Object : GraphService
|
|
2115
|
+
])
|
|
2116
|
+
], DatabaseChangeDetectorService);
|
|
2117
|
+
|
|
1963
2118
|
// src/sync/path-resolver.service.ts
|
|
1964
|
-
import { existsSync as
|
|
1965
|
-
import { isAbsolute, resolve as
|
|
2119
|
+
import { existsSync as existsSync5 } from "fs";
|
|
2120
|
+
import { isAbsolute, resolve as resolve3 } from "path";
|
|
1966
2121
|
import { Injectable as Injectable10 } from "@nestjs/common";
|
|
1967
2122
|
class PathResolverService {
|
|
1968
2123
|
docsPath;
|
|
@@ -1978,12 +2133,12 @@ class PathResolverService {
|
|
|
1978
2133
|
if (isAbsolute(userPath)) {
|
|
1979
2134
|
resolvedPath = userPath;
|
|
1980
2135
|
} else {
|
|
1981
|
-
resolvedPath =
|
|
2136
|
+
resolvedPath = resolve3(this.docsPath, userPath);
|
|
1982
2137
|
}
|
|
1983
2138
|
if (requireInDocs && !this.isUnderDocs(resolvedPath)) {
|
|
1984
2139
|
throw new Error(`Path "${userPath}" resolves to "${resolvedPath}" which is outside the docs directory (${this.docsPath})`);
|
|
1985
2140
|
}
|
|
1986
|
-
if (requireExists && !
|
|
2141
|
+
if (requireExists && !existsSync5(resolvedPath)) {
|
|
1987
2142
|
throw new Error(`Path "${userPath}" does not exist (resolved to: ${resolvedPath})`);
|
|
1988
2143
|
}
|
|
1989
2144
|
return resolvedPath;
|
|
@@ -2030,14 +2185,18 @@ class SyncService {
|
|
|
2030
2185
|
graph;
|
|
2031
2186
|
cascade;
|
|
2032
2187
|
pathResolver;
|
|
2188
|
+
dbChangeDetector;
|
|
2189
|
+
entityExtractor;
|
|
2033
2190
|
embeddingService;
|
|
2034
|
-
logger = new
|
|
2035
|
-
constructor(manifest, parser, graph, cascade, pathResolver, embeddingService) {
|
|
2191
|
+
logger = new Logger7(SyncService.name);
|
|
2192
|
+
constructor(manifest, parser, graph, cascade, pathResolver, dbChangeDetector, entityExtractor, embeddingService) {
|
|
2036
2193
|
this.manifest = manifest;
|
|
2037
2194
|
this.parser = parser;
|
|
2038
2195
|
this.graph = graph;
|
|
2039
2196
|
this.cascade = cascade;
|
|
2040
2197
|
this.pathResolver = pathResolver;
|
|
2198
|
+
this.dbChangeDetector = dbChangeDetector;
|
|
2199
|
+
this.entityExtractor = entityExtractor;
|
|
2041
2200
|
this.embeddingService = embeddingService;
|
|
2042
2201
|
}
|
|
2043
2202
|
async sync(options = {}) {
|
|
@@ -2054,8 +2213,16 @@ class SyncService {
|
|
|
2054
2213
|
embeddingsGenerated: 0,
|
|
2055
2214
|
entityEmbeddingsGenerated: 0
|
|
2056
2215
|
};
|
|
2216
|
+
const useDbDetection = options.legacy ? false : options.useDbChangeDetection ?? true;
|
|
2217
|
+
const useAiExtraction = options.legacy ? false : options.aiExtraction ?? true;
|
|
2057
2218
|
try {
|
|
2058
2219
|
await this.manifest.load();
|
|
2220
|
+
if (useDbDetection) {
|
|
2221
|
+
await this.dbChangeDetector.loadHashes();
|
|
2222
|
+
if (options.verbose) {
|
|
2223
|
+
this.logger.log(`v2 mode: Loaded ${this.dbChangeDetector.getCacheSize()} document hashes from database`);
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2059
2226
|
if (options.force) {
|
|
2060
2227
|
if (options.paths && options.paths.length > 0) {
|
|
2061
2228
|
if (options.verbose) {
|
|
@@ -2069,7 +2236,7 @@ class SyncService {
|
|
|
2069
2236
|
await this.clearManifest();
|
|
2070
2237
|
}
|
|
2071
2238
|
}
|
|
2072
|
-
const changes = await this.detectChanges(options.paths);
|
|
2239
|
+
const changes = await this.detectChanges(options.paths, useDbDetection);
|
|
2073
2240
|
result.changes = changes;
|
|
2074
2241
|
const docsToSync = [];
|
|
2075
2242
|
const docsByPath = new Map;
|
|
@@ -2086,15 +2253,42 @@ class SyncService {
|
|
|
2086
2253
|
}
|
|
2087
2254
|
}
|
|
2088
2255
|
}
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2256
|
+
if (useAiExtraction && docsToSync.length > 0) {
|
|
2257
|
+
if (options.verbose) {
|
|
2258
|
+
this.logger.log(`v2 AI extraction: Processing ${docsToSync.length} documents...`);
|
|
2259
|
+
}
|
|
2260
|
+
for (const doc of docsToSync) {
|
|
2261
|
+
try {
|
|
2262
|
+
const extraction = await this.entityExtractor.extractFromDocument(doc.path);
|
|
2263
|
+
if (extraction.success) {
|
|
2264
|
+
doc.entities = extraction.entities;
|
|
2265
|
+
doc.relationships = extraction.relationships;
|
|
2266
|
+
if (extraction.summary) {
|
|
2267
|
+
doc.summary = extraction.summary;
|
|
2268
|
+
}
|
|
2269
|
+
if (options.verbose) {
|
|
2270
|
+
this.logger.log(` Extracted ${extraction.entities.length} entities from ${doc.path}`);
|
|
2271
|
+
}
|
|
2272
|
+
} else {
|
|
2273
|
+
this.logger.warn(`AI extraction failed for ${doc.path}: ${extraction.error}`);
|
|
2274
|
+
}
|
|
2275
|
+
} catch (error) {
|
|
2276
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
2277
|
+
this.logger.warn(`AI extraction error for ${doc.path}: ${errorMsg}`);
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
if (!useAiExtraction) {
|
|
2282
|
+
const validationErrors = validateDocuments2(docsToSync);
|
|
2283
|
+
if (validationErrors.length > 0) {
|
|
2284
|
+
for (const err of validationErrors) {
|
|
2285
|
+
result.errors.push(err);
|
|
2286
|
+
this.logger.error(`Validation error in ${err.path}: ${err.error}`);
|
|
2287
|
+
}
|
|
2288
|
+
this.logger.error(`Sync aborted: ${validationErrors.length} validation error(s) found. Fix the errors and try again.`);
|
|
2289
|
+
result.duration = Date.now() - startTime;
|
|
2290
|
+
return result;
|
|
2094
2291
|
}
|
|
2095
|
-
this.logger.error(`Sync aborted: ${validationErrors.length} validation error(s) found. Fix the errors and try again.`);
|
|
2096
|
-
result.duration = Date.now() - startTime;
|
|
2097
|
-
return result;
|
|
2098
2292
|
}
|
|
2099
2293
|
const uniqueEntities = collectUniqueEntities(docsToSync);
|
|
2100
2294
|
if (options.verbose) {
|
|
@@ -2119,43 +2313,37 @@ class SyncService {
|
|
|
2119
2313
|
const CHECKPOINT_BATCH_SIZE = 10;
|
|
2120
2314
|
let processedCount = 0;
|
|
2121
2315
|
for (const change of changes) {
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2149
|
-
result.errors.push({ path: change.path, error: errorMessage });
|
|
2150
|
-
this.logger.warn(`Error processing ${change.path}: ${errorMessage}`);
|
|
2316
|
+
const doc = docsByPath.get(change.path);
|
|
2317
|
+
const cascadeWarnings = await this.processChange(change, options, doc);
|
|
2318
|
+
result.cascadeWarnings.push(...cascadeWarnings);
|
|
2319
|
+
switch (change.changeType) {
|
|
2320
|
+
case "new":
|
|
2321
|
+
result.added++;
|
|
2322
|
+
break;
|
|
2323
|
+
case "updated":
|
|
2324
|
+
result.updated++;
|
|
2325
|
+
break;
|
|
2326
|
+
case "deleted":
|
|
2327
|
+
result.deleted++;
|
|
2328
|
+
break;
|
|
2329
|
+
case "unchanged":
|
|
2330
|
+
result.unchanged++;
|
|
2331
|
+
break;
|
|
2332
|
+
}
|
|
2333
|
+
if (change.embeddingGenerated) {
|
|
2334
|
+
result.embeddingsGenerated++;
|
|
2335
|
+
}
|
|
2336
|
+
if (!options.dryRun && change.changeType !== "unchanged") {
|
|
2337
|
+
await this.manifest.save();
|
|
2338
|
+
}
|
|
2339
|
+
processedCount++;
|
|
2340
|
+
if (!options.dryRun && processedCount % CHECKPOINT_BATCH_SIZE === 0) {
|
|
2341
|
+
await this.graph.checkpoint();
|
|
2151
2342
|
}
|
|
2152
2343
|
}
|
|
2153
2344
|
if (!options.dryRun && processedCount > 0) {
|
|
2154
2345
|
await this.graph.checkpoint();
|
|
2155
2346
|
}
|
|
2156
|
-
if (!options.dryRun) {
|
|
2157
|
-
await this.manifest.save();
|
|
2158
|
-
}
|
|
2159
2347
|
} catch (error) {
|
|
2160
2348
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2161
2349
|
this.logger.error(`Sync failed: ${errorMessage}`);
|
|
@@ -2164,7 +2352,7 @@ class SyncService {
|
|
|
2164
2352
|
result.duration = Date.now() - startTime;
|
|
2165
2353
|
return result;
|
|
2166
2354
|
}
|
|
2167
|
-
async detectChanges(paths) {
|
|
2355
|
+
async detectChanges(paths, useDbDetection = false) {
|
|
2168
2356
|
const changes = [];
|
|
2169
2357
|
let allDocPaths = await this.parser.discoverDocuments();
|
|
2170
2358
|
if (paths && paths.length > 0) {
|
|
@@ -2175,11 +2363,11 @@ class SyncService {
|
|
|
2175
2363
|
const pathSet = new Set(normalizedPaths);
|
|
2176
2364
|
allDocPaths = allDocPaths.filter((p) => pathSet.has(p));
|
|
2177
2365
|
}
|
|
2178
|
-
const trackedPaths = new Set(this.manifest.getTrackedPaths());
|
|
2366
|
+
const trackedPaths = new Set(useDbDetection ? this.dbChangeDetector.getTrackedPaths() : this.manifest.getTrackedPaths());
|
|
2179
2367
|
for (const docPath of allDocPaths) {
|
|
2180
2368
|
try {
|
|
2181
2369
|
const doc = await this.parser.parseDocument(docPath);
|
|
2182
|
-
const changeType = this.manifest.detectChange(docPath, doc.contentHash, doc.frontmatterHash);
|
|
2370
|
+
const changeType = useDbDetection ? this.dbChangeDetector.detectChange(docPath, doc.contentHash) : this.manifest.detectChange(docPath, doc.contentHash, doc.frontmatterHash);
|
|
2183
2371
|
changes.push({
|
|
2184
2372
|
path: docPath,
|
|
2185
2373
|
changeType,
|
|
@@ -2245,7 +2433,7 @@ class SyncService {
|
|
|
2245
2433
|
}
|
|
2246
2434
|
} catch (error) {
|
|
2247
2435
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2248
|
-
|
|
2436
|
+
throw new Error(`Failed to generate embedding for ${doc.path}: ${errorMessage}`);
|
|
2249
2437
|
}
|
|
2250
2438
|
}
|
|
2251
2439
|
const entityTypeMap = new Map;
|
|
@@ -2336,6 +2524,11 @@ class SyncService {
|
|
|
2336
2524
|
change.embeddingGenerated = embeddingGenerated;
|
|
2337
2525
|
const currentDoc = await this.parser.parseDocument(change.path);
|
|
2338
2526
|
this.manifest.updateEntry(currentDoc.path, currentDoc.contentHash, currentDoc.frontmatterHash, currentDoc.entities.length, currentDoc.relationships.length);
|
|
2527
|
+
const shouldUpdateDbHashes = options.legacy ? false : options.useDbChangeDetection ?? true;
|
|
2528
|
+
if (shouldUpdateDbHashes) {
|
|
2529
|
+
const embeddingSourceHash = embeddingGenerated ? currentDoc.contentHash : undefined;
|
|
2530
|
+
await this.graph.updateDocumentHashes(currentDoc.path, currentDoc.contentHash, embeddingSourceHash);
|
|
2531
|
+
}
|
|
2339
2532
|
break;
|
|
2340
2533
|
}
|
|
2341
2534
|
case "deleted": {
|
|
@@ -2366,86 +2559,582 @@ class SyncService {
|
|
|
2366
2559
|
}
|
|
2367
2560
|
this.logger.log(`Marked ${normalizedPaths.length} document(s) for re-sync`);
|
|
2368
2561
|
}
|
|
2369
|
-
async getOldDocumentFromManifest(path2) {
|
|
2562
|
+
async getOldDocumentFromManifest(path2) {
|
|
2563
|
+
try {
|
|
2564
|
+
const manifest = await this.manifest.load();
|
|
2565
|
+
const entry = manifest.documents[path2];
|
|
2566
|
+
if (!entry) {
|
|
2567
|
+
return null;
|
|
2568
|
+
}
|
|
2569
|
+
try {
|
|
2570
|
+
return await this.parser.parseDocument(path2);
|
|
2571
|
+
} catch {
|
|
2572
|
+
return null;
|
|
2573
|
+
}
|
|
2574
|
+
} catch (error) {
|
|
2575
|
+
this.logger.warn(`Failed to retrieve old document for ${path2}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2576
|
+
return null;
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
async syncEntities(entities, options) {
|
|
2580
|
+
let embeddingsGenerated = 0;
|
|
2581
|
+
for (const [_key, entity] of entities) {
|
|
2582
|
+
const entityProps = {
|
|
2583
|
+
name: entity.name
|
|
2584
|
+
};
|
|
2585
|
+
if (entity.description) {
|
|
2586
|
+
entityProps.description = entity.description;
|
|
2587
|
+
}
|
|
2588
|
+
await this.graph.upsertNode(entity.type, entityProps);
|
|
2589
|
+
if (options.embeddings && !options.skipEmbeddings && this.embeddingService) {
|
|
2590
|
+
try {
|
|
2591
|
+
const text = composeEntityEmbeddingText(entity);
|
|
2592
|
+
const embedding = await this.embeddingService.generateEmbedding(text);
|
|
2593
|
+
await this.graph.updateNodeEmbedding(entity.type, entity.name, embedding);
|
|
2594
|
+
embeddingsGenerated++;
|
|
2595
|
+
this.logger.debug(`Generated embedding for ${entity.type}:${entity.name}`);
|
|
2596
|
+
} catch (error) {
|
|
2597
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2598
|
+
throw new Error(`Failed to generate embedding for ${entity.type}:${entity.name}: ${errorMessage}`);
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
return embeddingsGenerated;
|
|
2603
|
+
}
|
|
2604
|
+
async createEntityVectorIndices() {
|
|
2605
|
+
if (!this.embeddingService)
|
|
2606
|
+
return;
|
|
2607
|
+
const dimensions = this.embeddingService.getDimensions();
|
|
2608
|
+
for (const entityType of ENTITY_TYPES) {
|
|
2609
|
+
try {
|
|
2610
|
+
await this.graph.createVectorIndex(entityType, "embedding", dimensions);
|
|
2611
|
+
} catch (error) {
|
|
2612
|
+
this.logger.debug(`Vector index setup for ${entityType}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2613
|
+
}
|
|
2614
|
+
}
|
|
2615
|
+
}
|
|
2616
|
+
}
|
|
2617
|
+
SyncService = __legacyDecorateClassTS([
|
|
2618
|
+
Injectable11(),
|
|
2619
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
2620
|
+
typeof ManifestService === "undefined" ? Object : ManifestService,
|
|
2621
|
+
typeof DocumentParserService === "undefined" ? Object : DocumentParserService,
|
|
2622
|
+
typeof GraphService === "undefined" ? Object : GraphService,
|
|
2623
|
+
typeof CascadeService === "undefined" ? Object : CascadeService,
|
|
2624
|
+
typeof PathResolverService === "undefined" ? Object : PathResolverService,
|
|
2625
|
+
typeof DatabaseChangeDetectorService === "undefined" ? Object : DatabaseChangeDetectorService,
|
|
2626
|
+
typeof EntityExtractorService === "undefined" ? Object : EntityExtractorService,
|
|
2627
|
+
typeof EmbeddingService === "undefined" ? Object : EmbeddingService
|
|
2628
|
+
])
|
|
2629
|
+
], SyncService);
|
|
2630
|
+
|
|
2631
|
+
// src/commands/migrate.command.ts
|
|
2632
|
+
class MigrateCommand extends CommandRunner3 {
|
|
2633
|
+
graph;
|
|
2634
|
+
manifest;
|
|
2635
|
+
syncService;
|
|
2636
|
+
logger = new Logger8(MigrateCommand.name);
|
|
2637
|
+
constructor(graph, manifest, syncService) {
|
|
2638
|
+
super();
|
|
2639
|
+
this.graph = graph;
|
|
2640
|
+
this.manifest = manifest;
|
|
2641
|
+
this.syncService = syncService;
|
|
2642
|
+
}
|
|
2643
|
+
async run(_passedParams, options) {
|
|
2644
|
+
console.log(`
|
|
2645
|
+
\uD83D\uDD04 Migrating to Lattice v2...
|
|
2646
|
+
`);
|
|
2647
|
+
if (options.dryRun) {
|
|
2648
|
+
console.log(`\uD83D\uDCCB Dry run mode: No changes will be applied
|
|
2649
|
+
`);
|
|
2650
|
+
}
|
|
2651
|
+
try {
|
|
2652
|
+
console.log("\uD83D\uDCE6 Step 1/3: Applying v2 schema changes...");
|
|
2653
|
+
if (!options.dryRun) {
|
|
2654
|
+
await this.graph.runV2Migration();
|
|
2655
|
+
console.log(` \u2705 Schema updated with content_hash and embedding_source_hash columns
|
|
2656
|
+
`);
|
|
2657
|
+
} else {
|
|
2658
|
+
console.log(` [DRY-RUN] Would add content_hash and embedding_source_hash columns
|
|
2659
|
+
`);
|
|
2660
|
+
}
|
|
2661
|
+
console.log("\uD83D\uDCCB Step 2/3: Migrating manifest hashes to database...");
|
|
2662
|
+
const manifestStats = await this.migrateManifestHashes(options);
|
|
2663
|
+
if (manifestStats.total > 0) {
|
|
2664
|
+
if (!options.dryRun) {
|
|
2665
|
+
console.log(` \u2705 Migrated ${manifestStats.migrated}/${manifestStats.total} document hashes
|
|
2666
|
+
`);
|
|
2667
|
+
} else {
|
|
2668
|
+
console.log(` [DRY-RUN] Would migrate ${manifestStats.total} document hashes
|
|
2669
|
+
`);
|
|
2670
|
+
}
|
|
2671
|
+
} else {
|
|
2672
|
+
console.log(` \u2139\uFE0F No manifest found or manifest is empty (fresh install)
|
|
2673
|
+
`);
|
|
2674
|
+
}
|
|
2675
|
+
if (!options.skipSync) {
|
|
2676
|
+
console.log("\uD83E\uDD16 Step 3/3: Running full sync with AI entity extraction...");
|
|
2677
|
+
console.log(` This may take a while depending on the number of documents...
|
|
2678
|
+
`);
|
|
2679
|
+
if (!options.dryRun) {
|
|
2680
|
+
const result = await this.syncService.sync({
|
|
2681
|
+
force: false,
|
|
2682
|
+
useDbChangeDetection: true,
|
|
2683
|
+
aiExtraction: true,
|
|
2684
|
+
verbose: options.verbose,
|
|
2685
|
+
embeddings: true
|
|
2686
|
+
});
|
|
2687
|
+
console.log(`
|
|
2688
|
+
\uD83D\uDCCA Sync Results:`);
|
|
2689
|
+
console.log(` \u2705 Added: ${result.added}`);
|
|
2690
|
+
console.log(` \uD83D\uDD04 Updated: ${result.updated}`);
|
|
2691
|
+
console.log(` \uD83D\uDDD1\uFE0F Deleted: ${result.deleted}`);
|
|
2692
|
+
console.log(` \u23ED\uFE0F Unchanged: ${result.unchanged}`);
|
|
2693
|
+
if (result.errors.length > 0) {
|
|
2694
|
+
console.log(` \u274C Errors: ${result.errors.length}`);
|
|
2695
|
+
for (const err of result.errors) {
|
|
2696
|
+
console.log(` ${err.path}: ${err.error}`);
|
|
2697
|
+
}
|
|
2698
|
+
}
|
|
2699
|
+
console.log(` \u23F1\uFE0F Duration: ${result.duration}ms
|
|
2700
|
+
`);
|
|
2701
|
+
} else {
|
|
2702
|
+
console.log(` [DRY-RUN] Would run full sync with AI extraction
|
|
2703
|
+
`);
|
|
2704
|
+
}
|
|
2705
|
+
} else {
|
|
2706
|
+
console.log(`\u23ED\uFE0F Step 3/3: Skipping sync (--skip-sync flag)
|
|
2707
|
+
`);
|
|
2708
|
+
}
|
|
2709
|
+
console.log(`\u2705 Migration complete!
|
|
2710
|
+
`);
|
|
2711
|
+
console.log("Next steps:");
|
|
2712
|
+
console.log(" 1. Verify your graph: lattice status");
|
|
2713
|
+
console.log(" 2. Test semantic search: lattice search <query>");
|
|
2714
|
+
console.log(" 3. Future syncs will use v2 mode automatically");
|
|
2715
|
+
console.log(` 4. Optional: Delete ~/.lattice/.sync-manifest.json (no longer needed)
|
|
2716
|
+
`);
|
|
2717
|
+
} catch (error) {
|
|
2718
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
2719
|
+
console.error(`
|
|
2720
|
+
\u274C Migration failed: ${errorMsg}
|
|
2721
|
+
`);
|
|
2722
|
+
this.logger.error(`Migration failed: ${errorMsg}`, error instanceof Error ? error.stack : undefined);
|
|
2723
|
+
process.exit(1);
|
|
2724
|
+
}
|
|
2725
|
+
}
|
|
2726
|
+
async migrateManifestHashes(options) {
|
|
2727
|
+
const stats = { total: 0, migrated: 0, skipped: 0 };
|
|
2728
|
+
try {
|
|
2729
|
+
const manifestData = await this.manifest.load();
|
|
2730
|
+
const entries = Object.entries(manifestData.documents);
|
|
2731
|
+
stats.total = entries.length;
|
|
2732
|
+
if (stats.total === 0) {
|
|
2733
|
+
return stats;
|
|
2734
|
+
}
|
|
2735
|
+
for (const [path2, entry] of entries) {
|
|
2736
|
+
if (options.verbose) {
|
|
2737
|
+
console.log(` Processing: ${path2}`);
|
|
2738
|
+
}
|
|
2739
|
+
if (!options.dryRun) {
|
|
2740
|
+
try {
|
|
2741
|
+
await this.graph.updateDocumentHashes(path2, entry.contentHash);
|
|
2742
|
+
stats.migrated++;
|
|
2743
|
+
} catch (error) {
|
|
2744
|
+
stats.skipped++;
|
|
2745
|
+
if (options.verbose) {
|
|
2746
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
2747
|
+
console.log(` \u26A0\uFE0F Skipped ${path2}: ${errorMsg}`);
|
|
2748
|
+
}
|
|
2749
|
+
}
|
|
2750
|
+
} else {
|
|
2751
|
+
stats.migrated++;
|
|
2752
|
+
}
|
|
2753
|
+
}
|
|
2754
|
+
} catch (error) {
|
|
2755
|
+
if (options.verbose) {
|
|
2756
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
2757
|
+
this.logger.debug(`No manifest to migrate: ${errorMsg}`);
|
|
2758
|
+
}
|
|
2759
|
+
}
|
|
2760
|
+
return stats;
|
|
2761
|
+
}
|
|
2762
|
+
parseDryRun() {
|
|
2763
|
+
return true;
|
|
2764
|
+
}
|
|
2765
|
+
parseVerbose() {
|
|
2766
|
+
return true;
|
|
2767
|
+
}
|
|
2768
|
+
parseSkipSync() {
|
|
2769
|
+
return true;
|
|
2770
|
+
}
|
|
2771
|
+
}
|
|
2772
|
+
__legacyDecorateClassTS([
|
|
2773
|
+
Option2({
|
|
2774
|
+
flags: "--dry-run",
|
|
2775
|
+
description: "Show what would be done without making changes"
|
|
2776
|
+
}),
|
|
2777
|
+
__legacyMetadataTS("design:type", Function),
|
|
2778
|
+
__legacyMetadataTS("design:paramtypes", []),
|
|
2779
|
+
__legacyMetadataTS("design:returntype", Boolean)
|
|
2780
|
+
], MigrateCommand.prototype, "parseDryRun", null);
|
|
2781
|
+
__legacyDecorateClassTS([
|
|
2782
|
+
Option2({
|
|
2783
|
+
flags: "-v, --verbose",
|
|
2784
|
+
description: "Show detailed progress"
|
|
2785
|
+
}),
|
|
2786
|
+
__legacyMetadataTS("design:type", Function),
|
|
2787
|
+
__legacyMetadataTS("design:paramtypes", []),
|
|
2788
|
+
__legacyMetadataTS("design:returntype", Boolean)
|
|
2789
|
+
], MigrateCommand.prototype, "parseVerbose", null);
|
|
2790
|
+
__legacyDecorateClassTS([
|
|
2791
|
+
Option2({
|
|
2792
|
+
flags: "--skip-sync",
|
|
2793
|
+
description: "Skip the full sync step (only apply schema and migrate hashes)"
|
|
2794
|
+
}),
|
|
2795
|
+
__legacyMetadataTS("design:type", Function),
|
|
2796
|
+
__legacyMetadataTS("design:paramtypes", []),
|
|
2797
|
+
__legacyMetadataTS("design:returntype", Boolean)
|
|
2798
|
+
], MigrateCommand.prototype, "parseSkipSync", null);
|
|
2799
|
+
MigrateCommand = __legacyDecorateClassTS([
|
|
2800
|
+
Injectable12(),
|
|
2801
|
+
Command3({
|
|
2802
|
+
name: "migrate",
|
|
2803
|
+
description: "Migrate from v1 (manifest) to v2 (database-based) architecture"
|
|
2804
|
+
}),
|
|
2805
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
2806
|
+
typeof GraphService === "undefined" ? Object : GraphService,
|
|
2807
|
+
typeof ManifestService === "undefined" ? Object : ManifestService,
|
|
2808
|
+
typeof SyncService === "undefined" ? Object : SyncService
|
|
2809
|
+
])
|
|
2810
|
+
], MigrateCommand);
|
|
2811
|
+
// src/commands/ontology.command.ts
|
|
2812
|
+
import { Injectable as Injectable14 } from "@nestjs/common";
|
|
2813
|
+
import { Command as Command4, CommandRunner as CommandRunner4 } from "nest-commander";
|
|
2814
|
+
|
|
2815
|
+
// src/sync/ontology.service.ts
|
|
2816
|
+
import { Injectable as Injectable13 } from "@nestjs/common";
|
|
2817
|
+
class OntologyService {
|
|
2818
|
+
parser;
|
|
2819
|
+
constructor(parser) {
|
|
2820
|
+
this.parser = parser;
|
|
2821
|
+
}
|
|
2822
|
+
async deriveOntology() {
|
|
2823
|
+
const docs = await this.parser.parseAllDocuments();
|
|
2824
|
+
return this.deriveFromDocuments(docs);
|
|
2825
|
+
}
|
|
2826
|
+
deriveFromDocuments(docs) {
|
|
2827
|
+
const entityTypeSet = new Set;
|
|
2828
|
+
const relationshipTypeSet = new Set;
|
|
2829
|
+
const entityCounts = {};
|
|
2830
|
+
const relationshipCounts = {};
|
|
2831
|
+
const entityExamples = {};
|
|
2832
|
+
let documentsWithEntities = 0;
|
|
2833
|
+
let documentsWithoutEntities = 0;
|
|
2834
|
+
let totalRelationships = 0;
|
|
2835
|
+
for (const doc of docs) {
|
|
2836
|
+
if (doc.entities.length > 0) {
|
|
2837
|
+
documentsWithEntities++;
|
|
2838
|
+
} else {
|
|
2839
|
+
documentsWithoutEntities++;
|
|
2840
|
+
}
|
|
2841
|
+
for (const entity of doc.entities) {
|
|
2842
|
+
entityTypeSet.add(entity.type);
|
|
2843
|
+
entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
|
|
2844
|
+
if (!entityExamples[entity.name]) {
|
|
2845
|
+
entityExamples[entity.name] = { type: entity.type, documents: [] };
|
|
2846
|
+
}
|
|
2847
|
+
if (!entityExamples[entity.name].documents.includes(doc.path)) {
|
|
2848
|
+
entityExamples[entity.name].documents.push(doc.path);
|
|
2849
|
+
}
|
|
2850
|
+
}
|
|
2851
|
+
for (const rel of doc.relationships) {
|
|
2852
|
+
relationshipTypeSet.add(rel.relation);
|
|
2853
|
+
relationshipCounts[rel.relation] = (relationshipCounts[rel.relation] || 0) + 1;
|
|
2854
|
+
totalRelationships++;
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
return {
|
|
2858
|
+
entityTypes: Array.from(entityTypeSet).sort(),
|
|
2859
|
+
relationshipTypes: Array.from(relationshipTypeSet).sort(),
|
|
2860
|
+
entityCounts,
|
|
2861
|
+
relationshipCounts,
|
|
2862
|
+
totalEntities: Object.keys(entityExamples).length,
|
|
2863
|
+
totalRelationships,
|
|
2864
|
+
documentsWithEntities,
|
|
2865
|
+
documentsWithoutEntities,
|
|
2866
|
+
entityExamples
|
|
2867
|
+
};
|
|
2868
|
+
}
|
|
2869
|
+
printSummary(ontology) {
|
|
2870
|
+
console.log(`
|
|
2871
|
+
Derived Ontology Summary
|
|
2872
|
+
`);
|
|
2873
|
+
console.log(`Documents: ${ontology.documentsWithEntities} with entities, ${ontology.documentsWithoutEntities} without`);
|
|
2874
|
+
console.log(`Unique Entities: ${ontology.totalEntities}`);
|
|
2875
|
+
console.log(`Total Relationships: ${ontology.totalRelationships}`);
|
|
2876
|
+
console.log(`
|
|
2877
|
+
Entity Types:`);
|
|
2878
|
+
for (const type of ontology.entityTypes) {
|
|
2879
|
+
console.log(` ${type}: ${ontology.entityCounts[type]} instances`);
|
|
2880
|
+
}
|
|
2881
|
+
console.log(`
|
|
2882
|
+
Relationship Types:`);
|
|
2883
|
+
for (const type of ontology.relationshipTypes) {
|
|
2884
|
+
console.log(` ${type}: ${ontology.relationshipCounts[type]} instances`);
|
|
2885
|
+
}
|
|
2886
|
+
console.log(`
|
|
2887
|
+
Top Entities (by document count):`);
|
|
2888
|
+
const sorted = Object.entries(ontology.entityExamples).sort((a, b) => b[1].documents.length - a[1].documents.length).slice(0, 10);
|
|
2889
|
+
for (const [name, info] of sorted) {
|
|
2890
|
+
console.log(` ${name} (${info.type}): ${info.documents.length} docs`);
|
|
2891
|
+
}
|
|
2892
|
+
}
|
|
2893
|
+
}
|
|
2894
|
+
OntologyService = __legacyDecorateClassTS([
|
|
2895
|
+
Injectable13(),
|
|
2896
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
2897
|
+
typeof DocumentParserService === "undefined" ? Object : DocumentParserService
|
|
2898
|
+
])
|
|
2899
|
+
], OntologyService);
|
|
2900
|
+
|
|
2901
|
+
// src/commands/ontology.command.ts
|
|
2902
|
+
class OntologyCommand extends CommandRunner4 {
|
|
2903
|
+
ontologyService;
|
|
2904
|
+
constructor(ontologyService) {
|
|
2905
|
+
super();
|
|
2906
|
+
this.ontologyService = ontologyService;
|
|
2907
|
+
}
|
|
2908
|
+
async run() {
|
|
2909
|
+
try {
|
|
2910
|
+
const ontology = await this.ontologyService.deriveOntology();
|
|
2911
|
+
this.ontologyService.printSummary(ontology);
|
|
2912
|
+
process.exit(0);
|
|
2913
|
+
} catch (error) {
|
|
2914
|
+
console.error(`
|
|
2915
|
+
\u274C Ontology derivation failed:`, error instanceof Error ? error.message : String(error));
|
|
2916
|
+
process.exit(1);
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2920
|
+
OntologyCommand = __legacyDecorateClassTS([
|
|
2921
|
+
Injectable14(),
|
|
2922
|
+
Command4({
|
|
2923
|
+
name: "ontology",
|
|
2924
|
+
description: "Derive and display ontology from all documents"
|
|
2925
|
+
}),
|
|
2926
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
2927
|
+
typeof OntologyService === "undefined" ? Object : OntologyService
|
|
2928
|
+
])
|
|
2929
|
+
], OntologyCommand);
|
|
2930
|
+
// src/commands/query.command.ts
|
|
2931
|
+
import { Injectable as Injectable15 } from "@nestjs/common";
|
|
2932
|
+
import { Command as Command5, CommandRunner as CommandRunner5, Option as Option3 } from "nest-commander";
|
|
2933
|
+
class SearchCommand extends CommandRunner5 {
|
|
2934
|
+
graphService;
|
|
2935
|
+
embeddingService;
|
|
2936
|
+
constructor(graphService, embeddingService) {
|
|
2937
|
+
super();
|
|
2938
|
+
this.graphService = graphService;
|
|
2939
|
+
this.embeddingService = embeddingService;
|
|
2940
|
+
}
|
|
2941
|
+
async run(inputs, options) {
|
|
2942
|
+
const query3 = inputs[0];
|
|
2943
|
+
const limit = Math.min(parseInt(options.limit || "20", 10), 100);
|
|
2944
|
+
try {
|
|
2945
|
+
const queryEmbedding = await this.embeddingService.generateQueryEmbedding(query3);
|
|
2946
|
+
let results;
|
|
2947
|
+
if (options.label) {
|
|
2948
|
+
const labelResults = await this.graphService.vectorSearch(options.label, queryEmbedding, limit);
|
|
2949
|
+
results = labelResults.map((r) => ({
|
|
2950
|
+
name: r.name,
|
|
2951
|
+
label: options.label,
|
|
2952
|
+
title: r.title,
|
|
2953
|
+
score: r.score
|
|
2954
|
+
}));
|
|
2955
|
+
} else {
|
|
2956
|
+
results = await this.graphService.vectorSearchAll(queryEmbedding, limit);
|
|
2957
|
+
}
|
|
2958
|
+
const labelSuffix = options.label ? ` (${options.label})` : "";
|
|
2959
|
+
console.log(`
|
|
2960
|
+
=== Semantic Search Results for "${query3}"${labelSuffix} ===
|
|
2961
|
+
`);
|
|
2962
|
+
if (results.length === 0) {
|
|
2963
|
+
console.log(`No results found.
|
|
2964
|
+
`);
|
|
2965
|
+
if (options.label) {
|
|
2966
|
+
console.log(`Tip: Try without --label to search all entity types.
|
|
2967
|
+
`);
|
|
2968
|
+
}
|
|
2969
|
+
process.exit(0);
|
|
2970
|
+
}
|
|
2971
|
+
results.forEach((result, idx) => {
|
|
2972
|
+
console.log(`${idx + 1}. [${result.label}] ${result.name}`);
|
|
2973
|
+
if (result.title) {
|
|
2974
|
+
console.log(` Title: ${result.title}`);
|
|
2975
|
+
}
|
|
2976
|
+
if (result.description && result.label !== "Document") {
|
|
2977
|
+
const desc = result.description.length > 80 ? `${result.description.slice(0, 80)}...` : result.description;
|
|
2978
|
+
console.log(` ${desc}`);
|
|
2979
|
+
}
|
|
2980
|
+
console.log(` Similarity: ${(result.score * 100).toFixed(2)}%`);
|
|
2981
|
+
});
|
|
2982
|
+
console.log();
|
|
2983
|
+
process.exit(0);
|
|
2984
|
+
} catch (error) {
|
|
2985
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
2986
|
+
console.error("Error:", errorMsg);
|
|
2987
|
+
if (errorMsg.includes("no embeddings") || errorMsg.includes("vector")) {
|
|
2988
|
+
console.log(`
|
|
2989
|
+
Note: Semantic search requires embeddings to be generated first.`);
|
|
2990
|
+
console.log(`Run 'lattice sync' to generate embeddings for documents.
|
|
2991
|
+
`);
|
|
2992
|
+
}
|
|
2993
|
+
process.exit(1);
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
parseLabel(value) {
|
|
2997
|
+
return value;
|
|
2998
|
+
}
|
|
2999
|
+
parseLimit(value) {
|
|
3000
|
+
return value;
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
__legacyDecorateClassTS([
|
|
3004
|
+
Option3({
|
|
3005
|
+
flags: "-l, --label <label>",
|
|
3006
|
+
description: "Filter by entity label (e.g., Technology, Concept, Document)"
|
|
3007
|
+
}),
|
|
3008
|
+
__legacyMetadataTS("design:type", Function),
|
|
3009
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
3010
|
+
String
|
|
3011
|
+
]),
|
|
3012
|
+
__legacyMetadataTS("design:returntype", String)
|
|
3013
|
+
], SearchCommand.prototype, "parseLabel", null);
|
|
3014
|
+
__legacyDecorateClassTS([
|
|
3015
|
+
Option3({
|
|
3016
|
+
flags: "--limit <n>",
|
|
3017
|
+
description: "Limit results",
|
|
3018
|
+
defaultValue: "20"
|
|
3019
|
+
}),
|
|
3020
|
+
__legacyMetadataTS("design:type", Function),
|
|
3021
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
3022
|
+
String
|
|
3023
|
+
]),
|
|
3024
|
+
__legacyMetadataTS("design:returntype", String)
|
|
3025
|
+
], SearchCommand.prototype, "parseLimit", null);
|
|
3026
|
+
SearchCommand = __legacyDecorateClassTS([
|
|
3027
|
+
Injectable15(),
|
|
3028
|
+
Command5({
|
|
3029
|
+
name: "search",
|
|
3030
|
+
arguments: "<query>",
|
|
3031
|
+
description: "Semantic search across the knowledge graph"
|
|
3032
|
+
}),
|
|
3033
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
3034
|
+
typeof GraphService === "undefined" ? Object : GraphService,
|
|
3035
|
+
typeof EmbeddingService === "undefined" ? Object : EmbeddingService
|
|
3036
|
+
])
|
|
3037
|
+
], SearchCommand);
|
|
3038
|
+
|
|
3039
|
+
class RelsCommand extends CommandRunner5 {
|
|
3040
|
+
graphService;
|
|
3041
|
+
constructor(graphService) {
|
|
3042
|
+
super();
|
|
3043
|
+
this.graphService = graphService;
|
|
3044
|
+
}
|
|
3045
|
+
async run(inputs) {
|
|
3046
|
+
const name = inputs[0];
|
|
2370
3047
|
try {
|
|
2371
|
-
const
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
3048
|
+
const relationships = await this.graphService.findRelationships(name);
|
|
3049
|
+
console.log(`
|
|
3050
|
+
=== Relationships for "${name}" ===
|
|
3051
|
+
`);
|
|
3052
|
+
if (relationships.length === 0) {
|
|
3053
|
+
console.log(`No relationships found.
|
|
3054
|
+
`);
|
|
3055
|
+
process.exit(0);
|
|
2375
3056
|
}
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
3057
|
+
console.log("Relationships:");
|
|
3058
|
+
for (const rel of relationships) {
|
|
3059
|
+
const [relType, targetName] = rel;
|
|
3060
|
+
console.log(` -[${relType}]-> ${targetName}`);
|
|
2380
3061
|
}
|
|
3062
|
+
console.log();
|
|
3063
|
+
process.exit(0);
|
|
2381
3064
|
} catch (error) {
|
|
2382
|
-
|
|
2383
|
-
|
|
3065
|
+
console.error("Error:", error instanceof Error ? error.message : String(error));
|
|
3066
|
+
process.exit(1);
|
|
2384
3067
|
}
|
|
2385
3068
|
}
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2405
|
-
this.logger.warn(`Failed to generate embedding for ${entity.type}:${entity.name}: ${errorMessage}`);
|
|
2406
|
-
}
|
|
2407
|
-
}
|
|
2408
|
-
}
|
|
2409
|
-
return embeddingsGenerated;
|
|
3069
|
+
}
|
|
3070
|
+
RelsCommand = __legacyDecorateClassTS([
|
|
3071
|
+
Injectable15(),
|
|
3072
|
+
Command5({
|
|
3073
|
+
name: "rels",
|
|
3074
|
+
arguments: "<name>",
|
|
3075
|
+
description: "Show relationships for a node"
|
|
3076
|
+
}),
|
|
3077
|
+
__legacyMetadataTS("design:paramtypes", [
|
|
3078
|
+
typeof GraphService === "undefined" ? Object : GraphService
|
|
3079
|
+
])
|
|
3080
|
+
], RelsCommand);
|
|
3081
|
+
|
|
3082
|
+
class SqlCommand extends CommandRunner5 {
|
|
3083
|
+
graphService;
|
|
3084
|
+
constructor(graphService) {
|
|
3085
|
+
super();
|
|
3086
|
+
this.graphService = graphService;
|
|
2410
3087
|
}
|
|
2411
|
-
async
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
3088
|
+
async run(inputs) {
|
|
3089
|
+
const query3 = inputs[0];
|
|
3090
|
+
try {
|
|
3091
|
+
const result = await this.graphService.query(query3);
|
|
3092
|
+
console.log(`
|
|
3093
|
+
=== SQL Query Results ===
|
|
3094
|
+
`);
|
|
3095
|
+
const replacer = (_key, value) => typeof value === "bigint" ? Number(value) : value;
|
|
3096
|
+
console.log(JSON.stringify(result, replacer, 2));
|
|
3097
|
+
console.log();
|
|
3098
|
+
process.exit(0);
|
|
3099
|
+
} catch (error) {
|
|
3100
|
+
console.error("Error:", error instanceof Error ? error.message : String(error));
|
|
3101
|
+
process.exit(1);
|
|
2421
3102
|
}
|
|
2422
3103
|
}
|
|
2423
3104
|
}
|
|
2424
|
-
|
|
2425
|
-
|
|
3105
|
+
SqlCommand = __legacyDecorateClassTS([
|
|
3106
|
+
Injectable15(),
|
|
3107
|
+
Command5({
|
|
3108
|
+
name: "sql",
|
|
3109
|
+
arguments: "<query>",
|
|
3110
|
+
description: "Execute raw SQL query against DuckDB"
|
|
3111
|
+
}),
|
|
2426
3112
|
__legacyMetadataTS("design:paramtypes", [
|
|
2427
|
-
typeof
|
|
2428
|
-
typeof DocumentParserService === "undefined" ? Object : DocumentParserService,
|
|
2429
|
-
typeof GraphService === "undefined" ? Object : GraphService,
|
|
2430
|
-
typeof CascadeService === "undefined" ? Object : CascadeService,
|
|
2431
|
-
typeof PathResolverService === "undefined" ? Object : PathResolverService,
|
|
2432
|
-
typeof EmbeddingService === "undefined" ? Object : EmbeddingService
|
|
3113
|
+
typeof GraphService === "undefined" ? Object : GraphService
|
|
2433
3114
|
])
|
|
2434
|
-
],
|
|
2435
|
-
|
|
3115
|
+
], SqlCommand);
|
|
2436
3116
|
// src/commands/status.command.ts
|
|
2437
|
-
|
|
3117
|
+
import { Injectable as Injectable16 } from "@nestjs/common";
|
|
3118
|
+
import { Command as Command6, CommandRunner as CommandRunner6, Option as Option4 } from "nest-commander";
|
|
3119
|
+
class StatusCommand extends CommandRunner6 {
|
|
2438
3120
|
syncService;
|
|
2439
3121
|
manifestService;
|
|
2440
|
-
|
|
3122
|
+
dbChangeDetector;
|
|
3123
|
+
constructor(syncService, manifestService, dbChangeDetector) {
|
|
2441
3124
|
super();
|
|
2442
3125
|
this.syncService = syncService;
|
|
2443
3126
|
this.manifestService = manifestService;
|
|
3127
|
+
this.dbChangeDetector = dbChangeDetector;
|
|
2444
3128
|
}
|
|
2445
3129
|
async run(_inputs, options) {
|
|
2446
3130
|
try {
|
|
2447
|
-
|
|
2448
|
-
|
|
3131
|
+
const useDbDetection = !options.legacy;
|
|
3132
|
+
if (useDbDetection) {
|
|
3133
|
+
await this.dbChangeDetector.loadHashes();
|
|
3134
|
+
} else {
|
|
3135
|
+
await this.manifestService.load();
|
|
3136
|
+
}
|
|
3137
|
+
const changes = await this.syncService.detectChanges(undefined, useDbDetection);
|
|
2449
3138
|
const newDocs = changes.filter((c) => c.changeType === "new");
|
|
2450
3139
|
const updatedDocs = changes.filter((c) => c.changeType === "updated");
|
|
2451
3140
|
const deletedDocs = changes.filter((c) => c.changeType === "deleted");
|
|
@@ -2498,9 +3187,12 @@ class StatusCommand extends CommandRunner4 {
|
|
|
2498
3187
|
parseVerbose() {
|
|
2499
3188
|
return true;
|
|
2500
3189
|
}
|
|
3190
|
+
parseLegacy() {
|
|
3191
|
+
return true;
|
|
3192
|
+
}
|
|
2501
3193
|
}
|
|
2502
3194
|
__legacyDecorateClassTS([
|
|
2503
|
-
|
|
3195
|
+
Option4({
|
|
2504
3196
|
flags: "-v, --verbose",
|
|
2505
3197
|
description: "Show all documents including unchanged"
|
|
2506
3198
|
}),
|
|
@@ -2508,28 +3200,38 @@ __legacyDecorateClassTS([
|
|
|
2508
3200
|
__legacyMetadataTS("design:paramtypes", []),
|
|
2509
3201
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2510
3202
|
], StatusCommand.prototype, "parseVerbose", null);
|
|
3203
|
+
__legacyDecorateClassTS([
|
|
3204
|
+
Option4({
|
|
3205
|
+
flags: "--legacy",
|
|
3206
|
+
description: "Use legacy v1 mode: manifest-based change detection"
|
|
3207
|
+
}),
|
|
3208
|
+
__legacyMetadataTS("design:type", Function),
|
|
3209
|
+
__legacyMetadataTS("design:paramtypes", []),
|
|
3210
|
+
__legacyMetadataTS("design:returntype", Boolean)
|
|
3211
|
+
], StatusCommand.prototype, "parseLegacy", null);
|
|
2511
3212
|
StatusCommand = __legacyDecorateClassTS([
|
|
2512
|
-
|
|
2513
|
-
|
|
3213
|
+
Injectable16(),
|
|
3214
|
+
Command6({
|
|
2514
3215
|
name: "status",
|
|
2515
3216
|
description: "Show documents that need syncing (new or updated)"
|
|
2516
3217
|
}),
|
|
2517
3218
|
__legacyMetadataTS("design:paramtypes", [
|
|
2518
3219
|
typeof SyncService === "undefined" ? Object : SyncService,
|
|
2519
|
-
typeof ManifestService === "undefined" ? Object : ManifestService
|
|
3220
|
+
typeof ManifestService === "undefined" ? Object : ManifestService,
|
|
3221
|
+
typeof DatabaseChangeDetectorService === "undefined" ? Object : DatabaseChangeDetectorService
|
|
2520
3222
|
])
|
|
2521
3223
|
], StatusCommand);
|
|
2522
3224
|
// src/commands/sync.command.ts
|
|
2523
3225
|
import { watch } from "fs";
|
|
2524
3226
|
import { join as join3 } from "path";
|
|
2525
|
-
import { Injectable as
|
|
2526
|
-
import { Command as
|
|
3227
|
+
import { Injectable as Injectable18 } from "@nestjs/common";
|
|
3228
|
+
import { Command as Command7, CommandRunner as CommandRunner7, Option as Option5 } from "nest-commander";
|
|
2527
3229
|
|
|
2528
3230
|
// src/sync/graph-validator.service.ts
|
|
2529
|
-
import { Injectable as
|
|
3231
|
+
import { Injectable as Injectable17, Logger as Logger9 } from "@nestjs/common";
|
|
2530
3232
|
class GraphValidatorService {
|
|
2531
3233
|
graph;
|
|
2532
|
-
logger = new
|
|
3234
|
+
logger = new Logger9(GraphValidatorService.name);
|
|
2533
3235
|
constructor(graph) {
|
|
2534
3236
|
this.graph = graph;
|
|
2535
3237
|
}
|
|
@@ -2675,14 +3377,14 @@ class GraphValidatorService {
|
|
|
2675
3377
|
}
|
|
2676
3378
|
}
|
|
2677
3379
|
GraphValidatorService = __legacyDecorateClassTS([
|
|
2678
|
-
|
|
3380
|
+
Injectable17(),
|
|
2679
3381
|
__legacyMetadataTS("design:paramtypes", [
|
|
2680
3382
|
typeof GraphService === "undefined" ? Object : GraphService
|
|
2681
3383
|
])
|
|
2682
3384
|
], GraphValidatorService);
|
|
2683
3385
|
|
|
2684
3386
|
// src/commands/sync.command.ts
|
|
2685
|
-
class SyncCommand extends
|
|
3387
|
+
class SyncCommand extends CommandRunner7 {
|
|
2686
3388
|
syncService;
|
|
2687
3389
|
_graphValidator;
|
|
2688
3390
|
watcher = null;
|
|
@@ -2702,6 +3404,14 @@ class SyncCommand extends CommandRunner5 {
|
|
|
2702
3404
|
if (options.watch && options.force) {
|
|
2703
3405
|
console.log(`
|
|
2704
3406
|
\u26A0\uFE0F Watch mode is not compatible with --force mode (for safety)
|
|
3407
|
+
`);
|
|
3408
|
+
process.exit(1);
|
|
3409
|
+
}
|
|
3410
|
+
if (options.force && paths.length === 0) {
|
|
3411
|
+
console.log(`
|
|
3412
|
+
\u26A0\uFE0F --force requires specific paths to be specified.
|
|
3413
|
+
`);
|
|
3414
|
+
console.log(` Usage: lattice sync --force <path1> [path2] ...
|
|
2705
3415
|
`);
|
|
2706
3416
|
process.exit(1);
|
|
2707
3417
|
}
|
|
@@ -2711,19 +3421,16 @@ class SyncCommand extends CommandRunner5 {
|
|
|
2711
3421
|
verbose: options.verbose,
|
|
2712
3422
|
paths: paths.length > 0 ? paths : undefined,
|
|
2713
3423
|
skipCascade: options.skipCascade,
|
|
2714
|
-
embeddings: options.embeddings !== false
|
|
3424
|
+
embeddings: options.embeddings !== false,
|
|
3425
|
+
legacy: options.legacy,
|
|
3426
|
+
aiExtraction: !options.skipExtraction
|
|
2715
3427
|
};
|
|
2716
3428
|
console.log(`
|
|
2717
3429
|
\uD83D\uDD04 Graph Sync
|
|
2718
3430
|
`);
|
|
2719
3431
|
if (syncOptions.force) {
|
|
2720
|
-
|
|
2721
|
-
console.log(`\u26A0\uFE0F Force mode: ${syncOptions.paths.length} document(s) will be cleared and re-synced
|
|
3432
|
+
console.log(`\u26A0\uFE0F Force mode: ${syncOptions.paths?.length} document(s) will be cleared and re-synced
|
|
2722
3433
|
`);
|
|
2723
|
-
} else {
|
|
2724
|
-
console.log(`\u26A0\uFE0F Force mode: Entire graph will be cleared and rebuilt
|
|
2725
|
-
`);
|
|
2726
|
-
}
|
|
2727
3434
|
}
|
|
2728
3435
|
if (syncOptions.dryRun) {
|
|
2729
3436
|
console.log(`\uD83D\uDCCB Dry run mode: No changes will be applied
|
|
@@ -2737,6 +3444,15 @@ class SyncCommand extends CommandRunner5 {
|
|
|
2737
3444
|
console.log(`\uD83D\uDEAB Embedding generation disabled
|
|
2738
3445
|
`);
|
|
2739
3446
|
}
|
|
3447
|
+
if (syncOptions.legacy) {
|
|
3448
|
+
console.log(`\uD83D\uDCDC Legacy mode: Using manifest-based change detection
|
|
3449
|
+
`);
|
|
3450
|
+
} else {
|
|
3451
|
+
if (!syncOptions.aiExtraction) {
|
|
3452
|
+
console.log(`\u23ED\uFE0F AI entity extraction skipped (--skip-extraction)
|
|
3453
|
+
`);
|
|
3454
|
+
}
|
|
3455
|
+
}
|
|
2740
3456
|
if (syncOptions.paths) {
|
|
2741
3457
|
console.log(`\uD83D\uDCC1 Syncing specific paths: ${syncOptions.paths.join(", ")}
|
|
2742
3458
|
`);
|
|
@@ -2930,18 +3646,24 @@ class SyncCommand extends CommandRunner5 {
|
|
|
2930
3646
|
parseNoEmbeddings() {
|
|
2931
3647
|
return false;
|
|
2932
3648
|
}
|
|
3649
|
+
parseSkipExtraction() {
|
|
3650
|
+
return true;
|
|
3651
|
+
}
|
|
3652
|
+
parseLegacy() {
|
|
3653
|
+
return true;
|
|
3654
|
+
}
|
|
2933
3655
|
}
|
|
2934
3656
|
__legacyDecorateClassTS([
|
|
2935
|
-
|
|
3657
|
+
Option5({
|
|
2936
3658
|
flags: "-f, --force",
|
|
2937
|
-
description: "Force re-sync
|
|
3659
|
+
description: "Force re-sync specified documents (requires paths to be specified)"
|
|
2938
3660
|
}),
|
|
2939
3661
|
__legacyMetadataTS("design:type", Function),
|
|
2940
3662
|
__legacyMetadataTS("design:paramtypes", []),
|
|
2941
3663
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2942
3664
|
], SyncCommand.prototype, "parseForce", null);
|
|
2943
3665
|
__legacyDecorateClassTS([
|
|
2944
|
-
|
|
3666
|
+
Option5({
|
|
2945
3667
|
flags: "-d, --dry-run",
|
|
2946
3668
|
description: "Show what would change without applying"
|
|
2947
3669
|
}),
|
|
@@ -2950,7 +3672,7 @@ __legacyDecorateClassTS([
|
|
|
2950
3672
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2951
3673
|
], SyncCommand.prototype, "parseDryRun", null);
|
|
2952
3674
|
__legacyDecorateClassTS([
|
|
2953
|
-
|
|
3675
|
+
Option5({
|
|
2954
3676
|
flags: "-v, --verbose",
|
|
2955
3677
|
description: "Show detailed output"
|
|
2956
3678
|
}),
|
|
@@ -2959,7 +3681,7 @@ __legacyDecorateClassTS([
|
|
|
2959
3681
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2960
3682
|
], SyncCommand.prototype, "parseVerbose", null);
|
|
2961
3683
|
__legacyDecorateClassTS([
|
|
2962
|
-
|
|
3684
|
+
Option5({
|
|
2963
3685
|
flags: "-w, --watch",
|
|
2964
3686
|
description: "Watch for file changes and sync automatically"
|
|
2965
3687
|
}),
|
|
@@ -2968,7 +3690,7 @@ __legacyDecorateClassTS([
|
|
|
2968
3690
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2969
3691
|
], SyncCommand.prototype, "parseWatch", null);
|
|
2970
3692
|
__legacyDecorateClassTS([
|
|
2971
|
-
|
|
3693
|
+
Option5({
|
|
2972
3694
|
flags: "--diff",
|
|
2973
3695
|
description: "Show only changed documents (alias for --dry-run)"
|
|
2974
3696
|
}),
|
|
@@ -2977,7 +3699,7 @@ __legacyDecorateClassTS([
|
|
|
2977
3699
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2978
3700
|
], SyncCommand.prototype, "parseDiff", null);
|
|
2979
3701
|
__legacyDecorateClassTS([
|
|
2980
|
-
|
|
3702
|
+
Option5({
|
|
2981
3703
|
flags: "--skip-cascade",
|
|
2982
3704
|
description: "Skip cascade analysis (faster for large repos)"
|
|
2983
3705
|
}),
|
|
@@ -2986,7 +3708,7 @@ __legacyDecorateClassTS([
|
|
|
2986
3708
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2987
3709
|
], SyncCommand.prototype, "parseSkipCascade", null);
|
|
2988
3710
|
__legacyDecorateClassTS([
|
|
2989
|
-
|
|
3711
|
+
Option5({
|
|
2990
3712
|
flags: "--no-embeddings",
|
|
2991
3713
|
description: "Disable embedding generation during sync"
|
|
2992
3714
|
}),
|
|
@@ -2994,131 +3716,36 @@ __legacyDecorateClassTS([
|
|
|
2994
3716
|
__legacyMetadataTS("design:paramtypes", []),
|
|
2995
3717
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
2996
3718
|
], SyncCommand.prototype, "parseNoEmbeddings", null);
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
arguments: "[paths...]",
|
|
3002
|
-
description: "Synchronize documents to the knowledge graph"
|
|
3719
|
+
__legacyDecorateClassTS([
|
|
3720
|
+
Option5({
|
|
3721
|
+
flags: "--skip-extraction",
|
|
3722
|
+
description: "Skip AI entity extraction (sync without re-extracting entities)"
|
|
3003
3723
|
}),
|
|
3004
|
-
__legacyMetadataTS("design:
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
], SyncCommand);
|
|
3009
|
-
// src/commands/validate.command.ts
|
|
3010
|
-
import { Injectable as Injectable15 } from "@nestjs/common";
|
|
3011
|
-
import { Command as Command6, CommandRunner as CommandRunner6, Option as Option4 } from "nest-commander";
|
|
3012
|
-
class ValidateCommand extends CommandRunner6 {
|
|
3013
|
-
parserService;
|
|
3014
|
-
_graphValidator;
|
|
3015
|
-
constructor(parserService, _graphValidator) {
|
|
3016
|
-
super();
|
|
3017
|
-
this.parserService = parserService;
|
|
3018
|
-
this._graphValidator = _graphValidator;
|
|
3019
|
-
}
|
|
3020
|
-
async run(_inputs, options) {
|
|
3021
|
-
try {
|
|
3022
|
-
console.log(`=== Document Validation ===
|
|
3023
|
-
`);
|
|
3024
|
-
const { docs, errors: schemaErrors } = await this.parserService.parseAllDocumentsWithErrors();
|
|
3025
|
-
const issues = [];
|
|
3026
|
-
for (const schemaError of schemaErrors) {
|
|
3027
|
-
issues.push({
|
|
3028
|
-
type: "error",
|
|
3029
|
-
path: schemaError.path,
|
|
3030
|
-
message: schemaError.error
|
|
3031
|
-
});
|
|
3032
|
-
}
|
|
3033
|
-
const entityIndex = new Map;
|
|
3034
|
-
for (const doc of docs) {
|
|
3035
|
-
for (const entity of doc.entities) {
|
|
3036
|
-
let docPaths = entityIndex.get(entity.name);
|
|
3037
|
-
if (!docPaths) {
|
|
3038
|
-
docPaths = new Set;
|
|
3039
|
-
entityIndex.set(entity.name, docPaths);
|
|
3040
|
-
}
|
|
3041
|
-
docPaths.add(doc.path);
|
|
3042
|
-
}
|
|
3043
|
-
}
|
|
3044
|
-
const validationErrors = validateDocuments2(docs);
|
|
3045
|
-
for (const err of validationErrors) {
|
|
3046
|
-
issues.push({
|
|
3047
|
-
type: "error",
|
|
3048
|
-
path: err.path,
|
|
3049
|
-
message: err.error,
|
|
3050
|
-
suggestion: "Add entity definition or fix the reference"
|
|
3051
|
-
});
|
|
3052
|
-
}
|
|
3053
|
-
console.log(`Scanned ${docs.length} documents`);
|
|
3054
|
-
console.log(`Found ${entityIndex.size} unique entities
|
|
3055
|
-
`);
|
|
3056
|
-
if (issues.length > 0) {
|
|
3057
|
-
console.log(`Document Errors (${issues.length}):
|
|
3058
|
-
`);
|
|
3059
|
-
issues.forEach((i) => {
|
|
3060
|
-
console.log(` ${i.path}`);
|
|
3061
|
-
console.log(` Error: ${i.message}`);
|
|
3062
|
-
if (options.fix && i.suggestion) {
|
|
3063
|
-
console.log(` Suggestion: ${i.suggestion}`);
|
|
3064
|
-
}
|
|
3065
|
-
console.log("");
|
|
3066
|
-
});
|
|
3067
|
-
} else {
|
|
3068
|
-
console.log(`\u2713 Markdown files valid (schema + relationships)
|
|
3069
|
-
`);
|
|
3070
|
-
}
|
|
3071
|
-
const graphResult = {
|
|
3072
|
-
valid: true,
|
|
3073
|
-
issues: [],
|
|
3074
|
-
stats: {
|
|
3075
|
-
totalNodes: 0,
|
|
3076
|
-
documentsChecked: 0,
|
|
3077
|
-
entitiesChecked: 0,
|
|
3078
|
-
errorsFound: 0,
|
|
3079
|
-
warningsFound: 0
|
|
3080
|
-
}
|
|
3081
|
-
};
|
|
3082
|
-
const totalErrors = issues.length + graphResult.stats.errorsFound;
|
|
3083
|
-
const totalWarnings = graphResult.stats.warningsFound;
|
|
3084
|
-
console.log(`
|
|
3085
|
-
=== Validation Summary ===`);
|
|
3086
|
-
console.log(`Markdown files: ${issues.length === 0 ? "\u2713 PASSED" : `\u2717 ${issues.length} errors`}`);
|
|
3087
|
-
console.log(`Graph database: ${graphResult.stats.errorsFound === 0 ? "\u2713 PASSED" : `\u2717 ${graphResult.stats.errorsFound} errors`}`);
|
|
3088
|
-
console.log(`Warnings: ${totalWarnings}`);
|
|
3089
|
-
console.log(`
|
|
3090
|
-
Overall: ${totalErrors === 0 ? "\u2713 PASSED" : "\u2717 FAILED"}${totalWarnings > 0 ? ` (${totalWarnings} warnings)` : ""}
|
|
3091
|
-
`);
|
|
3092
|
-
process.exit(totalErrors > 0 ? 1 : 0);
|
|
3093
|
-
} catch (error) {
|
|
3094
|
-
console.error("Validation failed:", error instanceof Error ? error.message : String(error));
|
|
3095
|
-
process.exit(1);
|
|
3096
|
-
}
|
|
3097
|
-
}
|
|
3098
|
-
parseFix() {
|
|
3099
|
-
return true;
|
|
3100
|
-
}
|
|
3101
|
-
}
|
|
3724
|
+
__legacyMetadataTS("design:type", Function),
|
|
3725
|
+
__legacyMetadataTS("design:paramtypes", []),
|
|
3726
|
+
__legacyMetadataTS("design:returntype", Boolean)
|
|
3727
|
+
], SyncCommand.prototype, "parseSkipExtraction", null);
|
|
3102
3728
|
__legacyDecorateClassTS([
|
|
3103
|
-
|
|
3104
|
-
flags: "--
|
|
3105
|
-
description: "
|
|
3729
|
+
Option5({
|
|
3730
|
+
flags: "--legacy",
|
|
3731
|
+
description: "Use legacy v1 mode: manifest-based change detection, no AI extraction"
|
|
3106
3732
|
}),
|
|
3107
3733
|
__legacyMetadataTS("design:type", Function),
|
|
3108
3734
|
__legacyMetadataTS("design:paramtypes", []),
|
|
3109
3735
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
3110
|
-
],
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
name: "
|
|
3115
|
-
|
|
3736
|
+
], SyncCommand.prototype, "parseLegacy", null);
|
|
3737
|
+
SyncCommand = __legacyDecorateClassTS([
|
|
3738
|
+
Injectable18(),
|
|
3739
|
+
Command7({
|
|
3740
|
+
name: "sync",
|
|
3741
|
+
arguments: "[paths...]",
|
|
3742
|
+
description: "Synchronize documents to the knowledge graph"
|
|
3116
3743
|
}),
|
|
3117
3744
|
__legacyMetadataTS("design:paramtypes", [
|
|
3118
|
-
typeof
|
|
3745
|
+
typeof SyncService === "undefined" ? Object : SyncService,
|
|
3119
3746
|
typeof GraphValidatorService === "undefined" ? Object : GraphValidatorService
|
|
3120
3747
|
])
|
|
3121
|
-
],
|
|
3748
|
+
], SyncCommand);
|
|
3122
3749
|
// src/embedding/embedding.module.ts
|
|
3123
3750
|
import { Module } from "@nestjs/common";
|
|
3124
3751
|
import { ConfigModule } from "@nestjs/config";
|
|
@@ -3147,10 +3774,10 @@ GraphModule = __legacyDecorateClassTS([
|
|
|
3147
3774
|
import { Module as Module3 } from "@nestjs/common";
|
|
3148
3775
|
|
|
3149
3776
|
// src/query/query.service.ts
|
|
3150
|
-
import { Injectable as
|
|
3777
|
+
import { Injectable as Injectable19, Logger as Logger10 } from "@nestjs/common";
|
|
3151
3778
|
class QueryService {
|
|
3152
3779
|
graphService;
|
|
3153
|
-
logger = new
|
|
3780
|
+
logger = new Logger10(QueryService.name);
|
|
3154
3781
|
constructor(graphService) {
|
|
3155
3782
|
this.graphService = graphService;
|
|
3156
3783
|
}
|
|
@@ -3160,7 +3787,7 @@ class QueryService {
|
|
|
3160
3787
|
}
|
|
3161
3788
|
}
|
|
3162
3789
|
QueryService = __legacyDecorateClassTS([
|
|
3163
|
-
|
|
3790
|
+
Injectable19(),
|
|
3164
3791
|
__legacyMetadataTS("design:paramtypes", [
|
|
3165
3792
|
typeof GraphService === "undefined" ? Object : GraphService
|
|
3166
3793
|
])
|
|
@@ -3187,6 +3814,8 @@ SyncModule = __legacyDecorateClassTS([
|
|
|
3187
3814
|
providers: [
|
|
3188
3815
|
SyncService,
|
|
3189
3816
|
ManifestService,
|
|
3817
|
+
DatabaseChangeDetectorService,
|
|
3818
|
+
EntityExtractorService,
|
|
3190
3819
|
DocumentParserService,
|
|
3191
3820
|
OntologyService,
|
|
3192
3821
|
CascadeService,
|
|
@@ -3196,6 +3825,8 @@ SyncModule = __legacyDecorateClassTS([
|
|
|
3196
3825
|
exports: [
|
|
3197
3826
|
SyncService,
|
|
3198
3827
|
ManifestService,
|
|
3828
|
+
DatabaseChangeDetectorService,
|
|
3829
|
+
EntityExtractorService,
|
|
3199
3830
|
DocumentParserService,
|
|
3200
3831
|
OntologyService,
|
|
3201
3832
|
CascadeService,
|
|
@@ -3221,14 +3852,15 @@ AppModule = __legacyDecorateClassTS([
|
|
|
3221
3852
|
QueryModule
|
|
3222
3853
|
],
|
|
3223
3854
|
providers: [
|
|
3855
|
+
ExtractCommand,
|
|
3224
3856
|
SyncCommand,
|
|
3225
3857
|
StatusCommand,
|
|
3226
3858
|
SearchCommand,
|
|
3227
3859
|
RelsCommand,
|
|
3228
3860
|
SqlCommand,
|
|
3229
|
-
ValidateCommand,
|
|
3230
3861
|
OntologyCommand,
|
|
3231
|
-
InitCommand
|
|
3862
|
+
InitCommand,
|
|
3863
|
+
MigrateCommand
|
|
3232
3864
|
]
|
|
3233
3865
|
})
|
|
3234
3866
|
], AppModule);
|