@zabaca/lattice 1.0.11 → 1.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -23,17 +23,350 @@ import { CommandFactory } from "nest-commander";
23
23
  import { Module as Module5 } from "@nestjs/common";
24
24
  import { ConfigModule as ConfigModule2 } from "@nestjs/config";
25
25
 
26
+ // src/commands/extract.command.ts
27
+ import { existsSync, readFileSync } from "fs";
28
+ import { resolve } from "path";
29
+ import { query as query2 } from "@anthropic-ai/claude-agent-sdk";
30
+ import { Injectable as Injectable2 } from "@nestjs/common";
31
+ import { Command, CommandRunner, Option } from "nest-commander";
32
+
33
+ // src/sync/entity-extractor.service.ts
34
+ import { readFile } from "fs/promises";
35
+ import {
36
+ createSdkMcpServer,
37
+ query,
38
+ tool
39
+ } from "@anthropic-ai/claude-agent-sdk";
40
+ import { Injectable, Logger } from "@nestjs/common";
41
+ import { z } from "zod";
42
+ function validateExtraction(input, _filePath) {
43
+ const errors = [];
44
+ const { entities, relationships } = input ?? {};
45
+ if (!entities || !Array.isArray(entities)) {
46
+ return ["Entities array is missing or invalid"];
47
+ }
48
+ if (!relationships || !Array.isArray(relationships)) {
49
+ return ["Relationships array is missing or invalid"];
50
+ }
51
+ const entityNames = new Set(entities.map((e) => e.name));
52
+ for (const rel of relationships) {
53
+ if (rel.source !== "this" && !entityNames.has(rel.source)) {
54
+ errors.push(`Relationship source "${rel.source}" not found in extracted entities`);
55
+ }
56
+ if (!entityNames.has(rel.target)) {
57
+ errors.push(`Relationship target "${rel.target}" not found in extracted entities`);
58
+ }
59
+ }
60
+ return errors;
61
+ }
62
+ function createValidationServer(filePath) {
63
+ return createSdkMcpServer({
64
+ name: "entity-validator",
65
+ version: "1.0.0",
66
+ tools: [
67
+ tool("validate_extraction", "Validate your extracted entities and relationships. Call this to check your work before finishing.", {
68
+ entities: z.array(z.object({
69
+ name: z.string().min(1),
70
+ type: z.enum([
71
+ "Topic",
72
+ "Technology",
73
+ "Concept",
74
+ "Tool",
75
+ "Process",
76
+ "Person",
77
+ "Organization",
78
+ "Document"
79
+ ]),
80
+ description: z.string().min(1)
81
+ })),
82
+ relationships: z.array(z.object({
83
+ source: z.string().min(1),
84
+ relation: z.enum(["REFERENCES"]),
85
+ target: z.string().min(1)
86
+ })),
87
+ summary: z.string().min(10)
88
+ }, async (args) => {
89
+ const errors = validateExtraction(args, filePath);
90
+ if (errors.length === 0) {
91
+ return {
92
+ content: [
93
+ {
94
+ type: "text",
95
+ text: "\u2713 Validation passed. Your extraction is correct."
96
+ }
97
+ ]
98
+ };
99
+ }
100
+ return {
101
+ content: [
102
+ {
103
+ type: "text",
104
+ text: `\u2717 Validation failed:
105
+ ${errors.map((e) => `- ${e}`).join(`
106
+ `)}
107
+
108
+ Please fix these errors and call validate_extraction again.`
109
+ }
110
+ ]
111
+ };
112
+ })
113
+ ]
114
+ });
115
+ }
116
+
117
+ class EntityExtractorService {
118
+ logger = new Logger(EntityExtractorService.name);
119
+ lastExtractionTime = 0;
120
+ minIntervalMs = 500;
121
+ async extractFromDocument(filePath) {
122
+ await this.rateLimit();
123
+ try {
124
+ const content = await readFile(filePath, "utf-8");
125
+ const promptText = this.buildExtractionPrompt(filePath, content);
126
+ const validationServer = createValidationServer(filePath);
127
+ let lastValidExtraction = null;
128
+ for await (const message of query({
129
+ prompt: promptText,
130
+ options: {
131
+ maxTurns: 3,
132
+ model: "claude-haiku-4-5-20251001",
133
+ mcpServers: {
134
+ "entity-validator": validationServer
135
+ },
136
+ allowedTools: ["mcp__entity-validator__validate_extraction"],
137
+ permissionMode: "default"
138
+ }
139
+ })) {
140
+ if (message.type === "assistant") {
141
+ for (const block of message.message?.content ?? []) {
142
+ if (block.type === "tool_use" && block.name === "mcp__entity-validator__validate_extraction") {
143
+ const input = block.input;
144
+ const validationErrors = validateExtraction(input, filePath);
145
+ if (validationErrors.length === 0) {
146
+ lastValidExtraction = input;
147
+ }
148
+ }
149
+ }
150
+ } else if (message.type === "result") {
151
+ if (lastValidExtraction) {
152
+ this.logger.debug(`Extraction for ${filePath} completed in ${message.duration_ms}ms`);
153
+ return this.buildSuccessResult(lastValidExtraction, filePath);
154
+ }
155
+ const errorReason = message.subtype === "error_max_turns" ? "Max turns reached without valid extraction" : message.subtype === "error_during_execution" ? "Error during execution" : `Extraction failed: ${message.subtype}`;
156
+ return {
157
+ entities: [],
158
+ relationships: [],
159
+ summary: "",
160
+ success: false,
161
+ error: errorReason
162
+ };
163
+ }
164
+ }
165
+ throw new Error("No result received from SDK");
166
+ } catch (error) {
167
+ const errorMsg = error instanceof Error ? error.message : String(error);
168
+ this.logger.error(`Entity extraction failed for ${filePath}: ${errorMsg}`);
169
+ return {
170
+ entities: [],
171
+ relationships: [],
172
+ summary: "",
173
+ success: false,
174
+ error: errorMsg
175
+ };
176
+ }
177
+ }
178
+ async extractFromDocuments(filePaths, onProgress) {
179
+ const results = new Map;
180
+ for (let i = 0;i < filePaths.length; i++) {
181
+ const path = filePaths[i];
182
+ onProgress?.(i, filePaths.length, path);
183
+ const result = await this.extractFromDocument(path);
184
+ results.set(path, result);
185
+ }
186
+ return results;
187
+ }
188
+ async rateLimit() {
189
+ const now = Date.now();
190
+ const elapsed = now - this.lastExtractionTime;
191
+ if (elapsed < this.minIntervalMs) {
192
+ const waitTime = this.minIntervalMs - elapsed;
193
+ await new Promise((resolve) => setTimeout(resolve, waitTime));
194
+ }
195
+ this.lastExtractionTime = Date.now();
196
+ }
197
+ buildExtractionPrompt(filePath, content) {
198
+ return `Analyze this markdown document and extract entities, relationships, and a summary.
199
+
200
+ File: ${filePath}
201
+
202
+ <document>
203
+ ${content}
204
+ </document>
205
+
206
+ ## Instructions
207
+
208
+ Extract the following and call the validation tool with EXACTLY this schema:
209
+
210
+ ### 1. Entities (array of 3-10 objects)
211
+ Each entity must have:
212
+ - "name": string (entity name)
213
+ - "type": one of "Topic", "Technology", "Concept", "Tool", "Process", "Person", "Organization", "Document"
214
+ - "description": string (brief description)
215
+
216
+ ### 2. Relationships (array of objects)
217
+ Each relationship must have:
218
+ - "source": "this" (for document-to-entity) or an entity name
219
+ - "relation": "REFERENCES" (IMPORTANT: use "relation", not "type")
220
+ - "target": an entity name from your entities list
221
+
222
+ ### 3. Summary
223
+ A 50-100 word summary of the document's main purpose and key concepts.
224
+
225
+ ## IMPORTANT: Validation Required
226
+
227
+ You MUST call mcp__entity-validator__validate_extraction tool.
228
+ Pass the three fields DIRECTLY as top-level arguments (NOT wrapped in an "extraction" object):
229
+ - entities: your entities array
230
+ - relationships: your relationships array
231
+ - summary: your summary string
232
+
233
+ Example tool call structure:
234
+ {
235
+ "entities": [...],
236
+ "relationships": [...],
237
+ "summary": "..."
238
+ }
239
+
240
+ If validation fails, fix the errors and call the tool again.
241
+ Only finish after validation passes.`;
242
+ }
243
+ buildSuccessResult(extraction, filePath) {
244
+ const relationships = extraction.relationships.map((rel) => ({
245
+ ...rel,
246
+ source: rel.source === "this" ? filePath : rel.source
247
+ }));
248
+ return {
249
+ entities: extraction.entities,
250
+ relationships,
251
+ summary: extraction.summary,
252
+ success: true
253
+ };
254
+ }
255
+ }
256
+ EntityExtractorService = __legacyDecorateClassTS([
257
+ Injectable()
258
+ ], EntityExtractorService);
259
+
260
+ // src/commands/extract.command.ts
261
+ class ExtractCommand extends CommandRunner {
262
+ entityExtractor;
263
+ constructor(entityExtractor) {
264
+ super();
265
+ this.entityExtractor = entityExtractor;
266
+ }
267
+ async run(inputs, options) {
268
+ const [filePath] = inputs;
269
+ if (!filePath) {
270
+ console.error("Error: File path is required");
271
+ console.error("Usage: lattice extract <file>");
272
+ process.exit(1);
273
+ }
274
+ const absolutePath = resolve(process.cwd(), filePath);
275
+ if (!existsSync(absolutePath)) {
276
+ console.error(`Error: File not found: ${absolutePath}`);
277
+ process.exit(1);
278
+ }
279
+ try {
280
+ console.error(`Extracting entities from: ${absolutePath}
281
+ `);
282
+ if (options.raw) {
283
+ await this.extractRaw(absolutePath);
284
+ } else {
285
+ const result = await this.entityExtractor.extractFromDocument(absolutePath);
286
+ const output = options.pretty ? JSON.stringify(result, null, 2) : JSON.stringify(result);
287
+ console.log(output);
288
+ process.exit(result.success ? 0 : 1);
289
+ }
290
+ } catch (error) {
291
+ console.error("Error:", error instanceof Error ? error.message : String(error));
292
+ process.exit(1);
293
+ }
294
+ }
295
+ async extractRaw(filePath) {
296
+ const content = readFileSync(filePath, "utf-8");
297
+ const prompt = this.entityExtractor.buildExtractionPrompt(filePath, content);
298
+ console.error("--- Prompt ---");
299
+ console.error(`${prompt.substring(0, 500)}...
300
+ `);
301
+ console.error("--- Raw Response ---");
302
+ let rawResponse = "";
303
+ for await (const message of query2({
304
+ prompt,
305
+ options: {
306
+ maxTurns: 5,
307
+ model: "claude-haiku-4-5-20251001",
308
+ allowedTools: [],
309
+ permissionMode: "default"
310
+ }
311
+ })) {
312
+ if (message.type === "assistant" && message.message?.content) {
313
+ for (const block of message.message.content) {
314
+ if ("text" in block) {
315
+ rawResponse += block.text;
316
+ }
317
+ }
318
+ }
319
+ }
320
+ console.log(rawResponse);
321
+ process.exit(0);
322
+ }
323
+ parsePretty() {
324
+ return true;
325
+ }
326
+ parseRaw() {
327
+ return true;
328
+ }
329
+ }
330
+ __legacyDecorateClassTS([
331
+ Option({
332
+ flags: "-p, --pretty",
333
+ description: "Pretty-print JSON output"
334
+ }),
335
+ __legacyMetadataTS("design:type", Function),
336
+ __legacyMetadataTS("design:paramtypes", []),
337
+ __legacyMetadataTS("design:returntype", Boolean)
338
+ ], ExtractCommand.prototype, "parsePretty", null);
339
+ __legacyDecorateClassTS([
340
+ Option({
341
+ flags: "-r, --raw",
342
+ description: "Show raw Claude response (for debugging parse errors)"
343
+ }),
344
+ __legacyMetadataTS("design:type", Function),
345
+ __legacyMetadataTS("design:paramtypes", []),
346
+ __legacyMetadataTS("design:returntype", Boolean)
347
+ ], ExtractCommand.prototype, "parseRaw", null);
348
+ ExtractCommand = __legacyDecorateClassTS([
349
+ Injectable2(),
350
+ Command({
351
+ name: "extract",
352
+ description: "Extract entities from a document (debug tool)",
353
+ arguments: "<file>"
354
+ }),
355
+ __legacyMetadataTS("design:paramtypes", [
356
+ typeof EntityExtractorService === "undefined" ? Object : EntityExtractorService
357
+ ])
358
+ ], ExtractCommand);
26
359
  // src/commands/init.command.ts
27
- import { existsSync as existsSync2, writeFileSync } from "fs";
360
+ import { existsSync as existsSync3, writeFileSync } from "fs";
28
361
  import * as fs from "fs/promises";
29
362
  import { homedir as homedir2 } from "os";
30
363
  import * as path from "path";
31
364
  import { fileURLToPath } from "url";
32
- import { Injectable } from "@nestjs/common";
33
- import { Command, CommandRunner } from "nest-commander";
365
+ import { Injectable as Injectable3 } from "@nestjs/common";
366
+ import { Command as Command2, CommandRunner as CommandRunner2 } from "nest-commander";
34
367
 
35
368
  // src/utils/paths.ts
36
- import { existsSync, mkdirSync } from "fs";
369
+ import { existsSync as existsSync2, mkdirSync } from "fs";
37
370
  import { homedir } from "os";
38
371
  import { join } from "path";
39
372
  var latticeHomeOverride = null;
@@ -60,11 +393,11 @@ function getEnvPath() {
60
393
  }
61
394
  function ensureLatticeHome() {
62
395
  const home = getLatticeHomeInternal();
63
- if (!existsSync(home)) {
396
+ if (!existsSync2(home)) {
64
397
  mkdirSync(home, { recursive: true });
65
398
  }
66
399
  const docsPath = getDocsPath();
67
- if (!existsSync(docsPath)) {
400
+ if (!existsSync2(docsPath)) {
68
401
  mkdirSync(docsPath, { recursive: true });
69
402
  }
70
403
  }
@@ -74,12 +407,12 @@ var __filename2 = fileURLToPath(import.meta.url);
74
407
  var __dirname2 = path.dirname(__filename2);
75
408
  var COMMANDS = ["research.md", "graph-sync.md", "entity-extract.md"];
76
409
 
77
- class InitCommand extends CommandRunner {
410
+ class InitCommand extends CommandRunner2 {
78
411
  async run(_inputs, _options) {
79
412
  try {
80
413
  ensureLatticeHome();
81
414
  const envPath = getEnvPath();
82
- if (!existsSync2(envPath)) {
415
+ if (!existsSync3(envPath)) {
83
416
  writeFileSync(envPath, `# Lattice Configuration
84
417
  # Get your API key from: https://www.voyageai.com/
85
418
 
@@ -158,1504 +491,1224 @@ VOYAGE_API_KEY=
158
491
  }
159
492
  }
160
493
  InitCommand = __legacyDecorateClassTS([
161
- Injectable(),
162
- Command({
494
+ Injectable3(),
495
+ Command2({
163
496
  name: "init",
164
497
  description: "Install Claude Code slash commands for Lattice"
165
498
  })
166
499
  ], InitCommand);
167
- // src/commands/ontology.command.ts
168
- import { Injectable as Injectable4 } from "@nestjs/common";
169
- import { Command as Command2, CommandRunner as CommandRunner2 } from "nest-commander";
170
-
171
- // src/sync/ontology.service.ts
172
- import { Injectable as Injectable3 } from "@nestjs/common";
173
-
174
- // src/sync/document-parser.service.ts
175
- import { createHash } from "crypto";
176
- import { readFile as readFile2 } from "fs/promises";
177
- import { Injectable as Injectable2, Logger } from "@nestjs/common";
178
- import { glob } from "glob";
500
+ // src/commands/migrate.command.ts
501
+ import { Injectable as Injectable12, Logger as Logger8 } from "@nestjs/common";
502
+ import { Command as Command3, CommandRunner as CommandRunner3, Option as Option2 } from "nest-commander";
179
503
 
180
- // src/utils/frontmatter.ts
181
- import matter from "gray-matter";
182
- import { z } from "zod";
183
- var EntityTypeSchema = z.enum([
184
- "Topic",
185
- "Technology",
186
- "Concept",
187
- "Tool",
188
- "Process",
189
- "Person",
190
- "Organization",
191
- "Document"
192
- ]);
193
- var RelationTypeSchema = z.enum(["REFERENCES"]);
194
- var EntitySchema = z.object({
195
- name: z.string().min(1),
196
- type: EntityTypeSchema,
197
- description: z.string().min(1)
198
- });
199
- var RelationshipSchema = z.object({
200
- source: z.string().min(1),
201
- relation: RelationTypeSchema,
202
- target: z.string().min(1)
203
- });
204
- var GraphMetadataSchema = z.object({
205
- importance: z.enum(["high", "medium", "low"]).optional(),
206
- domain: z.string().optional()
207
- });
208
- var validateDateFormat = (dateStr) => {
209
- const match = dateStr.match(/^(\d{4})-(\d{2})-(\d{2})$/);
210
- if (!match)
211
- return false;
212
- const [, yearStr, monthStr, dayStr] = match;
213
- const year = parseInt(yearStr, 10);
214
- const month = parseInt(monthStr, 10);
215
- const day = parseInt(dayStr, 10);
216
- if (month < 1 || month > 12)
217
- return false;
218
- if (day < 1 || day > 31)
219
- return false;
220
- const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
221
- if (year % 4 === 0 && year % 100 !== 0 || year % 400 === 0) {
222
- daysInMonth[1] = 29;
504
+ // src/graph/graph.service.ts
505
+ import { DuckDBInstance } from "@duckdb/node-api";
506
+ import { Injectable as Injectable4, Logger as Logger2 } from "@nestjs/common";
507
+ import { ConfigService } from "@nestjs/config";
508
+ class GraphService {
509
+ configService;
510
+ logger = new Logger2(GraphService.name);
511
+ instance = null;
512
+ connection = null;
513
+ dbPath;
514
+ connecting = null;
515
+ vectorIndexes = new Set;
516
+ embeddingDimensions;
517
+ signalHandlersRegistered = false;
518
+ constructor(configService) {
519
+ this.configService = configService;
520
+ ensureLatticeHome();
521
+ this.dbPath = getDatabasePath();
522
+ this.embeddingDimensions = this.configService.get("EMBEDDING_DIMENSIONS") || 512;
523
+ this.registerSignalHandlers();
223
524
  }
224
- return day <= daysInMonth[month - 1];
225
- };
226
- var FrontmatterSchema = z.object({
227
- created: z.string().refine(validateDateFormat, "Date must be in YYYY-MM-DD format"),
228
- updated: z.string().refine(validateDateFormat, "Date must be in YYYY-MM-DD format"),
229
- status: z.enum(["draft", "ongoing", "complete"]).optional(),
230
- topic: z.string().optional(),
231
- tags: z.array(z.string()).optional(),
232
- summary: z.string().optional(),
233
- entities: z.array(EntitySchema).optional(),
234
- relationships: z.array(RelationshipSchema).optional(),
235
- graph: GraphMetadataSchema.optional()
236
- }).passthrough();
237
- function parseFrontmatter(content) {
238
- try {
239
- const { data, content: markdown } = matter(content);
240
- if (Object.keys(data).length === 0) {
241
- return {
242
- frontmatter: null,
243
- content: markdown.trim(),
244
- raw: content
245
- };
246
- }
247
- const normalizedData = normalizeData(data);
248
- const validated = FrontmatterSchema.safeParse(normalizedData);
249
- return {
250
- frontmatter: validated.success ? validated.data : normalizedData,
251
- content: markdown.trim(),
252
- raw: content
525
+ registerSignalHandlers() {
526
+ if (this.signalHandlersRegistered)
527
+ return;
528
+ this.signalHandlersRegistered = true;
529
+ const gracefulShutdown = async (signal) => {
530
+ this.logger.log(`Received ${signal}, checkpointing before exit...`);
531
+ try {
532
+ await this.checkpoint();
533
+ await this.disconnect();
534
+ } catch (error) {
535
+ this.logger.error(`Error during graceful shutdown: ${error instanceof Error ? error.message : String(error)}`);
536
+ }
537
+ process.exit(0);
253
538
  };
254
- } catch (error) {
255
- const errorMessage = error instanceof Error ? error.message : String(error);
256
- throw new Error(`YAML parsing error: ${errorMessage}`);
257
- }
258
- }
259
- function normalizeData(data) {
260
- if (data instanceof Date) {
261
- return data.toISOString().split("T")[0];
539
+ process.on("SIGINT", () => gracefulShutdown("SIGINT"));
540
+ process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
541
+ process.on("beforeExit", async () => {
542
+ if (this.connection) {
543
+ await this.checkpoint();
544
+ }
545
+ });
262
546
  }
263
- if (Array.isArray(data)) {
264
- return data.map(normalizeData);
547
+ async onModuleDestroy() {
548
+ await this.disconnect();
265
549
  }
266
- if (data !== null && typeof data === "object") {
267
- const normalized = {};
268
- for (const [key, value] of Object.entries(data)) {
269
- normalized[key] = normalizeData(value);
550
+ async ensureConnected() {
551
+ if (this.connection) {
552
+ return this.connection;
270
553
  }
271
- return normalized;
272
- }
273
- return data;
274
- }
275
-
276
- // src/sync/document-parser.service.ts
277
- class DocumentParserService {
278
- logger = new Logger(DocumentParserService.name);
279
- docsPath;
280
- constructor() {
281
- ensureLatticeHome();
282
- this.docsPath = getDocsPath();
283
- }
284
- getDocsPath() {
285
- return this.docsPath;
554
+ if (this.connecting) {
555
+ await this.connecting;
556
+ if (!this.connection) {
557
+ throw new Error("Connection failed to establish");
558
+ }
559
+ return this.connection;
560
+ }
561
+ this.connecting = this.connect();
562
+ await this.connecting;
563
+ this.connecting = null;
564
+ if (!this.connection) {
565
+ throw new Error("Connection failed to establish");
566
+ }
567
+ return this.connection;
286
568
  }
287
- async discoverDocuments() {
288
- const pattern = `${this.docsPath}/**/*.md`;
289
- const files = await glob(pattern, {
290
- ignore: ["**/node_modules/**", "**/.git/**"]
291
- });
292
- return files.sort();
293
- }
294
- async parseDocument(filePath) {
295
- const content = await readFile2(filePath, "utf-8");
296
- const parsed = parseFrontmatter(content);
297
- const title = this.extractTitle(content, filePath);
298
- const contentHash = this.computeHash(content);
299
- const frontmatterHash = this.computeHash(JSON.stringify(parsed.frontmatter || {}));
300
- const entities = this.extractEntities(parsed.frontmatter, filePath);
301
- const relationships = this.extractRelationships(parsed.frontmatter, filePath);
302
- const graphMetadata = this.extractGraphMetadata(parsed.frontmatter);
303
- return {
304
- path: filePath,
305
- title,
306
- content: parsed.content,
307
- contentHash,
308
- frontmatterHash,
309
- summary: parsed.frontmatter?.summary,
310
- topic: parsed.frontmatter?.topic,
311
- entities,
312
- relationships,
313
- graphMetadata,
314
- tags: parsed.frontmatter?.tags || [],
315
- created: parsed.frontmatter?.created,
316
- updated: parsed.frontmatter?.updated,
317
- status: parsed.frontmatter?.status
318
- };
319
- }
320
- async parseAllDocuments() {
321
- const { docs } = await this.parseAllDocumentsWithErrors();
322
- return docs;
323
- }
324
- async parseAllDocumentsWithErrors() {
325
- const files = await this.discoverDocuments();
326
- const docs = [];
327
- const errors = [];
328
- for (const file of files) {
569
+ async connect() {
570
+ try {
571
+ this.instance = await DuckDBInstance.create(":memory:", {
572
+ allow_unsigned_extensions: "true"
573
+ });
574
+ this.connection = await this.instance.connect();
575
+ await this.connection.run("INSTALL vss; LOAD vss;");
576
+ await this.connection.run("SET hnsw_enable_experimental_persistence = true;");
329
577
  try {
330
- const parsed = await this.parseDocument(file);
331
- docs.push(parsed);
332
- } catch (error) {
333
- const errorMsg = error instanceof Error ? error.message : String(error);
334
- errors.push({ path: file, error: errorMsg });
335
- this.logger.warn(`Failed to parse ${file}: ${error}`);
578
+ await this.connection.run("SET custom_extension_repository = 'http://duckpgq.s3.eu-north-1.amazonaws.com';");
579
+ await this.connection.run("FORCE INSTALL 'duckpgq';");
580
+ await this.connection.run("LOAD 'duckpgq';");
581
+ this.logger.log("DuckPGQ extension loaded successfully");
582
+ } catch (e) {
583
+ this.logger.warn(`DuckPGQ extension not available: ${e instanceof Error ? e.message : String(e)}`);
336
584
  }
585
+ await this.connection.run(`ATTACH '${this.dbPath}' AS lattice (READ_WRITE);`);
586
+ await this.connection.run("USE lattice;");
587
+ await this.initializeSchema();
588
+ await this.connection.run("FORCE CHECKPOINT lattice;");
589
+ this.logger.log(`Connected to DuckDB (in-memory + ATTACH) at ${this.dbPath}`);
590
+ } catch (error) {
591
+ this.connection = null;
592
+ this.instance = null;
593
+ this.logger.error(`Failed to connect to DuckDB: ${error instanceof Error ? error.message : String(error)}`);
594
+ throw error;
337
595
  }
338
- return { docs, errors };
339
- }
340
- extractTitle(content, filePath) {
341
- const h1Match = content.match(/^#\s+(.+)$/m);
342
- if (h1Match) {
343
- return h1Match[1];
344
- }
345
- const parts = filePath.split("/");
346
- return parts[parts.length - 1].replace(".md", "");
347
596
  }
348
- extractEntities(frontmatter, docPath) {
349
- const fm = frontmatter;
350
- if (!fm?.entities || !Array.isArray(fm.entities)) {
351
- return [];
352
- }
353
- const validEntities = [];
354
- const errors = [];
355
- for (let i = 0;i < fm.entities.length; i++) {
356
- const e = fm.entities[i];
357
- const result = EntitySchema.safeParse(e);
358
- if (result.success) {
359
- validEntities.push(result.data);
360
- } else {
361
- const entityPreview = typeof e === "string" ? `"${e}"` : JSON.stringify(e);
362
- const zodErrors = result.error.issues.map((issue) => {
363
- const path2 = issue.path.length > 0 ? issue.path.join(".") : "root";
364
- return `${path2}: ${issue.message}`;
365
- }).join("; ");
366
- errors.push(`Entity[${i}]: ${entityPreview} - ${zodErrors}`);
367
- }
597
+ async disconnect() {
598
+ if (this.connection) {
599
+ await this.checkpoint();
600
+ this.connection.closeSync();
601
+ this.connection = null;
602
+ this.logger.log("Disconnected from DuckDB");
368
603
  }
369
- if (errors.length > 0) {
370
- const errorMsg = `Invalid entity schema in ${docPath}:
371
- ${errors.join(`
372
- `)}`;
373
- throw new Error(errorMsg);
604
+ if (this.instance) {
605
+ this.instance = null;
374
606
  }
375
- return validEntities;
376
607
  }
377
- extractRelationships(frontmatter, docPath) {
378
- const fm = frontmatter;
379
- if (!fm?.relationships || !Array.isArray(fm.relationships)) {
380
- return [];
381
- }
382
- const validRelationships = [];
383
- const errors = [];
384
- const validRelationTypes = RelationTypeSchema.options;
385
- for (let i = 0;i < fm.relationships.length; i++) {
386
- const r = fm.relationships[i];
387
- const result = RelationshipSchema.safeParse(r);
388
- if (result.success) {
389
- const rel = result.data;
390
- if (rel.source === "this") {
391
- rel.source = docPath;
392
- }
393
- if (rel.target === "this") {
394
- rel.target = docPath;
395
- }
396
- validRelationships.push(rel);
397
- } else {
398
- if (typeof r === "string") {
399
- errors.push(`Relationship[${i}]: "${r}" - Expected object with {source, relation, target}, got string`);
400
- } else if (typeof r === "object" && r !== null) {
401
- const issues = [];
402
- if (!r.source)
403
- issues.push("missing source");
404
- if (!r.target)
405
- issues.push("missing target");
406
- if (!r.relation) {
407
- issues.push("missing relation");
408
- } else if (!validRelationTypes.includes(r.relation)) {
409
- issues.push(`invalid relation "${r.relation}" (allowed: ${validRelationTypes.join(", ")})`);
410
- }
411
- errors.push(`Relationship[${i}]: ${issues.join(", ")}`);
412
- } else {
413
- errors.push(`Relationship[${i}]: Expected object, got ${typeof r}`);
414
- }
415
- }
608
+ async checkpoint() {
609
+ if (!this.connection) {
610
+ return;
416
611
  }
417
- if (errors.length > 0) {
418
- const errorMsg = `Invalid relationship schema in ${docPath}:
419
- ${errors.join(`
420
- `)}`;
421
- throw new Error(errorMsg);
612
+ try {
613
+ await this.connection.run("FORCE CHECKPOINT lattice;");
614
+ this.logger.debug("Checkpoint completed");
615
+ } catch (error) {
616
+ this.logger.warn(`Checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
422
617
  }
423
- return validRelationships;
424
618
  }
425
- extractGraphMetadata(frontmatter) {
426
- const fm = frontmatter;
427
- if (!fm?.graph) {
428
- return;
619
+ async initializeSchema() {
620
+ if (!this.connection) {
621
+ throw new Error("Cannot initialize schema: not connected");
429
622
  }
430
- const result = GraphMetadataSchema.safeParse(fm.graph);
431
- return result.success ? result.data : undefined;
432
- }
433
- computeHash(content) {
434
- return createHash("sha256").update(content).digest("hex");
623
+ const conn = this.connection;
624
+ await conn.run(`
625
+ CREATE TABLE IF NOT EXISTS nodes (
626
+ label VARCHAR NOT NULL,
627
+ name VARCHAR NOT NULL,
628
+ properties JSON,
629
+ embedding FLOAT[${this.embeddingDimensions}],
630
+ created_at TIMESTAMP DEFAULT NOW(),
631
+ updated_at TIMESTAMP DEFAULT NOW(),
632
+ PRIMARY KEY(label, name)
633
+ )
634
+ `);
635
+ await conn.run(`
636
+ CREATE TABLE IF NOT EXISTS relationships (
637
+ source_label VARCHAR NOT NULL,
638
+ source_name VARCHAR NOT NULL,
639
+ relation_type VARCHAR NOT NULL,
640
+ target_label VARCHAR NOT NULL,
641
+ target_name VARCHAR NOT NULL,
642
+ properties JSON,
643
+ created_at TIMESTAMP DEFAULT NOW(),
644
+ PRIMARY KEY(source_label, source_name, relation_type, target_label, target_name)
645
+ )
646
+ `);
647
+ await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label)");
648
+ await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label_name ON nodes(label, name)");
649
+ await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_source ON relationships(source_label, source_name)");
650
+ await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_target ON relationships(target_label, target_name)");
651
+ await this.applyV2SchemaMigration(conn);
435
652
  }
436
- }
437
- DocumentParserService = __legacyDecorateClassTS([
438
- Injectable2(),
439
- __legacyMetadataTS("design:paramtypes", [])
440
- ], DocumentParserService);
441
-
442
- // src/sync/ontology.service.ts
443
- class OntologyService {
444
- parser;
445
- constructor(parser) {
446
- this.parser = parser;
653
+ async applyV2SchemaMigration(conn) {
654
+ try {
655
+ await conn.run("ALTER TABLE nodes ADD COLUMN IF NOT EXISTS content_hash VARCHAR");
656
+ } catch {}
657
+ try {
658
+ await conn.run("ALTER TABLE nodes ADD COLUMN IF NOT EXISTS embedding_source_hash VARCHAR");
659
+ } catch {}
660
+ try {
661
+ await conn.run("ALTER TABLE nodes ADD COLUMN IF NOT EXISTS extraction_method VARCHAR DEFAULT 'frontmatter'");
662
+ } catch {}
663
+ try {
664
+ await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_content_hash ON nodes(content_hash) WHERE label = 'Document'");
665
+ } catch {}
447
666
  }
448
- async deriveOntology() {
449
- const docs = await this.parser.parseAllDocuments();
450
- return this.deriveFromDocuments(docs);
667
+ async runV2Migration() {
668
+ const conn = await this.ensureConnected();
669
+ await this.applyV2SchemaMigration(conn);
670
+ this.logger.log("V2 schema migration completed");
451
671
  }
452
- deriveFromDocuments(docs) {
453
- const entityTypeSet = new Set;
454
- const relationshipTypeSet = new Set;
455
- const entityCounts = {};
456
- const relationshipCounts = {};
457
- const entityExamples = {};
458
- let documentsWithEntities = 0;
459
- let documentsWithoutEntities = 0;
460
- let totalRelationships = 0;
461
- for (const doc of docs) {
462
- if (doc.entities.length > 0) {
463
- documentsWithEntities++;
464
- } else {
465
- documentsWithoutEntities++;
466
- }
467
- for (const entity of doc.entities) {
468
- entityTypeSet.add(entity.type);
469
- entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
470
- if (!entityExamples[entity.name]) {
471
- entityExamples[entity.name] = { type: entity.type, documents: [] };
472
- }
473
- if (!entityExamples[entity.name].documents.includes(doc.path)) {
474
- entityExamples[entity.name].documents.push(doc.path);
475
- }
476
- }
477
- for (const rel of doc.relationships) {
478
- relationshipTypeSet.add(rel.relation);
479
- relationshipCounts[rel.relation] = (relationshipCounts[rel.relation] || 0) + 1;
480
- totalRelationships++;
481
- }
672
+ async query(sql, _params) {
673
+ try {
674
+ const conn = await this.ensureConnected();
675
+ const reader = await conn.runAndReadAll(sql);
676
+ const rows = reader.getRows();
677
+ return {
678
+ resultSet: rows,
679
+ stats: undefined
680
+ };
681
+ } catch (error) {
682
+ this.logger.error(`Query failed: ${error instanceof Error ? error.message : String(error)}`);
683
+ throw error;
482
684
  }
483
- return {
484
- entityTypes: Array.from(entityTypeSet).sort(),
485
- relationshipTypes: Array.from(relationshipTypeSet).sort(),
486
- entityCounts,
487
- relationshipCounts,
488
- totalEntities: Object.keys(entityExamples).length,
489
- totalRelationships,
490
- documentsWithEntities,
491
- documentsWithoutEntities,
492
- entityExamples
493
- };
494
685
  }
495
- printSummary(ontology) {
496
- console.log(`
497
- Derived Ontology Summary
498
- `);
499
- console.log(`Documents: ${ontology.documentsWithEntities} with entities, ${ontology.documentsWithoutEntities} without`);
500
- console.log(`Unique Entities: ${ontology.totalEntities}`);
501
- console.log(`Total Relationships: ${ontology.totalRelationships}`);
502
- console.log(`
503
- Entity Types:`);
504
- for (const type of ontology.entityTypes) {
505
- console.log(` ${type}: ${ontology.entityCounts[type]} instances`);
506
- }
507
- console.log(`
508
- Relationship Types:`);
509
- for (const type of ontology.relationshipTypes) {
510
- console.log(` ${type}: ${ontology.relationshipCounts[type]} instances`);
511
- }
512
- console.log(`
513
- Top Entities (by document count):`);
514
- const sorted = Object.entries(ontology.entityExamples).sort((a, b) => b[1].documents.length - a[1].documents.length).slice(0, 10);
515
- for (const [name, info] of sorted) {
516
- console.log(` ${name} (${info.type}): ${info.documents.length} docs`);
686
+ async upsertNode(label, properties) {
687
+ try {
688
+ const { name, ...otherProps } = properties;
689
+ if (!name) {
690
+ throw new Error("Node must have a 'name' property");
691
+ }
692
+ const conn = await this.ensureConnected();
693
+ const propsJson = JSON.stringify(otherProps);
694
+ await conn.run(`
695
+ INSERT INTO nodes (label, name, properties)
696
+ VALUES ('${this.escape(String(label))}', '${this.escape(String(name))}', '${this.escape(propsJson)}')
697
+ ON CONFLICT (label, name) DO UPDATE SET
698
+ properties = EXCLUDED.properties,
699
+ updated_at = NOW()
700
+ `);
701
+ } catch (error) {
702
+ this.logger.error(`Failed to upsert node: ${error instanceof Error ? error.message : String(error)}`);
703
+ throw error;
517
704
  }
518
705
  }
519
- }
520
- OntologyService = __legacyDecorateClassTS([
521
- Injectable3(),
522
- __legacyMetadataTS("design:paramtypes", [
523
- typeof DocumentParserService === "undefined" ? Object : DocumentParserService
524
- ])
525
- ], OntologyService);
526
-
527
- // src/commands/ontology.command.ts
528
- class OntologyCommand extends CommandRunner2 {
529
- ontologyService;
530
- constructor(ontologyService) {
531
- super();
532
- this.ontologyService = ontologyService;
533
- }
534
- async run() {
706
+ async upsertRelationship(sourceLabel, sourceName, relation, targetLabel, targetName, properties) {
535
707
  try {
536
- const ontology = await this.ontologyService.deriveOntology();
537
- this.ontologyService.printSummary(ontology);
538
- process.exit(0);
708
+ const conn = await this.ensureConnected();
709
+ await conn.run(`
710
+ INSERT INTO nodes (label, name, properties)
711
+ VALUES ('${this.escape(sourceLabel)}', '${this.escape(sourceName)}', '{}')
712
+ ON CONFLICT (label, name) DO NOTHING
713
+ `);
714
+ await conn.run(`
715
+ INSERT INTO nodes (label, name, properties)
716
+ VALUES ('${this.escape(targetLabel)}', '${this.escape(targetName)}', '{}')
717
+ ON CONFLICT (label, name) DO NOTHING
718
+ `);
719
+ const propsJson = properties ? JSON.stringify(properties) : "{}";
720
+ await conn.run(`
721
+ INSERT INTO relationships (source_label, source_name, relation_type, target_label, target_name, properties)
722
+ VALUES (
723
+ '${this.escape(sourceLabel)}',
724
+ '${this.escape(sourceName)}',
725
+ '${this.escape(relation)}',
726
+ '${this.escape(targetLabel)}',
727
+ '${this.escape(targetName)}',
728
+ '${this.escape(propsJson)}'
729
+ )
730
+ ON CONFLICT (source_label, source_name, relation_type, target_label, target_name) DO UPDATE SET
731
+ properties = EXCLUDED.properties
732
+ `);
539
733
  } catch (error) {
540
- console.error(`
541
- \u274C Ontology derivation failed:`, error instanceof Error ? error.message : String(error));
542
- process.exit(1);
734
+ this.logger.error(`Failed to upsert relationship: ${error instanceof Error ? error.message : String(error)}`);
735
+ throw error;
543
736
  }
544
737
  }
545
- }
546
- OntologyCommand = __legacyDecorateClassTS([
547
- Injectable4(),
548
- Command2({
549
- name: "ontology",
550
- description: "Derive and display ontology from all documents"
551
- }),
552
- __legacyMetadataTS("design:paramtypes", [
553
- typeof OntologyService === "undefined" ? Object : OntologyService
554
- ])
555
- ], OntologyCommand);
556
- // src/commands/query.command.ts
557
- import { Injectable as Injectable7 } from "@nestjs/common";
558
- import { Command as Command3, CommandRunner as CommandRunner3, Option } from "nest-commander";
559
-
560
- // src/embedding/embedding.service.ts
561
- import { Injectable as Injectable5, Logger as Logger2 } from "@nestjs/common";
562
- import { ConfigService } from "@nestjs/config";
563
-
564
- // src/schemas/config.schemas.ts
565
- import { z as z2 } from "zod";
566
- var DuckDBConfigSchema = z2.object({
567
- embeddingDimensions: z2.coerce.number().int().positive().default(512)
568
- });
569
- var EmbeddingConfigSchema = z2.object({
570
- provider: z2.enum(["openai", "voyage", "nomic", "mock"]).default("voyage"),
571
- apiKey: z2.string().optional(),
572
- model: z2.string().min(1).default("voyage-3.5-lite"),
573
- dimensions: z2.coerce.number().int().positive().default(512)
574
- });
575
-
576
- // src/embedding/embedding.types.ts
577
- var DEFAULT_EMBEDDING_CONFIG = {
578
- provider: "voyage",
579
- model: "voyage-3.5-lite",
580
- dimensions: 512
581
- };
582
-
583
- // src/embedding/providers/mock.provider.ts
584
- class MockEmbeddingProvider {
585
- name = "mock";
586
- dimensions;
587
- constructor(dimensions = 1536) {
588
- this.dimensions = dimensions;
589
- }
590
- async generateEmbedding(text) {
591
- return this.generateDeterministicEmbedding(text);
592
- }
593
- async generateEmbeddings(texts) {
594
- return Promise.all(texts.map((text) => this.generateEmbedding(text)));
595
- }
596
- generateDeterministicEmbedding(text) {
597
- const embedding = new Array(this.dimensions);
598
- let hash = 0;
599
- for (let i = 0;i < text.length; i++) {
600
- hash = (hash << 5) - hash + text.charCodeAt(i);
601
- hash = hash & hash;
602
- }
603
- for (let i = 0;i < this.dimensions; i++) {
604
- const seed = hash * (i + 1) ^ i * 31;
605
- embedding[i] = (seed % 2000 - 1000) / 1000;
738
+ async deleteNode(label, name) {
739
+ try {
740
+ const conn = await this.ensureConnected();
741
+ await conn.run(`
742
+ DELETE FROM relationships
743
+ WHERE (source_label = '${this.escape(label)}' AND source_name = '${this.escape(name)}')
744
+ OR (target_label = '${this.escape(label)}' AND target_name = '${this.escape(name)}')
745
+ `);
746
+ await conn.run(`
747
+ DELETE FROM nodes
748
+ WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
749
+ `);
750
+ } catch (error) {
751
+ this.logger.error(`Failed to delete node: ${error instanceof Error ? error.message : String(error)}`);
752
+ throw error;
606
753
  }
607
- const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
608
- return embedding.map((val) => val / magnitude);
609
754
  }
610
- }
611
-
612
- // src/embedding/providers/openai.provider.ts
613
- class OpenAIEmbeddingProvider {
614
- name = "openai";
615
- dimensions;
616
- model;
617
- apiKey;
618
- baseUrl = "https://api.openai.com/v1";
619
- constructor(config) {
620
- const apiKey = config?.apiKey || process.env.OPENAI_API_KEY;
621
- if (!apiKey) {
622
- throw new Error("OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass apiKey in config.");
755
+ async deleteDocumentRelationships(documentPath) {
756
+ try {
757
+ const conn = await this.ensureConnected();
758
+ await conn.run(`
759
+ DELETE FROM relationships
760
+ WHERE properties->>'documentPath' = '${this.escape(documentPath)}'
761
+ `);
762
+ } catch (error) {
763
+ this.logger.error(`Failed to delete document relationships: ${error instanceof Error ? error.message : String(error)}`);
764
+ throw error;
623
765
  }
624
- this.apiKey = apiKey;
625
- this.model = config?.model || "text-embedding-3-small";
626
- this.dimensions = config?.dimensions || 1536;
627
- }
628
- async generateEmbedding(text) {
629
- const embeddings = await this.generateEmbeddings([text]);
630
- return embeddings[0];
631
766
  }
632
- async generateEmbeddings(texts) {
633
- if (!texts || texts.length === 0) {
767
+ async findNodesByLabel(label, limit) {
768
+ try {
769
+ const conn = await this.ensureConnected();
770
+ const limitClause = limit ? ` LIMIT ${limit}` : "";
771
+ const reader = await conn.runAndReadAll(`
772
+ SELECT name, properties
773
+ FROM nodes
774
+ WHERE label = '${this.escape(label)}'${limitClause}
775
+ `);
776
+ return reader.getRows().map((row) => {
777
+ const [name, properties] = row;
778
+ const props = properties ? JSON.parse(properties) : {};
779
+ return { name, ...props };
780
+ });
781
+ } catch (error) {
782
+ this.logger.error(`Failed to find nodes by label: ${error instanceof Error ? error.message : String(error)}`);
634
783
  return [];
635
784
  }
785
+ }
786
+ async findRelationships(nodeName) {
636
787
  try {
637
- const response = await fetch(`${this.baseUrl}/embeddings`, {
638
- method: "POST",
639
- headers: {
640
- "Content-Type": "application/json",
641
- Authorization: `Bearer ${this.apiKey}`
642
- },
643
- body: JSON.stringify({
644
- model: this.model,
645
- input: texts,
646
- dimensions: this.dimensions
647
- })
788
+ const conn = await this.ensureConnected();
789
+ const reader = await conn.runAndReadAll(`
790
+ SELECT relation_type, target_name, source_name
791
+ FROM relationships
792
+ WHERE source_name = '${this.escape(nodeName)}'
793
+ OR target_name = '${this.escape(nodeName)}'
794
+ `);
795
+ return reader.getRows().map((row) => {
796
+ const [relType, targetName, sourceName] = row;
797
+ return [relType, sourceName === nodeName ? targetName : sourceName];
648
798
  });
649
- if (!response.ok) {
650
- const error = await response.json().catch(() => ({}));
651
- throw new Error(`OpenAI API error: ${response.status} ${JSON.stringify(error)}`);
799
+ } catch (error) {
800
+ this.logger.error(`Failed to find relationships: ${error instanceof Error ? error.message : String(error)}`);
801
+ return [];
802
+ }
803
+ }
804
+ async createVectorIndex(label, property, dimensions) {
805
+ try {
806
+ const indexKey = `${label}_${property}`;
807
+ if (this.vectorIndexes.has(indexKey)) {
808
+ return;
652
809
  }
653
- const data = await response.json();
654
- const sortedData = data.data.sort((a, b) => a.index - b.index);
655
- return sortedData.map((item) => item.embedding);
810
+ const conn = await this.ensureConnected();
811
+ try {
812
+ await conn.run(`
813
+ CREATE INDEX idx_embedding_${this.escape(label)}
814
+ ON nodes USING HNSW (embedding)
815
+ WITH (metric = 'cosine')
816
+ `);
817
+ } catch {
818
+ this.logger.debug(`Vector index on ${label}.${property} already exists`);
819
+ }
820
+ this.vectorIndexes.add(indexKey);
821
+ this.logger.log(`Created vector index on ${label}.${property} with ${dimensions} dimensions`);
656
822
  } catch (error) {
657
- if (error instanceof Error) {
658
- throw new Error(`Failed to generate embeddings: ${error.message}`);
823
+ const errorMessage = error instanceof Error ? error.message : String(error);
824
+ if (!errorMessage.includes("already exists")) {
825
+ this.logger.error(`Failed to create vector index: ${errorMessage}`);
826
+ throw error;
659
827
  }
660
- throw error;
661
828
  }
662
829
  }
663
- }
664
-
665
- // src/schemas/embedding.schemas.ts
666
- import { z as z3 } from "zod";
667
- var VoyageEmbeddingResponseSchema = z3.object({
668
- object: z3.string(),
669
- data: z3.array(z3.object({
670
- object: z3.string(),
671
- embedding: z3.array(z3.number()),
672
- index: z3.number().int().nonnegative()
673
- })),
674
- model: z3.string(),
675
- usage: z3.object({
676
- total_tokens: z3.number().int().nonnegative()
677
- })
678
- });
679
-
680
- // src/embedding/providers/voyage.provider.ts
681
- class VoyageEmbeddingProvider {
682
- name = "voyage";
683
- dimensions;
684
- model;
685
- apiKey;
686
- inputType;
687
- baseUrl = "https://api.voyageai.com/v1";
688
- constructor(config) {
689
- const apiKey = config?.apiKey || process.env.VOYAGE_API_KEY;
690
- if (!apiKey) {
691
- throw new Error("Voyage API key is required. Set VOYAGE_API_KEY environment variable or pass apiKey in config.");
830
+ async updateNodeEmbedding(label, name, embedding) {
831
+ try {
832
+ const conn = await this.ensureConnected();
833
+ const vectorStr = `[${embedding.join(", ")}]`;
834
+ await conn.run(`
835
+ UPDATE nodes
836
+ SET embedding = ${vectorStr}::FLOAT[${this.embeddingDimensions}]
837
+ WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
838
+ `);
839
+ } catch (error) {
840
+ this.logger.error(`Failed to update node embedding: ${error instanceof Error ? error.message : String(error)}`);
841
+ throw error;
692
842
  }
693
- this.apiKey = apiKey;
694
- this.model = config?.model || "voyage-3.5-lite";
695
- this.dimensions = config?.dimensions || 512;
696
- this.inputType = config?.inputType || "document";
697
843
  }
698
- async generateEmbedding(text) {
699
- const embeddings = await this.generateEmbeddings([text]);
700
- return embeddings[0];
844
+ async vectorSearch(label, queryVector, k = 10) {
845
+ try {
846
+ const conn = await this.ensureConnected();
847
+ const vectorStr = `[${queryVector.join(", ")}]`;
848
+ const reader = await conn.runAndReadAll(`
849
+ SELECT
850
+ name,
851
+ properties->>'title' as title,
852
+ array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
853
+ FROM nodes
854
+ WHERE label = '${this.escape(label)}'
855
+ AND embedding IS NOT NULL
856
+ ORDER BY similarity DESC
857
+ LIMIT ${k}
858
+ `);
859
+ return reader.getRows().map((row) => {
860
+ const [name, title, similarity] = row;
861
+ return {
862
+ name,
863
+ title: title || undefined,
864
+ score: similarity
865
+ };
866
+ });
867
+ } catch (error) {
868
+ this.logger.error(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
869
+ throw error;
870
+ }
701
871
  }
702
- async generateQueryEmbedding(text) {
703
- const embeddings = await this.generateEmbeddingsWithType([text], "query");
704
- return embeddings[0];
872
+ async vectorSearchAll(queryVector, k = 10) {
873
+ const allResults = [];
874
+ const conn = await this.ensureConnected();
875
+ const vectorStr = `[${queryVector.join(", ")}]`;
876
+ try {
877
+ const reader = await conn.runAndReadAll(`
878
+ SELECT
879
+ name,
880
+ label,
881
+ properties->>'title' as title,
882
+ properties->>'description' as description,
883
+ array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
884
+ FROM nodes
885
+ WHERE embedding IS NOT NULL
886
+ ORDER BY similarity DESC
887
+ LIMIT ${k}
888
+ `);
889
+ for (const row of reader.getRows()) {
890
+ const [name, label, title, description, similarity] = row;
891
+ allResults.push({
892
+ name,
893
+ label,
894
+ title: title || undefined,
895
+ description: description || undefined,
896
+ score: similarity
897
+ });
898
+ }
899
+ } catch (error) {
900
+ this.logger.debug(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
901
+ }
902
+ return allResults.sort((a, b) => b.score - a.score).slice(0, k);
705
903
  }
706
- async generateEmbeddings(texts) {
707
- return this.generateEmbeddingsWithType(texts, this.inputType);
904
+ escape(value) {
905
+ return value.replace(/'/g, "''");
708
906
  }
709
- async generateEmbeddingsWithType(texts, inputType) {
710
- if (!texts || texts.length === 0) {
711
- return [];
907
+ async loadAllDocumentHashes() {
908
+ try {
909
+ const conn = await this.ensureConnected();
910
+ const reader = await conn.runAndReadAll(`
911
+ SELECT name, content_hash, embedding_source_hash
912
+ FROM nodes
913
+ WHERE label = 'Document'
914
+ `);
915
+ const hashMap = new Map;
916
+ for (const row of reader.getRows()) {
917
+ const [name, contentHash, embeddingSourceHash] = row;
918
+ hashMap.set(name, { contentHash, embeddingSourceHash });
919
+ }
920
+ return hashMap;
921
+ } catch (error) {
922
+ this.logger.error(`Failed to load document hashes: ${error instanceof Error ? error.message : String(error)}`);
923
+ return new Map;
712
924
  }
925
+ }
926
+ async updateDocumentHashes(path2, contentHash, embeddingSourceHash) {
713
927
  try {
714
- const response = await fetch(`${this.baseUrl}/embeddings`, {
715
- method: "POST",
716
- headers: {
717
- "Content-Type": "application/json",
718
- Authorization: `Bearer ${this.apiKey}`
719
- },
720
- body: JSON.stringify({
721
- model: this.model,
722
- input: texts,
723
- output_dimension: this.dimensions,
724
- input_type: inputType
725
- })
726
- });
727
- if (!response.ok) {
728
- const error = await response.json().catch(() => ({}));
729
- throw new Error(`Voyage API error: ${response.status} ${JSON.stringify(error)}`);
928
+ const conn = await this.ensureConnected();
929
+ if (embeddingSourceHash) {
930
+ await conn.run(`
931
+ UPDATE nodes
932
+ SET content_hash = '${this.escape(contentHash)}',
933
+ embedding_source_hash = '${this.escape(embeddingSourceHash)}',
934
+ updated_at = NOW()
935
+ WHERE label = 'Document' AND name = '${this.escape(path2)}'
936
+ `);
937
+ } else {
938
+ await conn.run(`
939
+ UPDATE nodes
940
+ SET content_hash = '${this.escape(contentHash)}',
941
+ updated_at = NOW()
942
+ WHERE label = 'Document' AND name = '${this.escape(path2)}'
943
+ `);
730
944
  }
731
- const data = VoyageEmbeddingResponseSchema.parse(await response.json());
732
- const sortedData = data.data.sort((a, b) => a.index - b.index);
733
- return sortedData.map((item) => item.embedding);
734
945
  } catch (error) {
735
- if (error instanceof Error) {
736
- throw new Error(`Failed to generate embeddings: ${error.message}`);
946
+ this.logger.error(`Failed to update document hashes: ${error instanceof Error ? error.message : String(error)}`);
947
+ throw error;
948
+ }
949
+ }
950
+ async batchUpdateDocumentHashes(updates) {
951
+ if (updates.length === 0)
952
+ return;
953
+ try {
954
+ const conn = await this.ensureConnected();
955
+ await conn.run("BEGIN TRANSACTION");
956
+ for (const { path: path2, contentHash, embeddingSourceHash } of updates) {
957
+ if (embeddingSourceHash) {
958
+ await conn.run(`
959
+ UPDATE nodes
960
+ SET content_hash = '${this.escape(contentHash)}',
961
+ embedding_source_hash = '${this.escape(embeddingSourceHash)}',
962
+ updated_at = NOW()
963
+ WHERE label = 'Document' AND name = '${this.escape(path2)}'
964
+ `);
965
+ } else {
966
+ await conn.run(`
967
+ UPDATE nodes
968
+ SET content_hash = '${this.escape(contentHash)}',
969
+ updated_at = NOW()
970
+ WHERE label = 'Document' AND name = '${this.escape(path2)}'
971
+ `);
972
+ }
737
973
  }
974
+ await conn.run("COMMIT");
975
+ } catch (error) {
976
+ this.logger.error(`Failed to batch update document hashes: ${error instanceof Error ? error.message : String(error)}`);
738
977
  throw error;
739
978
  }
740
979
  }
980
+ async getTrackedDocumentPaths() {
981
+ try {
982
+ const conn = await this.ensureConnected();
983
+ const reader = await conn.runAndReadAll(`
984
+ SELECT name FROM nodes WHERE label = 'Document'
985
+ `);
986
+ return reader.getRows().map((row) => row[0]);
987
+ } catch (error) {
988
+ this.logger.error(`Failed to get tracked document paths: ${error instanceof Error ? error.message : String(error)}`);
989
+ return [];
990
+ }
991
+ }
741
992
  }
993
+ GraphService = __legacyDecorateClassTS([
994
+ Injectable4(),
995
+ __legacyMetadataTS("design:paramtypes", [
996
+ typeof ConfigService === "undefined" ? Object : ConfigService
997
+ ])
998
+ ], GraphService);
742
999
 
743
- // src/embedding/embedding.service.ts
744
- class EmbeddingService {
745
- configService;
746
- logger = new Logger2(EmbeddingService.name);
747
- provider;
748
- config;
749
- constructor(configService) {
750
- this.configService = configService;
751
- this.config = this.loadConfig();
752
- this.provider = this.createProvider();
753
- this.logger.log(`Initialized embedding service with provider: ${this.provider.name}`);
1000
+ // src/sync/manifest.service.ts
1001
+ import { createHash } from "crypto";
1002
+ import { existsSync as existsSync4 } from "fs";
1003
+ import { readFile as readFile3, writeFile } from "fs/promises";
1004
+ import { Injectable as Injectable5 } from "@nestjs/common";
1005
+
1006
+ // src/schemas/manifest.schemas.ts
1007
+ import { z as z2 } from "zod";
1008
+ var ManifestEntrySchema = z2.object({
1009
+ contentHash: z2.string(),
1010
+ frontmatterHash: z2.string(),
1011
+ lastSynced: z2.string(),
1012
+ entityCount: z2.number().int().nonnegative(),
1013
+ relationshipCount: z2.number().int().nonnegative()
1014
+ });
1015
+ var SyncManifestSchema = z2.object({
1016
+ version: z2.string(),
1017
+ lastSync: z2.string(),
1018
+ documents: z2.record(z2.string(), ManifestEntrySchema)
1019
+ });
1020
+
1021
+ // src/sync/manifest.service.ts
1022
+ class ManifestService {
1023
+ manifestPath;
1024
+ manifest = null;
1025
+ constructor() {
1026
+ this.manifestPath = getManifestPath();
754
1027
  }
755
- loadConfig() {
756
- const providerEnv = this.configService.get("EMBEDDING_PROVIDER");
757
- const provider = providerEnv ?? DEFAULT_EMBEDDING_CONFIG.provider;
758
- let apiKey;
759
- if (provider === "voyage") {
760
- apiKey = this.configService.get("VOYAGE_API_KEY");
761
- } else if (provider === "openai") {
762
- apiKey = this.configService.get("OPENAI_API_KEY");
1028
+ async load() {
1029
+ try {
1030
+ if (existsSync4(this.manifestPath)) {
1031
+ const content = await readFile3(this.manifestPath, "utf-8");
1032
+ this.manifest = SyncManifestSchema.parse(JSON.parse(content));
1033
+ } else {
1034
+ this.manifest = this.createEmptyManifest();
1035
+ }
1036
+ } catch (_error) {
1037
+ this.manifest = this.createEmptyManifest();
763
1038
  }
764
- return EmbeddingConfigSchema.parse({
765
- provider: providerEnv,
766
- apiKey,
767
- model: this.configService.get("EMBEDDING_MODEL"),
768
- dimensions: this.configService.get("EMBEDDING_DIMENSIONS")
769
- });
1039
+ return this.manifest;
770
1040
  }
771
- createProvider() {
772
- switch (this.config.provider) {
773
- case "openai":
774
- if (!this.config.apiKey) {
775
- throw new Error("OPENAI_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
776
- }
777
- return new OpenAIEmbeddingProvider({
778
- apiKey: this.config.apiKey,
779
- model: this.config.model,
780
- dimensions: this.config.dimensions
781
- });
782
- case "mock":
783
- return new MockEmbeddingProvider(this.config.dimensions);
784
- case "voyage":
785
- if (!this.config.apiKey) {
786
- throw new Error("VOYAGE_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
787
- }
788
- return new VoyageEmbeddingProvider({
789
- apiKey: this.config.apiKey,
790
- model: this.config.model,
791
- dimensions: this.config.dimensions
792
- });
793
- case "nomic":
794
- throw new Error(`Provider ${this.config.provider} not yet implemented. Use 'voyage', 'openai', or 'mock'.`);
795
- default:
796
- throw new Error(`Unknown embedding provider: ${this.config.provider}. Use 'voyage', 'openai', or 'mock'.`);
1041
+ async save() {
1042
+ if (!this.manifest) {
1043
+ throw new Error("Manifest not loaded. Call load() first.");
797
1044
  }
1045
+ this.manifest.lastSync = new Date().toISOString();
1046
+ const content = JSON.stringify(this.manifest, null, 2);
1047
+ await writeFile(this.manifestPath, content, "utf-8");
798
1048
  }
799
- getProviderName() {
800
- return this.provider.name;
801
- }
802
- getDimensions() {
803
- return this.provider.dimensions;
1049
+ getContentHash(content) {
1050
+ return createHash("sha256").update(content).digest("hex");
804
1051
  }
805
- async generateEmbedding(text) {
806
- if (!text || text.trim().length === 0) {
807
- throw new Error("Cannot generate embedding for empty text");
1052
+ detectChange(path2, contentHash, frontmatterHash) {
1053
+ if (!this.manifest) {
1054
+ throw new Error("Manifest not loaded. Call load() first.");
808
1055
  }
809
- return this.provider.generateEmbedding(text);
810
- }
811
- async generateQueryEmbedding(text) {
812
- if (!text || text.trim().length === 0) {
813
- throw new Error("Cannot generate embedding for empty text");
1056
+ const existing = this.manifest.documents[path2];
1057
+ if (!existing) {
1058
+ return "new";
814
1059
  }
815
- if (this.provider.generateQueryEmbedding) {
816
- return this.provider.generateQueryEmbedding(text);
1060
+ if (existing.contentHash === contentHash && existing.frontmatterHash === frontmatterHash) {
1061
+ return "unchanged";
817
1062
  }
818
- return this.provider.generateEmbedding(text);
1063
+ return "updated";
819
1064
  }
820
- async generateEmbeddings(texts) {
821
- const validTexts = texts.filter((t) => t && t.trim().length > 0);
822
- if (validTexts.length === 0) {
823
- return [];
1065
+ updateEntry(path2, contentHash, frontmatterHash, entityCount, relationshipCount) {
1066
+ if (!this.manifest) {
1067
+ throw new Error("Manifest not loaded. Call load() first.");
824
1068
  }
825
- return this.provider.generateEmbeddings(validTexts);
1069
+ this.manifest.documents[path2] = {
1070
+ contentHash,
1071
+ frontmatterHash,
1072
+ lastSynced: new Date().toISOString(),
1073
+ entityCount,
1074
+ relationshipCount
1075
+ };
826
1076
  }
827
- isRealProvider() {
828
- return this.provider.name !== "mock";
1077
+ removeEntry(path2) {
1078
+ if (!this.manifest) {
1079
+ throw new Error("Manifest not loaded. Call load() first.");
1080
+ }
1081
+ delete this.manifest.documents[path2];
1082
+ }
1083
+ getTrackedPaths() {
1084
+ if (!this.manifest) {
1085
+ throw new Error("Manifest not loaded. Call load() first.");
1086
+ }
1087
+ return Object.keys(this.manifest.documents);
1088
+ }
1089
+ createEmptyManifest() {
1090
+ return {
1091
+ version: "1.0",
1092
+ lastSync: new Date().toISOString(),
1093
+ documents: {}
1094
+ };
829
1095
  }
830
1096
  }
831
- EmbeddingService = __legacyDecorateClassTS([
1097
+ ManifestService = __legacyDecorateClassTS([
832
1098
  Injectable5(),
833
- __legacyMetadataTS("design:paramtypes", [
834
- typeof ConfigService === "undefined" ? Object : ConfigService
835
- ])
836
- ], EmbeddingService);
1099
+ __legacyMetadataTS("design:paramtypes", [])
1100
+ ], ManifestService);
837
1101
 
838
- // src/graph/graph.service.ts
839
- import { DuckDBInstance } from "@duckdb/node-api";
1102
+ // src/sync/sync.service.ts
1103
+ import { Injectable as Injectable11, Logger as Logger7 } from "@nestjs/common";
1104
+
1105
+ // src/embedding/embedding.service.ts
840
1106
  import { Injectable as Injectable6, Logger as Logger3 } from "@nestjs/common";
841
1107
  import { ConfigService as ConfigService2 } from "@nestjs/config";
842
- class GraphService {
843
- configService;
844
- logger = new Logger3(GraphService.name);
845
- instance = null;
846
- connection = null;
847
- dbPath;
848
- connecting = null;
849
- vectorIndexes = new Set;
850
- embeddingDimensions;
851
- signalHandlersRegistered = false;
852
- constructor(configService) {
853
- this.configService = configService;
854
- ensureLatticeHome();
855
- this.dbPath = getDatabasePath();
856
- this.embeddingDimensions = this.configService.get("EMBEDDING_DIMENSIONS") || 512;
857
- this.registerSignalHandlers();
1108
+
1109
+ // src/schemas/config.schemas.ts
1110
+ import { z as z3 } from "zod";
1111
+ var DuckDBConfigSchema = z3.object({
1112
+ embeddingDimensions: z3.coerce.number().int().positive().default(512)
1113
+ });
1114
+ var EmbeddingConfigSchema = z3.object({
1115
+ provider: z3.enum(["openai", "voyage", "nomic", "mock"]).default("voyage"),
1116
+ apiKey: z3.string().optional(),
1117
+ model: z3.string().min(1).default("voyage-3.5-lite"),
1118
+ dimensions: z3.coerce.number().int().positive().default(512)
1119
+ });
1120
+
1121
+ // src/embedding/embedding.types.ts
1122
+ var DEFAULT_EMBEDDING_CONFIG = {
1123
+ provider: "voyage",
1124
+ model: "voyage-3.5-lite",
1125
+ dimensions: 512
1126
+ };
1127
+
1128
+ // src/embedding/providers/mock.provider.ts
1129
+ class MockEmbeddingProvider {
1130
+ name = "mock";
1131
+ dimensions;
1132
+ constructor(dimensions = 1536) {
1133
+ this.dimensions = dimensions;
858
1134
  }
859
- registerSignalHandlers() {
860
- if (this.signalHandlersRegistered)
861
- return;
862
- this.signalHandlersRegistered = true;
863
- const gracefulShutdown = async (signal) => {
864
- this.logger.log(`Received ${signal}, checkpointing before exit...`);
865
- try {
866
- await this.checkpoint();
867
- await this.disconnect();
868
- } catch (error) {
869
- this.logger.error(`Error during graceful shutdown: ${error instanceof Error ? error.message : String(error)}`);
870
- }
871
- process.exit(0);
872
- };
873
- process.on("SIGINT", () => gracefulShutdown("SIGINT"));
874
- process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
875
- process.on("beforeExit", async () => {
876
- if (this.connection) {
877
- await this.checkpoint();
878
- }
879
- });
1135
+ async generateEmbedding(text) {
1136
+ return this.generateDeterministicEmbedding(text);
880
1137
  }
881
- async onModuleDestroy() {
882
- await this.disconnect();
1138
+ async generateEmbeddings(texts) {
1139
+ return Promise.all(texts.map((text) => this.generateEmbedding(text)));
883
1140
  }
884
- async ensureConnected() {
885
- if (this.connection) {
886
- return this.connection;
1141
+ generateDeterministicEmbedding(text) {
1142
+ const embedding = new Array(this.dimensions);
1143
+ let hash = 0;
1144
+ for (let i = 0;i < text.length; i++) {
1145
+ hash = (hash << 5) - hash + text.charCodeAt(i);
1146
+ hash = hash & hash;
887
1147
  }
888
- if (this.connecting) {
889
- await this.connecting;
890
- if (!this.connection) {
891
- throw new Error("Connection failed to establish");
892
- }
893
- return this.connection;
1148
+ for (let i = 0;i < this.dimensions; i++) {
1149
+ const seed = hash * (i + 1) ^ i * 31;
1150
+ embedding[i] = (seed % 2000 - 1000) / 1000;
894
1151
  }
895
- this.connecting = this.connect();
896
- await this.connecting;
897
- this.connecting = null;
898
- if (!this.connection) {
899
- throw new Error("Connection failed to establish");
1152
+ const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
1153
+ return embedding.map((val) => val / magnitude);
1154
+ }
1155
+ }
1156
+
1157
+ // src/embedding/providers/openai.provider.ts
1158
+ class OpenAIEmbeddingProvider {
1159
+ name = "openai";
1160
+ dimensions;
1161
+ model;
1162
+ apiKey;
1163
+ baseUrl = "https://api.openai.com/v1";
1164
+ constructor(config) {
1165
+ const apiKey = config?.apiKey || process.env.OPENAI_API_KEY;
1166
+ if (!apiKey) {
1167
+ throw new Error("OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass apiKey in config.");
900
1168
  }
901
- return this.connection;
1169
+ this.apiKey = apiKey;
1170
+ this.model = config?.model || "text-embedding-3-small";
1171
+ this.dimensions = config?.dimensions || 1536;
902
1172
  }
903
- async connect() {
1173
+ async generateEmbedding(text) {
1174
+ const embeddings = await this.generateEmbeddings([text]);
1175
+ return embeddings[0];
1176
+ }
1177
+ async generateEmbeddings(texts) {
1178
+ if (!texts || texts.length === 0) {
1179
+ return [];
1180
+ }
904
1181
  try {
905
- this.instance = await DuckDBInstance.create(":memory:", {
906
- allow_unsigned_extensions: "true"
1182
+ const response = await fetch(`${this.baseUrl}/embeddings`, {
1183
+ method: "POST",
1184
+ headers: {
1185
+ "Content-Type": "application/json",
1186
+ Authorization: `Bearer ${this.apiKey}`
1187
+ },
1188
+ body: JSON.stringify({
1189
+ model: this.model,
1190
+ input: texts,
1191
+ dimensions: this.dimensions
1192
+ })
907
1193
  });
908
- this.connection = await this.instance.connect();
909
- await this.connection.run("INSTALL vss; LOAD vss;");
910
- await this.connection.run("SET hnsw_enable_experimental_persistence = true;");
911
- try {
912
- await this.connection.run("SET custom_extension_repository = 'http://duckpgq.s3.eu-north-1.amazonaws.com';");
913
- await this.connection.run("FORCE INSTALL 'duckpgq';");
914
- await this.connection.run("LOAD 'duckpgq';");
915
- this.logger.log("DuckPGQ extension loaded successfully");
916
- } catch (e) {
917
- this.logger.warn(`DuckPGQ extension not available: ${e instanceof Error ? e.message : String(e)}`);
1194
+ if (!response.ok) {
1195
+ const error = await response.json().catch(() => ({}));
1196
+ throw new Error(`OpenAI API error: ${response.status} ${JSON.stringify(error)}`);
918
1197
  }
919
- await this.connection.run(`ATTACH '${this.dbPath}' AS lattice (READ_WRITE);`);
920
- await this.connection.run("USE lattice;");
921
- await this.initializeSchema();
922
- this.logger.log(`Connected to DuckDB (in-memory + ATTACH) at ${this.dbPath}`);
1198
+ const data = await response.json();
1199
+ const sortedData = data.data.sort((a, b) => a.index - b.index);
1200
+ return sortedData.map((item) => item.embedding);
923
1201
  } catch (error) {
924
- this.connection = null;
925
- this.instance = null;
926
- this.logger.error(`Failed to connect to DuckDB: ${error instanceof Error ? error.message : String(error)}`);
1202
+ if (error instanceof Error) {
1203
+ throw new Error(`Failed to generate embeddings: ${error.message}`);
1204
+ }
927
1205
  throw error;
928
1206
  }
929
1207
  }
930
- async disconnect() {
931
- if (this.connection) {
932
- await this.checkpoint();
933
- this.connection.closeSync();
934
- this.connection = null;
935
- this.logger.log("Disconnected from DuckDB");
936
- }
937
- if (this.instance) {
938
- this.instance = null;
1208
+ }
1209
+
1210
+ // src/schemas/embedding.schemas.ts
1211
+ import { z as z4 } from "zod";
1212
+ var VoyageEmbeddingResponseSchema = z4.object({
1213
+ object: z4.string(),
1214
+ data: z4.array(z4.object({
1215
+ object: z4.string(),
1216
+ embedding: z4.array(z4.number()),
1217
+ index: z4.number().int().nonnegative()
1218
+ })),
1219
+ model: z4.string(),
1220
+ usage: z4.object({
1221
+ total_tokens: z4.number().int().nonnegative()
1222
+ })
1223
+ });
1224
+
1225
+ // src/embedding/providers/voyage.provider.ts
1226
+ class VoyageEmbeddingProvider {
1227
+ name = "voyage";
1228
+ dimensions;
1229
+ model;
1230
+ apiKey;
1231
+ inputType;
1232
+ baseUrl = "https://api.voyageai.com/v1";
1233
+ constructor(config) {
1234
+ const apiKey = config?.apiKey || process.env.VOYAGE_API_KEY;
1235
+ if (!apiKey) {
1236
+ throw new Error("Voyage API key is required. Set VOYAGE_API_KEY environment variable or pass apiKey in config.");
939
1237
  }
1238
+ this.apiKey = apiKey;
1239
+ this.model = config?.model || "voyage-3.5-lite";
1240
+ this.dimensions = config?.dimensions || 512;
1241
+ this.inputType = config?.inputType || "document";
940
1242
  }
941
- async checkpoint() {
942
- if (!this.connection) {
943
- return;
1243
+ async generateEmbedding(text) {
1244
+ const embeddings = await this.generateEmbeddings([text]);
1245
+ return embeddings[0];
1246
+ }
1247
+ async generateQueryEmbedding(text) {
1248
+ const embeddings = await this.generateEmbeddingsWithType([text], "query");
1249
+ return embeddings[0];
1250
+ }
1251
+ async generateEmbeddings(texts) {
1252
+ return this.generateEmbeddingsWithType(texts, this.inputType);
1253
+ }
1254
+ async generateEmbeddingsWithType(texts, inputType) {
1255
+ if (!texts || texts.length === 0) {
1256
+ return [];
944
1257
  }
945
1258
  try {
946
- await this.connection.run("CHECKPOINT;");
947
- this.logger.debug("Checkpoint completed");
1259
+ const response = await fetch(`${this.baseUrl}/embeddings`, {
1260
+ method: "POST",
1261
+ headers: {
1262
+ "Content-Type": "application/json",
1263
+ Authorization: `Bearer ${this.apiKey}`
1264
+ },
1265
+ body: JSON.stringify({
1266
+ model: this.model,
1267
+ input: texts,
1268
+ output_dimension: this.dimensions,
1269
+ input_type: inputType
1270
+ })
1271
+ });
1272
+ if (!response.ok) {
1273
+ const error = await response.json().catch(() => ({}));
1274
+ throw new Error(`Voyage API error: ${response.status} ${JSON.stringify(error)}`);
1275
+ }
1276
+ const data = VoyageEmbeddingResponseSchema.parse(await response.json());
1277
+ const sortedData = data.data.sort((a, b) => a.index - b.index);
1278
+ return sortedData.map((item) => item.embedding);
948
1279
  } catch (error) {
949
- this.logger.warn(`Checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
1280
+ if (error instanceof Error) {
1281
+ throw new Error(`Failed to generate embeddings: ${error.message}`);
1282
+ }
1283
+ throw error;
950
1284
  }
951
1285
  }
952
- async initializeSchema() {
953
- if (!this.connection) {
954
- throw new Error("Cannot initialize schema: not connected");
955
- }
956
- const conn = this.connection;
957
- await conn.run(`
958
- CREATE TABLE IF NOT EXISTS nodes (
959
- label VARCHAR NOT NULL,
960
- name VARCHAR NOT NULL,
961
- properties JSON,
962
- embedding FLOAT[${this.embeddingDimensions}],
963
- created_at TIMESTAMP DEFAULT NOW(),
964
- updated_at TIMESTAMP DEFAULT NOW(),
965
- PRIMARY KEY(label, name)
966
- )
967
- `);
968
- await conn.run(`
969
- CREATE TABLE IF NOT EXISTS relationships (
970
- source_label VARCHAR NOT NULL,
971
- source_name VARCHAR NOT NULL,
972
- relation_type VARCHAR NOT NULL,
973
- target_label VARCHAR NOT NULL,
974
- target_name VARCHAR NOT NULL,
975
- properties JSON,
976
- created_at TIMESTAMP DEFAULT NOW(),
977
- PRIMARY KEY(source_label, source_name, relation_type, target_label, target_name)
978
- )
979
- `);
980
- await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label)");
981
- await conn.run("CREATE INDEX IF NOT EXISTS idx_nodes_label_name ON nodes(label, name)");
982
- await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_source ON relationships(source_label, source_name)");
983
- await conn.run("CREATE INDEX IF NOT EXISTS idx_rels_target ON relationships(target_label, target_name)");
1286
+ }
1287
+
1288
+ // src/embedding/embedding.service.ts
1289
+ class EmbeddingService {
1290
+ configService;
1291
+ logger = new Logger3(EmbeddingService.name);
1292
+ provider;
1293
+ config;
1294
+ constructor(configService) {
1295
+ this.configService = configService;
1296
+ this.config = this.loadConfig();
1297
+ this.provider = this.createProvider();
1298
+ this.logger.log(`Initialized embedding service with provider: ${this.provider.name}`);
984
1299
  }
985
- async query(sql, _params) {
986
- try {
987
- const conn = await this.ensureConnected();
988
- const reader = await conn.runAndReadAll(sql);
989
- const rows = reader.getRows();
990
- return {
991
- resultSet: rows,
992
- stats: undefined
993
- };
994
- } catch (error) {
995
- this.logger.error(`Query failed: ${error instanceof Error ? error.message : String(error)}`);
996
- throw error;
1300
+ loadConfig() {
1301
+ const providerEnv = this.configService.get("EMBEDDING_PROVIDER");
1302
+ const provider = providerEnv ?? DEFAULT_EMBEDDING_CONFIG.provider;
1303
+ let apiKey;
1304
+ if (provider === "voyage") {
1305
+ apiKey = this.configService.get("VOYAGE_API_KEY");
1306
+ } else if (provider === "openai") {
1307
+ apiKey = this.configService.get("OPENAI_API_KEY");
997
1308
  }
1309
+ return EmbeddingConfigSchema.parse({
1310
+ provider: providerEnv,
1311
+ apiKey,
1312
+ model: this.configService.get("EMBEDDING_MODEL"),
1313
+ dimensions: this.configService.get("EMBEDDING_DIMENSIONS")
1314
+ });
998
1315
  }
999
- async upsertNode(label, properties) {
1000
- try {
1001
- const { name, ...otherProps } = properties;
1002
- if (!name) {
1003
- throw new Error("Node must have a 'name' property");
1004
- }
1005
- const conn = await this.ensureConnected();
1006
- const propsJson = JSON.stringify(otherProps);
1007
- await conn.run(`
1008
- INSERT INTO nodes (label, name, properties)
1009
- VALUES ('${this.escape(String(label))}', '${this.escape(String(name))}', '${this.escape(propsJson)}')
1010
- ON CONFLICT (label, name) DO UPDATE SET
1011
- properties = EXCLUDED.properties,
1012
- updated_at = NOW()
1013
- `);
1014
- } catch (error) {
1015
- this.logger.error(`Failed to upsert node: ${error instanceof Error ? error.message : String(error)}`);
1016
- throw error;
1316
+ createProvider() {
1317
+ switch (this.config.provider) {
1318
+ case "openai":
1319
+ if (!this.config.apiKey) {
1320
+ throw new Error("OPENAI_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
1321
+ }
1322
+ return new OpenAIEmbeddingProvider({
1323
+ apiKey: this.config.apiKey,
1324
+ model: this.config.model,
1325
+ dimensions: this.config.dimensions
1326
+ });
1327
+ case "mock":
1328
+ return new MockEmbeddingProvider(this.config.dimensions);
1329
+ case "voyage":
1330
+ if (!this.config.apiKey) {
1331
+ throw new Error("VOYAGE_API_KEY environment variable is required for embeddings. " + "Set it in .env or use --no-embeddings to skip embedding generation.");
1332
+ }
1333
+ return new VoyageEmbeddingProvider({
1334
+ apiKey: this.config.apiKey,
1335
+ model: this.config.model,
1336
+ dimensions: this.config.dimensions
1337
+ });
1338
+ case "nomic":
1339
+ throw new Error(`Provider ${this.config.provider} not yet implemented. Use 'voyage', 'openai', or 'mock'.`);
1340
+ default:
1341
+ throw new Error(`Unknown embedding provider: ${this.config.provider}. Use 'voyage', 'openai', or 'mock'.`);
1017
1342
  }
1018
1343
  }
1019
- async upsertRelationship(sourceLabel, sourceName, relation, targetLabel, targetName, properties) {
1020
- try {
1021
- const conn = await this.ensureConnected();
1022
- await conn.run(`
1023
- INSERT INTO nodes (label, name, properties)
1024
- VALUES ('${this.escape(sourceLabel)}', '${this.escape(sourceName)}', '{}')
1025
- ON CONFLICT (label, name) DO NOTHING
1026
- `);
1027
- await conn.run(`
1028
- INSERT INTO nodes (label, name, properties)
1029
- VALUES ('${this.escape(targetLabel)}', '${this.escape(targetName)}', '{}')
1030
- ON CONFLICT (label, name) DO NOTHING
1031
- `);
1032
- const propsJson = properties ? JSON.stringify(properties) : "{}";
1033
- await conn.run(`
1034
- INSERT INTO relationships (source_label, source_name, relation_type, target_label, target_name, properties)
1035
- VALUES (
1036
- '${this.escape(sourceLabel)}',
1037
- '${this.escape(sourceName)}',
1038
- '${this.escape(relation)}',
1039
- '${this.escape(targetLabel)}',
1040
- '${this.escape(targetName)}',
1041
- '${this.escape(propsJson)}'
1042
- )
1043
- ON CONFLICT (source_label, source_name, relation_type, target_label, target_name) DO UPDATE SET
1044
- properties = EXCLUDED.properties
1045
- `);
1046
- } catch (error) {
1047
- this.logger.error(`Failed to upsert relationship: ${error instanceof Error ? error.message : String(error)}`);
1048
- throw error;
1049
- }
1344
+ getProviderName() {
1345
+ return this.provider.name;
1050
1346
  }
1051
- async deleteNode(label, name) {
1052
- try {
1053
- const conn = await this.ensureConnected();
1054
- await conn.run(`
1055
- DELETE FROM relationships
1056
- WHERE (source_label = '${this.escape(label)}' AND source_name = '${this.escape(name)}')
1057
- OR (target_label = '${this.escape(label)}' AND target_name = '${this.escape(name)}')
1058
- `);
1059
- await conn.run(`
1060
- DELETE FROM nodes
1061
- WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
1062
- `);
1063
- } catch (error) {
1064
- this.logger.error(`Failed to delete node: ${error instanceof Error ? error.message : String(error)}`);
1065
- throw error;
1066
- }
1347
+ getDimensions() {
1348
+ return this.provider.dimensions;
1067
1349
  }
1068
- async deleteDocumentRelationships(documentPath) {
1069
- try {
1070
- const conn = await this.ensureConnected();
1071
- await conn.run(`
1072
- DELETE FROM relationships
1073
- WHERE properties->>'documentPath' = '${this.escape(documentPath)}'
1074
- `);
1075
- } catch (error) {
1076
- this.logger.error(`Failed to delete document relationships: ${error instanceof Error ? error.message : String(error)}`);
1077
- throw error;
1350
+ async generateEmbedding(text) {
1351
+ if (!text || text.trim().length === 0) {
1352
+ throw new Error("Cannot generate embedding for empty text");
1078
1353
  }
1354
+ return this.provider.generateEmbedding(text);
1079
1355
  }
1080
- async findNodesByLabel(label, limit) {
1081
- try {
1082
- const conn = await this.ensureConnected();
1083
- const limitClause = limit ? ` LIMIT ${limit}` : "";
1084
- const reader = await conn.runAndReadAll(`
1085
- SELECT name, properties
1086
- FROM nodes
1087
- WHERE label = '${this.escape(label)}'${limitClause}
1088
- `);
1089
- return reader.getRows().map((row) => {
1090
- const [name, properties] = row;
1091
- const props = properties ? JSON.parse(properties) : {};
1092
- return { name, ...props };
1093
- });
1094
- } catch (error) {
1095
- this.logger.error(`Failed to find nodes by label: ${error instanceof Error ? error.message : String(error)}`);
1096
- return [];
1356
+ async generateQueryEmbedding(text) {
1357
+ if (!text || text.trim().length === 0) {
1358
+ throw new Error("Cannot generate embedding for empty text");
1097
1359
  }
1360
+ if (this.provider.generateQueryEmbedding) {
1361
+ return this.provider.generateQueryEmbedding(text);
1362
+ }
1363
+ return this.provider.generateEmbedding(text);
1098
1364
  }
1099
- async findRelationships(nodeName) {
1100
- try {
1101
- const conn = await this.ensureConnected();
1102
- const reader = await conn.runAndReadAll(`
1103
- SELECT relation_type, target_name, source_name
1104
- FROM relationships
1105
- WHERE source_name = '${this.escape(nodeName)}'
1106
- OR target_name = '${this.escape(nodeName)}'
1107
- `);
1108
- return reader.getRows().map((row) => {
1109
- const [relType, targetName, sourceName] = row;
1110
- return [relType, sourceName === nodeName ? targetName : sourceName];
1111
- });
1112
- } catch (error) {
1113
- this.logger.error(`Failed to find relationships: ${error instanceof Error ? error.message : String(error)}`);
1365
+ async generateEmbeddings(texts) {
1366
+ const validTexts = texts.filter((t) => t && t.trim().length > 0);
1367
+ if (validTexts.length === 0) {
1114
1368
  return [];
1115
1369
  }
1370
+ return this.provider.generateEmbeddings(validTexts);
1116
1371
  }
1117
- async createVectorIndex(label, property, dimensions) {
1118
- try {
1119
- const indexKey = `${label}_${property}`;
1120
- if (this.vectorIndexes.has(indexKey)) {
1121
- return;
1122
- }
1123
- const conn = await this.ensureConnected();
1124
- try {
1125
- await conn.run(`
1126
- CREATE INDEX idx_embedding_${this.escape(label)}
1127
- ON nodes USING HNSW (embedding)
1128
- WITH (metric = 'cosine')
1129
- `);
1130
- } catch {
1131
- this.logger.debug(`Vector index on ${label}.${property} already exists`);
1132
- }
1133
- this.vectorIndexes.add(indexKey);
1134
- this.logger.log(`Created vector index on ${label}.${property} with ${dimensions} dimensions`);
1135
- } catch (error) {
1136
- const errorMessage = error instanceof Error ? error.message : String(error);
1137
- if (!errorMessage.includes("already exists")) {
1138
- this.logger.error(`Failed to create vector index: ${errorMessage}`);
1139
- throw error;
1140
- }
1141
- }
1142
- }
1143
- async updateNodeEmbedding(label, name, embedding) {
1144
- try {
1145
- const conn = await this.ensureConnected();
1146
- const vectorStr = `[${embedding.join(", ")}]`;
1147
- await conn.run(`
1148
- UPDATE nodes
1149
- SET embedding = ${vectorStr}::FLOAT[${this.embeddingDimensions}]
1150
- WHERE label = '${this.escape(label)}' AND name = '${this.escape(name)}'
1151
- `);
1152
- } catch (error) {
1153
- this.logger.error(`Failed to update node embedding: ${error instanceof Error ? error.message : String(error)}`);
1154
- throw error;
1155
- }
1156
- }
1157
- async vectorSearch(label, queryVector, k = 10) {
1158
- try {
1159
- const conn = await this.ensureConnected();
1160
- const vectorStr = `[${queryVector.join(", ")}]`;
1161
- const reader = await conn.runAndReadAll(`
1162
- SELECT
1163
- name,
1164
- properties->>'title' as title,
1165
- array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
1166
- FROM nodes
1167
- WHERE label = '${this.escape(label)}'
1168
- AND embedding IS NOT NULL
1169
- ORDER BY similarity DESC
1170
- LIMIT ${k}
1171
- `);
1172
- return reader.getRows().map((row) => {
1173
- const [name, title, similarity] = row;
1174
- return {
1175
- name,
1176
- title: title || undefined,
1177
- score: similarity
1178
- };
1179
- });
1180
- } catch (error) {
1181
- this.logger.error(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
1182
- throw error;
1183
- }
1184
- }
1185
- async vectorSearchAll(queryVector, k = 10) {
1186
- const allResults = [];
1187
- const conn = await this.ensureConnected();
1188
- const vectorStr = `[${queryVector.join(", ")}]`;
1189
- try {
1190
- const reader = await conn.runAndReadAll(`
1191
- SELECT
1192
- name,
1193
- label,
1194
- properties->>'title' as title,
1195
- properties->>'description' as description,
1196
- array_cosine_similarity(embedding, ${vectorStr}::FLOAT[${this.embeddingDimensions}]) as similarity
1197
- FROM nodes
1198
- WHERE embedding IS NOT NULL
1199
- ORDER BY similarity DESC
1200
- LIMIT ${k}
1201
- `);
1202
- for (const row of reader.getRows()) {
1203
- const [name, label, title, description, similarity] = row;
1204
- allResults.push({
1205
- name,
1206
- label,
1207
- title: title || undefined,
1208
- description: description || undefined,
1209
- score: similarity
1210
- });
1211
- }
1212
- } catch (error) {
1213
- this.logger.debug(`Vector search failed: ${error instanceof Error ? error.message : String(error)}`);
1214
- }
1215
- return allResults.sort((a, b) => b.score - a.score).slice(0, k);
1216
- }
1217
- escape(value) {
1218
- return value.replace(/'/g, "''");
1372
+ isRealProvider() {
1373
+ return this.provider.name !== "mock";
1219
1374
  }
1220
1375
  }
1221
- GraphService = __legacyDecorateClassTS([
1376
+ EmbeddingService = __legacyDecorateClassTS([
1222
1377
  Injectable6(),
1223
1378
  __legacyMetadataTS("design:paramtypes", [
1224
1379
  typeof ConfigService2 === "undefined" ? Object : ConfigService2
1225
1380
  ])
1226
- ], GraphService);
1381
+ ], EmbeddingService);
1227
1382
 
1228
- // src/commands/query.command.ts
1229
- class SearchCommand extends CommandRunner3 {
1230
- graphService;
1231
- embeddingService;
1232
- constructor(graphService, embeddingService) {
1233
- super();
1234
- this.graphService = graphService;
1235
- this.embeddingService = embeddingService;
1383
+ // src/pure/embedding-text.ts
1384
+ function composeDocumentEmbeddingText(doc) {
1385
+ const parts = [];
1386
+ if (doc.title) {
1387
+ parts.push(`Title: ${doc.title}`);
1236
1388
  }
1237
- async run(inputs, options) {
1238
- const query = inputs[0];
1239
- const limit = Math.min(parseInt(options.limit || "20", 10), 100);
1240
- try {
1241
- const queryEmbedding = await this.embeddingService.generateQueryEmbedding(query);
1242
- let results;
1243
- if (options.label) {
1244
- const labelResults = await this.graphService.vectorSearch(options.label, queryEmbedding, limit);
1245
- results = labelResults.map((r) => ({
1246
- name: r.name,
1247
- label: options.label,
1248
- title: r.title,
1249
- score: r.score
1250
- }));
1389
+ if (doc.topic) {
1390
+ parts.push(`Topic: ${doc.topic}`);
1391
+ }
1392
+ if (doc.tags && doc.tags.length > 0) {
1393
+ parts.push(`Tags: ${doc.tags.join(", ")}`);
1394
+ }
1395
+ if (doc.entities && doc.entities.length > 0) {
1396
+ const entityNames = doc.entities.map((e) => e.name).join(", ");
1397
+ parts.push(`Entities: ${entityNames}`);
1398
+ }
1399
+ if (doc.summary) {
1400
+ parts.push(doc.summary);
1401
+ } else {
1402
+ parts.push(doc.content.slice(0, 500));
1403
+ }
1404
+ return parts.join(" | ");
1405
+ }
1406
+ function composeEntityEmbeddingText(entity) {
1407
+ const parts = [`${entity.type}: ${entity.name}`];
1408
+ if (entity.description) {
1409
+ parts.push(entity.description);
1410
+ }
1411
+ return parts.join(". ");
1412
+ }
1413
+ function collectUniqueEntities(docs) {
1414
+ const entities = new Map;
1415
+ for (const doc of docs) {
1416
+ for (const entity of doc.entities) {
1417
+ const key = `${entity.type}:${entity.name}`;
1418
+ const existing = entities.get(key);
1419
+ if (!existing) {
1420
+ entities.set(key, {
1421
+ type: entity.type,
1422
+ name: entity.name,
1423
+ description: entity.description,
1424
+ documentPaths: [doc.path]
1425
+ });
1251
1426
  } else {
1252
- results = await this.graphService.vectorSearchAll(queryEmbedding, limit);
1253
- }
1254
- const labelSuffix = options.label ? ` (${options.label})` : "";
1255
- console.log(`
1256
- === Semantic Search Results for "${query}"${labelSuffix} ===
1257
- `);
1258
- if (results.length === 0) {
1259
- console.log(`No results found.
1260
- `);
1261
- if (options.label) {
1262
- console.log(`Tip: Try without --label to search all entity types.
1263
- `);
1427
+ existing.documentPaths.push(doc.path);
1428
+ if (entity.description && (!existing.description || entity.description.length > existing.description.length)) {
1429
+ existing.description = entity.description;
1264
1430
  }
1265
- process.exit(0);
1266
1431
  }
1267
- results.forEach((result, idx) => {
1268
- console.log(`${idx + 1}. [${result.label}] ${result.name}`);
1269
- if (result.title) {
1270
- console.log(` Title: ${result.title}`);
1271
- }
1272
- if (result.description && result.label !== "Document") {
1273
- const desc = result.description.length > 80 ? `${result.description.slice(0, 80)}...` : result.description;
1274
- console.log(` ${desc}`);
1275
- }
1276
- console.log(` Similarity: ${(result.score * 100).toFixed(2)}%`);
1432
+ }
1433
+ }
1434
+ return entities;
1435
+ }
1436
+ // src/pure/validation.ts
1437
+ function validateDocuments(docs) {
1438
+ const errors = [];
1439
+ for (const doc of docs) {
1440
+ if (!doc.title || doc.title.trim() === "") {
1441
+ errors.push({
1442
+ path: doc.path,
1443
+ error: "Missing required field: title"
1277
1444
  });
1278
- console.log();
1279
- process.exit(0);
1280
- } catch (error) {
1281
- const errorMsg = error instanceof Error ? error.message : String(error);
1282
- console.error("Error:", errorMsg);
1283
- if (errorMsg.includes("no embeddings") || errorMsg.includes("vector")) {
1284
- console.log(`
1285
- Note: Semantic search requires embeddings to be generated first.`);
1286
- console.log(`Run 'lattice sync' to generate embeddings for documents.
1287
- `);
1445
+ }
1446
+ if (!doc.summary || doc.summary.trim() === "") {
1447
+ errors.push({
1448
+ path: doc.path,
1449
+ error: "Missing required field: summary"
1450
+ });
1451
+ }
1452
+ if (!doc.created) {
1453
+ errors.push({
1454
+ path: doc.path,
1455
+ error: "Missing required field: created"
1456
+ });
1457
+ }
1458
+ if (!doc.updated) {
1459
+ errors.push({
1460
+ path: doc.path,
1461
+ error: "Missing required field: updated"
1462
+ });
1463
+ }
1464
+ if (!doc.status) {
1465
+ errors.push({
1466
+ path: doc.path,
1467
+ error: "Missing required field: status"
1468
+ });
1469
+ }
1470
+ }
1471
+ const entityIndex = new Map;
1472
+ for (const doc of docs) {
1473
+ for (const entity of doc.entities) {
1474
+ let docSet = entityIndex.get(entity.name);
1475
+ if (!docSet) {
1476
+ docSet = new Set;
1477
+ entityIndex.set(entity.name, docSet);
1288
1478
  }
1289
- process.exit(1);
1479
+ docSet.add(doc.path);
1290
1480
  }
1291
1481
  }
1292
- parseLabel(value) {
1293
- return value;
1482
+ for (const doc of docs) {
1483
+ for (const rel of doc.relationships) {
1484
+ if (rel.source !== doc.path && !entityIndex.has(rel.source)) {
1485
+ errors.push({
1486
+ path: doc.path,
1487
+ error: `Relationship source "${rel.source}" not found in any document`
1488
+ });
1489
+ }
1490
+ const isDocPath = rel.target.endsWith(".md");
1491
+ const isKnownEntity = entityIndex.has(rel.target);
1492
+ const isSelfReference = rel.target === doc.path;
1493
+ if (!isDocPath && !isKnownEntity && !isSelfReference) {
1494
+ errors.push({
1495
+ path: doc.path,
1496
+ error: `Relationship target "${rel.target}" not found as entity`
1497
+ });
1498
+ }
1499
+ }
1294
1500
  }
1295
- parseLimit(value) {
1296
- return value;
1501
+ return errors;
1502
+ }
1503
+ function getChangeReason(changeType) {
1504
+ switch (changeType) {
1505
+ case "new":
1506
+ return "New document";
1507
+ case "updated":
1508
+ return "Content or frontmatter changed";
1509
+ case "deleted":
1510
+ return "File no longer exists";
1511
+ case "unchanged":
1512
+ return "No changes detected";
1297
1513
  }
1298
1514
  }
1299
- __legacyDecorateClassTS([
1300
- Option({
1301
- flags: "-l, --label <label>",
1302
- description: "Filter by entity label (e.g., Technology, Concept, Document)"
1303
- }),
1304
- __legacyMetadataTS("design:type", Function),
1305
- __legacyMetadataTS("design:paramtypes", [
1306
- String
1307
- ]),
1308
- __legacyMetadataTS("design:returntype", String)
1309
- ], SearchCommand.prototype, "parseLabel", null);
1310
- __legacyDecorateClassTS([
1311
- Option({
1312
- flags: "--limit <n>",
1313
- description: "Limit results",
1314
- defaultValue: "20"
1315
- }),
1316
- __legacyMetadataTS("design:type", Function),
1317
- __legacyMetadataTS("design:paramtypes", [
1318
- String
1319
- ]),
1320
- __legacyMetadataTS("design:returntype", String)
1321
- ], SearchCommand.prototype, "parseLimit", null);
1322
- SearchCommand = __legacyDecorateClassTS([
1323
- Injectable7(),
1324
- Command3({
1325
- name: "search",
1326
- arguments: "<query>",
1327
- description: "Semantic search across the knowledge graph"
1328
- }),
1329
- __legacyMetadataTS("design:paramtypes", [
1330
- typeof GraphService === "undefined" ? Object : GraphService,
1331
- typeof EmbeddingService === "undefined" ? Object : EmbeddingService
1332
- ])
1333
- ], SearchCommand);
1515
+ // src/sync/cascade.service.ts
1516
+ import { Injectable as Injectable8, Logger as Logger5 } from "@nestjs/common";
1334
1517
 
1335
- class RelsCommand extends CommandRunner3 {
1336
- graphService;
1337
- constructor(graphService) {
1338
- super();
1339
- this.graphService = graphService;
1518
+ // src/sync/document-parser.service.ts
1519
+ import { createHash as createHash2 } from "crypto";
1520
+ import { readFile as readFile4 } from "fs/promises";
1521
+ import { Injectable as Injectable7, Logger as Logger4 } from "@nestjs/common";
1522
+ import { glob } from "glob";
1523
+
1524
+ // src/utils/frontmatter.ts
1525
+ import matter from "gray-matter";
1526
+ import { z as z5 } from "zod";
1527
+ var EntityTypeSchema = z5.enum([
1528
+ "Topic",
1529
+ "Technology",
1530
+ "Concept",
1531
+ "Tool",
1532
+ "Process",
1533
+ "Person",
1534
+ "Organization",
1535
+ "Document"
1536
+ ]);
1537
+ var RelationTypeSchema = z5.enum(["REFERENCES"]);
1538
+ var EntitySchema = z5.object({
1539
+ name: z5.string().min(1),
1540
+ type: EntityTypeSchema,
1541
+ description: z5.string().min(1)
1542
+ });
1543
+ var RelationshipSchema = z5.object({
1544
+ source: z5.string().min(1),
1545
+ relation: RelationTypeSchema,
1546
+ target: z5.string().min(1)
1547
+ });
1548
+ var GraphMetadataSchema = z5.object({
1549
+ importance: z5.enum(["high", "medium", "low"]).optional(),
1550
+ domain: z5.string().optional()
1551
+ });
1552
+ var validateDateFormat = (dateStr) => {
1553
+ const match = dateStr.match(/^(\d{4})-(\d{2})-(\d{2})$/);
1554
+ if (!match)
1555
+ return false;
1556
+ const [, yearStr, monthStr, dayStr] = match;
1557
+ const year = parseInt(yearStr, 10);
1558
+ const month = parseInt(monthStr, 10);
1559
+ const day = parseInt(dayStr, 10);
1560
+ if (month < 1 || month > 12)
1561
+ return false;
1562
+ if (day < 1 || day > 31)
1563
+ return false;
1564
+ const daysInMonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
1565
+ if (year % 4 === 0 && year % 100 !== 0 || year % 400 === 0) {
1566
+ daysInMonth[1] = 29;
1340
1567
  }
1341
- async run(inputs) {
1342
- const name = inputs[0];
1343
- try {
1344
- const relationships = await this.graphService.findRelationships(name);
1345
- console.log(`
1346
- === Relationships for "${name}" ===
1347
- `);
1348
- if (relationships.length === 0) {
1349
- console.log(`No relationships found.
1350
- `);
1351
- process.exit(0);
1352
- }
1353
- console.log("Relationships:");
1354
- for (const rel of relationships) {
1355
- const [relType, targetName] = rel;
1356
- console.log(` -[${relType}]-> ${targetName}`);
1357
- }
1358
- console.log();
1359
- process.exit(0);
1360
- } catch (error) {
1361
- console.error("Error:", error instanceof Error ? error.message : String(error));
1362
- process.exit(1);
1568
+ return day <= daysInMonth[month - 1];
1569
+ };
1570
+ var FrontmatterSchema = z5.object({
1571
+ created: z5.string().refine(validateDateFormat, "Date must be in YYYY-MM-DD format"),
1572
+ updated: z5.string().refine(validateDateFormat, "Date must be in YYYY-MM-DD format"),
1573
+ status: z5.enum(["draft", "ongoing", "complete"]).optional(),
1574
+ topic: z5.string().optional(),
1575
+ tags: z5.array(z5.string()).optional(),
1576
+ summary: z5.string().optional(),
1577
+ entities: z5.array(EntitySchema).optional(),
1578
+ relationships: z5.array(RelationshipSchema).optional(),
1579
+ graph: GraphMetadataSchema.optional()
1580
+ }).passthrough();
1581
+ function parseFrontmatter(content) {
1582
+ try {
1583
+ const { data, content: markdown } = matter(content);
1584
+ if (Object.keys(data).length === 0) {
1585
+ return {
1586
+ frontmatter: null,
1587
+ content: markdown.trim(),
1588
+ raw: content
1589
+ };
1363
1590
  }
1591
+ const normalizedData = normalizeData(data);
1592
+ const validated = FrontmatterSchema.safeParse(normalizedData);
1593
+ return {
1594
+ frontmatter: validated.success ? validated.data : normalizedData,
1595
+ content: markdown.trim(),
1596
+ raw: content
1597
+ };
1598
+ } catch (error) {
1599
+ const errorMessage = error instanceof Error ? error.message : String(error);
1600
+ throw new Error(`YAML parsing error: ${errorMessage}`);
1364
1601
  }
1365
1602
  }
1366
- RelsCommand = __legacyDecorateClassTS([
1367
- Injectable7(),
1368
- Command3({
1369
- name: "rels",
1370
- arguments: "<name>",
1371
- description: "Show relationships for a node"
1372
- }),
1373
- __legacyMetadataTS("design:paramtypes", [
1374
- typeof GraphService === "undefined" ? Object : GraphService
1375
- ])
1376
- ], RelsCommand);
1377
-
1378
- class SqlCommand extends CommandRunner3 {
1379
- graphService;
1380
- constructor(graphService) {
1381
- super();
1382
- this.graphService = graphService;
1603
+ function normalizeData(data) {
1604
+ if (data instanceof Date) {
1605
+ return data.toISOString().split("T")[0];
1383
1606
  }
1384
- async run(inputs) {
1385
- const query = inputs[0];
1386
- try {
1387
- const result = await this.graphService.query(query);
1388
- console.log(`
1389
- === SQL Query Results ===
1390
- `);
1391
- const replacer = (_key, value) => typeof value === "bigint" ? Number(value) : value;
1392
- console.log(JSON.stringify(result, replacer, 2));
1393
- console.log();
1394
- process.exit(0);
1395
- } catch (error) {
1396
- console.error("Error:", error instanceof Error ? error.message : String(error));
1397
- process.exit(1);
1607
+ if (Array.isArray(data)) {
1608
+ return data.map(normalizeData);
1609
+ }
1610
+ if (data !== null && typeof data === "object") {
1611
+ const normalized = {};
1612
+ for (const [key, value] of Object.entries(data)) {
1613
+ normalized[key] = normalizeData(value);
1398
1614
  }
1615
+ return normalized;
1399
1616
  }
1617
+ return data;
1400
1618
  }
1401
- SqlCommand = __legacyDecorateClassTS([
1402
- Injectable7(),
1403
- Command3({
1404
- name: "sql",
1405
- arguments: "<query>",
1406
- description: "Execute raw SQL query against DuckDB"
1407
- }),
1408
- __legacyMetadataTS("design:paramtypes", [
1409
- typeof GraphService === "undefined" ? Object : GraphService
1410
- ])
1411
- ], SqlCommand);
1412
- // src/commands/status.command.ts
1413
- import { Injectable as Injectable12 } from "@nestjs/common";
1414
- import { Command as Command4, CommandRunner as CommandRunner4, Option as Option2 } from "nest-commander";
1415
-
1416
- // src/sync/manifest.service.ts
1417
- import { createHash as createHash2 } from "crypto";
1418
- import { existsSync as existsSync3 } from "fs";
1419
- import { readFile as readFile3, writeFile } from "fs/promises";
1420
- import { Injectable as Injectable8 } from "@nestjs/common";
1421
1619
 
1422
- // src/schemas/manifest.schemas.ts
1423
- import { z as z4 } from "zod";
1424
- var ManifestEntrySchema = z4.object({
1425
- contentHash: z4.string(),
1426
- frontmatterHash: z4.string(),
1427
- lastSynced: z4.string(),
1428
- entityCount: z4.number().int().nonnegative(),
1429
- relationshipCount: z4.number().int().nonnegative()
1430
- });
1431
- var SyncManifestSchema = z4.object({
1432
- version: z4.string(),
1433
- lastSync: z4.string(),
1434
- documents: z4.record(z4.string(), ManifestEntrySchema)
1435
- });
1436
-
1437
- // src/sync/manifest.service.ts
1438
- class ManifestService {
1439
- manifestPath;
1440
- manifest = null;
1620
+ // src/sync/document-parser.service.ts
1621
+ class DocumentParserService {
1622
+ logger = new Logger4(DocumentParserService.name);
1623
+ docsPath;
1441
1624
  constructor() {
1442
- this.manifestPath = getManifestPath();
1443
- }
1444
- async load() {
1445
- try {
1446
- if (existsSync3(this.manifestPath)) {
1447
- const content = await readFile3(this.manifestPath, "utf-8");
1448
- this.manifest = SyncManifestSchema.parse(JSON.parse(content));
1449
- } else {
1450
- this.manifest = this.createEmptyManifest();
1451
- }
1452
- } catch (_error) {
1453
- this.manifest = this.createEmptyManifest();
1454
- }
1455
- return this.manifest;
1456
- }
1457
- async save() {
1458
- if (!this.manifest) {
1459
- throw new Error("Manifest not loaded. Call load() first.");
1460
- }
1461
- this.manifest.lastSync = new Date().toISOString();
1462
- const content = JSON.stringify(this.manifest, null, 2);
1463
- await writeFile(this.manifestPath, content, "utf-8");
1625
+ ensureLatticeHome();
1626
+ this.docsPath = getDocsPath();
1464
1627
  }
1465
- getContentHash(content) {
1466
- return createHash2("sha256").update(content).digest("hex");
1628
+ getDocsPath() {
1629
+ return this.docsPath;
1467
1630
  }
1468
- detectChange(path2, contentHash, frontmatterHash) {
1469
- if (!this.manifest) {
1470
- throw new Error("Manifest not loaded. Call load() first.");
1471
- }
1472
- const existing = this.manifest.documents[path2];
1473
- if (!existing) {
1474
- return "new";
1475
- }
1476
- if (existing.contentHash === contentHash && existing.frontmatterHash === frontmatterHash) {
1477
- return "unchanged";
1478
- }
1479
- return "updated";
1631
+ async discoverDocuments() {
1632
+ const pattern = `${this.docsPath}/**/*.md`;
1633
+ const files = await glob(pattern, {
1634
+ ignore: ["**/node_modules/**", "**/.git/**"]
1635
+ });
1636
+ return files.sort();
1480
1637
  }
1481
- updateEntry(path2, contentHash, frontmatterHash, entityCount, relationshipCount) {
1482
- if (!this.manifest) {
1483
- throw new Error("Manifest not loaded. Call load() first.");
1484
- }
1485
- this.manifest.documents[path2] = {
1638
+ async parseDocument(filePath) {
1639
+ const content = await readFile4(filePath, "utf-8");
1640
+ const parsed = parseFrontmatter(content);
1641
+ const title = this.extractTitle(content, filePath);
1642
+ const contentHash = this.computeHash(content);
1643
+ const frontmatterHash = this.computeHash(JSON.stringify(parsed.frontmatter || {}));
1644
+ const graphMetadata = this.extractGraphMetadata(parsed.frontmatter);
1645
+ return {
1646
+ path: filePath,
1647
+ title,
1648
+ content: parsed.content,
1486
1649
  contentHash,
1487
1650
  frontmatterHash,
1488
- lastSynced: new Date().toISOString(),
1489
- entityCount,
1490
- relationshipCount
1491
- };
1492
- }
1493
- removeEntry(path2) {
1494
- if (!this.manifest) {
1495
- throw new Error("Manifest not loaded. Call load() first.");
1496
- }
1497
- delete this.manifest.documents[path2];
1498
- }
1499
- getTrackedPaths() {
1500
- if (!this.manifest) {
1501
- throw new Error("Manifest not loaded. Call load() first.");
1502
- }
1503
- return Object.keys(this.manifest.documents);
1504
- }
1505
- createEmptyManifest() {
1506
- return {
1507
- version: "1.0",
1508
- lastSync: new Date().toISOString(),
1509
- documents: {}
1651
+ summary: parsed.frontmatter?.summary,
1652
+ topic: parsed.frontmatter?.topic,
1653
+ entities: [],
1654
+ relationships: [],
1655
+ graphMetadata,
1656
+ tags: parsed.frontmatter?.tags || [],
1657
+ created: parsed.frontmatter?.created,
1658
+ updated: parsed.frontmatter?.updated,
1659
+ status: parsed.frontmatter?.status
1510
1660
  };
1511
1661
  }
1512
- }
1513
- ManifestService = __legacyDecorateClassTS([
1514
- Injectable8(),
1515
- __legacyMetadataTS("design:paramtypes", [])
1516
- ], ManifestService);
1517
-
1518
- // src/sync/sync.service.ts
1519
- import { Injectable as Injectable11, Logger as Logger5 } from "@nestjs/common";
1520
-
1521
- // src/pure/embedding-text.ts
1522
- function composeDocumentEmbeddingText(doc) {
1523
- const parts = [];
1524
- if (doc.title) {
1525
- parts.push(`Title: ${doc.title}`);
1526
- }
1527
- if (doc.topic) {
1528
- parts.push(`Topic: ${doc.topic}`);
1529
- }
1530
- if (doc.tags && doc.tags.length > 0) {
1531
- parts.push(`Tags: ${doc.tags.join(", ")}`);
1532
- }
1533
- if (doc.entities && doc.entities.length > 0) {
1534
- const entityNames = doc.entities.map((e) => e.name).join(", ");
1535
- parts.push(`Entities: ${entityNames}`);
1536
- }
1537
- if (doc.summary) {
1538
- parts.push(doc.summary);
1539
- } else {
1540
- parts.push(doc.content.slice(0, 500));
1541
- }
1542
- return parts.join(" | ");
1543
- }
1544
- function composeEntityEmbeddingText(entity) {
1545
- const parts = [`${entity.type}: ${entity.name}`];
1546
- if (entity.description) {
1547
- parts.push(entity.description);
1662
+ async parseAllDocuments() {
1663
+ const { docs } = await this.parseAllDocumentsWithErrors();
1664
+ return docs;
1548
1665
  }
1549
- return parts.join(". ");
1550
- }
1551
- function collectUniqueEntities(docs) {
1552
- const entities = new Map;
1553
- for (const doc of docs) {
1554
- for (const entity of doc.entities) {
1555
- const key = `${entity.type}:${entity.name}`;
1556
- const existing = entities.get(key);
1557
- if (!existing) {
1558
- entities.set(key, {
1559
- type: entity.type,
1560
- name: entity.name,
1561
- description: entity.description,
1562
- documentPaths: [doc.path]
1563
- });
1564
- } else {
1565
- existing.documentPaths.push(doc.path);
1566
- if (entity.description && (!existing.description || entity.description.length > existing.description.length)) {
1567
- existing.description = entity.description;
1568
- }
1666
+ async parseAllDocumentsWithErrors() {
1667
+ const files = await this.discoverDocuments();
1668
+ const docs = [];
1669
+ const errors = [];
1670
+ for (const file of files) {
1671
+ try {
1672
+ const parsed = await this.parseDocument(file);
1673
+ docs.push(parsed);
1674
+ } catch (error) {
1675
+ const errorMsg = error instanceof Error ? error.message : String(error);
1676
+ errors.push({ path: file, error: errorMsg });
1677
+ this.logger.warn(`Failed to parse ${file}: ${error}`);
1569
1678
  }
1570
1679
  }
1680
+ return { docs, errors };
1571
1681
  }
1572
- return entities;
1573
- }
1574
- // src/pure/validation.ts
1575
- function validateDocuments(docs) {
1576
- const errors = [];
1577
- for (const doc of docs) {
1578
- if (!doc.title || doc.title.trim() === "") {
1579
- errors.push({
1580
- path: doc.path,
1581
- error: "Missing required field: title"
1582
- });
1583
- }
1584
- if (!doc.summary || doc.summary.trim() === "") {
1585
- errors.push({
1586
- path: doc.path,
1587
- error: "Missing required field: summary"
1588
- });
1589
- }
1590
- if (!doc.created) {
1591
- errors.push({
1592
- path: doc.path,
1593
- error: "Missing required field: created"
1594
- });
1595
- }
1596
- if (!doc.updated) {
1597
- errors.push({
1598
- path: doc.path,
1599
- error: "Missing required field: updated"
1600
- });
1601
- }
1602
- if (!doc.status) {
1603
- errors.push({
1604
- path: doc.path,
1605
- error: "Missing required field: status"
1606
- });
1607
- }
1608
- }
1609
- const entityIndex = new Map;
1610
- for (const doc of docs) {
1611
- for (const entity of doc.entities) {
1612
- let docSet = entityIndex.get(entity.name);
1613
- if (!docSet) {
1614
- docSet = new Set;
1615
- entityIndex.set(entity.name, docSet);
1616
- }
1617
- docSet.add(doc.path);
1682
+ extractTitle(content, filePath) {
1683
+ const h1Match = content.match(/^#\s+(.+)$/m);
1684
+ if (h1Match) {
1685
+ return h1Match[1];
1618
1686
  }
1687
+ const parts = filePath.split("/");
1688
+ return parts[parts.length - 1].replace(".md", "");
1619
1689
  }
1620
- for (const doc of docs) {
1621
- for (const rel of doc.relationships) {
1622
- if (rel.source !== doc.path && !entityIndex.has(rel.source)) {
1623
- errors.push({
1624
- path: doc.path,
1625
- error: `Relationship source "${rel.source}" not found in any document`
1626
- });
1627
- }
1628
- const isDocPath = rel.target.endsWith(".md");
1629
- const isKnownEntity = entityIndex.has(rel.target);
1630
- const isSelfReference = rel.target === doc.path;
1631
- if (!isDocPath && !isKnownEntity && !isSelfReference) {
1632
- errors.push({
1633
- path: doc.path,
1634
- error: `Relationship target "${rel.target}" not found as entity`
1635
- });
1636
- }
1690
+ extractGraphMetadata(frontmatter) {
1691
+ const fm = frontmatter;
1692
+ if (!fm?.graph) {
1693
+ return;
1637
1694
  }
1695
+ const result = GraphMetadataSchema.safeParse(fm.graph);
1696
+ return result.success ? result.data : undefined;
1638
1697
  }
1639
- return errors;
1640
- }
1641
- function getChangeReason(changeType) {
1642
- switch (changeType) {
1643
- case "new":
1644
- return "New document";
1645
- case "updated":
1646
- return "Content or frontmatter changed";
1647
- case "deleted":
1648
- return "File no longer exists";
1649
- case "unchanged":
1650
- return "No changes detected";
1698
+ computeHash(content) {
1699
+ return createHash2("sha256").update(content).digest("hex");
1651
1700
  }
1652
1701
  }
1702
+ DocumentParserService = __legacyDecorateClassTS([
1703
+ Injectable7(),
1704
+ __legacyMetadataTS("design:paramtypes", [])
1705
+ ], DocumentParserService);
1706
+
1653
1707
  // src/sync/cascade.service.ts
1654
- import { Injectable as Injectable9, Logger as Logger4 } from "@nestjs/common";
1655
1708
  class CascadeService {
1656
1709
  graph;
1657
1710
  _parser;
1658
- logger = new Logger4(CascadeService.name);
1711
+ logger = new Logger5(CascadeService.name);
1659
1712
  constructor(graph, _parser) {
1660
1713
  this.graph = graph;
1661
1714
  this._parser = _parser;
@@ -1764,14 +1817,14 @@ class CascadeService {
1764
1817
  async findAffectedByRename(entityName, _newName) {
1765
1818
  try {
1766
1819
  const escapedName = this.escapeForSql(entityName);
1767
- const query = `
1820
+ const query3 = `
1768
1821
  SELECT DISTINCT n.name, n.properties->>'title' as title
1769
1822
  FROM nodes n
1770
1823
  INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
1771
1824
  WHERE r.source_name = '${escapedName}'
1772
1825
  AND n.label = 'Document'
1773
1826
  `.trim();
1774
- const result = await this.graph.query(query);
1827
+ const result = await this.graph.query(query3);
1775
1828
  return (result.resultSet || []).map((row) => ({
1776
1829
  path: row[0],
1777
1830
  reason: `References "${entityName}" in entities`,
@@ -1787,14 +1840,14 @@ class CascadeService {
1787
1840
  async findAffectedByDeletion(entityName) {
1788
1841
  try {
1789
1842
  const escapedName = this.escapeForSql(entityName);
1790
- const query = `
1843
+ const query3 = `
1791
1844
  SELECT DISTINCT n.name, n.properties->>'title' as title
1792
1845
  FROM nodes n
1793
1846
  INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
1794
1847
  WHERE r.source_name = '${escapedName}'
1795
1848
  AND n.label = 'Document'
1796
1849
  `.trim();
1797
- const result = await this.graph.query(query);
1850
+ const result = await this.graph.query(query3);
1798
1851
  return (result.resultSet || []).map((row) => ({
1799
1852
  path: row[0],
1800
1853
  reason: `References deleted entity "${entityName}"`,
@@ -1810,14 +1863,14 @@ class CascadeService {
1810
1863
  async findAffectedByTypeChange(entityName, oldType, newType) {
1811
1864
  try {
1812
1865
  const escapedName = this.escapeForSql(entityName);
1813
- const query = `
1866
+ const query3 = `
1814
1867
  SELECT DISTINCT n.name, n.properties->>'title' as title
1815
1868
  FROM nodes n
1816
1869
  INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
1817
1870
  WHERE r.source_name = '${escapedName}'
1818
1871
  AND n.label = 'Document'
1819
1872
  `.trim();
1820
- const result = await this.graph.query(query);
1873
+ const result = await this.graph.query(query3);
1821
1874
  return (result.resultSet || []).map((row) => ({
1822
1875
  path: row[0],
1823
1876
  reason: `References "${entityName}" with type "${oldType}" (now "${newType}")`,
@@ -1833,14 +1886,14 @@ class CascadeService {
1833
1886
  async findAffectedByRelationshipChange(entityName) {
1834
1887
  try {
1835
1888
  const escapedName = this.escapeForSql(entityName);
1836
- const query = `
1889
+ const query3 = `
1837
1890
  SELECT DISTINCT n.name, n.properties->>'title' as title, r.relation_type
1838
1891
  FROM nodes n
1839
1892
  INNER JOIN relationships r ON r.target_label = n.label AND r.target_name = n.name
1840
1893
  WHERE r.source_name = '${escapedName}'
1841
1894
  AND n.label = 'Document'
1842
1895
  `.trim();
1843
- const result = await this.graph.query(query);
1896
+ const result = await this.graph.query(query3);
1844
1897
  return (result.resultSet || []).map((row) => ({
1845
1898
  path: row[0],
1846
1899
  reason: `Has relationship with "${entityName}"`,
@@ -1856,14 +1909,14 @@ class CascadeService {
1856
1909
  async findAffectedByDocumentDeletion(documentPath) {
1857
1910
  try {
1858
1911
  const escapedPath = this.escapeForSql(documentPath);
1859
- const query = `
1912
+ const query3 = `
1860
1913
  SELECT DISTINCT n.name, r.relation_type
1861
1914
  FROM nodes n
1862
1915
  INNER JOIN relationships r ON r.source_label = n.label AND r.source_name = n.name
1863
1916
  WHERE r.target_name = '${escapedPath}'
1864
1917
  AND n.label = 'Document'
1865
1918
  `.trim();
1866
- const result = await this.graph.query(query);
1919
+ const result = await this.graph.query(query3);
1867
1920
  return (result.resultSet || []).map((row) => ({
1868
1921
  path: row[0],
1869
1922
  reason: `Links to deleted document "${documentPath}"`,
@@ -1953,16 +2006,118 @@ class CascadeService {
1953
2006
  }
1954
2007
  }
1955
2008
  CascadeService = __legacyDecorateClassTS([
1956
- Injectable9(),
2009
+ Injectable8(),
1957
2010
  __legacyMetadataTS("design:paramtypes", [
1958
2011
  typeof GraphService === "undefined" ? Object : GraphService,
1959
2012
  typeof DocumentParserService === "undefined" ? Object : DocumentParserService
1960
2013
  ])
1961
2014
  ], CascadeService);
1962
2015
 
2016
+ // src/sync/database-change-detector.service.ts
2017
+ import { createHash as createHash3 } from "crypto";
2018
+ import { Injectable as Injectable9, Logger as Logger6 } from "@nestjs/common";
2019
+ class DatabaseChangeDetectorService {
2020
+ graph;
2021
+ logger = new Logger6(DatabaseChangeDetectorService.name);
2022
+ hashCache = new Map;
2023
+ loaded = false;
2024
+ constructor(graph) {
2025
+ this.graph = graph;
2026
+ }
2027
+ async loadHashes() {
2028
+ this.hashCache = await this.graph.loadAllDocumentHashes();
2029
+ this.loaded = true;
2030
+ this.logger.debug(`Loaded ${this.hashCache.size} document hashes from DB`);
2031
+ }
2032
+ isLoaded() {
2033
+ return this.loaded;
2034
+ }
2035
+ reset() {
2036
+ this.hashCache.clear();
2037
+ this.loaded = false;
2038
+ }
2039
+ getContentHash(content) {
2040
+ return createHash3("sha256").update(content).digest("hex");
2041
+ }
2042
+ detectChange(path2, currentContentHash) {
2043
+ if (!this.loaded) {
2044
+ throw new Error("Hashes not loaded. Call loadHashes() before detectChange().");
2045
+ }
2046
+ const cached = this.hashCache.get(path2);
2047
+ if (!cached) {
2048
+ return "new";
2049
+ }
2050
+ if (!cached.contentHash) {
2051
+ return "updated";
2052
+ }
2053
+ return cached.contentHash === currentContentHash ? "unchanged" : "updated";
2054
+ }
2055
+ detectChangeWithReason(path2, currentContentHash) {
2056
+ const changeType = this.detectChange(path2, currentContentHash);
2057
+ let reason;
2058
+ switch (changeType) {
2059
+ case "new":
2060
+ reason = "New document not in database";
2061
+ break;
2062
+ case "updated":
2063
+ reason = "Content hash changed";
2064
+ break;
2065
+ case "unchanged":
2066
+ reason = "Content unchanged";
2067
+ break;
2068
+ default:
2069
+ reason = "Unknown";
2070
+ }
2071
+ return { path: path2, changeType, reason };
2072
+ }
2073
+ getTrackedPaths() {
2074
+ if (!this.loaded) {
2075
+ throw new Error("Hashes not loaded. Call loadHashes() before getTrackedPaths().");
2076
+ }
2077
+ return Array.from(this.hashCache.keys());
2078
+ }
2079
+ isEmbeddingStale(path2, currentSourceHash) {
2080
+ if (!this.loaded) {
2081
+ throw new Error("Hashes not loaded. Call loadHashes() before isEmbeddingStale().");
2082
+ }
2083
+ const cached = this.hashCache.get(path2);
2084
+ if (!cached?.embeddingSourceHash) {
2085
+ return true;
2086
+ }
2087
+ return cached.embeddingSourceHash !== currentSourceHash;
2088
+ }
2089
+ getCachedEntry(path2) {
2090
+ if (!this.loaded) {
2091
+ throw new Error("Hashes not loaded. Call loadHashes() before getCachedEntry().");
2092
+ }
2093
+ return this.hashCache.get(path2);
2094
+ }
2095
+ getCacheSize() {
2096
+ return this.hashCache.size;
2097
+ }
2098
+ findDocumentsNeedingEmbeddings() {
2099
+ if (!this.loaded) {
2100
+ throw new Error("Hashes not loaded. Call loadHashes() before findDocumentsNeedingEmbeddings().");
2101
+ }
2102
+ const needsEmbedding = [];
2103
+ for (const [path2, entry] of this.hashCache) {
2104
+ if (!entry.embeddingSourceHash) {
2105
+ needsEmbedding.push(path2);
2106
+ }
2107
+ }
2108
+ return needsEmbedding;
2109
+ }
2110
+ }
2111
+ DatabaseChangeDetectorService = __legacyDecorateClassTS([
2112
+ Injectable9(),
2113
+ __legacyMetadataTS("design:paramtypes", [
2114
+ typeof GraphService === "undefined" ? Object : GraphService
2115
+ ])
2116
+ ], DatabaseChangeDetectorService);
2117
+
1963
2118
  // src/sync/path-resolver.service.ts
1964
- import { existsSync as existsSync4 } from "fs";
1965
- import { isAbsolute, resolve as resolve2 } from "path";
2119
+ import { existsSync as existsSync5 } from "fs";
2120
+ import { isAbsolute, resolve as resolve3 } from "path";
1966
2121
  import { Injectable as Injectable10 } from "@nestjs/common";
1967
2122
  class PathResolverService {
1968
2123
  docsPath;
@@ -1978,12 +2133,12 @@ class PathResolverService {
1978
2133
  if (isAbsolute(userPath)) {
1979
2134
  resolvedPath = userPath;
1980
2135
  } else {
1981
- resolvedPath = resolve2(this.docsPath, userPath);
2136
+ resolvedPath = resolve3(this.docsPath, userPath);
1982
2137
  }
1983
2138
  if (requireInDocs && !this.isUnderDocs(resolvedPath)) {
1984
2139
  throw new Error(`Path "${userPath}" resolves to "${resolvedPath}" which is outside the docs directory (${this.docsPath})`);
1985
2140
  }
1986
- if (requireExists && !existsSync4(resolvedPath)) {
2141
+ if (requireExists && !existsSync5(resolvedPath)) {
1987
2142
  throw new Error(`Path "${userPath}" does not exist (resolved to: ${resolvedPath})`);
1988
2143
  }
1989
2144
  return resolvedPath;
@@ -2030,14 +2185,18 @@ class SyncService {
2030
2185
  graph;
2031
2186
  cascade;
2032
2187
  pathResolver;
2188
+ dbChangeDetector;
2189
+ entityExtractor;
2033
2190
  embeddingService;
2034
- logger = new Logger5(SyncService.name);
2035
- constructor(manifest, parser, graph, cascade, pathResolver, embeddingService) {
2191
+ logger = new Logger7(SyncService.name);
2192
+ constructor(manifest, parser, graph, cascade, pathResolver, dbChangeDetector, entityExtractor, embeddingService) {
2036
2193
  this.manifest = manifest;
2037
2194
  this.parser = parser;
2038
2195
  this.graph = graph;
2039
2196
  this.cascade = cascade;
2040
2197
  this.pathResolver = pathResolver;
2198
+ this.dbChangeDetector = dbChangeDetector;
2199
+ this.entityExtractor = entityExtractor;
2041
2200
  this.embeddingService = embeddingService;
2042
2201
  }
2043
2202
  async sync(options = {}) {
@@ -2054,8 +2213,16 @@ class SyncService {
2054
2213
  embeddingsGenerated: 0,
2055
2214
  entityEmbeddingsGenerated: 0
2056
2215
  };
2216
+ const useDbDetection = options.legacy ? false : options.useDbChangeDetection ?? true;
2217
+ const useAiExtraction = options.legacy ? false : options.aiExtraction ?? true;
2057
2218
  try {
2058
2219
  await this.manifest.load();
2220
+ if (useDbDetection) {
2221
+ await this.dbChangeDetector.loadHashes();
2222
+ if (options.verbose) {
2223
+ this.logger.log(`v2 mode: Loaded ${this.dbChangeDetector.getCacheSize()} document hashes from database`);
2224
+ }
2225
+ }
2059
2226
  if (options.force) {
2060
2227
  if (options.paths && options.paths.length > 0) {
2061
2228
  if (options.verbose) {
@@ -2069,7 +2236,7 @@ class SyncService {
2069
2236
  await this.clearManifest();
2070
2237
  }
2071
2238
  }
2072
- const changes = await this.detectChanges(options.paths);
2239
+ const changes = await this.detectChanges(options.paths, useDbDetection);
2073
2240
  result.changes = changes;
2074
2241
  const docsToSync = [];
2075
2242
  const docsByPath = new Map;
@@ -2086,15 +2253,42 @@ class SyncService {
2086
2253
  }
2087
2254
  }
2088
2255
  }
2089
- const validationErrors = validateDocuments2(docsToSync);
2090
- if (validationErrors.length > 0) {
2091
- for (const err of validationErrors) {
2092
- result.errors.push(err);
2093
- this.logger.error(`Validation error in ${err.path}: ${err.error}`);
2256
+ if (useAiExtraction && docsToSync.length > 0) {
2257
+ if (options.verbose) {
2258
+ this.logger.log(`v2 AI extraction: Processing ${docsToSync.length} documents...`);
2259
+ }
2260
+ for (const doc of docsToSync) {
2261
+ try {
2262
+ const extraction = await this.entityExtractor.extractFromDocument(doc.path);
2263
+ if (extraction.success) {
2264
+ doc.entities = extraction.entities;
2265
+ doc.relationships = extraction.relationships;
2266
+ if (extraction.summary) {
2267
+ doc.summary = extraction.summary;
2268
+ }
2269
+ if (options.verbose) {
2270
+ this.logger.log(` Extracted ${extraction.entities.length} entities from ${doc.path}`);
2271
+ }
2272
+ } else {
2273
+ this.logger.warn(`AI extraction failed for ${doc.path}: ${extraction.error}`);
2274
+ }
2275
+ } catch (error) {
2276
+ const errorMsg = error instanceof Error ? error.message : String(error);
2277
+ this.logger.warn(`AI extraction error for ${doc.path}: ${errorMsg}`);
2278
+ }
2279
+ }
2280
+ }
2281
+ if (!useAiExtraction) {
2282
+ const validationErrors = validateDocuments2(docsToSync);
2283
+ if (validationErrors.length > 0) {
2284
+ for (const err of validationErrors) {
2285
+ result.errors.push(err);
2286
+ this.logger.error(`Validation error in ${err.path}: ${err.error}`);
2287
+ }
2288
+ this.logger.error(`Sync aborted: ${validationErrors.length} validation error(s) found. Fix the errors and try again.`);
2289
+ result.duration = Date.now() - startTime;
2290
+ return result;
2094
2291
  }
2095
- this.logger.error(`Sync aborted: ${validationErrors.length} validation error(s) found. Fix the errors and try again.`);
2096
- result.duration = Date.now() - startTime;
2097
- return result;
2098
2292
  }
2099
2293
  const uniqueEntities = collectUniqueEntities(docsToSync);
2100
2294
  if (options.verbose) {
@@ -2119,43 +2313,37 @@ class SyncService {
2119
2313
  const CHECKPOINT_BATCH_SIZE = 10;
2120
2314
  let processedCount = 0;
2121
2315
  for (const change of changes) {
2122
- try {
2123
- const doc = docsByPath.get(change.path);
2124
- const cascadeWarnings = await this.processChange(change, options, doc);
2125
- result.cascadeWarnings.push(...cascadeWarnings);
2126
- switch (change.changeType) {
2127
- case "new":
2128
- result.added++;
2129
- break;
2130
- case "updated":
2131
- result.updated++;
2132
- break;
2133
- case "deleted":
2134
- result.deleted++;
2135
- break;
2136
- case "unchanged":
2137
- result.unchanged++;
2138
- break;
2139
- }
2140
- if (change.embeddingGenerated) {
2141
- result.embeddingsGenerated++;
2142
- }
2143
- processedCount++;
2144
- if (!options.dryRun && processedCount % CHECKPOINT_BATCH_SIZE === 0) {
2145
- await this.graph.checkpoint();
2146
- }
2147
- } catch (error) {
2148
- const errorMessage = error instanceof Error ? error.message : String(error);
2149
- result.errors.push({ path: change.path, error: errorMessage });
2150
- this.logger.warn(`Error processing ${change.path}: ${errorMessage}`);
2316
+ const doc = docsByPath.get(change.path);
2317
+ const cascadeWarnings = await this.processChange(change, options, doc);
2318
+ result.cascadeWarnings.push(...cascadeWarnings);
2319
+ switch (change.changeType) {
2320
+ case "new":
2321
+ result.added++;
2322
+ break;
2323
+ case "updated":
2324
+ result.updated++;
2325
+ break;
2326
+ case "deleted":
2327
+ result.deleted++;
2328
+ break;
2329
+ case "unchanged":
2330
+ result.unchanged++;
2331
+ break;
2332
+ }
2333
+ if (change.embeddingGenerated) {
2334
+ result.embeddingsGenerated++;
2335
+ }
2336
+ if (!options.dryRun && change.changeType !== "unchanged") {
2337
+ await this.manifest.save();
2338
+ }
2339
+ processedCount++;
2340
+ if (!options.dryRun && processedCount % CHECKPOINT_BATCH_SIZE === 0) {
2341
+ await this.graph.checkpoint();
2151
2342
  }
2152
2343
  }
2153
2344
  if (!options.dryRun && processedCount > 0) {
2154
2345
  await this.graph.checkpoint();
2155
2346
  }
2156
- if (!options.dryRun) {
2157
- await this.manifest.save();
2158
- }
2159
2347
  } catch (error) {
2160
2348
  const errorMessage = error instanceof Error ? error.message : String(error);
2161
2349
  this.logger.error(`Sync failed: ${errorMessage}`);
@@ -2164,7 +2352,7 @@ class SyncService {
2164
2352
  result.duration = Date.now() - startTime;
2165
2353
  return result;
2166
2354
  }
2167
- async detectChanges(paths) {
2355
+ async detectChanges(paths, useDbDetection = false) {
2168
2356
  const changes = [];
2169
2357
  let allDocPaths = await this.parser.discoverDocuments();
2170
2358
  if (paths && paths.length > 0) {
@@ -2175,11 +2363,11 @@ class SyncService {
2175
2363
  const pathSet = new Set(normalizedPaths);
2176
2364
  allDocPaths = allDocPaths.filter((p) => pathSet.has(p));
2177
2365
  }
2178
- const trackedPaths = new Set(this.manifest.getTrackedPaths());
2366
+ const trackedPaths = new Set(useDbDetection ? this.dbChangeDetector.getTrackedPaths() : this.manifest.getTrackedPaths());
2179
2367
  for (const docPath of allDocPaths) {
2180
2368
  try {
2181
2369
  const doc = await this.parser.parseDocument(docPath);
2182
- const changeType = this.manifest.detectChange(docPath, doc.contentHash, doc.frontmatterHash);
2370
+ const changeType = useDbDetection ? this.dbChangeDetector.detectChange(docPath, doc.contentHash) : this.manifest.detectChange(docPath, doc.contentHash, doc.frontmatterHash);
2183
2371
  changes.push({
2184
2372
  path: docPath,
2185
2373
  changeType,
@@ -2245,7 +2433,7 @@ class SyncService {
2245
2433
  }
2246
2434
  } catch (error) {
2247
2435
  const errorMessage = error instanceof Error ? error.message : String(error);
2248
- this.logger.warn(`Failed to generate embedding for ${doc.path}: ${errorMessage}`);
2436
+ throw new Error(`Failed to generate embedding for ${doc.path}: ${errorMessage}`);
2249
2437
  }
2250
2438
  }
2251
2439
  const entityTypeMap = new Map;
@@ -2336,6 +2524,11 @@ class SyncService {
2336
2524
  change.embeddingGenerated = embeddingGenerated;
2337
2525
  const currentDoc = await this.parser.parseDocument(change.path);
2338
2526
  this.manifest.updateEntry(currentDoc.path, currentDoc.contentHash, currentDoc.frontmatterHash, currentDoc.entities.length, currentDoc.relationships.length);
2527
+ const shouldUpdateDbHashes = options.legacy ? false : options.useDbChangeDetection ?? true;
2528
+ if (shouldUpdateDbHashes) {
2529
+ const embeddingSourceHash = embeddingGenerated ? currentDoc.contentHash : undefined;
2530
+ await this.graph.updateDocumentHashes(currentDoc.path, currentDoc.contentHash, embeddingSourceHash);
2531
+ }
2339
2532
  break;
2340
2533
  }
2341
2534
  case "deleted": {
@@ -2366,86 +2559,582 @@ class SyncService {
2366
2559
  }
2367
2560
  this.logger.log(`Marked ${normalizedPaths.length} document(s) for re-sync`);
2368
2561
  }
2369
- async getOldDocumentFromManifest(path2) {
2562
+ async getOldDocumentFromManifest(path2) {
2563
+ try {
2564
+ const manifest = await this.manifest.load();
2565
+ const entry = manifest.documents[path2];
2566
+ if (!entry) {
2567
+ return null;
2568
+ }
2569
+ try {
2570
+ return await this.parser.parseDocument(path2);
2571
+ } catch {
2572
+ return null;
2573
+ }
2574
+ } catch (error) {
2575
+ this.logger.warn(`Failed to retrieve old document for ${path2}: ${error instanceof Error ? error.message : String(error)}`);
2576
+ return null;
2577
+ }
2578
+ }
2579
+ async syncEntities(entities, options) {
2580
+ let embeddingsGenerated = 0;
2581
+ for (const [_key, entity] of entities) {
2582
+ const entityProps = {
2583
+ name: entity.name
2584
+ };
2585
+ if (entity.description) {
2586
+ entityProps.description = entity.description;
2587
+ }
2588
+ await this.graph.upsertNode(entity.type, entityProps);
2589
+ if (options.embeddings && !options.skipEmbeddings && this.embeddingService) {
2590
+ try {
2591
+ const text = composeEntityEmbeddingText(entity);
2592
+ const embedding = await this.embeddingService.generateEmbedding(text);
2593
+ await this.graph.updateNodeEmbedding(entity.type, entity.name, embedding);
2594
+ embeddingsGenerated++;
2595
+ this.logger.debug(`Generated embedding for ${entity.type}:${entity.name}`);
2596
+ } catch (error) {
2597
+ const errorMessage = error instanceof Error ? error.message : String(error);
2598
+ throw new Error(`Failed to generate embedding for ${entity.type}:${entity.name}: ${errorMessage}`);
2599
+ }
2600
+ }
2601
+ }
2602
+ return embeddingsGenerated;
2603
+ }
2604
+ async createEntityVectorIndices() {
2605
+ if (!this.embeddingService)
2606
+ return;
2607
+ const dimensions = this.embeddingService.getDimensions();
2608
+ for (const entityType of ENTITY_TYPES) {
2609
+ try {
2610
+ await this.graph.createVectorIndex(entityType, "embedding", dimensions);
2611
+ } catch (error) {
2612
+ this.logger.debug(`Vector index setup for ${entityType}: ${error instanceof Error ? error.message : String(error)}`);
2613
+ }
2614
+ }
2615
+ }
2616
+ }
2617
+ SyncService = __legacyDecorateClassTS([
2618
+ Injectable11(),
2619
+ __legacyMetadataTS("design:paramtypes", [
2620
+ typeof ManifestService === "undefined" ? Object : ManifestService,
2621
+ typeof DocumentParserService === "undefined" ? Object : DocumentParserService,
2622
+ typeof GraphService === "undefined" ? Object : GraphService,
2623
+ typeof CascadeService === "undefined" ? Object : CascadeService,
2624
+ typeof PathResolverService === "undefined" ? Object : PathResolverService,
2625
+ typeof DatabaseChangeDetectorService === "undefined" ? Object : DatabaseChangeDetectorService,
2626
+ typeof EntityExtractorService === "undefined" ? Object : EntityExtractorService,
2627
+ typeof EmbeddingService === "undefined" ? Object : EmbeddingService
2628
+ ])
2629
+ ], SyncService);
2630
+
2631
+ // src/commands/migrate.command.ts
2632
+ class MigrateCommand extends CommandRunner3 {
2633
+ graph;
2634
+ manifest;
2635
+ syncService;
2636
+ logger = new Logger8(MigrateCommand.name);
2637
+ constructor(graph, manifest, syncService) {
2638
+ super();
2639
+ this.graph = graph;
2640
+ this.manifest = manifest;
2641
+ this.syncService = syncService;
2642
+ }
2643
+ async run(_passedParams, options) {
2644
+ console.log(`
2645
+ \uD83D\uDD04 Migrating to Lattice v2...
2646
+ `);
2647
+ if (options.dryRun) {
2648
+ console.log(`\uD83D\uDCCB Dry run mode: No changes will be applied
2649
+ `);
2650
+ }
2651
+ try {
2652
+ console.log("\uD83D\uDCE6 Step 1/3: Applying v2 schema changes...");
2653
+ if (!options.dryRun) {
2654
+ await this.graph.runV2Migration();
2655
+ console.log(` \u2705 Schema updated with content_hash and embedding_source_hash columns
2656
+ `);
2657
+ } else {
2658
+ console.log(` [DRY-RUN] Would add content_hash and embedding_source_hash columns
2659
+ `);
2660
+ }
2661
+ console.log("\uD83D\uDCCB Step 2/3: Migrating manifest hashes to database...");
2662
+ const manifestStats = await this.migrateManifestHashes(options);
2663
+ if (manifestStats.total > 0) {
2664
+ if (!options.dryRun) {
2665
+ console.log(` \u2705 Migrated ${manifestStats.migrated}/${manifestStats.total} document hashes
2666
+ `);
2667
+ } else {
2668
+ console.log(` [DRY-RUN] Would migrate ${manifestStats.total} document hashes
2669
+ `);
2670
+ }
2671
+ } else {
2672
+ console.log(` \u2139\uFE0F No manifest found or manifest is empty (fresh install)
2673
+ `);
2674
+ }
2675
+ if (!options.skipSync) {
2676
+ console.log("\uD83E\uDD16 Step 3/3: Running full sync with AI entity extraction...");
2677
+ console.log(` This may take a while depending on the number of documents...
2678
+ `);
2679
+ if (!options.dryRun) {
2680
+ const result = await this.syncService.sync({
2681
+ force: false,
2682
+ useDbChangeDetection: true,
2683
+ aiExtraction: true,
2684
+ verbose: options.verbose,
2685
+ embeddings: true
2686
+ });
2687
+ console.log(`
2688
+ \uD83D\uDCCA Sync Results:`);
2689
+ console.log(` \u2705 Added: ${result.added}`);
2690
+ console.log(` \uD83D\uDD04 Updated: ${result.updated}`);
2691
+ console.log(` \uD83D\uDDD1\uFE0F Deleted: ${result.deleted}`);
2692
+ console.log(` \u23ED\uFE0F Unchanged: ${result.unchanged}`);
2693
+ if (result.errors.length > 0) {
2694
+ console.log(` \u274C Errors: ${result.errors.length}`);
2695
+ for (const err of result.errors) {
2696
+ console.log(` ${err.path}: ${err.error}`);
2697
+ }
2698
+ }
2699
+ console.log(` \u23F1\uFE0F Duration: ${result.duration}ms
2700
+ `);
2701
+ } else {
2702
+ console.log(` [DRY-RUN] Would run full sync with AI extraction
2703
+ `);
2704
+ }
2705
+ } else {
2706
+ console.log(`\u23ED\uFE0F Step 3/3: Skipping sync (--skip-sync flag)
2707
+ `);
2708
+ }
2709
+ console.log(`\u2705 Migration complete!
2710
+ `);
2711
+ console.log("Next steps:");
2712
+ console.log(" 1. Verify your graph: lattice status");
2713
+ console.log(" 2. Test semantic search: lattice search <query>");
2714
+ console.log(" 3. Future syncs will use v2 mode automatically");
2715
+ console.log(` 4. Optional: Delete ~/.lattice/.sync-manifest.json (no longer needed)
2716
+ `);
2717
+ } catch (error) {
2718
+ const errorMsg = error instanceof Error ? error.message : String(error);
2719
+ console.error(`
2720
+ \u274C Migration failed: ${errorMsg}
2721
+ `);
2722
+ this.logger.error(`Migration failed: ${errorMsg}`, error instanceof Error ? error.stack : undefined);
2723
+ process.exit(1);
2724
+ }
2725
+ }
2726
+ async migrateManifestHashes(options) {
2727
+ const stats = { total: 0, migrated: 0, skipped: 0 };
2728
+ try {
2729
+ const manifestData = await this.manifest.load();
2730
+ const entries = Object.entries(manifestData.documents);
2731
+ stats.total = entries.length;
2732
+ if (stats.total === 0) {
2733
+ return stats;
2734
+ }
2735
+ for (const [path2, entry] of entries) {
2736
+ if (options.verbose) {
2737
+ console.log(` Processing: ${path2}`);
2738
+ }
2739
+ if (!options.dryRun) {
2740
+ try {
2741
+ await this.graph.updateDocumentHashes(path2, entry.contentHash);
2742
+ stats.migrated++;
2743
+ } catch (error) {
2744
+ stats.skipped++;
2745
+ if (options.verbose) {
2746
+ const errorMsg = error instanceof Error ? error.message : String(error);
2747
+ console.log(` \u26A0\uFE0F Skipped ${path2}: ${errorMsg}`);
2748
+ }
2749
+ }
2750
+ } else {
2751
+ stats.migrated++;
2752
+ }
2753
+ }
2754
+ } catch (error) {
2755
+ if (options.verbose) {
2756
+ const errorMsg = error instanceof Error ? error.message : String(error);
2757
+ this.logger.debug(`No manifest to migrate: ${errorMsg}`);
2758
+ }
2759
+ }
2760
+ return stats;
2761
+ }
2762
+ parseDryRun() {
2763
+ return true;
2764
+ }
2765
+ parseVerbose() {
2766
+ return true;
2767
+ }
2768
+ parseSkipSync() {
2769
+ return true;
2770
+ }
2771
+ }
2772
+ __legacyDecorateClassTS([
2773
+ Option2({
2774
+ flags: "--dry-run",
2775
+ description: "Show what would be done without making changes"
2776
+ }),
2777
+ __legacyMetadataTS("design:type", Function),
2778
+ __legacyMetadataTS("design:paramtypes", []),
2779
+ __legacyMetadataTS("design:returntype", Boolean)
2780
+ ], MigrateCommand.prototype, "parseDryRun", null);
2781
+ __legacyDecorateClassTS([
2782
+ Option2({
2783
+ flags: "-v, --verbose",
2784
+ description: "Show detailed progress"
2785
+ }),
2786
+ __legacyMetadataTS("design:type", Function),
2787
+ __legacyMetadataTS("design:paramtypes", []),
2788
+ __legacyMetadataTS("design:returntype", Boolean)
2789
+ ], MigrateCommand.prototype, "parseVerbose", null);
2790
+ __legacyDecorateClassTS([
2791
+ Option2({
2792
+ flags: "--skip-sync",
2793
+ description: "Skip the full sync step (only apply schema and migrate hashes)"
2794
+ }),
2795
+ __legacyMetadataTS("design:type", Function),
2796
+ __legacyMetadataTS("design:paramtypes", []),
2797
+ __legacyMetadataTS("design:returntype", Boolean)
2798
+ ], MigrateCommand.prototype, "parseSkipSync", null);
2799
+ MigrateCommand = __legacyDecorateClassTS([
2800
+ Injectable12(),
2801
+ Command3({
2802
+ name: "migrate",
2803
+ description: "Migrate from v1 (manifest) to v2 (database-based) architecture"
2804
+ }),
2805
+ __legacyMetadataTS("design:paramtypes", [
2806
+ typeof GraphService === "undefined" ? Object : GraphService,
2807
+ typeof ManifestService === "undefined" ? Object : ManifestService,
2808
+ typeof SyncService === "undefined" ? Object : SyncService
2809
+ ])
2810
+ ], MigrateCommand);
2811
+ // src/commands/ontology.command.ts
2812
+ import { Injectable as Injectable14 } from "@nestjs/common";
2813
+ import { Command as Command4, CommandRunner as CommandRunner4 } from "nest-commander";
2814
+
2815
+ // src/sync/ontology.service.ts
2816
+ import { Injectable as Injectable13 } from "@nestjs/common";
2817
+ class OntologyService {
2818
+ parser;
2819
+ constructor(parser) {
2820
+ this.parser = parser;
2821
+ }
2822
+ async deriveOntology() {
2823
+ const docs = await this.parser.parseAllDocuments();
2824
+ return this.deriveFromDocuments(docs);
2825
+ }
2826
+ deriveFromDocuments(docs) {
2827
+ const entityTypeSet = new Set;
2828
+ const relationshipTypeSet = new Set;
2829
+ const entityCounts = {};
2830
+ const relationshipCounts = {};
2831
+ const entityExamples = {};
2832
+ let documentsWithEntities = 0;
2833
+ let documentsWithoutEntities = 0;
2834
+ let totalRelationships = 0;
2835
+ for (const doc of docs) {
2836
+ if (doc.entities.length > 0) {
2837
+ documentsWithEntities++;
2838
+ } else {
2839
+ documentsWithoutEntities++;
2840
+ }
2841
+ for (const entity of doc.entities) {
2842
+ entityTypeSet.add(entity.type);
2843
+ entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
2844
+ if (!entityExamples[entity.name]) {
2845
+ entityExamples[entity.name] = { type: entity.type, documents: [] };
2846
+ }
2847
+ if (!entityExamples[entity.name].documents.includes(doc.path)) {
2848
+ entityExamples[entity.name].documents.push(doc.path);
2849
+ }
2850
+ }
2851
+ for (const rel of doc.relationships) {
2852
+ relationshipTypeSet.add(rel.relation);
2853
+ relationshipCounts[rel.relation] = (relationshipCounts[rel.relation] || 0) + 1;
2854
+ totalRelationships++;
2855
+ }
2856
+ }
2857
+ return {
2858
+ entityTypes: Array.from(entityTypeSet).sort(),
2859
+ relationshipTypes: Array.from(relationshipTypeSet).sort(),
2860
+ entityCounts,
2861
+ relationshipCounts,
2862
+ totalEntities: Object.keys(entityExamples).length,
2863
+ totalRelationships,
2864
+ documentsWithEntities,
2865
+ documentsWithoutEntities,
2866
+ entityExamples
2867
+ };
2868
+ }
2869
+ printSummary(ontology) {
2870
+ console.log(`
2871
+ Derived Ontology Summary
2872
+ `);
2873
+ console.log(`Documents: ${ontology.documentsWithEntities} with entities, ${ontology.documentsWithoutEntities} without`);
2874
+ console.log(`Unique Entities: ${ontology.totalEntities}`);
2875
+ console.log(`Total Relationships: ${ontology.totalRelationships}`);
2876
+ console.log(`
2877
+ Entity Types:`);
2878
+ for (const type of ontology.entityTypes) {
2879
+ console.log(` ${type}: ${ontology.entityCounts[type]} instances`);
2880
+ }
2881
+ console.log(`
2882
+ Relationship Types:`);
2883
+ for (const type of ontology.relationshipTypes) {
2884
+ console.log(` ${type}: ${ontology.relationshipCounts[type]} instances`);
2885
+ }
2886
+ console.log(`
2887
+ Top Entities (by document count):`);
2888
+ const sorted = Object.entries(ontology.entityExamples).sort((a, b) => b[1].documents.length - a[1].documents.length).slice(0, 10);
2889
+ for (const [name, info] of sorted) {
2890
+ console.log(` ${name} (${info.type}): ${info.documents.length} docs`);
2891
+ }
2892
+ }
2893
+ }
2894
+ OntologyService = __legacyDecorateClassTS([
2895
+ Injectable13(),
2896
+ __legacyMetadataTS("design:paramtypes", [
2897
+ typeof DocumentParserService === "undefined" ? Object : DocumentParserService
2898
+ ])
2899
+ ], OntologyService);
2900
+
2901
+ // src/commands/ontology.command.ts
2902
+ class OntologyCommand extends CommandRunner4 {
2903
+ ontologyService;
2904
+ constructor(ontologyService) {
2905
+ super();
2906
+ this.ontologyService = ontologyService;
2907
+ }
2908
+ async run() {
2909
+ try {
2910
+ const ontology = await this.ontologyService.deriveOntology();
2911
+ this.ontologyService.printSummary(ontology);
2912
+ process.exit(0);
2913
+ } catch (error) {
2914
+ console.error(`
2915
+ \u274C Ontology derivation failed:`, error instanceof Error ? error.message : String(error));
2916
+ process.exit(1);
2917
+ }
2918
+ }
2919
+ }
2920
+ OntologyCommand = __legacyDecorateClassTS([
2921
+ Injectable14(),
2922
+ Command4({
2923
+ name: "ontology",
2924
+ description: "Derive and display ontology from all documents"
2925
+ }),
2926
+ __legacyMetadataTS("design:paramtypes", [
2927
+ typeof OntologyService === "undefined" ? Object : OntologyService
2928
+ ])
2929
+ ], OntologyCommand);
2930
+ // src/commands/query.command.ts
2931
+ import { Injectable as Injectable15 } from "@nestjs/common";
2932
+ import { Command as Command5, CommandRunner as CommandRunner5, Option as Option3 } from "nest-commander";
2933
+ class SearchCommand extends CommandRunner5 {
2934
+ graphService;
2935
+ embeddingService;
2936
+ constructor(graphService, embeddingService) {
2937
+ super();
2938
+ this.graphService = graphService;
2939
+ this.embeddingService = embeddingService;
2940
+ }
2941
+ async run(inputs, options) {
2942
+ const query3 = inputs[0];
2943
+ const limit = Math.min(parseInt(options.limit || "20", 10), 100);
2944
+ try {
2945
+ const queryEmbedding = await this.embeddingService.generateQueryEmbedding(query3);
2946
+ let results;
2947
+ if (options.label) {
2948
+ const labelResults = await this.graphService.vectorSearch(options.label, queryEmbedding, limit);
2949
+ results = labelResults.map((r) => ({
2950
+ name: r.name,
2951
+ label: options.label,
2952
+ title: r.title,
2953
+ score: r.score
2954
+ }));
2955
+ } else {
2956
+ results = await this.graphService.vectorSearchAll(queryEmbedding, limit);
2957
+ }
2958
+ const labelSuffix = options.label ? ` (${options.label})` : "";
2959
+ console.log(`
2960
+ === Semantic Search Results for "${query3}"${labelSuffix} ===
2961
+ `);
2962
+ if (results.length === 0) {
2963
+ console.log(`No results found.
2964
+ `);
2965
+ if (options.label) {
2966
+ console.log(`Tip: Try without --label to search all entity types.
2967
+ `);
2968
+ }
2969
+ process.exit(0);
2970
+ }
2971
+ results.forEach((result, idx) => {
2972
+ console.log(`${idx + 1}. [${result.label}] ${result.name}`);
2973
+ if (result.title) {
2974
+ console.log(` Title: ${result.title}`);
2975
+ }
2976
+ if (result.description && result.label !== "Document") {
2977
+ const desc = result.description.length > 80 ? `${result.description.slice(0, 80)}...` : result.description;
2978
+ console.log(` ${desc}`);
2979
+ }
2980
+ console.log(` Similarity: ${(result.score * 100).toFixed(2)}%`);
2981
+ });
2982
+ console.log();
2983
+ process.exit(0);
2984
+ } catch (error) {
2985
+ const errorMsg = error instanceof Error ? error.message : String(error);
2986
+ console.error("Error:", errorMsg);
2987
+ if (errorMsg.includes("no embeddings") || errorMsg.includes("vector")) {
2988
+ console.log(`
2989
+ Note: Semantic search requires embeddings to be generated first.`);
2990
+ console.log(`Run 'lattice sync' to generate embeddings for documents.
2991
+ `);
2992
+ }
2993
+ process.exit(1);
2994
+ }
2995
+ }
2996
+ parseLabel(value) {
2997
+ return value;
2998
+ }
2999
+ parseLimit(value) {
3000
+ return value;
3001
+ }
3002
+ }
3003
+ __legacyDecorateClassTS([
3004
+ Option3({
3005
+ flags: "-l, --label <label>",
3006
+ description: "Filter by entity label (e.g., Technology, Concept, Document)"
3007
+ }),
3008
+ __legacyMetadataTS("design:type", Function),
3009
+ __legacyMetadataTS("design:paramtypes", [
3010
+ String
3011
+ ]),
3012
+ __legacyMetadataTS("design:returntype", String)
3013
+ ], SearchCommand.prototype, "parseLabel", null);
3014
+ __legacyDecorateClassTS([
3015
+ Option3({
3016
+ flags: "--limit <n>",
3017
+ description: "Limit results",
3018
+ defaultValue: "20"
3019
+ }),
3020
+ __legacyMetadataTS("design:type", Function),
3021
+ __legacyMetadataTS("design:paramtypes", [
3022
+ String
3023
+ ]),
3024
+ __legacyMetadataTS("design:returntype", String)
3025
+ ], SearchCommand.prototype, "parseLimit", null);
3026
+ SearchCommand = __legacyDecorateClassTS([
3027
+ Injectable15(),
3028
+ Command5({
3029
+ name: "search",
3030
+ arguments: "<query>",
3031
+ description: "Semantic search across the knowledge graph"
3032
+ }),
3033
+ __legacyMetadataTS("design:paramtypes", [
3034
+ typeof GraphService === "undefined" ? Object : GraphService,
3035
+ typeof EmbeddingService === "undefined" ? Object : EmbeddingService
3036
+ ])
3037
+ ], SearchCommand);
3038
+
3039
+ class RelsCommand extends CommandRunner5 {
3040
+ graphService;
3041
+ constructor(graphService) {
3042
+ super();
3043
+ this.graphService = graphService;
3044
+ }
3045
+ async run(inputs) {
3046
+ const name = inputs[0];
2370
3047
  try {
2371
- const manifest = await this.manifest.load();
2372
- const entry = manifest.documents[path2];
2373
- if (!entry) {
2374
- return null;
3048
+ const relationships = await this.graphService.findRelationships(name);
3049
+ console.log(`
3050
+ === Relationships for "${name}" ===
3051
+ `);
3052
+ if (relationships.length === 0) {
3053
+ console.log(`No relationships found.
3054
+ `);
3055
+ process.exit(0);
2375
3056
  }
2376
- try {
2377
- return await this.parser.parseDocument(path2);
2378
- } catch {
2379
- return null;
3057
+ console.log("Relationships:");
3058
+ for (const rel of relationships) {
3059
+ const [relType, targetName] = rel;
3060
+ console.log(` -[${relType}]-> ${targetName}`);
2380
3061
  }
3062
+ console.log();
3063
+ process.exit(0);
2381
3064
  } catch (error) {
2382
- this.logger.warn(`Failed to retrieve old document for ${path2}: ${error instanceof Error ? error.message : String(error)}`);
2383
- return null;
3065
+ console.error("Error:", error instanceof Error ? error.message : String(error));
3066
+ process.exit(1);
2384
3067
  }
2385
3068
  }
2386
- async syncEntities(entities, options) {
2387
- let embeddingsGenerated = 0;
2388
- for (const [_key, entity] of entities) {
2389
- const entityProps = {
2390
- name: entity.name
2391
- };
2392
- if (entity.description) {
2393
- entityProps.description = entity.description;
2394
- }
2395
- await this.graph.upsertNode(entity.type, entityProps);
2396
- if (options.embeddings && !options.skipEmbeddings && this.embeddingService) {
2397
- try {
2398
- const text = composeEntityEmbeddingText(entity);
2399
- const embedding = await this.embeddingService.generateEmbedding(text);
2400
- await this.graph.updateNodeEmbedding(entity.type, entity.name, embedding);
2401
- embeddingsGenerated++;
2402
- this.logger.debug(`Generated embedding for ${entity.type}:${entity.name}`);
2403
- } catch (error) {
2404
- const errorMessage = error instanceof Error ? error.message : String(error);
2405
- this.logger.warn(`Failed to generate embedding for ${entity.type}:${entity.name}: ${errorMessage}`);
2406
- }
2407
- }
2408
- }
2409
- return embeddingsGenerated;
3069
+ }
3070
+ RelsCommand = __legacyDecorateClassTS([
3071
+ Injectable15(),
3072
+ Command5({
3073
+ name: "rels",
3074
+ arguments: "<name>",
3075
+ description: "Show relationships for a node"
3076
+ }),
3077
+ __legacyMetadataTS("design:paramtypes", [
3078
+ typeof GraphService === "undefined" ? Object : GraphService
3079
+ ])
3080
+ ], RelsCommand);
3081
+
3082
+ class SqlCommand extends CommandRunner5 {
3083
+ graphService;
3084
+ constructor(graphService) {
3085
+ super();
3086
+ this.graphService = graphService;
2410
3087
  }
2411
- async createEntityVectorIndices() {
2412
- if (!this.embeddingService)
2413
- return;
2414
- const dimensions = this.embeddingService.getDimensions();
2415
- for (const entityType of ENTITY_TYPES) {
2416
- try {
2417
- await this.graph.createVectorIndex(entityType, "embedding", dimensions);
2418
- } catch (error) {
2419
- this.logger.debug(`Vector index setup for ${entityType}: ${error instanceof Error ? error.message : String(error)}`);
2420
- }
3088
+ async run(inputs) {
3089
+ const query3 = inputs[0];
3090
+ try {
3091
+ const result = await this.graphService.query(query3);
3092
+ console.log(`
3093
+ === SQL Query Results ===
3094
+ `);
3095
+ const replacer = (_key, value) => typeof value === "bigint" ? Number(value) : value;
3096
+ console.log(JSON.stringify(result, replacer, 2));
3097
+ console.log();
3098
+ process.exit(0);
3099
+ } catch (error) {
3100
+ console.error("Error:", error instanceof Error ? error.message : String(error));
3101
+ process.exit(1);
2421
3102
  }
2422
3103
  }
2423
3104
  }
2424
- SyncService = __legacyDecorateClassTS([
2425
- Injectable11(),
3105
+ SqlCommand = __legacyDecorateClassTS([
3106
+ Injectable15(),
3107
+ Command5({
3108
+ name: "sql",
3109
+ arguments: "<query>",
3110
+ description: "Execute raw SQL query against DuckDB"
3111
+ }),
2426
3112
  __legacyMetadataTS("design:paramtypes", [
2427
- typeof ManifestService === "undefined" ? Object : ManifestService,
2428
- typeof DocumentParserService === "undefined" ? Object : DocumentParserService,
2429
- typeof GraphService === "undefined" ? Object : GraphService,
2430
- typeof CascadeService === "undefined" ? Object : CascadeService,
2431
- typeof PathResolverService === "undefined" ? Object : PathResolverService,
2432
- typeof EmbeddingService === "undefined" ? Object : EmbeddingService
3113
+ typeof GraphService === "undefined" ? Object : GraphService
2433
3114
  ])
2434
- ], SyncService);
2435
-
3115
+ ], SqlCommand);
2436
3116
  // src/commands/status.command.ts
2437
- class StatusCommand extends CommandRunner4 {
3117
+ import { Injectable as Injectable16 } from "@nestjs/common";
3118
+ import { Command as Command6, CommandRunner as CommandRunner6, Option as Option4 } from "nest-commander";
3119
+ class StatusCommand extends CommandRunner6 {
2438
3120
  syncService;
2439
3121
  manifestService;
2440
- constructor(syncService, manifestService) {
3122
+ dbChangeDetector;
3123
+ constructor(syncService, manifestService, dbChangeDetector) {
2441
3124
  super();
2442
3125
  this.syncService = syncService;
2443
3126
  this.manifestService = manifestService;
3127
+ this.dbChangeDetector = dbChangeDetector;
2444
3128
  }
2445
3129
  async run(_inputs, options) {
2446
3130
  try {
2447
- await this.manifestService.load();
2448
- const changes = await this.syncService.detectChanges();
3131
+ const useDbDetection = !options.legacy;
3132
+ if (useDbDetection) {
3133
+ await this.dbChangeDetector.loadHashes();
3134
+ } else {
3135
+ await this.manifestService.load();
3136
+ }
3137
+ const changes = await this.syncService.detectChanges(undefined, useDbDetection);
2449
3138
  const newDocs = changes.filter((c) => c.changeType === "new");
2450
3139
  const updatedDocs = changes.filter((c) => c.changeType === "updated");
2451
3140
  const deletedDocs = changes.filter((c) => c.changeType === "deleted");
@@ -2498,9 +3187,12 @@ class StatusCommand extends CommandRunner4 {
2498
3187
  parseVerbose() {
2499
3188
  return true;
2500
3189
  }
3190
+ parseLegacy() {
3191
+ return true;
3192
+ }
2501
3193
  }
2502
3194
  __legacyDecorateClassTS([
2503
- Option2({
3195
+ Option4({
2504
3196
  flags: "-v, --verbose",
2505
3197
  description: "Show all documents including unchanged"
2506
3198
  }),
@@ -2508,28 +3200,38 @@ __legacyDecorateClassTS([
2508
3200
  __legacyMetadataTS("design:paramtypes", []),
2509
3201
  __legacyMetadataTS("design:returntype", Boolean)
2510
3202
  ], StatusCommand.prototype, "parseVerbose", null);
3203
+ __legacyDecorateClassTS([
3204
+ Option4({
3205
+ flags: "--legacy",
3206
+ description: "Use legacy v1 mode: manifest-based change detection"
3207
+ }),
3208
+ __legacyMetadataTS("design:type", Function),
3209
+ __legacyMetadataTS("design:paramtypes", []),
3210
+ __legacyMetadataTS("design:returntype", Boolean)
3211
+ ], StatusCommand.prototype, "parseLegacy", null);
2511
3212
  StatusCommand = __legacyDecorateClassTS([
2512
- Injectable12(),
2513
- Command4({
3213
+ Injectable16(),
3214
+ Command6({
2514
3215
  name: "status",
2515
3216
  description: "Show documents that need syncing (new or updated)"
2516
3217
  }),
2517
3218
  __legacyMetadataTS("design:paramtypes", [
2518
3219
  typeof SyncService === "undefined" ? Object : SyncService,
2519
- typeof ManifestService === "undefined" ? Object : ManifestService
3220
+ typeof ManifestService === "undefined" ? Object : ManifestService,
3221
+ typeof DatabaseChangeDetectorService === "undefined" ? Object : DatabaseChangeDetectorService
2520
3222
  ])
2521
3223
  ], StatusCommand);
2522
3224
  // src/commands/sync.command.ts
2523
3225
  import { watch } from "fs";
2524
3226
  import { join as join3 } from "path";
2525
- import { Injectable as Injectable14 } from "@nestjs/common";
2526
- import { Command as Command5, CommandRunner as CommandRunner5, Option as Option3 } from "nest-commander";
3227
+ import { Injectable as Injectable18 } from "@nestjs/common";
3228
+ import { Command as Command7, CommandRunner as CommandRunner7, Option as Option5 } from "nest-commander";
2527
3229
 
2528
3230
  // src/sync/graph-validator.service.ts
2529
- import { Injectable as Injectable13, Logger as Logger6 } from "@nestjs/common";
3231
+ import { Injectable as Injectable17, Logger as Logger9 } from "@nestjs/common";
2530
3232
  class GraphValidatorService {
2531
3233
  graph;
2532
- logger = new Logger6(GraphValidatorService.name);
3234
+ logger = new Logger9(GraphValidatorService.name);
2533
3235
  constructor(graph) {
2534
3236
  this.graph = graph;
2535
3237
  }
@@ -2675,14 +3377,14 @@ class GraphValidatorService {
2675
3377
  }
2676
3378
  }
2677
3379
  GraphValidatorService = __legacyDecorateClassTS([
2678
- Injectable13(),
3380
+ Injectable17(),
2679
3381
  __legacyMetadataTS("design:paramtypes", [
2680
3382
  typeof GraphService === "undefined" ? Object : GraphService
2681
3383
  ])
2682
3384
  ], GraphValidatorService);
2683
3385
 
2684
3386
  // src/commands/sync.command.ts
2685
- class SyncCommand extends CommandRunner5 {
3387
+ class SyncCommand extends CommandRunner7 {
2686
3388
  syncService;
2687
3389
  _graphValidator;
2688
3390
  watcher = null;
@@ -2702,6 +3404,14 @@ class SyncCommand extends CommandRunner5 {
2702
3404
  if (options.watch && options.force) {
2703
3405
  console.log(`
2704
3406
  \u26A0\uFE0F Watch mode is not compatible with --force mode (for safety)
3407
+ `);
3408
+ process.exit(1);
3409
+ }
3410
+ if (options.force && paths.length === 0) {
3411
+ console.log(`
3412
+ \u26A0\uFE0F --force requires specific paths to be specified.
3413
+ `);
3414
+ console.log(` Usage: lattice sync --force <path1> [path2] ...
2705
3415
  `);
2706
3416
  process.exit(1);
2707
3417
  }
@@ -2711,19 +3421,16 @@ class SyncCommand extends CommandRunner5 {
2711
3421
  verbose: options.verbose,
2712
3422
  paths: paths.length > 0 ? paths : undefined,
2713
3423
  skipCascade: options.skipCascade,
2714
- embeddings: options.embeddings !== false
3424
+ embeddings: options.embeddings !== false,
3425
+ legacy: options.legacy,
3426
+ aiExtraction: !options.skipExtraction
2715
3427
  };
2716
3428
  console.log(`
2717
3429
  \uD83D\uDD04 Graph Sync
2718
3430
  `);
2719
3431
  if (syncOptions.force) {
2720
- if (syncOptions.paths && syncOptions.paths.length > 0) {
2721
- console.log(`\u26A0\uFE0F Force mode: ${syncOptions.paths.length} document(s) will be cleared and re-synced
3432
+ console.log(`\u26A0\uFE0F Force mode: ${syncOptions.paths?.length} document(s) will be cleared and re-synced
2722
3433
  `);
2723
- } else {
2724
- console.log(`\u26A0\uFE0F Force mode: Entire graph will be cleared and rebuilt
2725
- `);
2726
- }
2727
3434
  }
2728
3435
  if (syncOptions.dryRun) {
2729
3436
  console.log(`\uD83D\uDCCB Dry run mode: No changes will be applied
@@ -2737,6 +3444,15 @@ class SyncCommand extends CommandRunner5 {
2737
3444
  console.log(`\uD83D\uDEAB Embedding generation disabled
2738
3445
  `);
2739
3446
  }
3447
+ if (syncOptions.legacy) {
3448
+ console.log(`\uD83D\uDCDC Legacy mode: Using manifest-based change detection
3449
+ `);
3450
+ } else {
3451
+ if (!syncOptions.aiExtraction) {
3452
+ console.log(`\u23ED\uFE0F AI entity extraction skipped (--skip-extraction)
3453
+ `);
3454
+ }
3455
+ }
2740
3456
  if (syncOptions.paths) {
2741
3457
  console.log(`\uD83D\uDCC1 Syncing specific paths: ${syncOptions.paths.join(", ")}
2742
3458
  `);
@@ -2930,18 +3646,24 @@ class SyncCommand extends CommandRunner5 {
2930
3646
  parseNoEmbeddings() {
2931
3647
  return false;
2932
3648
  }
3649
+ parseSkipExtraction() {
3650
+ return true;
3651
+ }
3652
+ parseLegacy() {
3653
+ return true;
3654
+ }
2933
3655
  }
2934
3656
  __legacyDecorateClassTS([
2935
- Option3({
3657
+ Option5({
2936
3658
  flags: "-f, --force",
2937
- description: "Force re-sync: with paths, clears only those docs; without paths, rebuilds entire graph"
3659
+ description: "Force re-sync specified documents (requires paths to be specified)"
2938
3660
  }),
2939
3661
  __legacyMetadataTS("design:type", Function),
2940
3662
  __legacyMetadataTS("design:paramtypes", []),
2941
3663
  __legacyMetadataTS("design:returntype", Boolean)
2942
3664
  ], SyncCommand.prototype, "parseForce", null);
2943
3665
  __legacyDecorateClassTS([
2944
- Option3({
3666
+ Option5({
2945
3667
  flags: "-d, --dry-run",
2946
3668
  description: "Show what would change without applying"
2947
3669
  }),
@@ -2950,7 +3672,7 @@ __legacyDecorateClassTS([
2950
3672
  __legacyMetadataTS("design:returntype", Boolean)
2951
3673
  ], SyncCommand.prototype, "parseDryRun", null);
2952
3674
  __legacyDecorateClassTS([
2953
- Option3({
3675
+ Option5({
2954
3676
  flags: "-v, --verbose",
2955
3677
  description: "Show detailed output"
2956
3678
  }),
@@ -2959,7 +3681,7 @@ __legacyDecorateClassTS([
2959
3681
  __legacyMetadataTS("design:returntype", Boolean)
2960
3682
  ], SyncCommand.prototype, "parseVerbose", null);
2961
3683
  __legacyDecorateClassTS([
2962
- Option3({
3684
+ Option5({
2963
3685
  flags: "-w, --watch",
2964
3686
  description: "Watch for file changes and sync automatically"
2965
3687
  }),
@@ -2968,7 +3690,7 @@ __legacyDecorateClassTS([
2968
3690
  __legacyMetadataTS("design:returntype", Boolean)
2969
3691
  ], SyncCommand.prototype, "parseWatch", null);
2970
3692
  __legacyDecorateClassTS([
2971
- Option3({
3693
+ Option5({
2972
3694
  flags: "--diff",
2973
3695
  description: "Show only changed documents (alias for --dry-run)"
2974
3696
  }),
@@ -2977,7 +3699,7 @@ __legacyDecorateClassTS([
2977
3699
  __legacyMetadataTS("design:returntype", Boolean)
2978
3700
  ], SyncCommand.prototype, "parseDiff", null);
2979
3701
  __legacyDecorateClassTS([
2980
- Option3({
3702
+ Option5({
2981
3703
  flags: "--skip-cascade",
2982
3704
  description: "Skip cascade analysis (faster for large repos)"
2983
3705
  }),
@@ -2986,7 +3708,7 @@ __legacyDecorateClassTS([
2986
3708
  __legacyMetadataTS("design:returntype", Boolean)
2987
3709
  ], SyncCommand.prototype, "parseSkipCascade", null);
2988
3710
  __legacyDecorateClassTS([
2989
- Option3({
3711
+ Option5({
2990
3712
  flags: "--no-embeddings",
2991
3713
  description: "Disable embedding generation during sync"
2992
3714
  }),
@@ -2994,131 +3716,36 @@ __legacyDecorateClassTS([
2994
3716
  __legacyMetadataTS("design:paramtypes", []),
2995
3717
  __legacyMetadataTS("design:returntype", Boolean)
2996
3718
  ], SyncCommand.prototype, "parseNoEmbeddings", null);
2997
- SyncCommand = __legacyDecorateClassTS([
2998
- Injectable14(),
2999
- Command5({
3000
- name: "sync",
3001
- arguments: "[paths...]",
3002
- description: "Synchronize documents to the knowledge graph"
3719
+ __legacyDecorateClassTS([
3720
+ Option5({
3721
+ flags: "--skip-extraction",
3722
+ description: "Skip AI entity extraction (sync without re-extracting entities)"
3003
3723
  }),
3004
- __legacyMetadataTS("design:paramtypes", [
3005
- typeof SyncService === "undefined" ? Object : SyncService,
3006
- typeof GraphValidatorService === "undefined" ? Object : GraphValidatorService
3007
- ])
3008
- ], SyncCommand);
3009
- // src/commands/validate.command.ts
3010
- import { Injectable as Injectable15 } from "@nestjs/common";
3011
- import { Command as Command6, CommandRunner as CommandRunner6, Option as Option4 } from "nest-commander";
3012
- class ValidateCommand extends CommandRunner6 {
3013
- parserService;
3014
- _graphValidator;
3015
- constructor(parserService, _graphValidator) {
3016
- super();
3017
- this.parserService = parserService;
3018
- this._graphValidator = _graphValidator;
3019
- }
3020
- async run(_inputs, options) {
3021
- try {
3022
- console.log(`=== Document Validation ===
3023
- `);
3024
- const { docs, errors: schemaErrors } = await this.parserService.parseAllDocumentsWithErrors();
3025
- const issues = [];
3026
- for (const schemaError of schemaErrors) {
3027
- issues.push({
3028
- type: "error",
3029
- path: schemaError.path,
3030
- message: schemaError.error
3031
- });
3032
- }
3033
- const entityIndex = new Map;
3034
- for (const doc of docs) {
3035
- for (const entity of doc.entities) {
3036
- let docPaths = entityIndex.get(entity.name);
3037
- if (!docPaths) {
3038
- docPaths = new Set;
3039
- entityIndex.set(entity.name, docPaths);
3040
- }
3041
- docPaths.add(doc.path);
3042
- }
3043
- }
3044
- const validationErrors = validateDocuments2(docs);
3045
- for (const err of validationErrors) {
3046
- issues.push({
3047
- type: "error",
3048
- path: err.path,
3049
- message: err.error,
3050
- suggestion: "Add entity definition or fix the reference"
3051
- });
3052
- }
3053
- console.log(`Scanned ${docs.length} documents`);
3054
- console.log(`Found ${entityIndex.size} unique entities
3055
- `);
3056
- if (issues.length > 0) {
3057
- console.log(`Document Errors (${issues.length}):
3058
- `);
3059
- issues.forEach((i) => {
3060
- console.log(` ${i.path}`);
3061
- console.log(` Error: ${i.message}`);
3062
- if (options.fix && i.suggestion) {
3063
- console.log(` Suggestion: ${i.suggestion}`);
3064
- }
3065
- console.log("");
3066
- });
3067
- } else {
3068
- console.log(`\u2713 Markdown files valid (schema + relationships)
3069
- `);
3070
- }
3071
- const graphResult = {
3072
- valid: true,
3073
- issues: [],
3074
- stats: {
3075
- totalNodes: 0,
3076
- documentsChecked: 0,
3077
- entitiesChecked: 0,
3078
- errorsFound: 0,
3079
- warningsFound: 0
3080
- }
3081
- };
3082
- const totalErrors = issues.length + graphResult.stats.errorsFound;
3083
- const totalWarnings = graphResult.stats.warningsFound;
3084
- console.log(`
3085
- === Validation Summary ===`);
3086
- console.log(`Markdown files: ${issues.length === 0 ? "\u2713 PASSED" : `\u2717 ${issues.length} errors`}`);
3087
- console.log(`Graph database: ${graphResult.stats.errorsFound === 0 ? "\u2713 PASSED" : `\u2717 ${graphResult.stats.errorsFound} errors`}`);
3088
- console.log(`Warnings: ${totalWarnings}`);
3089
- console.log(`
3090
- Overall: ${totalErrors === 0 ? "\u2713 PASSED" : "\u2717 FAILED"}${totalWarnings > 0 ? ` (${totalWarnings} warnings)` : ""}
3091
- `);
3092
- process.exit(totalErrors > 0 ? 1 : 0);
3093
- } catch (error) {
3094
- console.error("Validation failed:", error instanceof Error ? error.message : String(error));
3095
- process.exit(1);
3096
- }
3097
- }
3098
- parseFix() {
3099
- return true;
3100
- }
3101
- }
3724
+ __legacyMetadataTS("design:type", Function),
3725
+ __legacyMetadataTS("design:paramtypes", []),
3726
+ __legacyMetadataTS("design:returntype", Boolean)
3727
+ ], SyncCommand.prototype, "parseSkipExtraction", null);
3102
3728
  __legacyDecorateClassTS([
3103
- Option4({
3104
- flags: "--fix",
3105
- description: "Show suggestions for common issues"
3729
+ Option5({
3730
+ flags: "--legacy",
3731
+ description: "Use legacy v1 mode: manifest-based change detection, no AI extraction"
3106
3732
  }),
3107
3733
  __legacyMetadataTS("design:type", Function),
3108
3734
  __legacyMetadataTS("design:paramtypes", []),
3109
3735
  __legacyMetadataTS("design:returntype", Boolean)
3110
- ], ValidateCommand.prototype, "parseFix", null);
3111
- ValidateCommand = __legacyDecorateClassTS([
3112
- Injectable15(),
3113
- Command6({
3114
- name: "validate",
3115
- description: "Validate entity references and relationships across documents"
3736
+ ], SyncCommand.prototype, "parseLegacy", null);
3737
+ SyncCommand = __legacyDecorateClassTS([
3738
+ Injectable18(),
3739
+ Command7({
3740
+ name: "sync",
3741
+ arguments: "[paths...]",
3742
+ description: "Synchronize documents to the knowledge graph"
3116
3743
  }),
3117
3744
  __legacyMetadataTS("design:paramtypes", [
3118
- typeof DocumentParserService === "undefined" ? Object : DocumentParserService,
3745
+ typeof SyncService === "undefined" ? Object : SyncService,
3119
3746
  typeof GraphValidatorService === "undefined" ? Object : GraphValidatorService
3120
3747
  ])
3121
- ], ValidateCommand);
3748
+ ], SyncCommand);
3122
3749
  // src/embedding/embedding.module.ts
3123
3750
  import { Module } from "@nestjs/common";
3124
3751
  import { ConfigModule } from "@nestjs/config";
@@ -3147,10 +3774,10 @@ GraphModule = __legacyDecorateClassTS([
3147
3774
  import { Module as Module3 } from "@nestjs/common";
3148
3775
 
3149
3776
  // src/query/query.service.ts
3150
- import { Injectable as Injectable16, Logger as Logger7 } from "@nestjs/common";
3777
+ import { Injectable as Injectable19, Logger as Logger10 } from "@nestjs/common";
3151
3778
  class QueryService {
3152
3779
  graphService;
3153
- logger = new Logger7(QueryService.name);
3780
+ logger = new Logger10(QueryService.name);
3154
3781
  constructor(graphService) {
3155
3782
  this.graphService = graphService;
3156
3783
  }
@@ -3160,7 +3787,7 @@ class QueryService {
3160
3787
  }
3161
3788
  }
3162
3789
  QueryService = __legacyDecorateClassTS([
3163
- Injectable16(),
3790
+ Injectable19(),
3164
3791
  __legacyMetadataTS("design:paramtypes", [
3165
3792
  typeof GraphService === "undefined" ? Object : GraphService
3166
3793
  ])
@@ -3187,6 +3814,8 @@ SyncModule = __legacyDecorateClassTS([
3187
3814
  providers: [
3188
3815
  SyncService,
3189
3816
  ManifestService,
3817
+ DatabaseChangeDetectorService,
3818
+ EntityExtractorService,
3190
3819
  DocumentParserService,
3191
3820
  OntologyService,
3192
3821
  CascadeService,
@@ -3196,6 +3825,8 @@ SyncModule = __legacyDecorateClassTS([
3196
3825
  exports: [
3197
3826
  SyncService,
3198
3827
  ManifestService,
3828
+ DatabaseChangeDetectorService,
3829
+ EntityExtractorService,
3199
3830
  DocumentParserService,
3200
3831
  OntologyService,
3201
3832
  CascadeService,
@@ -3221,14 +3852,15 @@ AppModule = __legacyDecorateClassTS([
3221
3852
  QueryModule
3222
3853
  ],
3223
3854
  providers: [
3855
+ ExtractCommand,
3224
3856
  SyncCommand,
3225
3857
  StatusCommand,
3226
3858
  SearchCommand,
3227
3859
  RelsCommand,
3228
3860
  SqlCommand,
3229
- ValidateCommand,
3230
3861
  OntologyCommand,
3231
- InitCommand
3862
+ InitCommand,
3863
+ MigrateCommand
3232
3864
  ]
3233
3865
  })
3234
3866
  ], AppModule);