@zabaca/lattice 1.0.3 → 1.0.5

package/dist/main.js

@@ -711,7 +711,14 @@ class VoyageEmbeddingProvider {
     const embeddings = await this.generateEmbeddings([text]);
     return embeddings[0];
   }
+  async generateQueryEmbedding(text) {
+    const embeddings = await this.generateEmbeddingsWithType([text], "query");
+    return embeddings[0];
+  }
   async generateEmbeddings(texts) {
+    return this.generateEmbeddingsWithType(texts, this.inputType);
+  }
+  async generateEmbeddingsWithType(texts, inputType) {
     if (!texts || texts.length === 0) {
       return [];
     }
@@ -726,7 +733,7 @@ class VoyageEmbeddingProvider {
           model: this.model,
           input: texts,
           output_dimension: this.dimensions,
-          input_type: this.inputType
+          input_type: inputType
         })
       });
       if (!response.ok) {
@@ -813,6 +820,15 @@ class EmbeddingService {
     }
     return this.provider.generateEmbedding(text);
   }
+  async generateQueryEmbedding(text) {
+    if (!text || text.trim().length === 0) {
+      throw new Error("Cannot generate embedding for empty text");
+    }
+    if (this.provider.generateQueryEmbedding) {
+      return this.provider.generateQueryEmbedding(text);
+    }
+    return this.provider.generateEmbedding(text);
+  }
   async generateEmbeddings(texts) {
     const validTexts = texts.filter((t) => t && t.trim().length > 0);
     if (validTexts.length === 0) {
@@ -832,7 +848,6 @@ EmbeddingService = __legacyDecorateClassTS([
 ], EmbeddingService);
 
 // src/graph/graph.service.ts
-import { existsSync as existsSync3, unlinkSync } from "fs";
 import { DuckDBInstance } from "@duckdb/node-api";
 import { Injectable as Injectable6, Logger as Logger3 } from "@nestjs/common";
 import { ConfigService as ConfigService2 } from "@nestjs/config";
@@ -845,11 +860,35 @@ class GraphService {
   connecting = null;
   vectorIndexes = new Set;
   embeddingDimensions;
+  signalHandlersRegistered = false;
   constructor(configService) {
     this.configService = configService;
     ensureLatticeHome();
     this.dbPath = getDatabasePath();
     this.embeddingDimensions = this.configService.get("EMBEDDING_DIMENSIONS") || 512;
+    this.registerSignalHandlers();
+  }
+  registerSignalHandlers() {
+    if (this.signalHandlersRegistered)
+      return;
+    this.signalHandlersRegistered = true;
+    const gracefulShutdown = async (signal) => {
+      this.logger.log(`Received ${signal}, checkpointing before exit...`);
+      try {
+        await this.checkpoint();
+        await this.disconnect();
+      } catch (error) {
+        this.logger.error(`Error during graceful shutdown: ${error instanceof Error ? error.message : String(error)}`);
+      }
+      process.exit(0);
+    };
+    process.on("SIGINT", () => gracefulShutdown("SIGINT"));
+    process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
+    process.on("beforeExit", async () => {
+      if (this.connection) {
+        await this.checkpoint();
+      }
+    });
   }
   async onModuleDestroy() {
     await this.disconnect();
@@ -875,12 +914,7 @@ class GraphService {
   }
   async connect() {
     try {
-      const walPath = `${this.dbPath}.wal`;
-      if (existsSync3(walPath)) {
-        this.logger.warn(`Removing WAL file to prevent HNSW index replay failure`);
-        unlinkSync(walPath);
-      }
-      this.instance = await DuckDBInstance.create(this.dbPath, {
+      this.instance = await DuckDBInstance.create(":memory:", {
         allow_unsigned_extensions: "true"
       });
       this.connection = await this.instance.connect();
@@ -894,8 +928,10 @@ class GraphService {
       } catch (e) {
         this.logger.warn(`DuckPGQ extension not available: ${e instanceof Error ? e.message : String(e)}`);
       }
+      await this.connection.run(`ATTACH '${this.dbPath}' AS lattice (READ_WRITE);`);
+      await this.connection.run("USE lattice;");
       await this.initializeSchema();
-      this.logger.log(`Connected to DuckDB at ${this.dbPath}`);
+      this.logger.log(`Connected to DuckDB (in-memory + ATTACH) at ${this.dbPath}`);
     } catch (error) {
       this.connection = null;
       this.instance = null;
@@ -905,11 +941,7 @@ class GraphService {
   }
   async disconnect() {
     if (this.connection) {
-      try {
-        await this.connection.run("CHECKPOINT;");
-      } catch {
-        this.logger.debug("Checkpoint failed during disconnect");
-      }
+      await this.checkpoint();
       this.connection.closeSync();
       this.connection = null;
       this.logger.log("Disconnected from DuckDB");
@@ -918,6 +950,17 @@ class GraphService {
       this.instance = null;
     }
   }
+  async checkpoint() {
+    if (!this.connection) {
+      return;
+    }
+    try {
+      await this.connection.run("CHECKPOINT;");
+      this.logger.debug("Checkpoint completed");
+    } catch (error) {
+      this.logger.warn(`Checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
   async initializeSchema() {
     if (!this.connection) {
       throw new Error("Cannot initialize schema: not connected");
@@ -1207,7 +1250,7 @@ class SearchCommand extends CommandRunner3 {
     const query = inputs[0];
     const limit = Math.min(parseInt(options.limit || "20", 10), 100);
     try {
-      const queryEmbedding = await this.embeddingService.generateEmbedding(query);
+      const queryEmbedding = await this.embeddingService.generateQueryEmbedding(query);
       let results;
       if (options.label) {
         const labelResults = await this.graphService.vectorSearch(options.label, queryEmbedding, limit);
@@ -1384,7 +1427,7 @@ import { Command as Command4, CommandRunner as CommandRunner4, Option as Option3
 
 // src/sync/manifest.service.ts
 import { createHash as createHash2 } from "crypto";
-import { existsSync as existsSync4 } from "fs";
+import { existsSync as existsSync3 } from "fs";
 import { readFile as readFile3, writeFile } from "fs/promises";
 import { Injectable as Injectable8 } from "@nestjs/common";
 
@@ -1412,7 +1455,7 @@ class ManifestService {
   }
   async load() {
     try {
-      if (existsSync4(this.manifestPath)) {
+      if (existsSync3(this.manifestPath)) {
         const content = await readFile3(this.manifestPath, "utf-8");
         this.manifest = SyncManifestSchema.parse(JSON.parse(content));
       } else {
@@ -1898,7 +1941,7 @@ CascadeService = __legacyDecorateClassTS([
 ], CascadeService);
 
 // src/sync/path-resolver.service.ts
-import { existsSync as existsSync5 } from "fs";
+import { existsSync as existsSync4 } from "fs";
 import { isAbsolute, resolve as resolve2 } from "path";
 import { Injectable as Injectable10 } from "@nestjs/common";
 class PathResolverService {
@@ -1920,7 +1963,7 @@ class PathResolverService {
     if (requireInDocs && !this.isUnderDocs(resolvedPath)) {
       throw new Error(`Path "${userPath}" resolves to "${resolvedPath}" which is outside the docs directory (${this.docsPath})`);
     }
-    if (requireExists && !existsSync5(resolvedPath)) {
+    if (requireExists && !existsSync4(resolvedPath)) {
       throw new Error(`Path "${userPath}" does not exist (resolved to: ${resolvedPath})`);
     }
     return resolvedPath;
@@ -2048,10 +2091,13 @@ class SyncService {
       }
       if (!options.dryRun) {
         result.entityEmbeddingsGenerated = await this.syncEntities(uniqueEntities, options);
+        await this.graph.checkpoint();
         if (options.verbose) {
           this.logger.log(`Synced ${uniqueEntities.size} entities, generated ${result.entityEmbeddingsGenerated} embeddings`);
         }
       }
+      const CHECKPOINT_BATCH_SIZE = 10;
+      let processedCount = 0;
       for (const change of changes) {
         try {
           const doc = docsByPath.get(change.path);
@@ -2074,12 +2120,19 @@ class SyncService {
           if (change.embeddingGenerated) {
             result.embeddingsGenerated++;
           }
+          processedCount++;
+          if (!options.dryRun && processedCount % CHECKPOINT_BATCH_SIZE === 0) {
+            await this.graph.checkpoint();
+          }
         } catch (error) {
           const errorMessage = error instanceof Error ? error.message : String(error);
           result.errors.push({ path: change.path, error: errorMessage });
           this.logger.warn(`Error processing ${change.path}: ${errorMessage}`);
         }
       }
+      if (!options.dryRun && processedCount > 0) {
+        await this.graph.checkpoint();
+      }
       if (!options.dryRun) {
         await this.manifest.save();
       }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@zabaca/lattice",
-	"version": "1.0.3",
+	"version": "1.0.5",
 	"description": "Human-initiated, AI-powered knowledge graph for markdown documentation",
 	"type": "module",
 	"bin": {