@zabaca/lattice 1.0.20 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/main.js +25 -126
- package/package.json +1 -1
package/dist/main.js
CHANGED
|
@@ -989,29 +989,6 @@ class GraphService {
|
|
|
989
989
|
return [];
|
|
990
990
|
}
|
|
991
991
|
}
|
|
992
|
-
async findNodesWithMissingEmbeddings(labels) {
|
|
993
|
-
try {
|
|
994
|
-
const conn = await this.ensureConnected();
|
|
995
|
-
const labelFilter = labels.map((l) => `'${this.escape(l)}'`).join(", ");
|
|
996
|
-
const reader = await conn.runAndReadAll(`
|
|
997
|
-
SELECT label, name, properties->>'description' as description
|
|
998
|
-
FROM nodes
|
|
999
|
-
WHERE label IN (${labelFilter})
|
|
1000
|
-
AND embedding IS NULL
|
|
1001
|
-
`);
|
|
1002
|
-
return reader.getRows().map((row) => {
|
|
1003
|
-
const [label, name, description] = row;
|
|
1004
|
-
return {
|
|
1005
|
-
label,
|
|
1006
|
-
name,
|
|
1007
|
-
description: description || undefined
|
|
1008
|
-
};
|
|
1009
|
-
});
|
|
1010
|
-
} catch (error) {
|
|
1011
|
-
this.logger.error(`Failed to find nodes with missing embeddings: ${error instanceof Error ? error.message : String(error)}`);
|
|
1012
|
-
return [];
|
|
1013
|
-
}
|
|
1014
|
-
}
|
|
1015
992
|
}
|
|
1016
993
|
GraphService = __legacyDecorateClassTS([
|
|
1017
994
|
Injectable4(),
|
|
@@ -2059,6 +2036,11 @@ class DatabaseChangeDetectorService {
|
|
|
2059
2036
|
this.hashCache.clear();
|
|
2060
2037
|
this.loaded = false;
|
|
2061
2038
|
}
|
|
2039
|
+
clearEntries(paths) {
|
|
2040
|
+
for (const path2 of paths) {
|
|
2041
|
+
this.hashCache.delete(path2);
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2062
2044
|
getContentHash(content) {
|
|
2063
2045
|
return createHash3("sha256").update(content).digest("hex");
|
|
2064
2046
|
}
|
|
@@ -2236,30 +2218,30 @@ class SyncService {
|
|
|
2236
2218
|
embeddingsGenerated: 0,
|
|
2237
2219
|
entityEmbeddingsGenerated: 0
|
|
2238
2220
|
};
|
|
2239
|
-
const
|
|
2240
|
-
const useAiExtraction = options.legacy ? false : options.aiExtraction ?? true;
|
|
2221
|
+
const useAiExtraction = options.aiExtraction ?? true;
|
|
2241
2222
|
try {
|
|
2242
2223
|
await this.manifest.load();
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
this.logger.log(`v2 mode: Loaded ${this.dbChangeDetector.getCacheSize()} document hashes from database`);
|
|
2247
|
-
}
|
|
2224
|
+
await this.dbChangeDetector.loadHashes();
|
|
2225
|
+
if (options.verbose) {
|
|
2226
|
+
this.logger.log(`Loaded ${this.dbChangeDetector.getCacheSize()} document hashes from database`);
|
|
2248
2227
|
}
|
|
2249
2228
|
if (options.force) {
|
|
2250
2229
|
if (options.paths && options.paths.length > 0) {
|
|
2230
|
+
const normalizedPaths = this.pathResolver.resolveDocPaths(options.paths, { requireExists: true, requireInDocs: true });
|
|
2251
2231
|
if (options.verbose) {
|
|
2252
|
-
this.logger.log(`Force mode: marking ${
|
|
2232
|
+
this.logger.log(`Force mode: marking ${normalizedPaths.length} document(s) for re-sync`);
|
|
2253
2233
|
}
|
|
2254
2234
|
await this.clearManifestEntries(options.paths);
|
|
2235
|
+
this.dbChangeDetector.clearEntries(normalizedPaths);
|
|
2255
2236
|
} else {
|
|
2256
2237
|
if (options.verbose) {
|
|
2257
|
-
this.logger.log("Force mode: clearing
|
|
2238
|
+
this.logger.log("Force mode: clearing tracking data to force full re-sync");
|
|
2258
2239
|
}
|
|
2259
2240
|
await this.clearManifest();
|
|
2241
|
+
this.dbChangeDetector.reset();
|
|
2260
2242
|
}
|
|
2261
2243
|
}
|
|
2262
|
-
const changes = await this.detectChanges(options.paths
|
|
2244
|
+
const changes = await this.detectChanges(options.paths);
|
|
2263
2245
|
result.changes = changes;
|
|
2264
2246
|
const docsToSync = [];
|
|
2265
2247
|
const docsByPath = new Map;
|
|
@@ -2328,10 +2310,6 @@ class SyncService {
|
|
|
2328
2310
|
}
|
|
2329
2311
|
if (!options.dryRun) {
|
|
2330
2312
|
result.entityEmbeddingsGenerated = await this.syncEntities(uniqueEntities, options);
|
|
2331
|
-
if (options.embeddings && !options.skipEmbeddings && this.embeddingService) {
|
|
2332
|
-
const repairedCount = await this.repairMissingEntityEmbeddings(options);
|
|
2333
|
-
result.entityEmbeddingsGenerated += repairedCount;
|
|
2334
|
-
}
|
|
2335
2313
|
await this.graph.checkpoint();
|
|
2336
2314
|
if (options.verbose) {
|
|
2337
2315
|
this.logger.log(`Synced ${uniqueEntities.size} entities, generated ${result.entityEmbeddingsGenerated} embeddings`);
|
|
@@ -2379,7 +2357,7 @@ class SyncService {
|
|
|
2379
2357
|
result.duration = Date.now() - startTime;
|
|
2380
2358
|
return result;
|
|
2381
2359
|
}
|
|
2382
|
-
async detectChanges(paths
|
|
2360
|
+
async detectChanges(paths) {
|
|
2383
2361
|
const changes = [];
|
|
2384
2362
|
let allDocPaths = await this.parser.discoverDocuments();
|
|
2385
2363
|
if (paths && paths.length > 0) {
|
|
@@ -2390,11 +2368,11 @@ class SyncService {
|
|
|
2390
2368
|
const pathSet = new Set(normalizedPaths);
|
|
2391
2369
|
allDocPaths = allDocPaths.filter((p) => pathSet.has(p));
|
|
2392
2370
|
}
|
|
2393
|
-
const trackedPaths = new Set(
|
|
2371
|
+
const trackedPaths = new Set(this.dbChangeDetector.getTrackedPaths());
|
|
2394
2372
|
for (const docPath of allDocPaths) {
|
|
2395
2373
|
try {
|
|
2396
2374
|
const doc = await this.parser.parseDocument(docPath);
|
|
2397
|
-
const changeType =
|
|
2375
|
+
const changeType = this.dbChangeDetector.detectChange(docPath, doc.contentHash);
|
|
2398
2376
|
changes.push({
|
|
2399
2377
|
path: docPath,
|
|
2400
2378
|
changeType,
|
|
@@ -2551,11 +2529,8 @@ class SyncService {
|
|
|
2551
2529
|
change.embeddingGenerated = embeddingGenerated;
|
|
2552
2530
|
const currentDoc = await this.parser.parseDocument(change.path);
|
|
2553
2531
|
this.manifest.updateEntry(currentDoc.path, currentDoc.contentHash, currentDoc.frontmatterHash, currentDoc.entities.length, currentDoc.relationships.length);
|
|
2554
|
-
const
|
|
2555
|
-
|
|
2556
|
-
const embeddingSourceHash = embeddingGenerated ? currentDoc.contentHash : undefined;
|
|
2557
|
-
await this.graph.updateDocumentHashes(currentDoc.path, currentDoc.contentHash, embeddingSourceHash);
|
|
2558
|
-
}
|
|
2532
|
+
const embeddingSourceHash = embeddingGenerated ? currentDoc.contentHash : undefined;
|
|
2533
|
+
await this.graph.updateDocumentHashes(currentDoc.path, currentDoc.contentHash, embeddingSourceHash);
|
|
2559
2534
|
break;
|
|
2560
2535
|
}
|
|
2561
2536
|
case "deleted": {
|
|
@@ -2640,43 +2615,6 @@ class SyncService {
|
|
|
2640
2615
|
}
|
|
2641
2616
|
}
|
|
2642
2617
|
}
|
|
2643
|
-
async repairMissingEntityEmbeddings(options) {
|
|
2644
|
-
if (!this.embeddingService) {
|
|
2645
|
-
return 0;
|
|
2646
|
-
}
|
|
2647
|
-
const labelsToCheck = [...ENTITY_TYPES, "Document"];
|
|
2648
|
-
const nodesWithMissingEmbeddings = await this.graph.findNodesWithMissingEmbeddings(labelsToCheck);
|
|
2649
|
-
if (nodesWithMissingEmbeddings.length === 0) {
|
|
2650
|
-
return 0;
|
|
2651
|
-
}
|
|
2652
|
-
if (options.verbose) {
|
|
2653
|
-
this.logger.log(`Found ${nodesWithMissingEmbeddings.length} nodes with missing embeddings, repairing...`);
|
|
2654
|
-
}
|
|
2655
|
-
let repairedCount = 0;
|
|
2656
|
-
for (const node of nodesWithMissingEmbeddings) {
|
|
2657
|
-
try {
|
|
2658
|
-
const text = node.label === "Document" ? node.name : composeEntityEmbeddingText({
|
|
2659
|
-
type: node.label,
|
|
2660
|
-
name: node.name,
|
|
2661
|
-
description: node.description,
|
|
2662
|
-
documentPaths: []
|
|
2663
|
-
});
|
|
2664
|
-
if (text.trim()) {
|
|
2665
|
-
const embedding = await this.embeddingService.generateEmbedding(text);
|
|
2666
|
-
await this.graph.updateNodeEmbedding(node.label, node.name, embedding);
|
|
2667
|
-
repairedCount++;
|
|
2668
|
-
this.logger.debug(`Repaired embedding for ${node.label}:${node.name}`);
|
|
2669
|
-
}
|
|
2670
|
-
} catch (error) {
|
|
2671
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2672
|
-
this.logger.warn(`Failed to repair embedding for ${node.label}:${node.name}: ${errorMessage}`);
|
|
2673
|
-
}
|
|
2674
|
-
}
|
|
2675
|
-
if (options.verbose && repairedCount > 0) {
|
|
2676
|
-
this.logger.log(`Repaired ${repairedCount} missing embeddings`);
|
|
2677
|
-
}
|
|
2678
|
-
return repairedCount;
|
|
2679
|
-
}
|
|
2680
2618
|
}
|
|
2681
2619
|
SyncService = __legacyDecorateClassTS([
|
|
2682
2620
|
Injectable11(),
|
|
@@ -2743,7 +2681,6 @@ class MigrateCommand extends CommandRunner3 {
|
|
|
2743
2681
|
if (!options.dryRun) {
|
|
2744
2682
|
const result = await this.syncService.sync({
|
|
2745
2683
|
force: false,
|
|
2746
|
-
useDbChangeDetection: true,
|
|
2747
2684
|
aiExtraction: true,
|
|
2748
2685
|
verbose: options.verbose,
|
|
2749
2686
|
embeddings: true
|
|
@@ -3182,23 +3119,16 @@ import { Injectable as Injectable16 } from "@nestjs/common";
|
|
|
3182
3119
|
import { Command as Command6, CommandRunner as CommandRunner6, Option as Option4 } from "nest-commander";
|
|
3183
3120
|
class StatusCommand extends CommandRunner6 {
|
|
3184
3121
|
syncService;
|
|
3185
|
-
manifestService;
|
|
3186
3122
|
dbChangeDetector;
|
|
3187
|
-
constructor(syncService,
|
|
3123
|
+
constructor(syncService, dbChangeDetector) {
|
|
3188
3124
|
super();
|
|
3189
3125
|
this.syncService = syncService;
|
|
3190
|
-
this.manifestService = manifestService;
|
|
3191
3126
|
this.dbChangeDetector = dbChangeDetector;
|
|
3192
3127
|
}
|
|
3193
3128
|
async run(_inputs, options) {
|
|
3194
3129
|
try {
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
await this.dbChangeDetector.loadHashes();
|
|
3198
|
-
} else {
|
|
3199
|
-
await this.manifestService.load();
|
|
3200
|
-
}
|
|
3201
|
-
const changes = await this.syncService.detectChanges(undefined, useDbDetection);
|
|
3130
|
+
await this.dbChangeDetector.loadHashes();
|
|
3131
|
+
const changes = await this.syncService.detectChanges();
|
|
3202
3132
|
const newDocs = changes.filter((c) => c.changeType === "new");
|
|
3203
3133
|
const updatedDocs = changes.filter((c) => c.changeType === "updated");
|
|
3204
3134
|
const deletedDocs = changes.filter((c) => c.changeType === "deleted");
|
|
@@ -3251,9 +3181,6 @@ class StatusCommand extends CommandRunner6 {
|
|
|
3251
3181
|
parseVerbose() {
|
|
3252
3182
|
return true;
|
|
3253
3183
|
}
|
|
3254
|
-
parseLegacy() {
|
|
3255
|
-
return true;
|
|
3256
|
-
}
|
|
3257
3184
|
}
|
|
3258
3185
|
__legacyDecorateClassTS([
|
|
3259
3186
|
Option4({
|
|
@@ -3264,15 +3191,6 @@ __legacyDecorateClassTS([
|
|
|
3264
3191
|
__legacyMetadataTS("design:paramtypes", []),
|
|
3265
3192
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
3266
3193
|
], StatusCommand.prototype, "parseVerbose", null);
|
|
3267
|
-
__legacyDecorateClassTS([
|
|
3268
|
-
Option4({
|
|
3269
|
-
flags: "--legacy",
|
|
3270
|
-
description: "Use legacy v1 mode: manifest-based change detection"
|
|
3271
|
-
}),
|
|
3272
|
-
__legacyMetadataTS("design:type", Function),
|
|
3273
|
-
__legacyMetadataTS("design:paramtypes", []),
|
|
3274
|
-
__legacyMetadataTS("design:returntype", Boolean)
|
|
3275
|
-
], StatusCommand.prototype, "parseLegacy", null);
|
|
3276
3194
|
StatusCommand = __legacyDecorateClassTS([
|
|
3277
3195
|
Injectable16(),
|
|
3278
3196
|
Command6({
|
|
@@ -3281,7 +3199,6 @@ StatusCommand = __legacyDecorateClassTS([
|
|
|
3281
3199
|
}),
|
|
3282
3200
|
__legacyMetadataTS("design:paramtypes", [
|
|
3283
3201
|
typeof SyncService === "undefined" ? Object : SyncService,
|
|
3284
|
-
typeof ManifestService === "undefined" ? Object : ManifestService,
|
|
3285
3202
|
typeof DatabaseChangeDetectorService === "undefined" ? Object : DatabaseChangeDetectorService
|
|
3286
3203
|
])
|
|
3287
3204
|
], StatusCommand);
|
|
@@ -3486,7 +3403,6 @@ class SyncCommand extends CommandRunner7 {
|
|
|
3486
3403
|
paths: paths.length > 0 ? paths : undefined,
|
|
3487
3404
|
skipCascade: options.skipCascade,
|
|
3488
3405
|
embeddings: options.embeddings !== false,
|
|
3489
|
-
legacy: options.legacy,
|
|
3490
3406
|
aiExtraction: !options.skipExtraction
|
|
3491
3407
|
};
|
|
3492
3408
|
console.log(`
|
|
@@ -3508,14 +3424,9 @@ class SyncCommand extends CommandRunner7 {
|
|
|
3508
3424
|
console.log(`\uD83D\uDEAB Embedding generation disabled
|
|
3509
3425
|
`);
|
|
3510
3426
|
}
|
|
3511
|
-
if (syncOptions.
|
|
3512
|
-
console.log(`\
|
|
3427
|
+
if (!syncOptions.aiExtraction) {
|
|
3428
|
+
console.log(`\u23ED\uFE0F AI entity extraction skipped (--skip-extraction)
|
|
3513
3429
|
`);
|
|
3514
|
-
} else {
|
|
3515
|
-
if (!syncOptions.aiExtraction) {
|
|
3516
|
-
console.log(`\u23ED\uFE0F AI entity extraction skipped (--skip-extraction)
|
|
3517
|
-
`);
|
|
3518
|
-
}
|
|
3519
3430
|
}
|
|
3520
3431
|
if (syncOptions.paths) {
|
|
3521
3432
|
console.log(`\uD83D\uDCC1 Syncing specific paths: ${syncOptions.paths.join(", ")}
|
|
@@ -3713,9 +3624,6 @@ class SyncCommand extends CommandRunner7 {
|
|
|
3713
3624
|
parseSkipExtraction() {
|
|
3714
3625
|
return true;
|
|
3715
3626
|
}
|
|
3716
|
-
parseLegacy() {
|
|
3717
|
-
return true;
|
|
3718
|
-
}
|
|
3719
3627
|
}
|
|
3720
3628
|
__legacyDecorateClassTS([
|
|
3721
3629
|
Option5({
|
|
@@ -3789,15 +3697,6 @@ __legacyDecorateClassTS([
|
|
|
3789
3697
|
__legacyMetadataTS("design:paramtypes", []),
|
|
3790
3698
|
__legacyMetadataTS("design:returntype", Boolean)
|
|
3791
3699
|
], SyncCommand.prototype, "parseSkipExtraction", null);
|
|
3792
|
-
__legacyDecorateClassTS([
|
|
3793
|
-
Option5({
|
|
3794
|
-
flags: "--legacy",
|
|
3795
|
-
description: "Use legacy v1 mode: manifest-based change detection, no AI extraction"
|
|
3796
|
-
}),
|
|
3797
|
-
__legacyMetadataTS("design:type", Function),
|
|
3798
|
-
__legacyMetadataTS("design:paramtypes", []),
|
|
3799
|
-
__legacyMetadataTS("design:returntype", Boolean)
|
|
3800
|
-
], SyncCommand.prototype, "parseLegacy", null);
|
|
3801
3700
|
SyncCommand = __legacyDecorateClassTS([
|
|
3802
3701
|
Injectable18(),
|
|
3803
3702
|
Command7({
|