@utaba/deep-memory 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/{StorageProvider-BGtKZZt4.d.cts → StorageProvider-DDTLUcuj.d.ts} +32 -8
- package/dist/{StorageProvider-D_o2Hksf.d.ts → StorageProvider-DGH7bVuK.d.cts} +32 -8
- package/dist/index.cjs +380 -99
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +71 -13
- package/dist/index.d.ts +71 -13
- package/dist/index.js +375 -99
- package/dist/index.js.map +1 -1
- package/dist/{portability-P1hL-bOa.d.cts → portability-Dbm6vCdD.d.cts} +15 -3
- package/dist/{portability-P1hL-bOa.d.ts → portability-Dbm6vCdD.d.ts} +15 -3
- package/dist/providers/index.cjs.map +1 -1
- package/dist/providers/index.d.cts +3 -3
- package/dist/providers/index.d.ts +3 -3
- package/dist/testing/conformance.cjs +10 -0
- package/dist/testing/conformance.cjs.map +1 -1
- package/dist/testing/conformance.d.cts +2 -2
- package/dist/testing/conformance.d.ts +2 -2
- package/dist/testing/conformance.js +10 -0
- package/dist/testing/conformance.js.map +1 -1
- package/dist/{traversal-CBE5h5ob.d.cts → traversal-1f35drOx.d.cts} +24 -3
- package/dist/{traversal-B_84kGaO.d.ts → traversal-OkXViSTI.d.ts} +24 -3
- package/dist/types/index.d.cts +13 -4
- package/dist/types/index.d.ts +13 -4
- package/package.json +2 -3
package/dist/index.js
CHANGED
|
@@ -280,6 +280,14 @@ var TraversalTimeoutError = class extends DeepMemoryError {
|
|
|
280
280
|
this.timeoutMs = timeoutMs;
|
|
281
281
|
}
|
|
282
282
|
};
|
|
283
|
+
var UnsupportedQueryError = class extends DeepMemoryError {
|
|
284
|
+
provider;
|
|
285
|
+
constructor(provider, message, suggestion) {
|
|
286
|
+
super("UNSUPPORTED_QUERY", message, suggestion);
|
|
287
|
+
this.name = "UnsupportedQueryError";
|
|
288
|
+
this.provider = provider;
|
|
289
|
+
}
|
|
290
|
+
};
|
|
283
291
|
|
|
284
292
|
// src/core/DeepMemory.ts
|
|
285
293
|
import { randomUUID } from "crypto";
|
|
@@ -833,10 +841,15 @@ var EntityManager = class {
|
|
|
833
841
|
attempt++;
|
|
834
842
|
if (attempt > maxRetries) {
|
|
835
843
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
844
|
+
const failureMessage = `embedBatch failed after ${maxRetries} retries: ${errorMsg}`;
|
|
845
|
+
const failed = entries.map(([id]) => ({ entityId: id, error: failureMessage }));
|
|
846
|
+
for (const { entityId, error } of failed) {
|
|
847
|
+
await options?.onItemFailed?.(entityId, error);
|
|
848
|
+
}
|
|
836
849
|
return {
|
|
837
850
|
processed: 0,
|
|
838
851
|
failed: entries.length,
|
|
839
|
-
errors:
|
|
852
|
+
errors: failed,
|
|
840
853
|
modelId: this.embedding.modelId(),
|
|
841
854
|
dimensions: this.embedding.dimensions()
|
|
842
855
|
};
|
|
@@ -859,7 +872,9 @@ var EntityManager = class {
|
|
|
859
872
|
});
|
|
860
873
|
processed++;
|
|
861
874
|
} catch (err) {
|
|
862
|
-
|
|
875
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
876
|
+
errors.push({ entityId: id, error: errorMessage });
|
|
877
|
+
await options?.onItemFailed?.(id, errorMessage);
|
|
863
878
|
}
|
|
864
879
|
}
|
|
865
880
|
return {
|
|
@@ -891,7 +906,7 @@ var EntityManager = class {
|
|
|
891
906
|
let totalFailed = 0;
|
|
892
907
|
const allErrors = [];
|
|
893
908
|
let offset = 0;
|
|
894
|
-
while (offset < total) {
|
|
909
|
+
while (total === void 0 || offset < total) {
|
|
895
910
|
if (signal?.aborted) {
|
|
896
911
|
throw new OperationAbortedError("reembedAll");
|
|
897
912
|
}
|
|
@@ -902,7 +917,8 @@ var EntityManager = class {
|
|
|
902
917
|
if (page.items.length === 0) break;
|
|
903
918
|
const ids = page.items.map((e) => e.id);
|
|
904
919
|
const result = await this.reembedEntities(ids, {
|
|
905
|
-
maxRetries: options?.maxRetries
|
|
920
|
+
maxRetries: options?.maxRetries,
|
|
921
|
+
onItemFailed: options?.onItemFailed
|
|
906
922
|
});
|
|
907
923
|
totalProcessed += result.processed;
|
|
908
924
|
totalFailed += result.failed;
|
|
@@ -917,7 +933,8 @@ var EntityManager = class {
|
|
|
917
933
|
throw new OperationAbortedError("reembedAll");
|
|
918
934
|
}
|
|
919
935
|
const delayMs = options?.delayBetweenBatchesMs ?? 0;
|
|
920
|
-
|
|
936
|
+
const moreLikely = total === void 0 ? page.items.length === batchSize : offset < total;
|
|
937
|
+
if (delayMs > 0 && moreLikely) {
|
|
921
938
|
await new Promise((resolve) => {
|
|
922
939
|
setTimeout(resolve, delayMs);
|
|
923
940
|
});
|
|
@@ -1346,22 +1363,29 @@ async function executeFallbackTraversal(repositoryId, storage, spec) {
|
|
|
1346
1363
|
const dedup = spec.dedup !== false;
|
|
1347
1364
|
const limit = spec.limit ?? 50;
|
|
1348
1365
|
const offset = spec.offset ?? 0;
|
|
1349
|
-
let
|
|
1366
|
+
let paged;
|
|
1367
|
+
let pagedAllRels = [];
|
|
1368
|
+
let total;
|
|
1369
|
+
let unionFullLength = 0;
|
|
1350
1370
|
if (spec.returnMode === "all") {
|
|
1351
|
-
|
|
1371
|
+
const unionElements = buildAllModeUnion(allCollected);
|
|
1372
|
+
unionFullLength = unionElements.length;
|
|
1373
|
+
const pageSlice = unionElements.slice(offset, offset + limit);
|
|
1374
|
+
const pageEntries = [];
|
|
1375
|
+
for (const element of pageSlice) {
|
|
1376
|
+
if (element.kind === "entity") {
|
|
1377
|
+
pageEntries.push(element.entry);
|
|
1378
|
+
} else {
|
|
1379
|
+
pagedAllRels.push(element.rel);
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
paged = pageEntries;
|
|
1383
|
+
total = pageEntries.length + pagedAllRels.length;
|
|
1352
1384
|
} else {
|
|
1353
|
-
resultEntries = frontier;
|
|
1385
|
+
const resultEntries = dedup && spec.returnMode !== "path" ? dedupEntriesById(frontier) : frontier;
|
|
1386
|
+
total = resultEntries.length;
|
|
1387
|
+
paged = resultEntries.slice(offset, offset + limit);
|
|
1354
1388
|
}
|
|
1355
|
-
if (dedup) {
|
|
1356
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1357
|
-
resultEntries = resultEntries.filter((entry) => {
|
|
1358
|
-
if (seen.has(entry.entity.id)) return false;
|
|
1359
|
-
seen.add(entry.entity.id);
|
|
1360
|
-
return true;
|
|
1361
|
-
});
|
|
1362
|
-
}
|
|
1363
|
-
const total = resultEntries.length;
|
|
1364
|
-
const paged = resultEntries.slice(offset, offset + limit);
|
|
1365
1389
|
const suppressEntities = spec.projection !== void 0 && spec.projection.includeEntities !== true;
|
|
1366
1390
|
let entities;
|
|
1367
1391
|
if (suppressEntities) {
|
|
@@ -1402,7 +1426,8 @@ async function executeFallbackTraversal(repositoryId, storage, spec) {
|
|
|
1402
1426
|
const propNames = spec.projection.properties;
|
|
1403
1427
|
const mode = spec.projection.mode ?? "values";
|
|
1404
1428
|
const distinct = spec.projection.distinct ?? false;
|
|
1405
|
-
const
|
|
1429
|
+
const aggregateDedup = spec.returnMode === "all" || dedup && spec.returnMode !== "path";
|
|
1430
|
+
const sourceEntries = aggregateDedup ? (() => {
|
|
1406
1431
|
const seen = /* @__PURE__ */ new Set();
|
|
1407
1432
|
return (spec.returnMode === "all" ? allCollected : frontier).filter((e) => {
|
|
1408
1433
|
if (seen.has(e.entity.id)) return false;
|
|
@@ -1444,17 +1469,28 @@ async function executeFallbackTraversal(repositoryId, storage, spec) {
|
|
|
1444
1469
|
}
|
|
1445
1470
|
let relationships;
|
|
1446
1471
|
let paths;
|
|
1447
|
-
if (!suppressEntities &&
|
|
1472
|
+
if (!suppressEntities && spec.returnMode === "all") {
|
|
1473
|
+
relationships = pagedAllRels.map((rel) => ({
|
|
1474
|
+
id: rel.id,
|
|
1475
|
+
type: rel.relationshipType,
|
|
1476
|
+
sourceEntityId: rel.sourceEntityId,
|
|
1477
|
+
targetEntityId: rel.targetEntityId,
|
|
1478
|
+
direction: "outbound",
|
|
1479
|
+
properties: rel.properties
|
|
1480
|
+
}));
|
|
1481
|
+
} else if (!suppressEntities && spec.returnMode === "path") {
|
|
1448
1482
|
const relMap = /* @__PURE__ */ new Map();
|
|
1449
1483
|
for (const entry of paged) {
|
|
1450
|
-
for (
|
|
1484
|
+
for (let i = 0; i < entry.relationshipPath.length; i++) {
|
|
1485
|
+
const rel = entry.relationshipPath[i];
|
|
1451
1486
|
if (!relMap.has(rel.id)) {
|
|
1487
|
+
const fromId = entry.entityPath[i];
|
|
1452
1488
|
relMap.set(rel.id, {
|
|
1453
1489
|
id: rel.id,
|
|
1454
1490
|
type: rel.relationshipType,
|
|
1455
1491
|
sourceEntityId: rel.sourceEntityId,
|
|
1456
1492
|
targetEntityId: rel.targetEntityId,
|
|
1457
|
-
direction: "outbound",
|
|
1493
|
+
direction: fromId === rel.sourceEntityId ? "outbound" : "inbound",
|
|
1458
1494
|
properties: rel.properties
|
|
1459
1495
|
});
|
|
1460
1496
|
}
|
|
@@ -1463,39 +1499,61 @@ async function executeFallbackTraversal(repositoryId, storage, spec) {
|
|
|
1463
1499
|
relationships = Array.from(relMap.values());
|
|
1464
1500
|
}
|
|
1465
1501
|
if (!suppressEntities && spec.returnMode === "path") {
|
|
1502
|
+
const pathEntityIds = /* @__PURE__ */ new Set();
|
|
1503
|
+
for (const entry of paged) {
|
|
1504
|
+
for (const id of entry.entityPath) {
|
|
1505
|
+
pathEntityIds.add(id);
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
const pathEntityMap = pathEntityIds.size > 0 ? await storage.getEntities(repositoryId, [...pathEntityIds]) : /* @__PURE__ */ new Map();
|
|
1509
|
+
if (pathEntityIds.size > 0) {
|
|
1510
|
+
storageCalls++;
|
|
1511
|
+
}
|
|
1466
1512
|
paths = paged.map((entry) => ({
|
|
1467
1513
|
length: entry.entityPath.length - 1,
|
|
1468
|
-
entities:
|
|
1469
|
-
const
|
|
1470
|
-
if (!
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1514
|
+
entities: entry.entityPath.map((id) => {
|
|
1515
|
+
const stored = pathEntityMap.get(id);
|
|
1516
|
+
if (!stored) {
|
|
1517
|
+
throw new EntityNotFoundError(id);
|
|
1518
|
+
}
|
|
1519
|
+
const projected = projectEntity(stored, detailLevel);
|
|
1520
|
+
if (!spec.includeProvenance) {
|
|
1521
|
+
delete projected["provenance"];
|
|
1522
|
+
}
|
|
1523
|
+
return projected;
|
|
1524
|
+
}),
|
|
1525
|
+
relationships: entry.relationshipPath.map((rel, i) => {
|
|
1526
|
+
const fromId = entry.entityPath[i];
|
|
1527
|
+
return {
|
|
1528
|
+
id: rel.id,
|
|
1529
|
+
type: rel.relationshipType,
|
|
1530
|
+
sourceEntityId: rel.sourceEntityId,
|
|
1531
|
+
targetEntityId: rel.targetEntityId,
|
|
1532
|
+
direction: fromId === rel.sourceEntityId ? "outbound" : "inbound",
|
|
1533
|
+
properties: rel.properties
|
|
1534
|
+
};
|
|
1535
|
+
})
|
|
1481
1536
|
}));
|
|
1482
1537
|
}
|
|
1483
1538
|
const executionTimeMs = Date.now() - startTime;
|
|
1539
|
+
const returned = spec.returnMode === "all" ? total : paged.length;
|
|
1540
|
+
const hasMore = spec.returnMode === "all" ? offset + limit < unionFullLength : offset + limit < total;
|
|
1541
|
+
const truncated = spec.returnMode === "all" ? unionFullLength > limit + offset : total > limit + offset;
|
|
1484
1542
|
return {
|
|
1485
1543
|
entities,
|
|
1486
1544
|
relationships,
|
|
1487
1545
|
paths,
|
|
1488
1546
|
aggregations,
|
|
1489
1547
|
total,
|
|
1490
|
-
returned
|
|
1491
|
-
hasMore
|
|
1548
|
+
returned,
|
|
1549
|
+
hasMore,
|
|
1492
1550
|
queryMetadata: {
|
|
1493
1551
|
executionTimeMs,
|
|
1494
1552
|
appliedLimits: {
|
|
1495
1553
|
maxResults: limit
|
|
1496
1554
|
},
|
|
1497
|
-
truncated
|
|
1498
|
-
truncationReason:
|
|
1555
|
+
truncated,
|
|
1556
|
+
truncationReason: truncated ? "result_limit" : void 0
|
|
1499
1557
|
}
|
|
1500
1558
|
};
|
|
1501
1559
|
}
|
|
@@ -1611,6 +1669,58 @@ function getTargetId(rel, currentEntityId) {
|
|
|
1611
1669
|
}
|
|
1612
1670
|
return rel.sourceEntityId;
|
|
1613
1671
|
}
|
|
1672
|
+
function dedupEntriesById(entries) {
|
|
1673
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1674
|
+
const result = [];
|
|
1675
|
+
for (const entry of entries) {
|
|
1676
|
+
if (!seen.has(entry.entity.id)) {
|
|
1677
|
+
seen.add(entry.entity.id);
|
|
1678
|
+
result.push(entry);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
return result;
|
|
1682
|
+
}
|
|
1683
|
+
function buildAllModeUnion(entries) {
|
|
1684
|
+
if (entries.length === 0) return [];
|
|
1685
|
+
let maxDepth = 0;
|
|
1686
|
+
const byDepth = /* @__PURE__ */ new Map();
|
|
1687
|
+
for (const entry of entries) {
|
|
1688
|
+
const depth = entry.relationshipPath.length;
|
|
1689
|
+
if (depth > maxDepth) maxDepth = depth;
|
|
1690
|
+
let bucket = byDepth.get(depth);
|
|
1691
|
+
if (!bucket) {
|
|
1692
|
+
bucket = [];
|
|
1693
|
+
byDepth.set(depth, bucket);
|
|
1694
|
+
}
|
|
1695
|
+
bucket.push(entry);
|
|
1696
|
+
}
|
|
1697
|
+
const seenEntities = /* @__PURE__ */ new Set();
|
|
1698
|
+
const seenRels = /* @__PURE__ */ new Set();
|
|
1699
|
+
const result = [];
|
|
1700
|
+
for (const entry of byDepth.get(0) ?? []) {
|
|
1701
|
+
if (!seenEntities.has(entry.entity.id)) {
|
|
1702
|
+
seenEntities.add(entry.entity.id);
|
|
1703
|
+
result.push({ kind: "entity", entry });
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
for (let depth = 1; depth <= maxDepth; depth++) {
|
|
1707
|
+
const bucket = byDepth.get(depth) ?? [];
|
|
1708
|
+
for (const entry of bucket) {
|
|
1709
|
+
const newRel = entry.relationshipPath[depth - 1];
|
|
1710
|
+
if (newRel && !seenRels.has(newRel.id)) {
|
|
1711
|
+
seenRels.add(newRel.id);
|
|
1712
|
+
result.push({ kind: "relationship", rel: newRel });
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
for (const entry of bucket) {
|
|
1716
|
+
if (!seenEntities.has(entry.entity.id)) {
|
|
1717
|
+
seenEntities.add(entry.entity.id);
|
|
1718
|
+
result.push({ kind: "entity", entry });
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
return result;
|
|
1723
|
+
}
|
|
1614
1724
|
|
|
1615
1725
|
// src/relationships/GraphTraversal.ts
|
|
1616
1726
|
var GraphTraversal = class {
|
|
@@ -1871,11 +1981,15 @@ var SearchOrchestrator = class {
|
|
|
1871
1981
|
const offset = options?.offset ?? 0;
|
|
1872
1982
|
const threshold = options?.similarityThreshold ?? this.defaultSimilarityThreshold;
|
|
1873
1983
|
const queryEmbedding = await this.embedding.embed(query);
|
|
1874
|
-
const allEntities = await this.storage.findEntities(
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1984
|
+
const allEntities = await this.storage.findEntities(
|
|
1985
|
+
this.repositoryId,
|
|
1986
|
+
{
|
|
1987
|
+
entityTypes: options?.entityTypes,
|
|
1988
|
+
limit: this.conceptSearchScanLimit,
|
|
1989
|
+
offset: 0
|
|
1990
|
+
},
|
|
1991
|
+
{ loadEmbeddings: true }
|
|
1992
|
+
);
|
|
1879
1993
|
const scored = [];
|
|
1880
1994
|
for (const entity of allEntities.items) {
|
|
1881
1995
|
let score;
|
|
@@ -2331,6 +2445,9 @@ var MemoryRepository = class {
|
|
|
2331
2445
|
signal: options?.signal,
|
|
2332
2446
|
onProgress: async (processed, total, failed) => {
|
|
2333
2447
|
await options?.onProgress?.({ processed, totalEntities: total, failed });
|
|
2448
|
+
},
|
|
2449
|
+
onItemFailed: async (entityId, error) => {
|
|
2450
|
+
await options?.onItemFailed?.({ entityId, error });
|
|
2334
2451
|
}
|
|
2335
2452
|
});
|
|
2336
2453
|
await this.storage.updateRepository(this.repositoryId, {
|
|
@@ -3255,7 +3372,14 @@ var VocabularyEngine = class {
|
|
|
3255
3372
|
async proposeExtension(proposal, proposedBy) {
|
|
3256
3373
|
return this.proposeChange(proposal, proposedBy);
|
|
3257
3374
|
}
|
|
3258
|
-
/**
|
|
3375
|
+
/**
|
|
3376
|
+
* Cascade-delete all data for a deleted vocabulary type.
|
|
3377
|
+
*
|
|
3378
|
+
* `deletedRelationships` may be `undefined` when the underlying provider
|
|
3379
|
+
* does not count cascaded edges (see StorageProvider.deleteEntitiesByType).
|
|
3380
|
+
* The return value is currently discarded by the only caller; the type is
|
|
3381
|
+
* preserved for symmetry with the storage contract.
|
|
3382
|
+
*/
|
|
3259
3383
|
async cascadeDeleteData(proposal) {
|
|
3260
3384
|
if (proposal.proposalType === "delete_entity_type" && proposal.deleteEntityType) {
|
|
3261
3385
|
return this.storage.deleteEntitiesByType(
|
|
@@ -3453,7 +3577,7 @@ var EventBus = class {
|
|
|
3453
3577
|
};
|
|
3454
3578
|
|
|
3455
3579
|
// src/portability/RepositoryExporter.ts
|
|
3456
|
-
var LIBRARY_VERSION = true ? "0.
|
|
3580
|
+
var LIBRARY_VERSION = true ? "0.17.0" : "0.1.0";
|
|
3457
3581
|
var RepositoryExporter = class {
|
|
3458
3582
|
storage;
|
|
3459
3583
|
provenance;
|
|
@@ -3807,12 +3931,14 @@ var RepositoryImporter = class {
|
|
|
3807
3931
|
storage;
|
|
3808
3932
|
actorId;
|
|
3809
3933
|
onProgress;
|
|
3934
|
+
onItemFailed;
|
|
3810
3935
|
signal;
|
|
3811
3936
|
migrationEngine = new MigrationEngine();
|
|
3812
3937
|
constructor(config) {
|
|
3813
3938
|
this.storage = config.storage;
|
|
3814
3939
|
this.actorId = config.actorId;
|
|
3815
3940
|
this.onProgress = config.onProgress;
|
|
3941
|
+
this.onItemFailed = config.onItemFailed;
|
|
3816
3942
|
this.signal = config.signal;
|
|
3817
3943
|
}
|
|
3818
3944
|
throwIfAborted() {
|
|
@@ -3909,6 +4035,8 @@ var RepositoryImporter = class {
|
|
|
3909
4035
|
message: `Failed to import ${e.item}: ${e.error}`,
|
|
3910
4036
|
id: e.item
|
|
3911
4037
|
});
|
|
4038
|
+
const itemType = chunk.entities?.some((x) => x.id === e.item) ? "entity" : chunk.relationships?.some((x) => x.id === e.item) ? "relationship" : "entity";
|
|
4039
|
+
await this.onItemFailed?.({ itemId: e.item, itemType, error: e.error });
|
|
3912
4040
|
}
|
|
3913
4041
|
await this.onProgress?.({
|
|
3914
4042
|
repositoryId: target.repositoryId,
|
|
@@ -4045,11 +4173,13 @@ var RepositoryImporter = class {
|
|
|
4045
4173
|
const targetExists = await this.storage.getEntity(repositoryId, rel.targetEntityId);
|
|
4046
4174
|
if (!sourceExists || !targetExists) {
|
|
4047
4175
|
relationshipsSkipped++;
|
|
4176
|
+
const errorMsg = `source or target entity missing`;
|
|
4048
4177
|
warnings.push({
|
|
4049
4178
|
code: "relationship_orphaned",
|
|
4050
|
-
message: `Relationship "${rel.id}" skipped \u2014
|
|
4179
|
+
message: `Relationship "${rel.id}" skipped \u2014 ${errorMsg}`,
|
|
4051
4180
|
relationshipId: rel.id
|
|
4052
4181
|
});
|
|
4182
|
+
await this.onItemFailed?.({ itemId: rel.id, itemType: "relationship", error: errorMsg });
|
|
4053
4183
|
} else {
|
|
4054
4184
|
try {
|
|
4055
4185
|
await this.storage.createRelationship(repositoryId, rel);
|
|
@@ -4396,6 +4526,12 @@ var DeepMemory = class {
|
|
|
4396
4526
|
storage: this.storage,
|
|
4397
4527
|
actorId: this.provenance.getContext().actorId,
|
|
4398
4528
|
onProgress: (progress) => this.globalEventBus.emit("import:progress", progress),
|
|
4529
|
+
onItemFailed: (failure) => this.globalEventBus.emit("import:item-failed", {
|
|
4530
|
+
repositoryId: options.target.repositoryId,
|
|
4531
|
+
itemId: failure.itemId,
|
|
4532
|
+
itemType: failure.itemType,
|
|
4533
|
+
error: failure.error
|
|
4534
|
+
}),
|
|
4399
4535
|
signal: options.signal
|
|
4400
4536
|
});
|
|
4401
4537
|
try {
|
|
@@ -4489,6 +4625,12 @@ var DeepMemory = class {
|
|
|
4489
4625
|
storage: this.storage,
|
|
4490
4626
|
actorId: this.provenance.getContext().actorId,
|
|
4491
4627
|
onProgress: (progress) => this.globalEventBus.emit("import:progress", progress),
|
|
4628
|
+
onItemFailed: (failure) => this.globalEventBus.emit("import:item-failed", {
|
|
4629
|
+
repositoryId: options.target.repositoryId,
|
|
4630
|
+
itemId: failure.itemId,
|
|
4631
|
+
itemType: failure.itemType,
|
|
4632
|
+
error: failure.error
|
|
4633
|
+
}),
|
|
4492
4634
|
signal: options.signal
|
|
4493
4635
|
});
|
|
4494
4636
|
await this.globalEventBus.emit("import:started", { repositoryId: options.target.repositoryId });
|
|
@@ -4538,11 +4680,21 @@ var DeepMemory = class {
|
|
|
4538
4680
|
try {
|
|
4539
4681
|
const result = await repo.reembedAll({
|
|
4540
4682
|
...options,
|
|
4683
|
+
// The repo signature widens `totalEntities` to `number | undefined`
|
|
4684
|
+
// because PaginatedResult.total may be undefined under some provider/
|
|
4685
|
+
// query combinations. The `reembed:progress` event contract is
|
|
4686
|
+
// `totalEntities: number`, so fall back to the cached stats count
|
|
4687
|
+
// when the inner layer doesn't supply one.
|
|
4541
4688
|
onProgress: (progress) => this.globalEventBus.emit("reembed:progress", {
|
|
4542
4689
|
repositoryId,
|
|
4543
4690
|
processed: progress.processed,
|
|
4544
|
-
totalEntities: progress.totalEntities,
|
|
4691
|
+
totalEntities: progress.totalEntities ?? stats.entityCount,
|
|
4545
4692
|
failed: progress.failed
|
|
4693
|
+
}),
|
|
4694
|
+
onItemFailed: (failure) => this.globalEventBus.emit("reembed:item-failed", {
|
|
4695
|
+
repositoryId,
|
|
4696
|
+
entityId: failure.entityId,
|
|
4697
|
+
error: failure.error
|
|
4546
4698
|
})
|
|
4547
4699
|
});
|
|
4548
4700
|
await this.globalEventBus.emit("reembed:completed", {
|
|
@@ -4595,6 +4747,69 @@ function createSafeSink(sink) {
|
|
|
4595
4747
|
|
|
4596
4748
|
// src/relationships/compilers/GremlinCompiler.ts
|
|
4597
4749
|
var DEFAULT_ESTIMATED_FANOUT_PER_HOP = 10;
|
|
4750
|
+
var VERTEX_PROJECTION = [
|
|
4751
|
+
["__kind", `.by(constant('v'))`],
|
|
4752
|
+
["id", `.by(id)`],
|
|
4753
|
+
["entityType", `.by('entityType')`],
|
|
4754
|
+
["entityLabel", `.by('entityLabel')`],
|
|
4755
|
+
["slug", `.by('slug')`],
|
|
4756
|
+
["summary", `.by(coalesce(values('summary'), constant('')))`],
|
|
4757
|
+
["properties", `.by(coalesce(values('properties'), constant('{}')))`],
|
|
4758
|
+
["data", `.by(coalesce(values('data'), constant('')))`],
|
|
4759
|
+
["dataFormat", `.by(coalesce(values('dataFormat'), constant('')))`],
|
|
4760
|
+
["createdBy", `.by('createdBy')`],
|
|
4761
|
+
["createdByType", `.by('createdByType')`],
|
|
4762
|
+
["createdAt", `.by('createdAt')`],
|
|
4763
|
+
["createdInConversation", `.by(coalesce(values('createdInConversation'), constant('')))`],
|
|
4764
|
+
["createdFromMessage", `.by(coalesce(values('createdFromMessage'), constant('')))`],
|
|
4765
|
+
["modifiedBy", `.by('modifiedBy')`],
|
|
4766
|
+
["modifiedByType", `.by('modifiedByType')`],
|
|
4767
|
+
["modifiedAt", `.by('modifiedAt')`],
|
|
4768
|
+
["modifiedInConversation", `.by(coalesce(values('modifiedInConversation'), constant('')))`],
|
|
4769
|
+
["modifiedFromMessage", `.by(coalesce(values('modifiedFromMessage'), constant('')))`]
|
|
4770
|
+
];
|
|
4771
|
+
var EDGE_PROJECTION = [
|
|
4772
|
+
["__kind", `.by(constant('e'))`],
|
|
4773
|
+
["id", `.by(id)`],
|
|
4774
|
+
["relationshipType", `.by('relationshipType')`],
|
|
4775
|
+
["sourceEntityId", `.by('sourceEntityId')`],
|
|
4776
|
+
["targetEntityId", `.by('targetEntityId')`],
|
|
4777
|
+
["properties", `.by(coalesce(values('properties'), constant('{}')))`],
|
|
4778
|
+
["bidirectional", `.by(coalesce(values('bidirectional'), constant(false)))`],
|
|
4779
|
+
["createdBy", `.by('createdBy')`],
|
|
4780
|
+
["createdByType", `.by('createdByType')`],
|
|
4781
|
+
["createdAt", `.by('createdAt')`],
|
|
4782
|
+
["createdInConversation", `.by(coalesce(values('createdInConversation'), constant('')))`],
|
|
4783
|
+
["createdFromMessage", `.by(coalesce(values('createdFromMessage'), constant('')))`],
|
|
4784
|
+
["modifiedBy", `.by('modifiedBy')`],
|
|
4785
|
+
["modifiedByType", `.by('modifiedByType')`],
|
|
4786
|
+
["modifiedAt", `.by('modifiedAt')`],
|
|
4787
|
+
["modifiedInConversation", `.by(coalesce(values('modifiedInConversation'), constant('')))`],
|
|
4788
|
+
["modifiedFromMessage", `.by(coalesce(values('modifiedFromMessage'), constant('')))`]
|
|
4789
|
+
];
|
|
4790
|
+
var EMBEDDING_PROJECTION_ENTRY = [
|
|
4791
|
+
"embedding",
|
|
4792
|
+
`.by(coalesce(values('embedding'), constant('')))`
|
|
4793
|
+
];
|
|
4794
|
+
function buildProjectExpression(entries) {
|
|
4795
|
+
const keys = entries.map(([k]) => `'${k}'`).join(",");
|
|
4796
|
+
const bys = entries.map(([, by]) => by).join("");
|
|
4797
|
+
return `project(${keys})${bys}`;
|
|
4798
|
+
}
|
|
4799
|
+
var VERTEX_PROJECT_EXPR = buildProjectExpression(VERTEX_PROJECTION);
|
|
4800
|
+
var VERTEX_PROJECT_EXPR_WITH_EMBEDDING = buildProjectExpression([
|
|
4801
|
+
...VERTEX_PROJECTION,
|
|
4802
|
+
EMBEDDING_PROJECTION_ENTRY
|
|
4803
|
+
]);
|
|
4804
|
+
var EDGE_PROJECT_EXPR = buildProjectExpression(EDGE_PROJECTION);
|
|
4805
|
+
var GREMLIN_VERTEX_PROJECTION_FIELDS = VERTEX_PROJECTION.map(([k]) => k).filter((k) => k !== "__kind");
|
|
4806
|
+
var GREMLIN_EDGE_PROJECTION_FIELDS = EDGE_PROJECTION.map(([k]) => k).filter((k) => k !== "__kind");
|
|
4807
|
+
function buildVertexProjectChain(opts) {
|
|
4808
|
+
return opts?.withEmbedding ? VERTEX_PROJECT_EXPR_WITH_EMBEDDING : VERTEX_PROJECT_EXPR;
|
|
4809
|
+
}
|
|
4810
|
+
function buildEdgeProjectChain() {
|
|
4811
|
+
return EDGE_PROJECT_EXPR;
|
|
4812
|
+
}
|
|
4598
4813
|
var GremlinCompiler = class {
|
|
4599
4814
|
language = "gremlin";
|
|
4600
4815
|
compile(spec, _vocabulary) {
|
|
@@ -4610,7 +4825,7 @@ var GremlinCompiler = class {
|
|
|
4610
4825
|
parts.push("g.V()");
|
|
4611
4826
|
if (spec.start.entityId) {
|
|
4612
4827
|
const p = nextParam(spec.start.entityId);
|
|
4613
|
-
parts.push(`.
|
|
4828
|
+
parts.push(`.hasId(${p})`);
|
|
4614
4829
|
} else if (spec.start.entityType) {
|
|
4615
4830
|
const p = nextParam(spec.start.entityType);
|
|
4616
4831
|
parts.push(`.has('entityType', ${p})`);
|
|
@@ -4622,44 +4837,79 @@ var GremlinCompiler = class {
|
|
|
4622
4837
|
}
|
|
4623
4838
|
}
|
|
4624
4839
|
const steps = spec.steps ?? [];
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
estimatedFanOut *= DEFAULT_ESTIMATED_FANOUT_PER_HOP;
|
|
4632
|
-
} else {
|
|
4633
|
-
parts.push(compileSimpleStep(step, nextParam));
|
|
4634
|
-
estimatedFanOut *= DEFAULT_ESTIMATED_FANOUT_PER_HOP;
|
|
4840
|
+
const returnMode = spec.returnMode ?? "terminal";
|
|
4841
|
+
if (returnMode === "all") {
|
|
4842
|
+
if (steps.some((s) => s.repeat)) {
|
|
4843
|
+
throw new Error(
|
|
4844
|
+
"GremlinCompiler: 'all' returnMode does not support repeat steps. Use 'terminal' or 'path' mode, or unroll the repeat into explicit steps."
|
|
4845
|
+
);
|
|
4635
4846
|
}
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
4847
|
+
const compiledSteps = steps.map((step) => ({
|
|
4848
|
+
edge: compileEdgeOnly(step, nextParam),
|
|
4849
|
+
vertex: compileVertexHop(step),
|
|
4850
|
+
// Per plan §3: entity-type/property filters apply only on branches
|
|
4851
|
+
// ending at that depth's vertex. They are NOT included in the prefix
|
|
4852
|
+
// that deeper branches traverse through.
|
|
4853
|
+
entityFilters: compileEntityFilters(step, nextParam)
|
|
4854
|
+
}));
|
|
4855
|
+
const branches = [`__.identity().${VERTEX_PROJECT_EXPR}`];
|
|
4856
|
+
let prefix = "";
|
|
4857
|
+
for (const { edge, vertex, entityFilters } of compiledSteps) {
|
|
4858
|
+
branches.push(`__${prefix}${edge}.${EDGE_PROJECT_EXPR}`);
|
|
4859
|
+
branches.push(`__${prefix}${edge}${vertex}${entityFilters}.${VERTEX_PROJECT_EXPR}`);
|
|
4860
|
+
prefix = `${prefix}${edge}${vertex}`;
|
|
4861
|
+
estimatedFanOut *= DEFAULT_ESTIMATED_FANOUT_PER_HOP;
|
|
4639
4862
|
}
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4863
|
+
parts.push(`.union(${branches.join(", ")})`);
|
|
4864
|
+
parts.push(`.dedup().by(select('id'))`);
|
|
4865
|
+
const limit = spec.limit ?? 50;
|
|
4866
|
+
const offset = spec.offset ?? 0;
|
|
4867
|
+
const pOffset = nextParam(offset);
|
|
4868
|
+
const pEnd = nextParam(offset + limit);
|
|
4869
|
+
parts.push(`.range(${pOffset}, ${pEnd})`);
|
|
4870
|
+
params["_limit"] = limit;
|
|
4871
|
+
params["_offset"] = offset;
|
|
4872
|
+
} else {
|
|
4873
|
+
const useEdgeEmission = returnMode === "path";
|
|
4874
|
+
for (const step of steps) {
|
|
4875
|
+
if (step.repeat) {
|
|
4876
|
+
parts.push(compileRepeatStep(step, nextParam, useEdgeEmission));
|
|
4877
|
+
estimatedFanOut *= step.repeat.maxDepth * DEFAULT_ESTIMATED_FANOUT_PER_HOP;
|
|
4878
|
+
} else if (useEdgeEmission || step.relationshipFilter && step.relationshipFilter.length > 0) {
|
|
4879
|
+
parts.push(compileEdgeStep(step, nextParam));
|
|
4880
|
+
estimatedFanOut *= DEFAULT_ESTIMATED_FANOUT_PER_HOP;
|
|
4881
|
+
} else {
|
|
4882
|
+
parts.push(compileSimpleStep(step, nextParam));
|
|
4883
|
+
estimatedFanOut *= DEFAULT_ESTIMATED_FANOUT_PER_HOP;
|
|
4884
|
+
}
|
|
4885
|
+
if (step.entityTypes && step.entityTypes.length > 0) {
|
|
4886
|
+
const typeParams = step.entityTypes.map((t) => nextParam(t));
|
|
4887
|
+
parts.push(`.has('entityType', within(${typeParams.join(", ")}))`);
|
|
4888
|
+
}
|
|
4889
|
+
if (step.entityFilter) {
|
|
4890
|
+
for (const f of step.entityFilter) {
|
|
4891
|
+
parts.push(compilePropertyFilter(f, nextParam));
|
|
4892
|
+
}
|
|
4643
4893
|
}
|
|
4644
4894
|
}
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
|
|
4895
|
+
if (returnMode === "terminal" && spec.dedup !== false) {
|
|
4896
|
+
parts.push(".dedup()");
|
|
4897
|
+
}
|
|
4898
|
+
if (returnMode === "path") {
|
|
4899
|
+
parts.push(".simplePath()");
|
|
4900
|
+
}
|
|
4901
|
+
const limit = spec.limit ?? 50;
|
|
4902
|
+
const offset = spec.offset ?? 0;
|
|
4903
|
+
const pOffset = nextParam(offset);
|
|
4904
|
+
const pEnd = nextParam(offset + limit);
|
|
4905
|
+
parts.push(`.range(${pOffset}, ${pEnd})`);
|
|
4906
|
+
params["_limit"] = limit;
|
|
4907
|
+
params["_offset"] = offset;
|
|
4908
|
+
if (returnMode === "terminal") {
|
|
4909
|
+
parts.push(`.${VERTEX_PROJECT_EXPR}`);
|
|
4910
|
+
} else {
|
|
4911
|
+
parts.push(`.path().by(${VERTEX_PROJECT_EXPR}).by(${EDGE_PROJECT_EXPR})`);
|
|
4912
|
+
}
|
|
4663
4913
|
}
|
|
4664
4914
|
return {
|
|
4665
4915
|
query: parts.join(""),
|
|
@@ -4680,7 +4930,7 @@ function compileSimpleStep(step, nextParam) {
|
|
|
4680
4930
|
return types ? `.both(${typeArgs})` : ".both()";
|
|
4681
4931
|
}
|
|
4682
4932
|
}
|
|
4683
|
-
function
|
|
4933
|
+
function compileEdgeOnly(step, nextParam) {
|
|
4684
4934
|
const parts = [];
|
|
4685
4935
|
const types = step.relationshipTypes;
|
|
4686
4936
|
const typeArgs = types ? types.map((t) => nextParam(t)).join(", ") : "";
|
|
@@ -4700,22 +4950,37 @@ function compileEdgeStep(step, nextParam) {
|
|
|
4700
4950
|
parts.push(compilePropertyFilter(f, nextParam));
|
|
4701
4951
|
}
|
|
4702
4952
|
}
|
|
4953
|
+
return parts.join("");
|
|
4954
|
+
}
|
|
4955
|
+
function compileVertexHop(step) {
|
|
4703
4956
|
switch (step.direction) {
|
|
4704
4957
|
case "out":
|
|
4705
|
-
|
|
4706
|
-
break;
|
|
4958
|
+
return ".inV()";
|
|
4707
4959
|
case "in":
|
|
4708
|
-
|
|
4709
|
-
break;
|
|
4960
|
+
return ".outV()";
|
|
4710
4961
|
case "both":
|
|
4711
|
-
|
|
4712
|
-
|
|
4962
|
+
return ".otherV()";
|
|
4963
|
+
}
|
|
4964
|
+
}
|
|
4965
|
+
function compileEntityFilters(step, nextParam) {
|
|
4966
|
+
const parts = [];
|
|
4967
|
+
if (step.entityTypes && step.entityTypes.length > 0) {
|
|
4968
|
+
const typeParams = step.entityTypes.map((t) => nextParam(t));
|
|
4969
|
+
parts.push(`.has('entityType', within(${typeParams.join(", ")}))`);
|
|
4970
|
+
}
|
|
4971
|
+
if (step.entityFilter) {
|
|
4972
|
+
for (const f of step.entityFilter) {
|
|
4973
|
+
parts.push(compilePropertyFilter(f, nextParam));
|
|
4974
|
+
}
|
|
4713
4975
|
}
|
|
4714
4976
|
return parts.join("");
|
|
4715
4977
|
}
|
|
4716
|
-
function
|
|
4978
|
+
function compileEdgeStep(step, nextParam) {
|
|
4979
|
+
return compileEdgeOnly(step, nextParam) + compileVertexHop(step);
|
|
4980
|
+
}
|
|
4981
|
+
function compileRepeatStep(step, nextParam, useEdgeEmission) {
|
|
4717
4982
|
const parts = [];
|
|
4718
|
-
const innerStep = step.relationshipFilter?.length ? compileEdgeStep({ ...step, repeat: void 0 }, nextParam) : compileSimpleStep(step, nextParam);
|
|
4983
|
+
const innerStep = useEdgeEmission || step.relationshipFilter?.length ? compileEdgeStep({ ...step, repeat: void 0 }, nextParam) : compileSimpleStep(step, nextParam);
|
|
4719
4984
|
if (step.repeat?.emitIntermediates !== false) {
|
|
4720
4985
|
parts.push(".emit()");
|
|
4721
4986
|
}
|
|
@@ -4724,10 +4989,13 @@ function compileRepeatStep(step, nextParam) {
|
|
|
4724
4989
|
const untilParts = step.repeat.until.map((f) => compilePropertyFilter(f, nextParam));
|
|
4725
4990
|
parts.push(`.until(${untilParts.join("")})`);
|
|
4726
4991
|
}
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4992
|
+
const n = step.repeat.maxDepth;
|
|
4993
|
+
if (!Number.isInteger(n) || n < 1) {
|
|
4994
|
+
throw new Error(
|
|
4995
|
+
`GremlinCompiler: repeat.maxDepth must be a positive integer; got ${n}`
|
|
4996
|
+
);
|
|
4730
4997
|
}
|
|
4998
|
+
parts.push(`.times(${n})`);
|
|
4731
4999
|
if (step.repeat?.emitIntermediates === false) {
|
|
4732
5000
|
parts.push(".emit()");
|
|
4733
5001
|
}
|
|
@@ -5046,17 +5314,20 @@ var InMemoryStorageProvider = class {
|
|
|
5046
5314
|
store.slugIndex.set(entity.slug, entity.id);
|
|
5047
5315
|
return entity;
|
|
5048
5316
|
}
|
|
5049
|
-
|
|
5317
|
+
// The in-memory provider always carries the full entity (embedding and all).
|
|
5318
|
+
// The `loadEmbeddings` option is accepted for interface symmetry but ignored —
|
|
5319
|
+
// there is no light/full split to switch between when entities live in a Map.
|
|
5320
|
+
async getEntity(repositoryId, entityId, _options) {
|
|
5050
5321
|
const store = this.getStore(repositoryId);
|
|
5051
5322
|
return store.entities.get(entityId) ?? null;
|
|
5052
5323
|
}
|
|
5053
|
-
async getEntityBySlug(repositoryId, slug) {
|
|
5324
|
+
async getEntityBySlug(repositoryId, slug, _options) {
|
|
5054
5325
|
const store = this.getStore(repositoryId);
|
|
5055
5326
|
const id = store.slugIndex.get(slug);
|
|
5056
5327
|
if (!id) return null;
|
|
5057
5328
|
return store.entities.get(id) ?? null;
|
|
5058
5329
|
}
|
|
5059
|
-
async getEntities(repositoryId, entityIds) {
|
|
5330
|
+
async getEntities(repositoryId, entityIds, _options) {
|
|
5060
5331
|
const store = this.getStore(repositoryId);
|
|
5061
5332
|
const result = /* @__PURE__ */ new Map();
|
|
5062
5333
|
for (const id of entityIds) {
|
|
@@ -5127,7 +5398,7 @@ var InMemoryStorageProvider = class {
|
|
|
5127
5398
|
}
|
|
5128
5399
|
return { deletedEntities: entityIds.size, deletedRelationships };
|
|
5129
5400
|
}
|
|
5130
|
-
async findEntities(repositoryId, query) {
|
|
5401
|
+
async findEntities(repositoryId, query, _options) {
|
|
5131
5402
|
const store = this.getStore(repositoryId);
|
|
5132
5403
|
let matches = Array.from(store.entities.values());
|
|
5133
5404
|
if (query.entityTypes && query.entityTypes.length > 0) {
|
|
@@ -5649,6 +5920,8 @@ export {
|
|
|
5649
5920
|
EmbeddingProviderRequiredError,
|
|
5650
5921
|
EntityNotFoundError,
|
|
5651
5922
|
ExportError,
|
|
5923
|
+
GREMLIN_EDGE_PROJECTION_FIELDS,
|
|
5924
|
+
GREMLIN_VERTEX_PROJECTION_FIELDS,
|
|
5652
5925
|
GovernanceDeniedError,
|
|
5653
5926
|
GraphTraversalProviderRequiredError,
|
|
5654
5927
|
GremlinCompiler,
|
|
@@ -5670,7 +5943,10 @@ export {
|
|
|
5670
5943
|
TraversalTimeoutError,
|
|
5671
5944
|
TraversalValidationError,
|
|
5672
5945
|
TraversalVocabularyError,
|
|
5946
|
+
UnsupportedQueryError,
|
|
5673
5947
|
VocabularyValidationError,
|
|
5948
|
+
buildEdgeProjectChain,
|
|
5949
|
+
buildVertexProjectChain,
|
|
5674
5950
|
createSafeSink,
|
|
5675
5951
|
generateId,
|
|
5676
5952
|
isValidUuid,
|