@opum-ai/lore 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +33 -22
  2. package/bin/lore.cjs +35 -7
  3. package/package.json +17 -17
  4. package/src/adapters/backlog.ts +0 -1084
  5. package/src/adapters/git.ts +0 -221
  6. package/src/cli.ts +0 -667
  7. package/src/commands/agent.ts +0 -301
  8. package/src/commands/agents.ts +0 -302
  9. package/src/commands/args.ts +0 -209
  10. package/src/commands/changed.ts +0 -70
  11. package/src/commands/check.ts +0 -1031
  12. package/src/commands/codex-bridge.ts +0 -49
  13. package/src/commands/concurrency.ts +0 -48
  14. package/src/commands/context.ts +0 -292
  15. package/src/commands/discover.ts +0 -89
  16. package/src/commands/explorer.ts +0 -253
  17. package/src/commands/export.ts +0 -93
  18. package/src/commands/fswrite.ts +0 -928
  19. package/src/commands/graph.ts +0 -291
  20. package/src/commands/help.ts +0 -151
  21. package/src/commands/impact.ts +0 -59
  22. package/src/commands/init.ts +0 -583
  23. package/src/commands/instructions.ts +0 -91
  24. package/src/commands/link.ts +0 -929
  25. package/src/commands/new.ts +0 -476
  26. package/src/commands/orphans.ts +0 -457
  27. package/src/commands/path.ts +0 -67
  28. package/src/commands/provenance.ts +0 -68
  29. package/src/commands/query.ts +0 -312
  30. package/src/commands/reconcile-shared.ts +0 -280
  31. package/src/commands/rename.ts +0 -585
  32. package/src/commands/replace.ts +0 -320
  33. package/src/commands/scaffold.ts +0 -346
  34. package/src/commands/schema.ts +0 -293
  35. package/src/commands/snapshot.ts +0 -130
  36. package/src/commands/supersede.ts +0 -400
  37. package/src/commands/sync.ts +0 -371
  38. package/src/commands/tasks.ts +0 -271
  39. package/src/commands/traversal.ts +0 -151
  40. package/src/commands/validate.ts +0 -226
  41. package/src/config.ts +0 -598
  42. package/src/core/agent-bridge.ts +0 -287
  43. package/src/core/agent-context.ts +0 -498
  44. package/src/core/agent-profile.ts +0 -447
  45. package/src/core/bundle.ts +0 -893
  46. package/src/core/check.ts +0 -853
  47. package/src/core/codex-bridge.ts +0 -100
  48. package/src/core/concept.ts +0 -597
  49. package/src/core/consumer-scaffold.ts +0 -433
  50. package/src/core/context.ts +0 -271
  51. package/src/core/explorer-contract.ts +0 -441
  52. package/src/core/explorer-qualification.ts +0 -58
  53. package/src/core/explorer.ts +0 -518
  54. package/src/core/finding.ts +0 -31
  55. package/src/core/graph.ts +0 -201
  56. package/src/core/indexes.ts +0 -436
  57. package/src/core/instructions.ts +0 -209
  58. package/src/core/ladybug-driver.ts +0 -1795
  59. package/src/core/ladybug-lifecycle.ts +0 -1178
  60. package/src/core/ladybug-native.ts +0 -95
  61. package/src/core/ladybug-source.ts +0 -667
  62. package/src/core/links.ts +0 -681
  63. package/src/core/log.ts +0 -253
  64. package/src/core/managed-block.ts +0 -540
  65. package/src/core/manifest.ts +0 -718
  66. package/src/core/order.ts +0 -13
  67. package/src/core/profile.ts +0 -1007
  68. package/src/core/projection.ts +0 -195
  69. package/src/core/query.ts +0 -542
  70. package/src/core/reconcile.ts +0 -236
  71. package/src/core/replace.ts +0 -419
  72. package/src/core/retrieval.ts +0 -213
  73. package/src/core/rewrite.ts +0 -940
  74. package/src/core/scaffold.ts +0 -255
  75. package/src/core/schema.ts +0 -366
  76. package/src/core/snapshot-runtime.ts +0 -52
  77. package/src/core/snapshot-store.ts +0 -287
  78. package/src/core/snapshot.ts +0 -711
  79. package/src/core/template.ts +0 -429
  80. package/src/core/traversal.ts +0 -487
  81. package/src/core/validate.ts +0 -517
  82. package/src/core/workspace-contract.ts +0 -473
  83. package/src/core/workspace-projection.ts +0 -365
  84. package/src/core/workspace-retrieval.ts +0 -196
  85. package/src/core/workspace-source.ts +0 -174
  86. package/src/errors.ts +0 -697
  87. package/src/meta.ts +0 -7
  88. package/src/output.ts +0 -589
  89. package/src/scripts/upstream-backlog-watch.ts +0 -288
  90. package/src/state.ts +0 -390
@@ -1,1795 +0,0 @@
1
- /**
2
- * Private @ladybugdb/core boundary for ladybug-projection/1.
3
- *
4
- * Physical tables and Cypher remain internal implementation details. Callers
5
- * provide validated export-source records and receive only Lore identities and
6
- * structural verification facts.
7
- */
8
-
9
- import { createHash, randomUUID } from "node:crypto";
10
- import { closeSync, openSync, unlinkSync, writeSync } from "node:fs";
11
- import { dirname, join } from "node:path";
12
- import ladybug, { Connection, Database, type LbugValue, type QueryResult } from "@ladybugdb/core";
13
- import { LoreError } from "../errors";
14
- import { type BundleGraph, type Edge, type EdgeKind, frontmatterScalar } from "./bundle";
15
- import type { Concept } from "./concept";
16
- import type { LadybugDatabaseVerification, LadybugIndexedReader } from "./ladybug-native";
17
- import {
18
- canonicalJson,
19
- LADYBUG_INDEX_FORMAT,
20
- type LadybugProjectionSource,
21
- type ProjectionConceptRecord,
22
- type ProjectionEdgeRecord,
23
- type ProjectionTaskRecord,
24
- } from "./ladybug-source";
25
- import {
26
- type Bm25Index,
27
- type QueryResult as LoreQueryResult,
28
- type QueryOptions,
29
- queryWithBm25Index,
30
- searchableConceptFields,
31
- tokenizeQueryText,
32
- } from "./query";
33
- import type { TraversalSourceRecords } from "./traversal";
34
-
35
- export const LADYBUG_VERSION = String(ladybug.VERSION);
36
- export const LADYBUG_STORAGE_VERSION = String(ladybug.STORAGE_VERSION);
37
-
38
- const WRITER_BUFFER_BYTES = 256 * 1024 * 1024;
39
- const READER_BUFFER_BYTES = 64 * 1024 * 1024;
40
- const FULL_SCAN_BUFFER_BYTES = 512 * 1024 * 1024;
41
- const VERIFIED_LEXICAL_LENGTHS = new WeakMap<LadybugProjectionSource, ReadonlyMap<string, number>>();
42
- const CONCEPT_BODY_QUERY = "MATCH (n:ConceptRecord {recordKey: $recordKey}) RETURN n.body AS body";
43
-
44
- const SCHEMA_QUERIES = [
45
- `CREATE NODE TABLE RepositoryProjection(
46
- repositoryScopeKey STRING,
47
- bundleId STRING,
48
- docsRoot STRING,
49
- PRIMARY KEY (repositoryScopeKey)
50
- )`,
51
- `CREATE NODE TABLE ProjectionSnapshot(
52
- snapshotKey STRING,
53
- indexFormatVersion STRING,
54
- projectionSchemaVersion STRING,
55
- normalizationVersion STRING,
56
- exporterName STRING,
57
- exporterVersion STRING,
58
- loreVersion STRING,
59
- ladybugVersion STRING,
60
- ladybugStorageVersion STRING,
61
- repositoryScopeKey STRING,
62
- bundleId STRING,
63
- okfVersion STRING,
64
- docsRoot STRING,
65
- gitCommit STRING,
66
- exportDigest STRING,
67
- taskSnapshotDigest STRING,
68
- sourceFingerprint STRING,
69
- sourceRecordsDigest STRING,
70
- recordKeysDigest STRING,
71
- lexicalDocumentCount INT64,
72
- lexicalTotalLength INT64,
73
- recordCount INT64,
74
- conceptCount INT64,
75
- taskCount INT64,
76
- authoredEdgeCount INT64,
77
- manifestJson STRING,
78
- trailerJson STRING,
79
- PRIMARY KEY (snapshotKey)
80
- )`,
81
- `CREATE NODE TABLE SourceCommit(
82
- commitKey STRING,
83
- repositoryScopeKey STRING,
84
- sha STRING,
85
- PRIMARY KEY (commitKey)
86
- )`,
87
- `CREATE NODE TABLE ConceptRecord(
88
- recordKey STRING,
89
- repositoryScopeKey STRING,
90
- snapshotKey STRING,
91
- bundleId STRING,
92
- gitCommit STRING,
93
- exportDigest STRING,
94
- conceptId STRING,
95
- path STRING,
96
- conceptType STRING,
97
- frontmatterJson STRING,
98
- title STRING,
99
- summary STRING,
100
- description STRING,
101
- tagsJson STRING,
102
- body STRING,
103
- contentHash STRING,
104
- tokenEstimate INT64,
105
- lexicalLength INT64,
106
- PRIMARY KEY (recordKey)
107
- )`,
108
- `CREATE NODE TABLE TaskRecord(
109
- recordKey STRING,
110
- repositoryScopeKey STRING,
111
- snapshotKey STRING,
112
- bundleId STRING,
113
- gitCommit STRING,
114
- exportDigest STRING,
115
- taskId STRING,
116
- title STRING,
117
- status STRING,
118
- labelsJson STRING,
119
- priority STRING,
120
- ordinal INT64,
121
- assigneesJson STRING,
122
- milestone STRING,
123
- parentTaskId STRING,
124
- sourceAdapterVersion STRING,
125
- sourceRecordJson STRING,
126
- PRIMARY KEY (recordKey)
127
- )`,
128
- `CREATE NODE TABLE AuthoredEdgeRecord(
129
- recordKey STRING,
130
- repositoryScopeKey STRING,
131
- snapshotKey STRING,
132
- bundleId STRING,
133
- gitCommit STRING,
134
- exportDigest STRING,
135
- fromRecordKey STRING,
136
- toRecordKey STRING,
137
- kind STRING,
138
- target STRING,
139
- ordinal INT64,
140
- dangling BOOL,
141
- sourceRecordJson STRING,
142
- PRIMARY KEY (recordKey)
143
- )`,
144
- `CREATE NODE TABLE LexicalTerm(
145
- termKey STRING,
146
- documentFrequency INT64,
147
- PRIMARY KEY (termKey)
148
- )`,
149
- "CREATE REL TABLE HAS_SNAPSHOT(FROM RepositoryProjection TO ProjectionSnapshot)",
150
- "CREATE REL TABLE AT_COMMIT(FROM ProjectionSnapshot TO SourceCommit)",
151
- "CREATE REL TABLE HAS_CONCEPT(FROM ProjectionSnapshot TO ConceptRecord)",
152
- "CREATE REL TABLE HAS_TASK(FROM ProjectionSnapshot TO TaskRecord)",
153
- "CREATE REL TABLE HAS_EDGE(FROM ProjectionSnapshot TO AuthoredEdgeRecord)",
154
- "CREATE REL TABLE EDGE_SOURCE(FROM ConceptRecord TO AuthoredEdgeRecord)",
155
- "CREATE REL TABLE EDGE_TASK_SOURCE(FROM TaskRecord TO AuthoredEdgeRecord)",
156
- "CREATE REL TABLE EDGE_CONCEPT_TARGET(FROM AuthoredEdgeRecord TO ConceptRecord)",
157
- "CREATE REL TABLE EDGE_TASK_TARGET(FROM AuthoredEdgeRecord TO TaskRecord)",
158
- "CREATE REL TABLE HAS_TERM(FROM ConceptRecord TO LexicalTerm, frequency INT64)",
159
- ] as const;
160
-
161
- /** Create and populate a new isolated database. The path must not be published. */
162
- export async function buildLadybugDatabase(databasePath: string, source: LadybugProjectionSource): Promise<void> {
163
- const database = writableDatabase(databasePath);
164
- const connection = new Connection(database);
165
- try {
166
- for (const query of SCHEMA_QUERIES) await executeQuery(connection, query);
167
- await insertProjection(connection, databasePath, source);
168
- await executeQuery(connection, "CHECKPOINT");
169
- } finally {
170
- await closeConnection(connection);
171
- await database.close();
172
- }
173
- }
174
-
175
- /**
176
- * Open an immutable generation read-only and prove that its promoted metadata,
177
- * source records, keys, counts, and structural relationship endpoints match the
178
- * validated export snapshot.
179
- */
180
- export async function verifyLadybugDatabase(
181
- databasePath: string,
182
- source: LadybugProjectionSource,
183
- ): Promise<LadybugDatabaseVerification> {
184
- const database = readOnlyDatabase(databasePath);
185
- const connection = new Connection(database);
186
- try {
187
- const snapshotRows = await queryRows(
188
- connection,
189
- `MATCH (s:ProjectionSnapshot {snapshotKey: $snapshotKey})
190
- RETURN s.snapshotKey AS snapshotKey,
191
- s.indexFormatVersion AS indexFormatVersion,
192
- s.projectionSchemaVersion AS projectionSchemaVersion,
193
- s.normalizationVersion AS normalizationVersion,
194
- s.exporterName AS exporterName,
195
- s.exporterVersion AS exporterVersion,
196
- s.loreVersion AS loreVersion,
197
- s.ladybugVersion AS ladybugVersion,
198
- s.ladybugStorageVersion AS ladybugStorageVersion,
199
- s.repositoryScopeKey AS repositoryScopeKey,
200
- s.bundleId AS bundleId,
201
- s.okfVersion AS okfVersion,
202
- s.docsRoot AS docsRoot,
203
- s.gitCommit AS gitCommit,
204
- s.exportDigest AS exportDigest,
205
- s.taskSnapshotDigest AS taskSnapshotDigest,
206
- s.sourceFingerprint AS sourceFingerprint,
207
- s.sourceRecordsDigest AS sourceRecordsDigest,
208
- s.recordKeysDigest AS recordKeysDigest,
209
- s.recordCount AS recordCount,
210
- s.conceptCount AS conceptCount,
211
- s.taskCount AS taskCount,
212
- s.authoredEdgeCount AS authoredEdgeCount,
213
- s.manifestJson AS manifestJson,
214
- s.trailerJson AS trailerJson`,
215
- { snapshotKey: source.snapshotKey },
216
- );
217
- if (snapshotRows.length !== 1) corrupt("projection snapshot metadata is missing or duplicated");
218
- const actual = snapshotRows[0] as Record<string, unknown>;
219
- const expected = snapshotMetadata(source);
220
- for (const [key, value] of Object.entries(expected)) {
221
- if (!sameDatabaseValue(actual[key], value)) {
222
- corrupt(`projection snapshot metadata differs for ${key}`);
223
- }
224
- }
225
-
226
- await verifyConceptRecordTable(connection, source);
227
- await verifyRecordTable(connection, "TaskRecord", source.tasks);
228
- await verifyRecordTable(connection, "AuthoredEdgeRecord", source.authoredEdges);
229
- await verifyPromotedSamples(connection, source);
230
-
231
- await expectCount(connection, "MATCH (n:RepositoryProjection) RETURN count(n) AS count", 1, "repository");
232
- await expectCount(connection, "MATCH (n:ProjectionSnapshot) RETURN count(n) AS count", 1, "snapshot");
233
- await expectCount(
234
- connection,
235
- "MATCH (n:SourceCommit) RETURN count(n) AS count",
236
- source.commitKey === null ? 0 : 1,
237
- "source commit",
238
- );
239
- await expectCount(connection, "MATCH ()-[r:HAS_SNAPSHOT]->() RETURN count(r) AS count", 1, "HAS_SNAPSHOT");
240
- await expectCount(
241
- connection,
242
- "MATCH ()-[r:AT_COMMIT]->() RETURN count(r) AS count",
243
- source.commitKey === null ? 0 : 1,
244
- "AT_COMMIT",
245
- );
246
- await expectCount(
247
- connection,
248
- "MATCH ()-[r:HAS_CONCEPT]->() RETURN count(r) AS count",
249
- source.counts.concepts,
250
- "HAS_CONCEPT",
251
- );
252
- await expectCount(
253
- connection,
254
- "MATCH ()-[r:HAS_TASK]->() RETURN count(r) AS count",
255
- source.counts.tasks,
256
- "HAS_TASK",
257
- );
258
- await expectCount(
259
- connection,
260
- "MATCH ()-[r:HAS_EDGE]->() RETURN count(r) AS count",
261
- source.counts.authoredEdges,
262
- "HAS_EDGE",
263
- );
264
- const conceptSources = source.authoredEdges.filter((edge) => edgeSourceKind(edge) === "concept");
265
- const taskSources = source.authoredEdges.filter((edge) => edgeSourceKind(edge) === "task");
266
- await expectCount(
267
- connection,
268
- "MATCH ()-[r:EDGE_SOURCE]->() RETURN count(r) AS count",
269
- conceptSources.length,
270
- "EDGE_SOURCE",
271
- );
272
- if (taskSources.length > 0) {
273
- await expectCount(
274
- connection,
275
- "MATCH ()-[r:EDGE_TASK_SOURCE]->() RETURN count(r) AS count",
276
- taskSources.length,
277
- "EDGE_TASK_SOURCE",
278
- );
279
- }
280
- const conceptTargets = source.authoredEdges.filter(
281
- (edge) => edgeTargetKind(edge) === "concept" && !edge.dangling,
282
- ).length;
283
- const taskTargets = source.authoredEdges.filter((edge) => edgeTargetKind(edge) === "task" && !edge.dangling).length;
284
- await expectCount(
285
- connection,
286
- "MATCH ()-[r:EDGE_CONCEPT_TARGET]->() RETURN count(r) AS count",
287
- conceptTargets,
288
- "EDGE_CONCEPT_TARGET",
289
- );
290
- await expectCount(
291
- connection,
292
- "MATCH ()-[r:EDGE_TASK_TARGET]->() RETURN count(r) AS count",
293
- taskTargets,
294
- "EDGE_TASK_TARGET",
295
- );
296
- await expectCount(
297
- connection,
298
- `MATCH (source:ConceptRecord)-[:EDGE_SOURCE]->(edge:AuthoredEdgeRecord)
299
- WHERE source.recordKey <> edge.fromRecordKey
300
- RETURN count(edge) AS count`,
301
- 0,
302
- "EDGE_SOURCE endpoint mismatch",
303
- );
304
- if (taskSources.length > 0) {
305
- await expectCount(
306
- connection,
307
- `MATCH (source:TaskRecord)-[:EDGE_TASK_SOURCE]->(edge:AuthoredEdgeRecord)
308
- WHERE source.recordKey <> edge.fromRecordKey
309
- RETURN count(edge) AS count`,
310
- 0,
311
- "EDGE_TASK_SOURCE endpoint mismatch",
312
- );
313
- }
314
- await expectCount(
315
- connection,
316
- `MATCH (edge:AuthoredEdgeRecord)-[:EDGE_CONCEPT_TARGET]->(target:ConceptRecord)
317
- WHERE edge.toRecordKey <> target.recordKey OR edge.dangling = true
318
- RETURN count(edge) AS count`,
319
- 0,
320
- "EDGE_CONCEPT_TARGET endpoint mismatch",
321
- );
322
- await expectCount(
323
- connection,
324
- `MATCH (edge:AuthoredEdgeRecord)-[:EDGE_TASK_TARGET]->(target:TaskRecord)
325
- WHERE edge.toRecordKey <> target.recordKey OR edge.dangling = true
326
- RETURN count(edge) AS count`,
327
- 0,
328
- "EDGE_TASK_TARGET endpoint mismatch",
329
- );
330
- await expectRelationshipPairs(
331
- connection,
332
- `MATCH (a:RepositoryProjection)-[:HAS_SNAPSHOT]->(b:ProjectionSnapshot)
333
- RETURN a.repositoryScopeKey AS fromKey, b.snapshotKey AS toKey`,
334
- [[source.repositoryScopeKey, source.snapshotKey]],
335
- "HAS_SNAPSHOT",
336
- );
337
- await expectRelationshipPairs(
338
- connection,
339
- `MATCH (a:ProjectionSnapshot)-[:AT_COMMIT]->(b:SourceCommit)
340
- RETURN a.snapshotKey AS fromKey, b.commitKey AS toKey`,
341
- source.commitKey === null ? [] : [[source.snapshotKey, source.commitKey]],
342
- "AT_COMMIT",
343
- );
344
- await expectRelationshipPairs(
345
- connection,
346
- `MATCH (a:ProjectionSnapshot)-[:HAS_CONCEPT]->(b:ConceptRecord)
347
- RETURN a.snapshotKey AS fromKey, b.recordKey AS toKey`,
348
- source.concepts.map((record) => [source.snapshotKey, record.key]),
349
- "HAS_CONCEPT",
350
- );
351
- await expectRelationshipPairs(
352
- connection,
353
- `MATCH (a:ProjectionSnapshot)-[:HAS_TASK]->(b:TaskRecord)
354
- RETURN a.snapshotKey AS fromKey, b.recordKey AS toKey`,
355
- source.tasks.map((record) => [source.snapshotKey, record.key]),
356
- "HAS_TASK",
357
- );
358
- await expectRelationshipPairs(
359
- connection,
360
- `MATCH (a:ProjectionSnapshot)-[:HAS_EDGE]->(b:AuthoredEdgeRecord)
361
- RETURN a.snapshotKey AS fromKey, b.recordKey AS toKey`,
362
- source.authoredEdges.map((record) => [source.snapshotKey, record.key]),
363
- "HAS_EDGE",
364
- );
365
- await expectRelationshipPairs(
366
- connection,
367
- `MATCH (a:ConceptRecord)-[:EDGE_SOURCE]->(b:AuthoredEdgeRecord)
368
- RETURN a.recordKey AS fromKey, b.recordKey AS toKey`,
369
- conceptSources.map((record) => [record.from, record.key]),
370
- "EDGE_SOURCE",
371
- );
372
- if (taskSources.length > 0) {
373
- await expectRelationshipPairs(
374
- connection,
375
- `MATCH (a:TaskRecord)-[:EDGE_TASK_SOURCE]->(b:AuthoredEdgeRecord)
376
- RETURN a.recordKey AS fromKey, b.recordKey AS toKey`,
377
- taskSources.map((record) => [record.from, record.key]),
378
- "EDGE_TASK_SOURCE",
379
- );
380
- }
381
- await expectRelationshipPairs(
382
- connection,
383
- `MATCH (a:AuthoredEdgeRecord)-[:EDGE_CONCEPT_TARGET]->(b:ConceptRecord)
384
- RETURN a.recordKey AS fromKey, b.recordKey AS toKey`,
385
- source.authoredEdges
386
- .filter((record) => edgeTargetKind(record) === "concept" && !record.dangling && record.to !== null)
387
- .map((record) => [record.key, record.to as string]),
388
- "EDGE_CONCEPT_TARGET",
389
- );
390
- await expectRelationshipPairs(
391
- connection,
392
- `MATCH (a:AuthoredEdgeRecord)-[:EDGE_TASK_TARGET]->(b:TaskRecord)
393
- RETURN a.recordKey AS fromKey, b.recordKey AS toKey`,
394
- source.authoredEdges
395
- .filter((record) => edgeTargetKind(record) === "task" && !record.dangling && record.to !== null)
396
- .map((record) => [record.key, record.to as string]),
397
- "EDGE_TASK_TARGET",
398
- );
399
-
400
- return {
401
- repositoryScopeKey: source.repositoryScopeKey,
402
- snapshotKey: source.snapshotKey,
403
- sourceFingerprint: source.sourceFingerprint,
404
- exportDigest: source.exportDigest,
405
- taskSnapshotDigest: source.taskSnapshotDigest,
406
- sourceRecordsDigest: source.sourceRecordsDigest,
407
- recordKeysDigest: source.recordKeysDigest,
408
- recordCount: source.trailer.recordCount,
409
- conceptCount: source.counts.concepts,
410
- taskCount: source.counts.tasks,
411
- authoredEdgeCount: source.counts.authoredEdges,
412
- };
413
- } finally {
414
- await closeConnection(connection);
415
- await database.close();
416
- }
417
- }
418
-
419
- /** Verify bounded promoted metadata after the immutable file digest has already matched. */
420
- export async function verifyLadybugDatabaseMetadata(
421
- databasePath: string,
422
- expected: LadybugDatabaseVerification,
423
- ): Promise<LadybugDatabaseVerification> {
424
- return withReadConnection(databasePath, async (connection) => {
425
- const rows = await queryRows(
426
- connection,
427
- `MATCH (s:ProjectionSnapshot {snapshotKey: $snapshotKey})
428
- RETURN s.repositoryScopeKey AS repositoryScopeKey, s.snapshotKey AS snapshotKey,
429
- s.sourceFingerprint AS sourceFingerprint, s.exportDigest AS exportDigest,
430
- s.taskSnapshotDigest AS taskSnapshotDigest, s.sourceRecordsDigest AS sourceRecordsDigest,
431
- s.recordKeysDigest AS recordKeysDigest, s.recordCount AS recordCount,
432
- s.conceptCount AS conceptCount, s.taskCount AS taskCount,
433
- s.authoredEdgeCount AS authoredEdgeCount`,
434
- { snapshotKey: expected.snapshotKey },
435
- );
436
- if (rows.length !== 1) corrupt("projection snapshot metadata is missing or duplicated");
437
- const row = rows[0] as Record<string, unknown>;
438
- for (const [key, value] of Object.entries(expected)) {
439
- if (!sameDatabaseValue(row[key], value)) corrupt(`projection snapshot metadata differs for ${key}`);
440
- }
441
- await expectCount(connection, "MATCH (n:ConceptRecord) RETURN count(n) AS count", expected.conceptCount, "concept");
442
- await expectCount(connection, "MATCH (n:TaskRecord) RETURN count(n) AS count", expected.taskCount, "task");
443
- await expectCount(
444
- connection,
445
- "MATCH (n:AuthoredEdgeRecord) RETURN count(n) AS count",
446
- expected.authoredEdgeCount,
447
- "authored edge",
448
- );
449
- return expected;
450
- });
451
- }
452
-
453
- /** Read canonical source records for deterministic tests and later bounded readers. */
454
- export async function readLadybugSourceRecords(databasePath: string): Promise<ReadonlyMap<string, string>> {
455
- const database = fullScanDatabase(databasePath);
456
- const connection = new Connection(database);
457
- try {
458
- const result = new Map<string, string>();
459
- const concepts = await queryRows(
460
- connection,
461
- `MATCH (n:ConceptRecord)
462
- RETURN n.recordKey AS recordKey, n.conceptId AS conceptId, n.path AS path,
463
- n.conceptType AS conceptType, n.frontmatterJson AS frontmatterJson,
464
- n.body AS body, n.contentHash AS contentHash, n.tokenEstimate AS tokenEstimate
465
- ORDER BY n.recordKey`,
466
- );
467
- for (const row of concepts) {
468
- const record = conceptRecordFromPromotedRow(row);
469
- result.set(record.key, JSON.stringify(record));
470
- }
471
- for (const table of ["TaskRecord", "AuthoredEdgeRecord"] as const) {
472
- const rows = await queryRows(
473
- connection,
474
- `MATCH (n:${table}) RETURN n.recordKey AS key, n.sourceRecordJson AS sourceRecordJson ORDER BY n.recordKey`,
475
- );
476
- for (const row of rows) {
477
- const key = row.key;
478
- const sourceRecordJson = row.sourceRecordJson;
479
- if (typeof key !== "string" || typeof sourceRecordJson !== "string" || result.has(key)) {
480
- corrupt(`invalid or duplicate source record in ${table}`);
481
- }
482
- result.set(key, sourceRecordJson);
483
- }
484
- }
485
- return result;
486
- } finally {
487
- await closeConnection(connection);
488
- await database.close();
489
- }
490
- }
491
-
492
- /**
493
- * Read the verified projection into Lore's existing deterministic graph model.
494
- *
495
- * The database remains an implementation detail: canonical export records are
496
- * the read boundary, source record keys are translated back to concept ids, and
497
- * neither Ladybug ids nor physical schema values enter the returned graph.
498
- */
499
- export async function readLadybugBundleGraph(
500
- databasePath: string,
501
- source: LadybugProjectionSource,
502
- ): Promise<BundleGraph> {
503
- const database = fullScanDatabase(databasePath);
504
- const connection = new Connection(database);
505
- try {
506
- const conceptRows = await queryRows(
507
- connection,
508
- `MATCH (n:ConceptRecord)
509
- RETURN n.recordKey AS recordKey, n.conceptId AS conceptId, n.path AS path,
510
- n.conceptType AS conceptType, n.frontmatterJson AS frontmatterJson,
511
- n.body AS body, n.contentHash AS contentHash, n.tokenEstimate AS tokenEstimate`,
512
- );
513
- const edgeRows = await queryRows(
514
- connection,
515
- "MATCH (n:AuthoredEdgeRecord) RETURN n.sourceRecordJson AS sourceRecordJson",
516
- );
517
- if (conceptRows.length !== source.counts.concepts || edgeRows.length !== source.counts.authoredEdges) {
518
- corrupt("indexed read counts differ from the verified source snapshot");
519
- }
520
-
521
- const records = conceptRows.map(conceptRecordFromPromotedRow);
522
- records.sort((a, b) => compare(a.id, b.id));
523
- const concepts = new Map<string, Concept>();
524
- const conceptIdsByRecordKey = new Map<string, string>();
525
- const tokenEstimates = new Map<string, number>();
526
- const docsPrefix = `${source.manifest.bundle.docsRoot}/`;
527
- for (const record of records) {
528
- if (concepts.has(record.id) || conceptIdsByRecordKey.has(record.key) || !record.path.startsWith(docsPrefix)) {
529
- corrupt("indexed concept identities are duplicated or outside the bundle root");
530
- }
531
- const concept: Concept = {
532
- id: record.id,
533
- path: record.path.slice(docsPrefix.length),
534
- type: record.type,
535
- frontmatter: record.frontmatter,
536
- body: record.body,
537
- };
538
- concepts.set(record.id, concept);
539
- conceptIdsByRecordKey.set(record.key, record.id);
540
- tokenEstimates.set(record.id, record.tokenEstimate);
541
- }
542
-
543
- const indexedEdges = edgeRows
544
- .map((row) => parseEdgeSourceRecord(row.sourceRecordJson))
545
- .filter((record) => edgeSourceKind(record) === "concept" && edgeTargetKind(record) === "concept")
546
- .map((record) => {
547
- const from = conceptIdsByRecordKey.get(record.from);
548
- const to = record.to === null ? null : conceptIdsByRecordKey.get(record.to);
549
- if (
550
- from === undefined ||
551
- (record.to !== null && to === undefined) ||
552
- record.dangling !== (record.to === null) ||
553
- !isEdgeKind(record.kind)
554
- ) {
555
- corrupt("indexed authored-edge identities or promoted fields disagree");
556
- }
557
- return { record, edge: { from, to: to ?? null, kind: record.kind, target: record.target } satisfies Edge };
558
- });
559
- indexedEdges.sort(
560
- (a, b) =>
561
- compare(a.edge.from, b.edge.from) || a.record.ordinal - b.record.ordinal || compare(a.record.key, b.record.key),
562
- );
563
- for (let index = 1; index < indexedEdges.length; index++) {
564
- const previous = indexedEdges[index - 1] as (typeof indexedEdges)[number];
565
- const current = indexedEdges[index] as (typeof indexedEdges)[number];
566
- if (previous.edge.from === current.edge.from && previous.record.ordinal === current.record.ordinal) {
567
- corrupt("indexed concept-edge ordinals are duplicated");
568
- }
569
- }
570
- const edges = indexedEdges.map(({ edge }) => edge);
571
-
572
- let total: number | undefined;
573
- const tokenEstimate = (id?: string): number => {
574
- if (id === undefined) {
575
- total ??= [...tokenEstimates.values()].reduce((sum, value) => sum + value, 0);
576
- return total;
577
- }
578
- const value = tokenEstimates.get(id);
579
- if (value === undefined) {
580
- throw new LoreError(
581
- "not_found",
582
- `concept "${id}" is not in the bundle`,
583
- "run `lore query` to find the right id, or check the path",
584
- { id },
585
- );
586
- }
587
- return value;
588
- };
589
- return { concepts, edges, tokenEstimate };
590
- } finally {
591
- await closeConnection(connection);
592
- await database.close();
593
- }
594
- }
595
-
596
- /** Open one shared read-only native session for bounded indexed operations in this CLI process. */
597
- export function openLadybugIndexedReader(databasePath: string, source: LadybugProjectionSource): LadybugIndexedReader {
598
- const database = readOnlyDatabase(databasePath);
599
- const connection = new Connection(database);
600
- // Prepare the primary-key body lookup once during warm open. Context calls on
601
- // this reader then bind and execute it without reparsing/replanning Cypher.
602
- const conceptBodyStatement = prepareSync(connection, CONCEPT_BODY_QUERY);
603
- const recordKeyById = new Map<string, string>();
604
- let closed = false;
605
- return {
606
- readBundleGraph: (bodyId?: string) =>
607
- readIndexedBundleGraph(databasePath, source, bodyId, connection, (recordIds) => {
608
- for (const [recordKey, id] of recordIds) recordKeyById.set(id, recordKey);
609
- }),
610
- readConceptBody: async (id: string) => {
611
- const recordKey = recordKeyById.get(id);
612
- if (recordKey === undefined) return undefined;
613
- const rows = await queryPreparedRows(connection, conceptBodyStatement, { recordKey });
614
- if (rows.length !== 1) corrupt("indexed concept body is missing or duplicated");
615
- const body = rows[0]?.body;
616
- if (body !== null && typeof body !== "string") corrupt("indexed concept body is invalid");
617
- return body ?? "";
618
- },
619
- readTraversalRecords: () => readIndexedTraversalRecords(source, connection),
620
- query: (options: QueryOptions) => queryIndexedDatabase(databasePath, source, options, connection),
621
- close: async () => {
622
- if (closed) return;
623
- closed = true;
624
- await closeConnection(connection);
625
- await database.close();
626
- },
627
- };
628
- }
629
-
630
- async function readIndexedTraversalRecords(
631
- source: LadybugProjectionSource,
632
- connection: Connection,
633
- ): Promise<TraversalSourceRecords> {
634
- const read = async <T extends ProjectionTaskRecord | ProjectionEdgeRecord>(
635
- table: "TaskRecord" | "AuthoredEdgeRecord",
636
- discriminator: T["record"],
637
- ): Promise<T[]> => {
638
- const rows = await queryRows(
639
- connection,
640
- `MATCH (n:${table}) RETURN n.sourceRecordJson AS sourceRecordJson ORDER BY n.recordKey`,
641
- );
642
- return rows.map((row) => {
643
- const record = parseSourceRecord(row.sourceRecordJson);
644
- if (record.record !== discriminator) corrupt(`invalid ${table} traversal source record`);
645
- return record as unknown as T;
646
- });
647
- };
648
- const conceptRows = await queryRows(
649
- connection,
650
- `MATCH (n:ConceptRecord)
651
- RETURN n.recordKey AS recordKey, n.conceptId AS conceptId, n.path AS path,
652
- n.conceptType AS conceptType, n.frontmatterJson AS frontmatterJson,
653
- n.body AS body, n.contentHash AS contentHash, n.tokenEstimate AS tokenEstimate
654
- ORDER BY n.recordKey`,
655
- );
656
- const concepts = conceptRows.map(conceptRecordFromPromotedRow);
657
- const tasks = await read<ProjectionTaskRecord>("TaskRecord", "task");
658
- const authoredEdges = await read<ProjectionEdgeRecord>("AuthoredEdgeRecord", "edge");
659
- if (
660
- concepts.length !== source.counts.concepts ||
661
- tasks.length !== source.counts.tasks ||
662
- authoredEdges.length !== source.counts.authoredEdges
663
- ) {
664
- corrupt("indexed traversal counts differ from the verified source snapshot");
665
- }
666
- return { concepts, tasks, authoredEdges };
667
- }
668
-
669
- async function readIndexedBundleGraph(
670
- databasePath: string,
671
- source: LadybugProjectionSource,
672
- bodyId?: string,
673
- existingConnection?: Connection,
674
- observeRecordIds?: (recordIds: ReadonlyMap<string, string>) => void,
675
- ): Promise<BundleGraph> {
676
- return useReadConnection(databasePath, existingConnection, async (connection) => {
677
- const conceptRows = await queryRows(
678
- connection,
679
- `MATCH (n:ConceptRecord)
680
- RETURN n.recordKey AS recordKey, n.conceptId AS conceptId, n.path AS path,
681
- n.conceptType AS conceptType, n.frontmatterJson AS frontmatterJson,
682
- n.tokenEstimate AS tokenEstimate
683
- ORDER BY n.conceptId`,
684
- );
685
- if (conceptRows.length !== source.counts.concepts) corrupt("indexed concept count differs");
686
- const bodies = new Map<string, string>();
687
- if (bodyId !== undefined) {
688
- const rows = await queryRows(
689
- connection,
690
- `MATCH (n:ConceptRecord) WHERE n.conceptId = $conceptId
691
- RETURN n.conceptId AS conceptId, n.body AS body`,
692
- { conceptId: bodyId },
693
- );
694
- if (rows.length > 1) corrupt("indexed concept id is duplicated");
695
- const row = rows[0];
696
- if (
697
- row !== undefined &&
698
- typeof row.conceptId === "string" &&
699
- (typeof row.body === "string" || row.body === null)
700
- ) {
701
- bodies.set(row.conceptId, row.body ?? "");
702
- }
703
- }
704
- const { concepts, recordIds, tokenEstimates } = conceptsFromRows(conceptRows, source, bodies);
705
- observeRecordIds?.(recordIds);
706
- const edgeRows = await queryRows(
707
- connection,
708
- `MATCH (n:AuthoredEdgeRecord) WHERE n.kind <> 'task'
709
- RETURN n.recordKey AS recordKey, n.fromRecordKey AS fromRecordKey,
710
- n.toRecordKey AS toRecordKey, n.kind AS kind, n.target AS target,
711
- n.ordinal AS ordinal, n.dangling AS dangling
712
- ORDER BY n.fromRecordKey, n.ordinal, n.recordKey`,
713
- );
714
- const edges = edgesFromRows(edgeRows, recordIds);
715
- return bundleGraph(concepts, edges, tokenEstimates);
716
- });
717
- }
718
-
719
- async function queryIndexedDatabase(
720
- databasePath: string,
721
- source: LadybugProjectionSource,
722
- options: QueryOptions,
723
- existingConnection?: Connection,
724
- ): Promise<LoreQueryResult> {
725
- return useReadConnection(databasePath, existingConnection, async (connection) => {
726
- const terms = [...new Set(tokenizeQueryText((options.text ?? "").trim()))];
727
- if (terms.length === 0) {
728
- const rows = await queryConceptMetadata(connection);
729
- const { concepts, tokenEstimates } = conceptsFromRows(rows, source);
730
- return queryWithBm25Index(bundleGraph(concepts, [], tokenEstimates), options);
731
- }
732
-
733
- const snapshotRows = await queryRows(
734
- connection,
735
- `MATCH (s:ProjectionSnapshot {snapshotKey: $snapshotKey})
736
- RETURN s.lexicalDocumentCount AS documentCount, s.lexicalTotalLength AS totalLength`,
737
- { snapshotKey: source.snapshotKey },
738
- );
739
- if (snapshotRows.length !== 1) corrupt("lexical snapshot metadata is missing or duplicated");
740
- const n = requiredNumber(snapshotRows[0]?.documentCount, "lexical document count");
741
- const totalLength = requiredNumber(snapshotRows[0]?.totalLength, "lexical total length");
742
- const rowById = new Map<string, Record<string, LbugValue>>();
743
- const tfById = new Map<string, Map<string, number>>();
744
- const df = new Map<string, number>();
745
- for (const term of terms) {
746
- const rows = await queryRows(
747
- connection,
748
- `MATCH (t:LexicalTerm {termKey: $termKey})<-[p:HAS_TERM]-(c:ConceptRecord)
749
- RETURN t.documentFrequency AS documentFrequency,
750
- c.recordKey AS recordKey, c.conceptId AS conceptId, c.path AS path,
751
- c.conceptType AS conceptType, c.frontmatterJson AS frontmatterJson,
752
- c.tokenEstimate AS tokenEstimate, c.lexicalLength AS lexicalLength,
753
- p.frequency AS frequency
754
- ORDER BY c.conceptId`,
755
- { termKey: lexicalTermKey(term) },
756
- );
757
- for (const row of rows) {
758
- const id = requiredString(row.conceptId, "lexical concept id");
759
- rowById.set(id, row);
760
- const frequency = requiredNumber(row.frequency, "lexical term frequency");
761
- let frequencies = tfById.get(id);
762
- if (frequencies === undefined) {
763
- frequencies = new Map();
764
- tfById.set(id, frequencies);
765
- }
766
- frequencies.set(term, frequency);
767
- const documentFrequency = requiredNumber(row.documentFrequency, "lexical document frequency");
768
- const previous = df.get(term);
769
- if (previous !== undefined && previous !== documentFrequency) corrupt("lexical document frequency differs");
770
- df.set(term, documentFrequency);
771
- }
772
- }
773
- const rows = [...rowById.values()].sort((a, b) =>
774
- compare(requiredString(a.conceptId, "concept id"), requiredString(b.conceptId, "concept id")),
775
- );
776
- const { concepts, tokenEstimates } = conceptsFromRows(rows, source);
777
- const docs = new Map<string, { tf: ReadonlyMap<string, number>; length: number }>();
778
- for (const row of rows) {
779
- const id = requiredString(row.conceptId, "lexical concept id");
780
- docs.set(id, {
781
- tf: tfById.get(id) ?? new Map(),
782
- length: requiredNumber(row.lexicalLength, "lexical document length"),
783
- });
784
- }
785
- const index: Bm25Index = { docs, df, n, avgdl: n === 0 ? 0 : totalLength / n };
786
- return queryWithBm25Index(bundleGraph(concepts, [], tokenEstimates), options, index);
787
- });
788
- }
789
-
790
- async function queryConceptMetadata(connection: Connection): Promise<Record<string, LbugValue>[]> {
791
- return queryRows(
792
- connection,
793
- `MATCH (n:ConceptRecord)
794
- RETURN n.recordKey AS recordKey, n.conceptId AS conceptId, n.path AS path,
795
- n.conceptType AS conceptType, n.frontmatterJson AS frontmatterJson,
796
- n.tokenEstimate AS tokenEstimate
797
- ORDER BY n.conceptId`,
798
- );
799
- }
800
-
801
- function conceptsFromRows(
802
- rows: readonly Record<string, LbugValue>[],
803
- source: LadybugProjectionSource,
804
- bodies: ReadonlyMap<string, string> = new Map(),
805
- ): {
806
- concepts: Map<string, Concept>;
807
- recordIds: Map<string, string>;
808
- tokenEstimates: Map<string, number>;
809
- } {
810
- const concepts = new Map<string, Concept>();
811
- const recordIds = new Map<string, string>();
812
- const tokenEstimates = new Map<string, number>();
813
- const docsPrefix = `${source.manifest.bundle.docsRoot}/`;
814
- for (const row of rows) {
815
- const recordKey = requiredString(row.recordKey, "concept record key");
816
- const id = requiredString(row.conceptId, "concept id");
817
- const path = requiredString(row.path, "concept path");
818
- const type = requiredString(row.conceptType, "concept type");
819
- if (!path.startsWith(docsPrefix) || concepts.has(id) || recordIds.has(recordKey)) {
820
- corrupt("indexed concept identities are duplicated or outside the bundle root");
821
- }
822
- const frontmatter = parseObjectJson(row.frontmatterJson, "concept frontmatter");
823
- concepts.set(id, { id, path: path.slice(docsPrefix.length), type, frontmatter, body: bodies.get(id) ?? "" });
824
- recordIds.set(recordKey, id);
825
- tokenEstimates.set(id, requiredNumber(row.tokenEstimate, "concept token estimate"));
826
- }
827
- return { concepts, recordIds, tokenEstimates };
828
- }
829
-
830
- function edgesFromRows(rows: readonly Record<string, LbugValue>[], recordIds: ReadonlyMap<string, string>): Edge[] {
831
- const indexed = rows.map((row) => {
832
- const fromKey = requiredString(row.fromRecordKey, "edge source key");
833
- const from = recordIds.get(fromKey);
834
- const toKey = row.toRecordKey;
835
- const to = typeof toKey === "string" ? recordIds.get(toKey) : undefined;
836
- const kind = requiredString(row.kind, "edge kind");
837
- const dangling = row.dangling;
838
- if (from === undefined || !isEdgeKind(kind) || typeof dangling !== "boolean") corrupt("indexed edge is invalid");
839
- if ((!dangling && (typeof toKey !== "string" || to === undefined)) || (dangling && toKey !== null)) {
840
- corrupt("indexed edge endpoint differs");
841
- }
842
- return {
843
- ordinal: requiredNumber(row.ordinal, "edge ordinal"),
844
- recordKey: requiredString(row.recordKey, "edge record key"),
845
- edge: { from, to: to ?? null, kind, target: requiredString(row.target, "edge target") } satisfies Edge,
846
- };
847
- });
848
- indexed.sort(
849
- (a, b) => compare(a.edge.from, b.edge.from) || a.ordinal - b.ordinal || compare(a.recordKey, b.recordKey),
850
- );
851
- return indexed.map(({ edge }) => edge);
852
- }
853
-
854
- function bundleGraph(
855
- concepts: ReadonlyMap<string, Concept>,
856
- edges: readonly Edge[],
857
- tokenEstimates: ReadonlyMap<string, number>,
858
- ): BundleGraph {
859
- const neighbors = buildNeighborIndex(edges);
860
- let total: number | undefined;
861
- return {
862
- concepts,
863
- edges,
864
- neighbors: (id) => neighbors.get(id) ?? EMPTY_NEIGHBORS,
865
- tokenEstimate(id?: string): number {
866
- if (id === undefined) {
867
- total ??= [...tokenEstimates.values()].reduce((sum, value) => sum + value, 0);
868
- return total;
869
- }
870
- const value = tokenEstimates.get(id);
871
- if (value === undefined) {
872
- throw new LoreError(
873
- "not_found",
874
- `concept "${id}" is not in the bundle`,
875
- "run `lore query` to find the right id, or check the path",
876
- { id },
877
- );
878
- }
879
- return value;
880
- },
881
- };
882
- }
883
-
884
- const EMPTY_NEIGHBORS: readonly string[] = [];
885
-
886
- function buildNeighborIndex(edges: readonly Edge[]): ReadonlyMap<string, ReadonlySet<string>> {
887
- const neighbors = new Map<string, Set<string>>();
888
- const connect = (from: string, to: string): void => {
889
- let adjacent = neighbors.get(from);
890
- if (adjacent === undefined) {
891
- adjacent = new Set<string>();
892
- neighbors.set(from, adjacent);
893
- }
894
- adjacent.add(to);
895
- };
896
- for (const edge of edges) {
897
- if (edge.to === null || edge.to === edge.from) continue;
898
- connect(edge.from, edge.to);
899
- connect(edge.to, edge.from);
900
- }
901
- return neighbors;
902
- }
903
-
904
- async function withReadConnection<T>(databasePath: string, fn: (connection: Connection) => Promise<T>): Promise<T> {
905
- const database = readOnlyDatabase(databasePath);
906
- const connection = new Connection(database);
907
- try {
908
- return await fn(connection);
909
- } finally {
910
- await closeConnection(connection);
911
- await database.close();
912
- }
913
- }
914
-
915
- function useReadConnection<T>(
916
- databasePath: string,
917
- existing: Connection | undefined,
918
- fn: (connection: Connection) => Promise<T>,
919
- ): Promise<T> {
920
- return existing === undefined ? withReadConnection(databasePath, fn) : fn(existing);
921
- }
922
-
923
- function parseObjectJson(value: unknown, label: string): Record<string, unknown> {
924
- if (typeof value !== "string") corrupt(`${label} JSON is missing`);
925
- let parsed: unknown;
926
- try {
927
- parsed = JSON.parse(value);
928
- } catch {
929
- corrupt(`${label} JSON is malformed`);
930
- }
931
- if (!isObject(parsed)) corrupt(`${label} JSON is not an object`);
932
- return parsed;
933
- }
934
-
935
- function requiredString(value: unknown, label: string): string {
936
- if (typeof value !== "string") corrupt(`${label} is invalid`);
937
- return value;
938
- }
939
-
940
- function requiredNumber(value: unknown, label: string): number {
941
- const number = numericValue(value);
942
- if (number === null || !Number.isSafeInteger(number) || number < 0) corrupt(`${label} is invalid`);
943
- return number;
944
- }
945
-
946
- function writableDatabase(path: string): Database {
947
- return new Database(path, WRITER_BUFFER_BYTES, true, false, 0, false, -1, true, true, true);
948
- }
949
-
950
- function readOnlyDatabase(path: string): Database {
951
- return new Database(path, READER_BUFFER_BYTES, true, true, 0, true, -1, true, true, true);
952
- }
953
-
954
- function fullScanDatabase(path: string): Database {
955
- return new Database(path, FULL_SCAN_BUFFER_BYTES, true, true, 0, true, -1, true, true, true);
956
- }
957
-
958
- async function insertProjection(
959
- connection: Connection,
960
- databasePath: string,
961
- source: LadybugProjectionSource,
962
- ): Promise<void> {
963
- const common = {
964
- repositoryScopeKey: source.repositoryScopeKey,
965
- snapshotKey: source.snapshotKey,
966
- bundleId: source.manifest.bundle.id,
967
- gitCommit: source.manifest.bundle.gitCommit,
968
- exportDigest: source.exportDigest,
969
- };
970
- const lexical = buildProjectionLexicalIndex(source);
971
- VERIFIED_LEXICAL_LENGTHS.set(source, new Map([...lexical.docs].map(([id, document]) => [id, document.length])));
972
- Bun.gc(true);
973
- await executePrepared(
974
- connection,
975
- `CREATE (n:RepositoryProjection {
976
- repositoryScopeKey: $repositoryScopeKey,
977
- bundleId: $bundleId,
978
- docsRoot: $docsRoot
979
- })`,
980
- {
981
- repositoryScopeKey: source.repositoryScopeKey,
982
- bundleId: source.manifest.bundle.id,
983
- docsRoot: source.manifest.bundle.docsRoot,
984
- },
985
- );
986
- await executePrepared(
987
- connection,
988
- `CREATE (n:ProjectionSnapshot {
989
- snapshotKey: $snapshotKey,
990
- indexFormatVersion: $indexFormatVersion,
991
- projectionSchemaVersion: $projectionSchemaVersion,
992
- normalizationVersion: $normalizationVersion,
993
- exporterName: $exporterName,
994
- exporterVersion: $exporterVersion,
995
- loreVersion: $loreVersion,
996
- ladybugVersion: $ladybugVersion,
997
- ladybugStorageVersion: $ladybugStorageVersion,
998
- repositoryScopeKey: $repositoryScopeKey,
999
- bundleId: $bundleId,
1000
- okfVersion: $okfVersion,
1001
- docsRoot: $docsRoot,
1002
- gitCommit: $gitCommit,
1003
- exportDigest: $exportDigest,
1004
- taskSnapshotDigest: $taskSnapshotDigest,
1005
- sourceFingerprint: $sourceFingerprint,
1006
- sourceRecordsDigest: $sourceRecordsDigest,
1007
- recordKeysDigest: $recordKeysDigest,
1008
- lexicalDocumentCount: $lexicalDocumentCount,
1009
- lexicalTotalLength: $lexicalTotalLength,
1010
- recordCount: $recordCount,
1011
- conceptCount: $conceptCount,
1012
- taskCount: $taskCount,
1013
- authoredEdgeCount: $authoredEdgeCount,
1014
- manifestJson: $manifestJson,
1015
- trailerJson: $trailerJson
1016
- })`,
1017
- snapshotMetadata(source, lexical),
1018
- );
1019
- if (source.commitKey !== null && source.manifest.bundle.gitCommit !== null) {
1020
- await executePrepared(
1021
- connection,
1022
- `CREATE (n:SourceCommit {
1023
- commitKey: $commitKey,
1024
- repositoryScopeKey: $repositoryScopeKey,
1025
- sha: $sha
1026
- })`,
1027
- {
1028
- commitKey: source.commitKey,
1029
- repositoryScopeKey: source.repositoryScopeKey,
1030
- sha: source.manifest.bundle.gitCommit,
1031
- },
1032
- );
1033
- }
1034
-
1035
- await copyRows(
1036
- connection,
1037
- databasePath,
1038
- "ConceptRecord",
1039
- (function* (): Generator<readonly CsvValue[]> {
1040
- for (const record of source.concepts) {
1041
- const doc = lexical.docs.get(record.id);
1042
- if (doc === undefined) corrupt(`lexical document is missing for ${record.id}`);
1043
- yield [
1044
- record.key,
1045
- common.repositoryScopeKey,
1046
- common.snapshotKey,
1047
- common.bundleId,
1048
- common.gitCommit,
1049
- common.exportDigest,
1050
- record.id,
1051
- record.path,
1052
- record.type,
1053
- JSON.stringify(record.frontmatter),
1054
- frontmatterScalar(record.frontmatter.title) ?? null,
1055
- frontmatterScalar(record.frontmatter.summary) ?? null,
1056
- frontmatterScalar(record.frontmatter.description) ?? null,
1057
- canonicalJson(record.frontmatter.tags ?? null),
1058
- record.body,
1059
- record.contentHash,
1060
- record.tokenEstimate,
1061
- doc.length,
1062
- ];
1063
- }
1064
- })(),
1065
- );
1066
- Bun.gc(true);
1067
- await executeQuery(connection, "CHECKPOINT");
1068
- await copyRows(
1069
- connection,
1070
- databasePath,
1071
- "TaskRecord",
1072
- source.tasks.map((record) => [
1073
- record.key,
1074
- common.repositoryScopeKey,
1075
- common.snapshotKey,
1076
- common.bundleId,
1077
- common.gitCommit,
1078
- common.exportDigest,
1079
- record.id,
1080
- record.title,
1081
- record.status,
1082
- canonicalJson(record.labels),
1083
- record.priority,
1084
- record.ordinal,
1085
- canonicalJson(record.assignees),
1086
- record.milestone,
1087
- record.parentTaskId,
1088
- record.sourceAdapterVersion,
1089
- JSON.stringify(record),
1090
- ]),
1091
- );
1092
- await copyRows(
1093
- connection,
1094
- databasePath,
1095
- "AuthoredEdgeRecord",
1096
- source.authoredEdges.map((record) => [
1097
- record.key,
1098
- common.repositoryScopeKey,
1099
- common.snapshotKey,
1100
- common.bundleId,
1101
- common.gitCommit,
1102
- common.exportDigest,
1103
- record.from,
1104
- record.to,
1105
- record.kind,
1106
- record.target,
1107
- record.ordinal,
1108
- record.dangling,
1109
- JSON.stringify(record),
1110
- ]),
1111
- );
1112
- await executeQuery(connection, "CHECKPOINT");
1113
-
1114
- const terms = [...lexical.df.entries()].sort(([a], [b]) => compare(a, b));
1115
- await copyRows(
1116
- connection,
1117
- databasePath,
1118
- "LexicalTerm",
1119
- terms.map(([termKey, documentFrequency]) => [termKey, documentFrequency]),
1120
- );
1121
-
1122
- await createRelationship(
1123
- connection,
1124
- "RepositoryProjection",
1125
- "repositoryScopeKey",
1126
- source.repositoryScopeKey,
1127
- "HAS_SNAPSHOT",
1128
- "ProjectionSnapshot",
1129
- "snapshotKey",
1130
- source.snapshotKey,
1131
- );
1132
- if (source.commitKey !== null) {
1133
- await createRelationship(
1134
- connection,
1135
- "ProjectionSnapshot",
1136
- "snapshotKey",
1137
- source.snapshotKey,
1138
- "AT_COMMIT",
1139
- "SourceCommit",
1140
- "commitKey",
1141
- source.commitKey,
1142
- );
1143
- }
1144
- await copyRows(
1145
- connection,
1146
- databasePath,
1147
- "HAS_CONCEPT",
1148
- source.concepts.map((record) => [source.snapshotKey, record.key]),
1149
- );
1150
- await copyRows(
1151
- connection,
1152
- databasePath,
1153
- "HAS_TASK",
1154
- source.tasks.map((record) => [source.snapshotKey, record.key]),
1155
- );
1156
- await copyRows(
1157
- connection,
1158
- databasePath,
1159
- "HAS_EDGE",
1160
- source.authoredEdges.map((record) => [source.snapshotKey, record.key]),
1161
- );
1162
- await copyRows(
1163
- connection,
1164
- databasePath,
1165
- "EDGE_SOURCE",
1166
- source.authoredEdges
1167
- .filter((record) => edgeSourceKind(record) === "concept")
1168
- .map((record) => [record.from, record.key]),
1169
- );
1170
- await copyRows(
1171
- connection,
1172
- databasePath,
1173
- "EDGE_TASK_SOURCE",
1174
- source.authoredEdges
1175
- .filter((record) => edgeSourceKind(record) === "task")
1176
- .map((record) => [record.from, record.key]),
1177
- );
1178
- await copyRows(
1179
- connection,
1180
- databasePath,
1181
- "EDGE_CONCEPT_TARGET",
1182
- source.authoredEdges
1183
- .filter((record) => edgeTargetKind(record) === "concept" && !record.dangling && record.to !== null)
1184
- .map((record) => [record.key, record.to]),
1185
- );
1186
- await copyRows(
1187
- connection,
1188
- databasePath,
1189
- "EDGE_TASK_TARGET",
1190
- source.authoredEdges
1191
- .filter((record) => edgeTargetKind(record) === "task" && !record.dangling && record.to !== null)
1192
- .map((record) => [record.key, record.to]),
1193
- );
1194
- const recordKeys = new Map(source.concepts.map((record) => [record.id, record.key]));
1195
- await copyRows(
1196
- connection,
1197
- databasePath,
1198
- "HAS_TERM",
1199
- (function* (): Generator<readonly CsvValue[]> {
1200
- for (const [id, doc] of lexical.docs) {
1201
- const recordKey = recordKeys.get(id);
1202
- if (recordKey === undefined) corrupt(`lexical record key is missing for ${id}`);
1203
- for (const [termKey, frequency] of doc.tf) yield [recordKey, termKey, frequency];
1204
- }
1205
- })(),
1206
- );
1207
- await executeQuery(connection, "CHECKPOINT");
1208
- }
1209
-
1210
- type CsvValue = string | number | boolean | null;
1211
-
1212
- function buildProjectionLexicalIndex(source: LadybugProjectionSource): Bm25Index {
1213
- const docsPrefix = `${source.manifest.bundle.docsRoot}/`;
1214
- const concepts = new Map<string, Concept>();
1215
- for (const record of source.concepts) {
1216
- if (!record.path.startsWith(docsPrefix)) corrupt("projection concept path is outside the bundle root");
1217
- concepts.set(record.id, {
1218
- id: record.id,
1219
- path: record.path.slice(docsPrefix.length),
1220
- type: record.type,
1221
- frontmatter: record.frontmatter,
1222
- body: record.body,
1223
- });
1224
- }
1225
- const docs = new Map<string, { tf: ReadonlyMap<string, number>; length: number }>();
1226
- const df = new Map<string, number>();
1227
- let totalLength = 0;
1228
- for (const concept of concepts.values()) {
1229
- const tf = new Map<string, number>();
1230
- let length = 0;
1231
- for (const field of searchableConceptFields(concept)) {
1232
- for (const match of field.matchAll(/[\p{L}\p{N}]+/gu)) {
1233
- const termKey = lexicalTermKey((match[0] as string).toLowerCase());
1234
- tf.set(termKey, (tf.get(termKey) ?? 0) + 1);
1235
- length++;
1236
- }
1237
- }
1238
- for (const termKey of tf.keys()) df.set(termKey, (df.get(termKey) ?? 0) + 1);
1239
- docs.set(concept.id, { tf, length });
1240
- totalLength += length;
1241
- }
1242
- const n = docs.size;
1243
- return { docs, df, n, avgdl: n === 0 ? 0 : totalLength / n };
1244
- }
1245
-
1246
- function lexicalTermKey(term: string): string {
1247
- return createHash("sha256").update(term).digest("hex");
1248
- }
1249
-
1250
- async function copyRows(
1251
- connection: Connection,
1252
- databasePath: string,
1253
- table: string,
1254
- rows: Iterable<readonly CsvValue[]>,
1255
- ): Promise<void> {
1256
- const maxBatchBytes = 16 * 1024 * 1024;
1257
- const iterator = rows[Symbol.iterator]();
1258
- let current = iterator.next();
1259
- while (!current.done) {
1260
- const path = join(dirname(databasePath), `.lore-import-${table}-${randomUUID()}.csv`);
1261
- const file = openSync(path, "wx", 0o600);
1262
- let batchBytes = 0;
1263
- try {
1264
- while (!current.done) {
1265
- const line = `${current.value.map(csvValue).join(",")}\n`;
1266
- const lineBytes = Buffer.byteLength(line);
1267
- if (batchBytes > 0 && batchBytes + lineBytes > maxBatchBytes) break;
1268
- writeSync(file, line);
1269
- batchBytes += lineBytes;
1270
- current = iterator.next();
1271
- }
1272
- } finally {
1273
- closeSync(file);
1274
- }
1275
- try {
1276
- await executeQuery(
1277
- connection,
1278
- `COPY ${table} FROM ${JSON.stringify(path)} (header=false, auto_detect=false, parallel=false, delim=",", quote="\\"", escape="\\"")`,
1279
- );
1280
- } finally {
1281
- try {
1282
- unlinkSync(path);
1283
- } catch {
1284
- // The generation staging directory remains disposable if cleanup races a failure.
1285
- }
1286
- }
1287
- }
1288
- }
1289
-
1290
- function csvValue(value: CsvValue): string {
1291
- if (value === null) return "";
1292
- if (typeof value === "boolean") return value ? "true" : "false";
1293
- if (typeof value === "number") return String(value);
1294
- return `"${value.replace(/"/g, '""')}"`;
1295
- }
1296
-
1297
- function snapshotMetadata(source: LadybugProjectionSource, lexical?: Bm25Index): Record<string, LbugValue> {
1298
- return {
1299
- snapshotKey: source.snapshotKey,
1300
- indexFormatVersion: LADYBUG_INDEX_FORMAT,
1301
- projectionSchemaVersion: source.manifest.schemaVersion,
1302
- normalizationVersion: source.manifest.normalizationVersion,
1303
- exporterName: source.manifest.exporter.name,
1304
- exporterVersion: source.manifest.exporter.version,
1305
- loreVersion: source.manifest.exporter.version,
1306
- ladybugVersion: source.ladybugVersion,
1307
- ladybugStorageVersion: source.ladybugStorageVersion,
1308
- repositoryScopeKey: source.repositoryScopeKey,
1309
- bundleId: source.manifest.bundle.id,
1310
- okfVersion: source.manifest.bundle.okfVersion,
1311
- docsRoot: source.manifest.bundle.docsRoot,
1312
- gitCommit: source.manifest.bundle.gitCommit,
1313
- exportDigest: source.exportDigest,
1314
- taskSnapshotDigest: source.taskSnapshotDigest,
1315
- sourceFingerprint: source.sourceFingerprint,
1316
- sourceRecordsDigest: source.sourceRecordsDigest,
1317
- recordKeysDigest: source.recordKeysDigest,
1318
- ...(lexical === undefined
1319
- ? {}
1320
- : {
1321
- lexicalDocumentCount: lexical.n,
1322
- lexicalTotalLength: [...lexical.docs.values()].reduce((sum, doc) => sum + doc.length, 0),
1323
- }),
1324
- recordCount: source.trailer.recordCount,
1325
- conceptCount: source.counts.concepts,
1326
- taskCount: source.counts.tasks,
1327
- authoredEdgeCount: source.counts.authoredEdges,
1328
- manifestJson: JSON.stringify(source.manifest),
1329
- trailerJson: JSON.stringify(source.trailer),
1330
- };
1331
- }
1332
-
1333
- async function verifyConceptRecordTable(connection: Connection, source: LadybugProjectionSource): Promise<void> {
1334
- const expectedRecords = source.concepts;
1335
- const lexicalLengths = VERIFIED_LEXICAL_LENGTHS.get(source);
1336
- const expected = new Map(expectedRecords.map((record) => [record.key, record]));
1337
- let count = 0;
1338
- await queryEachRow(
1339
- connection,
1340
- `MATCH (n:ConceptRecord)
1341
- RETURN n.recordKey AS recordKey, n.conceptId AS conceptId, n.path AS path,
1342
- n.conceptType AS conceptType, n.frontmatterJson AS frontmatterJson,
1343
- n.title AS title, n.summary AS summary, n.description AS description,
1344
- n.tagsJson AS tagsJson, n.contentHash AS contentHash,
1345
- n.tokenEstimate AS tokenEstimate, n.lexicalLength AS lexicalLength`,
1346
- (row) => {
1347
- count++;
1348
- const key = requiredString(row.recordKey, "concept record key");
1349
- const record = expected.get(key);
1350
- if (
1351
- record === undefined ||
1352
- row.conceptId !== record.id ||
1353
- row.path !== record.path ||
1354
- row.conceptType !== record.type ||
1355
- row.frontmatterJson !== JSON.stringify(record.frontmatter) ||
1356
- row.title !== (frontmatterScalar(record.frontmatter.title) ?? null) ||
1357
- row.summary !== (frontmatterScalar(record.frontmatter.summary) ?? null) ||
1358
- row.description !== (frontmatterScalar(record.frontmatter.description) ?? null) ||
1359
- row.tagsJson !== canonicalJson(record.frontmatter.tags ?? null) ||
1360
- row.contentHash !== record.contentHash ||
1361
- numericValue(row.tokenEstimate) !== record.tokenEstimate ||
1362
- numericValue(row.lexicalLength) !== expectedLexicalLength(record, lexicalLengths)
1363
- ) {
1364
- corrupt("ConceptRecord source record differs");
1365
- }
1366
- expected.delete(key);
1367
- },
1368
- );
1369
- if (count !== expectedRecords.length || expected.size !== 0) corrupt("ConceptRecord count differs");
1370
-
1371
- const expectedBodyDigests = new Map(
1372
- expectedRecords.map((record) => [
1373
- record.key,
1374
- record.body === "" ? null : createHash("sha256").update(record.body).digest("hex"),
1375
- ]),
1376
- );
1377
- await queryEachRow(
1378
- connection,
1379
- "MATCH (n:ConceptRecord) RETURN n.recordKey AS recordKey, sha256(n.body) AS bodyDigest",
1380
- (row) => {
1381
- const key = requiredString(row.recordKey, "concept body record key");
1382
- if (!expectedBodyDigests.has(key) || row.bodyDigest !== expectedBodyDigests.get(key)) {
1383
- corrupt("ConceptRecord body differs");
1384
- }
1385
- expectedBodyDigests.delete(key);
1386
- },
1387
- );
1388
- if (expectedBodyDigests.size !== 0) corrupt("ConceptRecord bodies are missing");
1389
- }
1390
-
1391
- function expectedLexicalLength(
1392
- record: ProjectionConceptRecord,
1393
- cached: ReadonlyMap<string, number> | undefined,
1394
- ): number {
1395
- const length = cached?.get(record.id);
1396
- if (length !== undefined) return length;
1397
- return searchableConceptFields({
1398
- id: record.id,
1399
- path: record.path,
1400
- type: record.type,
1401
- frontmatter: record.frontmatter,
1402
- body: record.body,
1403
- }).reduce((total, field) => total + tokenizeQueryText(field).length, 0);
1404
- }
1405
-
1406
- async function verifyRecordTable(
1407
- connection: Connection,
1408
- table: "ConceptRecord" | "TaskRecord" | "AuthoredEdgeRecord",
1409
- expectedRecords: readonly { readonly key: string; readonly record: string }[],
1410
- ): Promise<void> {
1411
- const rows = await queryRows(
1412
- connection,
1413
- `MATCH (n:${table}) RETURN n.recordKey AS key, n.sourceRecordJson AS sourceRecordJson ORDER BY n.recordKey`,
1414
- );
1415
- const expected = new Map(expectedRecords.map((record) => [record.key, JSON.stringify(record)]));
1416
- if (rows.length !== expected.size) corrupt(`${table} count differs`);
1417
- for (const row of rows) {
1418
- const key = row.key;
1419
- const sourceRecordJson = row.sourceRecordJson;
1420
- if (typeof key !== "string" || typeof sourceRecordJson !== "string" || expected.get(key) !== sourceRecordJson) {
1421
- corrupt(`${table} source record differs`);
1422
- }
1423
- expected.delete(key);
1424
- }
1425
- if (expected.size !== 0) corrupt(`${table} is missing source records`);
1426
- }
1427
-
1428
- async function verifyPromotedSamples(connection: Connection, source: LadybugProjectionSource): Promise<void> {
1429
- const concept = [...source.concepts].sort((a, b) => compare(a.key, b.key))[0];
1430
- await expectSample(
1431
- connection,
1432
- `MATCH (n:ConceptRecord {recordKey: $recordKey})
1433
- RETURN n.recordKey AS recordKey,
1434
- n.repositoryScopeKey AS repositoryScopeKey,
1435
- n.snapshotKey AS snapshotKey,
1436
- n.bundleId AS bundleId,
1437
- n.gitCommit AS gitCommit,
1438
- n.exportDigest AS exportDigest,
1439
- n.conceptId AS conceptId,
1440
- n.path AS path,
1441
- n.conceptType AS conceptType,
1442
- n.frontmatterJson AS frontmatterJson,
1443
- n.body AS body,
1444
- n.contentHash AS contentHash,
1445
- n.tokenEstimate AS tokenEstimate`,
1446
- concept === undefined
1447
- ? undefined
1448
- : {
1449
- ...commonExpected(source),
1450
- recordKey: concept.key,
1451
- conceptId: concept.id,
1452
- path: concept.path,
1453
- conceptType: concept.type,
1454
- frontmatterJson: JSON.stringify(concept.frontmatter),
1455
- body: concept.body === "" ? null : concept.body,
1456
- contentHash: concept.contentHash,
1457
- tokenEstimate: concept.tokenEstimate,
1458
- },
1459
- "ConceptRecord",
1460
- { recordKey: concept?.key ?? "" },
1461
- );
1462
-
1463
- const task = [...source.tasks].sort((a, b) => compare(a.key, b.key))[0];
1464
- await expectSample(
1465
- connection,
1466
- `MATCH (n:TaskRecord {recordKey: $recordKey})
1467
- RETURN n.recordKey AS recordKey,
1468
- n.repositoryScopeKey AS repositoryScopeKey,
1469
- n.snapshotKey AS snapshotKey,
1470
- n.bundleId AS bundleId,
1471
- n.gitCommit AS gitCommit,
1472
- n.exportDigest AS exportDigest,
1473
- n.taskId AS taskId,
1474
- n.title AS title,
1475
- n.status AS status,
1476
- n.labelsJson AS labelsJson,
1477
- n.priority AS priority,
1478
- n.ordinal AS ordinal,
1479
- n.assigneesJson AS assigneesJson,
1480
- n.milestone AS milestone,
1481
- n.parentTaskId AS parentTaskId,
1482
- n.sourceAdapterVersion AS sourceAdapterVersion`,
1483
- task === undefined
1484
- ? undefined
1485
- : {
1486
- ...commonExpected(source),
1487
- recordKey: task.key,
1488
- taskId: task.id,
1489
- title: task.title,
1490
- status: task.status,
1491
- labelsJson: canonicalJson(task.labels),
1492
- priority: task.priority,
1493
- ordinal: task.ordinal,
1494
- assigneesJson: canonicalJson(task.assignees),
1495
- milestone: task.milestone,
1496
- parentTaskId: task.parentTaskId,
1497
- sourceAdapterVersion: task.sourceAdapterVersion,
1498
- },
1499
- "TaskRecord",
1500
- { recordKey: task?.key ?? "" },
1501
- );
1502
-
1503
- const edge = [...source.authoredEdges].sort((a, b) => compare(a.key, b.key))[0];
1504
- await expectSample(
1505
- connection,
1506
- `MATCH (n:AuthoredEdgeRecord {recordKey: $recordKey})
1507
- RETURN n.recordKey AS recordKey,
1508
- n.repositoryScopeKey AS repositoryScopeKey,
1509
- n.snapshotKey AS snapshotKey,
1510
- n.bundleId AS bundleId,
1511
- n.gitCommit AS gitCommit,
1512
- n.exportDigest AS exportDigest,
1513
- n.fromRecordKey AS fromRecordKey,
1514
- n.toRecordKey AS toRecordKey,
1515
- n.kind AS kind,
1516
- n.target AS target,
1517
- n.ordinal AS ordinal,
1518
- n.dangling AS dangling`,
1519
- edge === undefined
1520
- ? undefined
1521
- : {
1522
- ...commonExpected(source),
1523
- recordKey: edge.key,
1524
- fromRecordKey: edge.from,
1525
- toRecordKey: edge.to,
1526
- kind: edge.kind,
1527
- target: edge.target,
1528
- ordinal: edge.ordinal,
1529
- dangling: edge.dangling,
1530
- },
1531
- "AuthoredEdgeRecord",
1532
- { recordKey: edge?.key ?? "" },
1533
- );
1534
- }
1535
-
1536
- function commonExpected(source: LadybugProjectionSource): Record<string, LbugValue> {
1537
- return {
1538
- repositoryScopeKey: source.repositoryScopeKey,
1539
- snapshotKey: source.snapshotKey,
1540
- bundleId: source.manifest.bundle.id,
1541
- gitCommit: source.manifest.bundle.gitCommit,
1542
- exportDigest: source.exportDigest,
1543
- };
1544
- }
1545
-
1546
- async function expectSample(
1547
- connection: Connection,
1548
- query: string,
1549
- expected: Record<string, LbugValue> | undefined,
1550
- label: string,
1551
- params: Record<string, LbugValue>,
1552
- ): Promise<void> {
1553
- const rows = await queryRows(connection, query, params);
1554
- if (expected === undefined) {
1555
- if (rows.length !== 0) corrupt(`${label} conformance sample should be empty`);
1556
- return;
1557
- }
1558
- if (rows.length !== 1) corrupt(`${label} conformance sample is missing or duplicated`);
1559
- const actual = rows[0] as Record<string, unknown>;
1560
- for (const [key, value] of Object.entries(expected)) {
1561
- if (!sameDatabaseValue(actual[key], value)) corrupt(`${label} conformance sample differs for ${key}`);
1562
- }
1563
- }
1564
-
1565
- async function expectRelationshipPairs(
1566
- connection: Connection,
1567
- query: string,
1568
- expected: readonly (readonly [string, string])[],
1569
- label: string,
1570
- ): Promise<void> {
1571
- const rows = await queryRows(connection, query);
1572
- const actualPairs = rows.map((row) => {
1573
- if (typeof row.fromKey !== "string" || typeof row.toKey !== "string") {
1574
- corrupt(`${label} contains an invalid endpoint`);
1575
- }
1576
- return `${row.fromKey}\0${row.toKey}`;
1577
- });
1578
- const expectedPairs = expected.map(([from, to]) => `${from}\0${to}`);
1579
- actualPairs.sort(compare);
1580
- expectedPairs.sort(compare);
1581
- if (actualPairs.length !== expectedPairs.length || actualPairs.some((pair, index) => pair !== expectedPairs[index])) {
1582
- corrupt(`${label} endpoints differ`);
1583
- }
1584
- }
1585
-
1586
- async function createRelationship(
1587
- connection: Connection,
1588
- fromTable: string,
1589
- fromProperty: string,
1590
- fromValue: string,
1591
- relationship: string,
1592
- toTable: string,
1593
- toProperty: string,
1594
- toValue: string,
1595
- ): Promise<void> {
1596
- await executePrepared(
1597
- connection,
1598
- `MATCH (a:${fromTable} {${fromProperty}: $from}), (b:${toTable} {${toProperty}: $to})
1599
- CREATE (a)-[:${relationship}]->(b)`,
1600
- { from: fromValue, to: toValue },
1601
- );
1602
- }
1603
-
1604
- async function expectCount(connection: Connection, query: string, expected: number, label: string): Promise<void> {
1605
- const rows = await queryRows(connection, query);
1606
- if (rows.length !== 1 || numericValue(rows[0]?.count) !== expected) {
1607
- corrupt(`${label} count differs`);
1608
- }
1609
- }
1610
-
1611
- async function executePrepared(
1612
- connection: Connection,
1613
- query: string,
1614
- params: Record<string, LbugValue>,
1615
- ): Promise<void> {
1616
- const statement = await prepare(connection, query);
1617
- await executeStatement(connection, statement, params);
1618
- }
1619
-
1620
- async function prepare(connection: Connection, query: string) {
1621
- const statement = await connection.prepare(query);
1622
- if (!statement.isSuccess()) throw new Error(statement.getErrorMessage());
1623
- return statement;
1624
- }
1625
-
1626
- function prepareSync(connection: Connection, query: string): ReturnType<Connection["prepareSync"]> {
1627
- const statement = connection.prepareSync(query);
1628
- if (!statement.isSuccess()) throw new Error(statement.getErrorMessage());
1629
- return statement;
1630
- }
1631
-
1632
- async function executeStatement(
1633
- connection: Connection,
1634
- statement: Awaited<ReturnType<Connection["prepare"]>>,
1635
- params: Record<string, LbugValue>,
1636
- ): Promise<void> {
1637
- const result = await connection.execute(statement, params);
1638
- closeResult(result);
1639
- }
1640
-
1641
- async function executeQuery(connection: Connection, query: string): Promise<void> {
1642
- const result = await connection.query(query);
1643
- closeResult(result);
1644
- }
1645
-
1646
- async function queryRows(
1647
- connection: Connection,
1648
- query: string,
1649
- params?: Record<string, LbugValue>,
1650
- ): Promise<Record<string, LbugValue>[]> {
1651
- const result =
1652
- params === undefined
1653
- ? await connection.query(query)
1654
- : await connection.execute(await prepare(connection, query), params);
1655
- return resultRows(result);
1656
- }
1657
-
1658
- async function queryPreparedRows(
1659
- connection: Connection,
1660
- statement: Awaited<ReturnType<Connection["prepare"]>>,
1661
- params: Record<string, LbugValue>,
1662
- ): Promise<Record<string, LbugValue>[]> {
1663
- return resultRows(await connection.execute(statement, params));
1664
- }
1665
-
1666
- async function resultRows(result: QueryResult | QueryResult[]): Promise<Record<string, LbugValue>[]> {
1667
- if (Array.isArray(result)) {
1668
- closeResult(result);
1669
- throw new Error("Ladybug returned multiple result sets for a single query");
1670
- }
1671
- try {
1672
- return await result.getAll();
1673
- } finally {
1674
- result.close();
1675
- }
1676
- }
1677
-
1678
- async function queryEachRow(
1679
- connection: Connection,
1680
- query: string,
1681
- visit: (row: Record<string, LbugValue>) => void,
1682
- ): Promise<void> {
1683
- const result = await connection.query(query);
1684
- if (Array.isArray(result)) {
1685
- closeResult(result);
1686
- throw new Error("Ladybug returned multiple result sets for a single query");
1687
- }
1688
- try {
1689
- while (result.hasNext()) {
1690
- const row = await result.getNext();
1691
- if (row !== null) visit(row);
1692
- }
1693
- } finally {
1694
- result.close();
1695
- }
1696
- }
1697
-
1698
- function closeResult(result: QueryResult | QueryResult[]): void {
1699
- if (Array.isArray(result)) {
1700
- for (const item of result) item.close();
1701
- } else {
1702
- result.close();
1703
- }
1704
- }
1705
-
1706
- async function closeConnection(connection: Connection): Promise<void> {
1707
- try {
1708
- await connection.close();
1709
- } catch {
1710
- // Database.close remains the final native resource boundary.
1711
- }
1712
- }
1713
-
1714
- function numericValue(value: unknown): number | null {
1715
- if (typeof value === "bigint") return Number(value);
1716
- return typeof value === "number" ? value : null;
1717
- }
1718
-
1719
- function sameDatabaseValue(actual: unknown, expected: unknown): boolean {
1720
- if (typeof actual === "bigint" && typeof expected === "number") return actual === BigInt(expected);
1721
- return actual === expected;
1722
- }
1723
-
1724
- function compare(a: string, b: string): number {
1725
- return a < b ? -1 : a > b ? 1 : 0;
1726
- }
1727
-
1728
- function conceptRecordFromPromotedRow(row: Record<string, LbugValue>): ProjectionConceptRecord {
1729
- return {
1730
- record: "concept",
1731
- key: requiredString(row.recordKey, "concept record key"),
1732
- id: requiredString(row.conceptId, "concept id"),
1733
- path: requiredString(row.path, "concept path"),
1734
- type: requiredString(row.conceptType, "concept type"),
1735
- frontmatter: parseObjectJson(row.frontmatterJson, "concept frontmatter"),
1736
- body: row.body === null ? "" : requiredString(row.body, "concept body"),
1737
- contentHash: requiredString(row.contentHash, "concept content hash"),
1738
- tokenEstimate: requiredNumber(row.tokenEstimate, "concept token estimate"),
1739
- };
1740
- }
1741
-
1742
- function parseEdgeSourceRecord(value: unknown): ProjectionEdgeRecord {
1743
- const record = parseSourceRecord(value);
1744
- if (
1745
- record.record !== "edge" ||
1746
- typeof record.key !== "string" ||
1747
- typeof record.from !== "string" ||
1748
- (record.to !== null && typeof record.to !== "string") ||
1749
- typeof record.kind !== "string" ||
1750
- typeof record.target !== "string" ||
1751
- typeof record.ordinal !== "number" ||
1752
- !Number.isSafeInteger(record.ordinal) ||
1753
- record.ordinal < 0 ||
1754
- typeof record.dangling !== "boolean"
1755
- ) {
1756
- corrupt("indexed authored-edge source record is invalid");
1757
- }
1758
- return record as ProjectionEdgeRecord;
1759
- }
1760
-
1761
- function parseSourceRecord(value: unknown): Record<string, unknown> {
1762
- if (typeof value !== "string") corrupt("indexed source record JSON is missing");
1763
- let parsed: unknown;
1764
- try {
1765
- parsed = JSON.parse(value);
1766
- } catch {
1767
- corrupt("indexed source record JSON is malformed");
1768
- }
1769
- if (!isObject(parsed)) corrupt("indexed source record JSON is not an object");
1770
- return parsed;
1771
- }
1772
-
1773
- function isObject(value: unknown): value is Record<string, unknown> {
1774
- return typeof value === "object" && value !== null && !Array.isArray(value);
1775
- }
1776
-
1777
- function isEdgeKind(value: string): value is EdgeKind {
1778
- return value === "link" || value === "specs" || value === "supersedes" || value === "superseded_by";
1779
- }
1780
-
1781
- function edgeSourceKind(record: ProjectionEdgeRecord): "concept" | "task" {
1782
- return record.workspaceFromKind ?? "concept";
1783
- }
1784
-
1785
- function edgeTargetKind(record: ProjectionEdgeRecord): "concept" | "task" {
1786
- return record.workspaceToKind ?? (record.kind === "task" ? "task" : "concept");
1787
- }
1788
-
1789
- function corrupt(message: string): never {
1790
- throw new LoreError(
1791
- "validation",
1792
- `Ladybug projection verification failed: ${message}`,
1793
- "quarantine and rebuild the disposable local projection under exclusive writer ownership",
1794
- );
1795
- }