@danielblomma/cortex-mcp 2.1.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +72 -24
  2. package/bin/cortex.mjs +50 -21
  3. package/mcp-registry-submission.json +63 -0
  4. package/package.json +19 -5
  5. package/scaffold/AGENTS.md +13 -9
  6. package/scaffold/CLAUDE.md +13 -9
  7. package/scaffold/docs/architecture.md +1 -1
  8. package/scaffold/mcp/build.mjs +92 -0
  9. package/scaffold/mcp/package.json +4 -4
  10. package/scaffold/mcp/src/cli/query.ts +455 -0
  11. package/scaffold/mcp/src/embed.ts +181 -77
  12. package/scaffold/mcp/src/embedScheduler.ts +509 -0
  13. package/scaffold/mcp/src/embeddings.ts +31 -30
  14. package/scaffold/mcp/src/graphCsv.ts +65 -0
  15. package/scaffold/mcp/src/jsonl.ts +69 -14
  16. package/scaffold/mcp/src/loadGraph.ts +276 -40
  17. package/scaffold/mcp/src/lruCache.ts +41 -0
  18. package/scaffold/mcp/src/searchCore.ts +1 -1
  19. package/scaffold/mcp/src/searchResults.ts +3 -3
  20. package/scaffold/mcp/src/types.ts +6 -1
  21. package/scaffold/mcp/tests/embed-scheduler.test.mjs +474 -0
  22. package/scaffold/mcp/tests/embedding-jsonl.test.mjs +39 -0
  23. package/scaffold/mcp/tests/graph-bulk-load.test.mjs +258 -0
  24. package/scaffold/mcp/tests/graph-csv.test.mjs +118 -0
  25. package/scaffold/mcp/tests/lru-cache.test.mjs +37 -0
  26. package/scaffold/mcp/tests/query-cli.test.mjs +115 -0
  27. package/scaffold/mcp/tests/vector-index.test.mjs +109 -0
  28. package/scaffold/mcp/tsconfig.json +3 -1
  29. package/scaffold/scripts/bootstrap.sh +37 -4
  30. package/scaffold/scripts/dashboard.mjs +1 -1
  31. package/scaffold/scripts/doctor.sh +9 -8
  32. package/scaffold/scripts/embed.sh +3 -2
  33. package/scaffold/scripts/ingest-parsers.mjs +387 -0
  34. package/scaffold/scripts/ingest-worker.mjs +52 -0
  35. package/scaffold/scripts/ingest.mjs +550 -378
  36. package/scaffold/scripts/load-ryu.sh +3 -2
  37. package/scaffold/scripts/memory-compile.mjs +5 -2
  38. package/scaffold/scripts/memory-lint.mjs +5 -2
  39. package/scaffold/scripts/parsers/csharp.mjs +2 -1
  40. package/scaffold/scripts/parsers/javascript/ast.mjs +160 -8
  41. package/scaffold/scripts/parsers/javascript/chunks.mjs +129 -22
  42. package/scaffold/scripts/parsers/javascript/imports.mjs +38 -4
  43. package/scaffold/scripts/parsers/vbnet.mjs +2 -1
  44. package/scaffold/scripts/status.sh +50 -14
  45. package/scaffold/scripts/parsers/node_modules/.package-lock.json +0 -56
  46. package/scaffold/scripts/parsers/node_modules/acorn/CHANGELOG.md +0 -972
  47. package/scaffold/scripts/parsers/node_modules/acorn/LICENSE +0 -21
  48. package/scaffold/scripts/parsers/node_modules/acorn/README.md +0 -301
  49. package/scaffold/scripts/parsers/node_modules/acorn/bin/acorn +0 -4
  50. package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.d.mts +0 -883
  51. package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.d.ts +0 -883
  52. package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.js +0 -6295
  53. package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.mjs +0 -6266
  54. package/scaffold/scripts/parsers/node_modules/acorn/dist/bin.js +0 -90
  55. package/scaffold/scripts/parsers/node_modules/acorn/package.json +0 -50
  56. package/scaffold/scripts/parsers/node_modules/acorn-typescript/CHANGELOG.md +0 -421
  57. package/scaffold/scripts/parsers/node_modules/acorn-typescript/LICENSE +0 -21
  58. package/scaffold/scripts/parsers/node_modules/acorn-typescript/README.md +0 -81
  59. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/error.d.ts +0 -103
  60. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/error.js +0 -78
  61. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/error.js.map +0 -1
  62. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/decorators.d.ts +0 -167
  63. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/decorators.js +0 -75
  64. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/decorators.js.map +0 -1
  65. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/import-assertions.d.ts +0 -177
  66. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/import-assertions.js +0 -56
  67. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/import-assertions.js.map +0 -1
  68. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/index.d.ts +0 -198
  69. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/index.js +0 -327
  70. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/index.js.map +0 -1
  71. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/xhtml.d.ts +0 -256
  72. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/xhtml.js +0 -256
  73. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/xhtml.js.map +0 -1
  74. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.d.ts +0 -472
  75. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.js +0 -1
  76. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.js.map +0 -1
  77. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.mjs +0 -1
  78. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/middleware.d.ts +0 -159
  79. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/middleware.js +0 -2
  80. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/middleware.js.map +0 -1
  81. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/parseutil.d.ts +0 -10
  82. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/parseutil.js +0 -38
  83. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/parseutil.js.map +0 -1
  84. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/scopeflags.d.ts +0 -12
  85. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/scopeflags.js +0 -29
  86. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/scopeflags.js.map +0 -1
  87. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/tokenType.d.ts +0 -2
  88. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/tokenType.js +0 -118
  89. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/tokenType.js.map +0 -1
  90. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/types.d.ts +0 -60
  91. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/types.js +0 -2
  92. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/types.js.map +0 -1
  93. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/whitespace.d.ts +0 -2
  94. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/whitespace.js +0 -19
  95. package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/whitespace.js.map +0 -1
  96. package/scaffold/scripts/parsers/node_modules/acorn-typescript/package.json +0 -53
  97. package/scaffold/scripts/parsers/node_modules/acorn-typescript/tsconfig.json +0 -19
  98. package/scaffold/scripts/parsers/node_modules/acorn-walk/CHANGELOG.md +0 -209
  99. package/scaffold/scripts/parsers/node_modules/acorn-walk/LICENSE +0 -21
  100. package/scaffold/scripts/parsers/node_modules/acorn-walk/README.md +0 -124
  101. package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.d.mts +0 -152
  102. package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.d.ts +0 -152
  103. package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.js +0 -485
  104. package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.mjs +0 -467
  105. package/scaffold/scripts/parsers/node_modules/acorn-walk/package.json +0 -50
  106. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/LICENSE +0 -24
  107. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/README.md +0 -23
  108. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-bash.wasm +0 -0
  109. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-c.wasm +0 -0
  110. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-c_sharp.wasm +0 -0
  111. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-cpp.wasm +0 -0
  112. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-css.wasm +0 -0
  113. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-dart.wasm +0 -0
  114. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-elisp.wasm +0 -0
  115. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-elixir.wasm +0 -0
  116. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-elm.wasm +0 -0
  117. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-embedded_template.wasm +0 -0
  118. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-go.wasm +0 -0
  119. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-html.wasm +0 -0
  120. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-java.wasm +0 -0
  121. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-javascript.wasm +0 -0
  122. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-json.wasm +0 -0
  123. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-kotlin.wasm +0 -0
  124. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-lua.wasm +0 -0
  125. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-objc.wasm +0 -0
  126. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-ocaml.wasm +0 -0
  127. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-php.wasm +0 -0
  128. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-python.wasm +0 -0
  129. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-ql.wasm +0 -0
  130. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-rescript.wasm +0 -0
  131. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-ruby.wasm +0 -0
  132. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-rust.wasm +0 -0
  133. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-scala.wasm +0 -0
  134. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-solidity.wasm +0 -0
  135. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-swift.wasm +0 -0
  136. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-systemrdl.wasm +0 -0
  137. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-tlaplus.wasm +0 -0
  138. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-toml.wasm +0 -0
  139. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-tsx.wasm +0 -0
  140. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-typescript.wasm +0 -0
  141. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-vue.wasm +0 -0
  142. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-yaml.wasm +0 -0
  143. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-zig.wasm +0 -0
  144. package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/package.json +0 -64
  145. package/scaffold/scripts/parsers/node_modules/web-tree-sitter/LICENSE +0 -21
  146. package/scaffold/scripts/parsers/node_modules/web-tree-sitter/README.md +0 -198
  147. package/scaffold/scripts/parsers/node_modules/web-tree-sitter/package.json +0 -37
  148. package/scaffold/scripts/parsers/node_modules/web-tree-sitter/tree-sitter-web.d.ts +0 -242
  149. package/scaffold/scripts/parsers/node_modules/web-tree-sitter/tree-sitter.js +0 -1
  150. package/scaffold/scripts/parsers/node_modules/web-tree-sitter/tree-sitter.wasm +0 -0
@@ -1,24 +1,79 @@
1
1
  import fs from "node:fs";
2
+ import { StringDecoder } from "node:string_decoder";
2
3
  import type { JsonObject, JsonValue } from "./types.js";
3
4
 
4
- export function readJsonl(filePath: string): JsonObject[] {
5
+ function parseJsonlLine(line: string): JsonObject | null {
6
+ const trimmed = line.trim();
7
+ if (!trimmed) {
8
+ return null;
9
+ }
10
+
11
+ try {
12
+ return JSON.parse(trimmed) as JsonObject;
13
+ } catch {
14
+ return null;
15
+ }
16
+ }
17
+
18
+ export function* readJsonlRecords(filePath: string): Generator<JsonObject> {
5
19
  if (!fs.existsSync(filePath)) {
6
- return [];
20
+ return;
7
21
  }
8
22
 
9
- return fs
10
- .readFileSync(filePath, "utf8")
11
- .split(/\r?\n/)
12
- .map((line) => line.trim())
13
- .filter(Boolean)
14
- .map((line) => {
15
- try {
16
- return JSON.parse(line) as JsonObject;
17
- } catch {
18
- return null;
23
+ const fd = fs.openSync(filePath, "r");
24
+ const decoder = new StringDecoder("utf8");
25
+ const buffer = Buffer.allocUnsafe(64 * 1024);
26
+ let carry = "";
27
+
28
+ try {
29
+ while (true) {
30
+ const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, null);
31
+ if (bytesRead === 0) {
32
+ break;
19
33
  }
20
- })
21
- .filter((value): value is JsonObject => value !== null);
34
+
35
+ carry += decoder.write(buffer.subarray(0, bytesRead));
36
+ let lineStart = 0;
37
+ let newlineIndex = carry.indexOf("\n", lineStart);
38
+ while (newlineIndex !== -1) {
39
+ const parsed = parseJsonlLine(carry.slice(lineStart, newlineIndex));
40
+ if (parsed) {
41
+ yield parsed;
42
+ }
43
+ lineStart = newlineIndex + 1;
44
+ newlineIndex = carry.indexOf("\n", lineStart);
45
+ }
46
+ carry = carry.slice(lineStart);
47
+ }
48
+
49
+ carry += decoder.end();
50
+ const parsed = parseJsonlLine(carry);
51
+ if (parsed) {
52
+ yield parsed;
53
+ }
54
+ } finally {
55
+ fs.closeSync(fd);
56
+ }
57
+ }
58
+
59
+ export function readJsonl(filePath: string): JsonObject[] {
60
+ return Array.from(readJsonlRecords(filePath));
61
+ }
62
+
63
+ export function writeJsonlRecords(filePath: string, records: Iterable<unknown>): number {
64
+ const fd = fs.openSync(filePath, "w");
65
+ let count = 0;
66
+
67
+ try {
68
+ for (const record of records) {
69
+ fs.writeSync(fd, `${JSON.stringify(record)}\n`, undefined, "utf8");
70
+ count += 1;
71
+ }
72
+ } finally {
73
+ fs.closeSync(fd);
74
+ }
75
+
76
+ return count;
22
77
  }
23
78
 
24
79
  export function asString(value: JsonValue | undefined, fallback = ""): string {
@@ -3,9 +3,11 @@ import path from "node:path";
3
3
  import ryugraph, { type Connection, type PreparedStatement, type QueryResult, type RyuValue } from "ryugraph";
4
4
  import { readJsonl, asString, asNumber, asBoolean } from "./jsonl.js";
5
5
  import { CACHE_DIR, CONTEXT_DIR, DB_PATH } from "./paths.js";
6
+ import { CSV_COPY_OPTIONS, writeCsv, toCopyPathLiteral, type CsvValue } from "./graphCsv.js";
6
7
  import type { JsonObject } from "./types.js";
7
8
 
8
9
  const ONTOLOGY_PATH = path.join(CONTEXT_DIR, "ontology.cypher");
10
+ const GRAPH_IMPORT_DIR = path.join(CACHE_DIR, "graph-import");
9
11
  const BATCH_SIZE = 50;
10
12
 
11
13
  async function executeBatch(
@@ -428,25 +430,244 @@ function warnIfOptionalFilesMissing(): void {
428
430
  );
429
431
  }
430
432
 
431
- async function main(): Promise<void> {
432
- const args = new Set(process.argv.slice(2));
433
- const reset = !args.has("--no-reset");
433
+ type GraphData = {
434
+ fileEntities: FileEntity[];
435
+ ruleEntities: RuleEntity[];
436
+ adrEntities: AdrEntity[];
437
+ chunkEntities: ChunkEntity[];
438
+ moduleEntities: ModuleEntity[];
439
+ projectEntities: ProjectEntity[];
440
+ constrains: Relation[];
441
+ implementsEdges: Relation[];
442
+ supersedes: Relation[];
443
+ defines: Relation[];
444
+ calls: CallRelation[];
445
+ callsSql: Relation[];
446
+ usesConfigKey: Relation[];
447
+ usesResourceKey: Relation[];
448
+ usesSettingKey: Relation[];
449
+ imports: ImportRelation[];
450
+ contains: Relation[];
451
+ containsModule: Relation[];
452
+ exports: Relation[];
453
+ includesFile: Relation[];
454
+ referencesProject: Relation[];
455
+ usesResource: Relation[];
456
+ usesSetting: Relation[];
457
+ usesConfig: Relation[];
458
+ transformsConfig: Relation[];
459
+ };
434
460
 
435
- await ensureRequiredFiles();
436
- warnIfOptionalFilesMissing();
461
+ function parseGraphData(): GraphData {
462
+ return {
463
+ fileEntities: parseFiles(readEntityFile("entities.file.jsonl")),
464
+ ruleEntities: parseRules(readEntityFile("entities.rule.jsonl")),
465
+ adrEntities: parseAdrs(readEntityFile("entities.adr.jsonl")),
466
+ chunkEntities: parseChunks(readEntityFile("entities.chunk.jsonl")),
467
+ moduleEntities: parseModules(readEntityFile("entities.module.jsonl")),
468
+ projectEntities: parseProjects(readEntityFile("entities.project.jsonl")),
469
+ constrains: parseRelations("relations.constrains.jsonl", "note"),
470
+ implementsEdges: parseRelations("relations.implements.jsonl", "note"),
471
+ supersedes: parseRelations("relations.supersedes.jsonl", "reason"),
472
+ defines: parseSimpleRelations("relations.defines.jsonl"),
473
+ calls: parseCallRelations("relations.calls.jsonl"),
474
+ callsSql: parseRelations("relations.calls_sql.jsonl", "note"),
475
+ usesConfigKey: parseRelations("relations.uses_config_key.jsonl", "note"),
476
+ usesResourceKey: parseRelations("relations.uses_resource_key.jsonl", "note"),
477
+ usesSettingKey: parseRelations("relations.uses_setting_key.jsonl", "note"),
478
+ imports: parseImportRelations("relations.imports.jsonl"),
479
+ contains: parseSimpleRelations("relations.contains.jsonl"),
480
+ containsModule: parseSimpleRelations("relations.contains_module.jsonl"),
481
+ exports: parseSimpleRelations("relations.exports.jsonl"),
482
+ includesFile: parseSimpleRelations("relations.includes_file.jsonl"),
483
+ referencesProject: parseRelations("relations.references_project.jsonl", "note"),
484
+ usesResource: parseRelations("relations.uses_resource.jsonl", "note"),
485
+ usesSetting: parseRelations("relations.uses_setting.jsonl", "note"),
486
+ usesConfig: parseRelations("relations.uses_config.jsonl", "note"),
487
+ transformsConfig: parseRelations("relations.transforms_config.jsonl", "note")
488
+ };
489
+ }
437
490
 
438
- if (reset) {
439
- fs.rmSync(DB_PATH, { recursive: true, force: true });
440
- fs.rmSync(`${DB_PATH}.wal`, { force: true });
441
- fs.rmSync(`${DB_PATH}.shm`, { force: true });
491
+ // Mirror the MATCH (...)-CREATE skip semantics of the prepared-statement
492
+ // loader: an edge whose endpoint node was never created is silently dropped.
493
+ // COPY into a rel table instead errors on an unknown primary key, so edges
494
+ // must be pre-filtered against the node-id sets to stay byte-identical.
495
+ function filterEdges<T extends { from: string; to: string }>(
496
+ edges: Iterable<T>,
497
+ fromSet: Set<string>,
498
+ toSet: Set<string>
499
+ ): Iterable<T> {
500
+ return (function* filteredEdges() {
501
+ for (const edge of edges) {
502
+ if (fromSet.has(edge.from) && toSet.has(edge.to)) {
503
+ yield edge;
504
+ }
505
+ }
506
+ })();
507
+ }
508
+
509
+ function* ids<T extends { id: string }>(items: Iterable<T>): Iterable<string> {
510
+ for (const item of items) {
511
+ yield item.id;
442
512
  }
443
- fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
513
+ }
444
514
 
445
- const db = new ryugraph.Database(DB_PATH);
446
- const conn = new ryugraph.Connection(db);
515
+ function* csvRows<T>(items: Iterable<T>, toRow: (item: T) => CsvValue[]): Iterable<CsvValue[]> {
516
+ for (const item of items) {
517
+ yield toRow(item);
518
+ }
519
+ }
447
520
 
448
- const ontologyStatements = parseOntologyStatements(fs.readFileSync(ONTOLOGY_PATH, "utf8"));
449
- await executeStatements(conn, ontologyStatements);
521
+ async function copyTable(
522
+ conn: Connection,
523
+ table: string,
524
+ header: string[],
525
+ rows: Iterable<CsvValue[]>
526
+ ): Promise<void> {
527
+ const csvPath = path.join(GRAPH_IMPORT_DIR, `${table}.csv`);
528
+ const rowCount = writeCsv(csvPath, header, rows);
529
+ if (rowCount === 0) {
530
+ fs.rmSync(csvPath, { force: true });
531
+ return;
532
+ }
533
+ // ryugraph parses COPY's path as a double-quoted literal; toCopyPathLiteral
534
+ // normalizes separators and escapes any embedded quote so a repo/cache path
535
+ // containing a double quote still bulk-loads instead of falling back.
536
+ const csvForCopy = toCopyPathLiteral(csvPath);
537
+ await conn.query(`COPY ${table} FROM "${csvForCopy}" ${CSV_COPY_OPTIONS};`);
538
+ }
539
+
540
+ // Bulk path: one COPY per table instead of a prepared statement per row.
541
+ // Only used on the reset path, where the freshly created tables are empty.
542
+ // Produces a byte-identical graph to rowByRowLoad (same parsed arrays, same
543
+ // node-then-edge order, same dangling-edge filtering).
544
+ async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
545
+ fs.rmSync(GRAPH_IMPORT_DIR, { recursive: true, force: true });
546
+ fs.mkdirSync(GRAPH_IMPORT_DIR, { recursive: true });
547
+
548
+ try {
549
+ const fileIds = new Set(ids(data.fileEntities));
550
+ const ruleIds = new Set(ids(data.ruleEntities));
551
+ const adrIds = new Set(ids(data.adrEntities));
552
+ const chunkIds = new Set(ids(data.chunkEntities));
553
+ const moduleIds = new Set(ids(data.moduleEntities));
554
+ const projectIds = new Set(ids(data.projectEntities));
555
+
556
+ // Nodes first — edges reference them by primary key.
557
+ await copyTable(
558
+ conn,
559
+ "File",
560
+ ["id", "path", "kind", "excerpt", "checksum", "updated_at", "source_of_truth", "trust_level", "status"],
561
+ csvRows(data.fileEntities, (e) => [
562
+ e.id, e.path, e.kind, e.excerpt, e.checksum, e.updated_at, e.source_of_truth, e.trust_level, e.status
563
+ ])
564
+ );
565
+ await copyTable(
566
+ conn,
567
+ "Rule",
568
+ ["id", "title", "body", "scope", "priority", "updated_at", "source_of_truth", "trust_level", "status"],
569
+ csvRows(data.ruleEntities, (e) => [
570
+ e.id, e.title, e.body, e.scope, e.priority, e.updated_at, e.source_of_truth, e.trust_level, e.status
571
+ ])
572
+ );
573
+ await copyTable(
574
+ conn,
575
+ "ADR",
576
+ ["id", "path", "title", "body", "decision_date", "supersedes_id", "source_of_truth", "trust_level", "status"],
577
+ csvRows(data.adrEntities, (e) => [
578
+ e.id, e.path, e.title, e.body, e.decision_date, e.supersedes_id, e.source_of_truth, e.trust_level, e.status
579
+ ])
580
+ );
581
+ await copyTable(
582
+ conn,
583
+ "Chunk",
584
+ [
585
+ "id", "file_id", "name", "kind", "signature", "body", "description", "start_line", "end_line",
586
+ "language", "exported", "checksum", "updated_at", "source_of_truth", "trust_level", "status"
587
+ ],
588
+ csvRows(data.chunkEntities, (e) => [
589
+ e.id, e.file_id, e.name, e.kind, e.signature, e.body, e.description, e.start_line, e.end_line,
590
+ e.language, e.exported, e.checksum, e.updated_at, e.source_of_truth, e.trust_level, e.status
591
+ ])
592
+ );
593
+ await copyTable(
594
+ conn,
595
+ "Module",
596
+ ["id", "path", "name", "summary", "file_count", "exported_symbols", "updated_at", "source_of_truth", "trust_level", "status"],
597
+ csvRows(data.moduleEntities, (e) => [
598
+ e.id, e.path, e.name, e.summary, e.file_count, e.exported_symbols, e.updated_at, e.source_of_truth, e.trust_level, e.status
599
+ ])
600
+ );
601
+ await copyTable(
602
+ conn,
603
+ "Project",
604
+ [
605
+ "id", "path", "name", "kind", "language", "target_framework", "summary", "file_count",
606
+ "updated_at", "source_of_truth", "trust_level", "status"
607
+ ],
608
+ csvRows(data.projectEntities, (e) => [
609
+ e.id, e.path, e.name, e.kind, e.language, e.target_framework, e.summary, e.file_count,
610
+ e.updated_at, e.source_of_truth, e.trust_level, e.status
611
+ ])
612
+ );
613
+
614
+ // Edges, filtered to existing endpoints. Header names are positional only
615
+ // (COPY into a rel table reads src key, dst key, then properties in schema
616
+ // order), but emitting real names keeps the CSVs self-describing.
617
+ await copyTable(conn, "CONSTRAINS", ["from", "to", "note"],
618
+ csvRows(filterEdges(data.constrains, ruleIds, fileIds), (e) => [e.from, e.to, e.note]));
619
+ await copyTable(conn, "IMPLEMENTS", ["from", "to", "note"],
620
+ csvRows(filterEdges(data.implementsEdges, fileIds, ruleIds), (e) => [e.from, e.to, e.note]));
621
+ await copyTable(conn, "SUPERSEDES", ["from", "to", "reason"],
622
+ csvRows(filterEdges(data.supersedes, adrIds, adrIds), (e) => [e.from, e.to, e.note]));
623
+ await copyTable(conn, "DEFINES", ["from", "to"],
624
+ csvRows(filterEdges(data.defines, fileIds, chunkIds), (e) => [e.from, e.to]));
625
+ await copyTable(conn, "CALLS", ["from", "to", "call_type"],
626
+ csvRows(filterEdges(data.calls, chunkIds, chunkIds), (e) => [e.from, e.to, e.call_type]));
627
+ await copyTable(conn, "IMPORTS", ["from", "to", "import_name"],
628
+ csvRows(filterEdges(data.imports, chunkIds, fileIds), (e) => [e.from, e.to, e.import_name]));
629
+ await copyTable(conn, "CALLS_SQL", ["from", "to", "note"],
630
+ csvRows(filterEdges(data.callsSql, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
631
+ await copyTable(conn, "USES_CONFIG_KEY", ["from", "to", "note"],
632
+ csvRows(filterEdges(data.usesConfigKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
633
+ await copyTable(conn, "USES_RESOURCE_KEY", ["from", "to", "note"],
634
+ csvRows(filterEdges(data.usesResourceKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
635
+ await copyTable(conn, "USES_SETTING_KEY", ["from", "to", "note"],
636
+ csvRows(filterEdges(data.usesSettingKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
637
+ await copyTable(conn, "CONTAINS", ["from", "to"],
638
+ csvRows(filterEdges(data.contains, moduleIds, fileIds), (e) => [e.from, e.to]));
639
+ await copyTable(conn, "CONTAINS_MODULE", ["from", "to"],
640
+ csvRows(filterEdges(data.containsModule, moduleIds, moduleIds), (e) => [e.from, e.to]));
641
+ await copyTable(conn, "EXPORTS", ["from", "to"],
642
+ csvRows(filterEdges(data.exports, moduleIds, chunkIds), (e) => [e.from, e.to]));
643
+ await copyTable(conn, "INCLUDES_FILE", ["from", "to"],
644
+ csvRows(filterEdges(data.includesFile, projectIds, fileIds), (e) => [e.from, e.to]));
645
+ await copyTable(conn, "REFERENCES_PROJECT", ["from", "to", "note"],
646
+ csvRows(filterEdges(data.referencesProject, projectIds, projectIds), (e) => [e.from, e.to, e.note]));
647
+ await copyTable(conn, "USES_RESOURCE", ["from", "to", "note"],
648
+ csvRows(filterEdges(data.usesResource, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
649
+ await copyTable(conn, "USES_SETTING", ["from", "to", "note"],
650
+ csvRows(filterEdges(data.usesSetting, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
651
+ await copyTable(conn, "USES_CONFIG", ["from", "to", "note"],
652
+ csvRows(filterEdges(data.usesConfig, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
653
+ await copyTable(conn, "TRANSFORMS_CONFIG", ["from", "to", "note"],
654
+ csvRows(filterEdges(data.transformsConfig, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
655
+ } finally {
656
+ fs.rmSync(GRAPH_IMPORT_DIR, { recursive: true, force: true });
657
+ }
658
+ }
659
+
660
+ // Original loader: clears every table, then inserts each node/edge through a
661
+ // prepared statement in batches. Used for --no-reset and as the fallback when
662
+ // bulk COPY fails. MATCH-based edge inserts skip dangling endpoints silently.
663
+ async function rowByRowLoad(conn: Connection, data: GraphData): Promise<void> {
664
+ const {
665
+ fileEntities, ruleEntities, adrEntities, chunkEntities, moduleEntities, projectEntities,
666
+ constrains, implementsEdges, supersedes, defines, calls, callsSql,
667
+ usesConfigKey, usesResourceKey, usesSettingKey, imports,
668
+ contains, containsModule, exports, includesFile, referencesProject,
669
+ usesResource, usesSetting, usesConfig, transformsConfig
670
+ } = data;
450
671
 
451
672
  // Delete all relations first, then all nodes, to avoid orphaned edges
452
673
  await conn.query("MATCH (a:ADR)-[r:SUPERSEDES]->(b:ADR) DELETE r;");
@@ -477,32 +698,6 @@ async function main(): Promise<void> {
477
698
  await conn.query("MATCH (n:Project) DELETE n;");
478
699
  await conn.query("MATCH (n:File) DELETE n;");
479
700
 
480
- const fileEntities = parseFiles(readEntityFile("entities.file.jsonl"));
481
- const ruleEntities = parseRules(readEntityFile("entities.rule.jsonl"));
482
- const adrEntities = parseAdrs(readEntityFile("entities.adr.jsonl"));
483
- const chunkEntities = parseChunks(readEntityFile("entities.chunk.jsonl"));
484
- const constrains = parseRelations("relations.constrains.jsonl", "note");
485
- const implementsEdges = parseRelations("relations.implements.jsonl", "note");
486
- const supersedes = parseRelations("relations.supersedes.jsonl", "reason");
487
- const defines = parseSimpleRelations("relations.defines.jsonl");
488
- const calls = parseCallRelations("relations.calls.jsonl");
489
- const callsSql = parseRelations("relations.calls_sql.jsonl", "note");
490
- const usesConfigKey = parseRelations("relations.uses_config_key.jsonl", "note");
491
- const usesResourceKey = parseRelations("relations.uses_resource_key.jsonl", "note");
492
- const usesSettingKey = parseRelations("relations.uses_setting_key.jsonl", "note");
493
- const imports = parseImportRelations("relations.imports.jsonl");
494
- const moduleEntities = parseModules(readEntityFile("entities.module.jsonl"));
495
- const projectEntities = parseProjects(readEntityFile("entities.project.jsonl"));
496
- const contains = parseSimpleRelations("relations.contains.jsonl");
497
- const containsModule = parseSimpleRelations("relations.contains_module.jsonl");
498
- const exports = parseSimpleRelations("relations.exports.jsonl");
499
- const includesFile = parseSimpleRelations("relations.includes_file.jsonl");
500
- const referencesProject = parseRelations("relations.references_project.jsonl", "note");
501
- const usesResource = parseRelations("relations.uses_resource.jsonl", "note");
502
- const usesSetting = parseRelations("relations.uses_setting.jsonl", "note");
503
- const usesConfig = parseRelations("relations.uses_config.jsonl", "note");
504
- const transformsConfig = parseRelations("relations.transforms_config.jsonl", "note");
505
-
506
701
  const insertFile = await conn.prepare(`
507
702
  CREATE (f:File {
508
703
  id: $id,
@@ -725,6 +920,47 @@ async function main(): Promise<void> {
725
920
  await executeBatch(conn, insertUsesSetting, usesSetting);
726
921
  await executeBatch(conn, insertUsesConfig, usesConfig);
727
922
  await executeBatch(conn, insertTransformsConfig, transformsConfig);
923
+ }
924
+
925
+ async function main(): Promise<void> {
926
+ const args = new Set(process.argv.slice(2));
927
+ const reset = !args.has("--no-reset");
928
+
929
+ await ensureRequiredFiles();
930
+ warnIfOptionalFilesMissing();
931
+
932
+ if (reset) {
933
+ fs.rmSync(DB_PATH, { recursive: true, force: true });
934
+ fs.rmSync(`${DB_PATH}.wal`, { force: true });
935
+ fs.rmSync(`${DB_PATH}.shm`, { force: true });
936
+ }
937
+ fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
938
+
939
+ const db = new ryugraph.Database(DB_PATH);
940
+ const conn = new ryugraph.Connection(db);
941
+
942
+ const ontologyStatements = parseOntologyStatements(fs.readFileSync(ONTOLOGY_PATH, "utf8"));
943
+ await executeStatements(conn, ontologyStatements);
944
+
945
+ const data = parseGraphData();
946
+
947
+ const bulkEnabled = reset && process.env.CORTEX_GRAPH_BULK_LOAD !== "never";
948
+ let usedBulk = false;
949
+ if (bulkEnabled) {
950
+ try {
951
+ await bulkLoad(conn, data);
952
+ usedBulk = true;
953
+ console.log("[graph-load] loaded via COPY bulk import");
954
+ } catch (error) {
955
+ const message = error instanceof Error ? error.message : String(error);
956
+ console.warn(
957
+ `[graph-load] bulk COPY load failed (${message}); falling back to row-by-row inserts`
958
+ );
959
+ }
960
+ }
961
+ if (!usedBulk) {
962
+ await rowByRowLoad(conn, data);
963
+ }
728
964
 
729
965
  const fileCount = await rows(await conn.query("MATCH (f:File) RETURN count(*) AS count;"));
730
966
  const ruleCount = await rows(await conn.query("MATCH (r:Rule) RETURN count(*) AS count;"));
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Minimal LRU cache on Map insertion order: reads refresh recency, writes
3
+ * evict the least-recently-used entry once capacity is reached.
4
+ */
5
+ export class LruCache<K, V> {
6
+ private readonly capacity: number;
7
+ private readonly entries = new Map<K, V>();
8
+
9
+ constructor(capacity: number) {
10
+ if (!Number.isInteger(capacity) || capacity < 1) {
11
+ throw new Error(`LruCache capacity must be a positive integer, got ${capacity}`);
12
+ }
13
+ this.capacity = capacity;
14
+ }
15
+
16
+ get(key: K): V | undefined {
17
+ if (!this.entries.has(key)) {
18
+ return undefined;
19
+ }
20
+ const value = this.entries.get(key) as V;
21
+ this.entries.delete(key);
22
+ this.entries.set(key, value);
23
+ return value;
24
+ }
25
+
26
+ set(key: K, value: V): void {
27
+ if (this.entries.has(key)) {
28
+ this.entries.delete(key);
29
+ } else if (this.entries.size >= this.capacity) {
30
+ const oldest = this.entries.keys().next();
31
+ if (!oldest.done) {
32
+ this.entries.delete(oldest.value);
33
+ }
34
+ }
35
+ this.entries.set(key, value);
36
+ }
37
+
38
+ get size(): number {
39
+ return this.entries.size;
40
+ }
41
+ }
@@ -251,7 +251,7 @@ export function legacyDataAccessBoost(entity: SearchEntity, queryTokens: string[
251
251
  return boost;
252
252
  }
253
253
 
254
- export function cosineSimilarity(a: number[], b: number[]): number {
254
+ export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
255
255
  if (a.length === 0 || b.length === 0 || a.length !== b.length) {
256
256
  return 0;
257
257
  }
@@ -23,13 +23,13 @@ export function buildSearchResults(params: {
23
23
  includeScores: boolean;
24
24
  includeMatchedRules: boolean;
25
25
  includeContent: boolean;
26
- queryVector: number[] | null;
27
- embeddingVectors: Map<string, number[]>;
26
+ queryVector: Float32Array | null;
27
+ embeddingVectors: Map<string, Float32Array>;
28
28
  topK: number;
29
29
  minLexicalRelevance: number;
30
30
  minVectorRelevance: number;
31
31
  semanticScorer: (queryTokens: string[], queryPhrase: string, text: string) => number;
32
- vectorScorer: (a: number[], b: number[]) => number;
32
+ vectorScorer: (a: Float32Array, b: Float32Array) => number;
33
33
  recencyScorer: (isoDate: string) => number;
34
34
  legacyDataAccessBooster: (entity: SearchEntity, queryTokens: string[], queryPhrase: string) => number;
35
35
  }): Record<string, unknown>[] {
@@ -151,7 +151,12 @@ export type SearchEntity = {
151
151
 
152
152
  export type EmbeddingIndex = {
153
153
  model: string | null;
154
- vectors: Map<string, number[]>;
154
+ // Stored as Float32Array (not number[]): ~4x less memory than boxed
155
+ // float64 arrays, and a monomorphic typed-array cosine loop. The JSONL on
156
+ // disk stays canonical 6-decimal text; the float32 re-encode perturbs
157
+ // values by ~1e-8, far below the rounding already applied and below the
158
+ // 4-decimal score rounding at the API boundary.
159
+ vectors: Map<string, Float32Array>;
155
160
  warning?: string;
156
161
  };
157
162