@danielblomma/cortex-mcp 2.1.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -24
- package/bin/cortex.mjs +50 -21
- package/mcp-registry-submission.json +63 -0
- package/package.json +19 -5
- package/scaffold/AGENTS.md +13 -9
- package/scaffold/CLAUDE.md +13 -9
- package/scaffold/docs/architecture.md +1 -1
- package/scaffold/mcp/build.mjs +92 -0
- package/scaffold/mcp/package.json +4 -4
- package/scaffold/mcp/src/cli/query.ts +455 -0
- package/scaffold/mcp/src/embed.ts +181 -77
- package/scaffold/mcp/src/embedScheduler.ts +509 -0
- package/scaffold/mcp/src/embeddings.ts +31 -30
- package/scaffold/mcp/src/graphCsv.ts +65 -0
- package/scaffold/mcp/src/jsonl.ts +69 -14
- package/scaffold/mcp/src/loadGraph.ts +276 -40
- package/scaffold/mcp/src/lruCache.ts +41 -0
- package/scaffold/mcp/src/searchCore.ts +1 -1
- package/scaffold/mcp/src/searchResults.ts +3 -3
- package/scaffold/mcp/src/types.ts +6 -1
- package/scaffold/mcp/tests/embed-scheduler.test.mjs +474 -0
- package/scaffold/mcp/tests/embedding-jsonl.test.mjs +39 -0
- package/scaffold/mcp/tests/graph-bulk-load.test.mjs +258 -0
- package/scaffold/mcp/tests/graph-csv.test.mjs +118 -0
- package/scaffold/mcp/tests/lru-cache.test.mjs +37 -0
- package/scaffold/mcp/tests/query-cli.test.mjs +115 -0
- package/scaffold/mcp/tests/vector-index.test.mjs +109 -0
- package/scaffold/mcp/tsconfig.json +3 -1
- package/scaffold/scripts/bootstrap.sh +37 -4
- package/scaffold/scripts/dashboard.mjs +1 -1
- package/scaffold/scripts/doctor.sh +9 -8
- package/scaffold/scripts/embed.sh +3 -2
- package/scaffold/scripts/ingest-parsers.mjs +387 -0
- package/scaffold/scripts/ingest-worker.mjs +52 -0
- package/scaffold/scripts/ingest.mjs +550 -378
- package/scaffold/scripts/load-ryu.sh +3 -2
- package/scaffold/scripts/memory-compile.mjs +5 -2
- package/scaffold/scripts/memory-lint.mjs +5 -2
- package/scaffold/scripts/parsers/csharp.mjs +2 -1
- package/scaffold/scripts/parsers/javascript/ast.mjs +160 -8
- package/scaffold/scripts/parsers/javascript/chunks.mjs +129 -22
- package/scaffold/scripts/parsers/javascript/imports.mjs +38 -4
- package/scaffold/scripts/parsers/vbnet.mjs +2 -1
- package/scaffold/scripts/status.sh +50 -14
- package/scaffold/scripts/parsers/node_modules/.package-lock.json +0 -56
- package/scaffold/scripts/parsers/node_modules/acorn/CHANGELOG.md +0 -972
- package/scaffold/scripts/parsers/node_modules/acorn/LICENSE +0 -21
- package/scaffold/scripts/parsers/node_modules/acorn/README.md +0 -301
- package/scaffold/scripts/parsers/node_modules/acorn/bin/acorn +0 -4
- package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.d.mts +0 -883
- package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.d.ts +0 -883
- package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.js +0 -6295
- package/scaffold/scripts/parsers/node_modules/acorn/dist/acorn.mjs +0 -6266
- package/scaffold/scripts/parsers/node_modules/acorn/dist/bin.js +0 -90
- package/scaffold/scripts/parsers/node_modules/acorn/package.json +0 -50
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/CHANGELOG.md +0 -421
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/LICENSE +0 -21
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/README.md +0 -81
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/error.d.ts +0 -103
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/error.js +0 -78
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/error.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/decorators.d.ts +0 -167
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/decorators.js +0 -75
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/decorators.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/import-assertions.d.ts +0 -177
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/import-assertions.js +0 -56
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/import-assertions.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/index.d.ts +0 -198
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/index.js +0 -327
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/index.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/xhtml.d.ts +0 -256
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/xhtml.js +0 -256
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/extentions/jsx/xhtml.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.d.ts +0 -472
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.js +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/index.mjs +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/middleware.d.ts +0 -159
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/middleware.js +0 -2
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/middleware.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/parseutil.d.ts +0 -10
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/parseutil.js +0 -38
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/parseutil.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/scopeflags.d.ts +0 -12
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/scopeflags.js +0 -29
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/scopeflags.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/tokenType.d.ts +0 -2
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/tokenType.js +0 -118
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/tokenType.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/types.d.ts +0 -60
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/types.js +0 -2
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/types.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/whitespace.d.ts +0 -2
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/whitespace.js +0 -19
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/lib/whitespace.js.map +0 -1
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/package.json +0 -53
- package/scaffold/scripts/parsers/node_modules/acorn-typescript/tsconfig.json +0 -19
- package/scaffold/scripts/parsers/node_modules/acorn-walk/CHANGELOG.md +0 -209
- package/scaffold/scripts/parsers/node_modules/acorn-walk/LICENSE +0 -21
- package/scaffold/scripts/parsers/node_modules/acorn-walk/README.md +0 -124
- package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.d.mts +0 -152
- package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.d.ts +0 -152
- package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.js +0 -485
- package/scaffold/scripts/parsers/node_modules/acorn-walk/dist/walk.mjs +0 -467
- package/scaffold/scripts/parsers/node_modules/acorn-walk/package.json +0 -50
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/LICENSE +0 -24
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/README.md +0 -23
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-bash.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-c.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-c_sharp.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-cpp.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-css.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-dart.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-elisp.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-elixir.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-elm.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-embedded_template.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-go.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-html.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-java.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-javascript.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-json.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-kotlin.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-lua.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-objc.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-ocaml.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-php.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-python.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-ql.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-rescript.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-ruby.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-rust.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-scala.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-solidity.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-swift.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-systemrdl.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-tlaplus.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-toml.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-tsx.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-typescript.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-vue.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-yaml.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/out/tree-sitter-zig.wasm +0 -0
- package/scaffold/scripts/parsers/node_modules/tree-sitter-wasms/package.json +0 -64
- package/scaffold/scripts/parsers/node_modules/web-tree-sitter/LICENSE +0 -21
- package/scaffold/scripts/parsers/node_modules/web-tree-sitter/README.md +0 -198
- package/scaffold/scripts/parsers/node_modules/web-tree-sitter/package.json +0 -37
- package/scaffold/scripts/parsers/node_modules/web-tree-sitter/tree-sitter-web.d.ts +0 -242
- package/scaffold/scripts/parsers/node_modules/web-tree-sitter/tree-sitter.js +0 -1
- package/scaffold/scripts/parsers/node_modules/web-tree-sitter/tree-sitter.wasm +0 -0
|
@@ -1,24 +1,79 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
+
import { StringDecoder } from "node:string_decoder";
|
|
2
3
|
import type { JsonObject, JsonValue } from "./types.js";
|
|
3
4
|
|
|
4
|
-
|
|
5
|
+
function parseJsonlLine(line: string): JsonObject | null {
|
|
6
|
+
const trimmed = line.trim();
|
|
7
|
+
if (!trimmed) {
|
|
8
|
+
return null;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
return JSON.parse(trimmed) as JsonObject;
|
|
13
|
+
} catch {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function* readJsonlRecords(filePath: string): Generator<JsonObject> {
|
|
5
19
|
if (!fs.existsSync(filePath)) {
|
|
6
|
-
return
|
|
20
|
+
return;
|
|
7
21
|
}
|
|
8
22
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
23
|
+
const fd = fs.openSync(filePath, "r");
|
|
24
|
+
const decoder = new StringDecoder("utf8");
|
|
25
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
26
|
+
let carry = "";
|
|
27
|
+
|
|
28
|
+
try {
|
|
29
|
+
while (true) {
|
|
30
|
+
const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, null);
|
|
31
|
+
if (bytesRead === 0) {
|
|
32
|
+
break;
|
|
19
33
|
}
|
|
20
|
-
|
|
21
|
-
|
|
34
|
+
|
|
35
|
+
carry += decoder.write(buffer.subarray(0, bytesRead));
|
|
36
|
+
let lineStart = 0;
|
|
37
|
+
let newlineIndex = carry.indexOf("\n", lineStart);
|
|
38
|
+
while (newlineIndex !== -1) {
|
|
39
|
+
const parsed = parseJsonlLine(carry.slice(lineStart, newlineIndex));
|
|
40
|
+
if (parsed) {
|
|
41
|
+
yield parsed;
|
|
42
|
+
}
|
|
43
|
+
lineStart = newlineIndex + 1;
|
|
44
|
+
newlineIndex = carry.indexOf("\n", lineStart);
|
|
45
|
+
}
|
|
46
|
+
carry = carry.slice(lineStart);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
carry += decoder.end();
|
|
50
|
+
const parsed = parseJsonlLine(carry);
|
|
51
|
+
if (parsed) {
|
|
52
|
+
yield parsed;
|
|
53
|
+
}
|
|
54
|
+
} finally {
|
|
55
|
+
fs.closeSync(fd);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function readJsonl(filePath: string): JsonObject[] {
|
|
60
|
+
return Array.from(readJsonlRecords(filePath));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function writeJsonlRecords(filePath: string, records: Iterable<unknown>): number {
|
|
64
|
+
const fd = fs.openSync(filePath, "w");
|
|
65
|
+
let count = 0;
|
|
66
|
+
|
|
67
|
+
try {
|
|
68
|
+
for (const record of records) {
|
|
69
|
+
fs.writeSync(fd, `${JSON.stringify(record)}\n`, undefined, "utf8");
|
|
70
|
+
count += 1;
|
|
71
|
+
}
|
|
72
|
+
} finally {
|
|
73
|
+
fs.closeSync(fd);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return count;
|
|
22
77
|
}
|
|
23
78
|
|
|
24
79
|
export function asString(value: JsonValue | undefined, fallback = ""): string {
|
|
@@ -3,9 +3,11 @@ import path from "node:path";
|
|
|
3
3
|
import ryugraph, { type Connection, type PreparedStatement, type QueryResult, type RyuValue } from "ryugraph";
|
|
4
4
|
import { readJsonl, asString, asNumber, asBoolean } from "./jsonl.js";
|
|
5
5
|
import { CACHE_DIR, CONTEXT_DIR, DB_PATH } from "./paths.js";
|
|
6
|
+
import { CSV_COPY_OPTIONS, writeCsv, toCopyPathLiteral, type CsvValue } from "./graphCsv.js";
|
|
6
7
|
import type { JsonObject } from "./types.js";
|
|
7
8
|
|
|
8
9
|
const ONTOLOGY_PATH = path.join(CONTEXT_DIR, "ontology.cypher");
|
|
10
|
+
const GRAPH_IMPORT_DIR = path.join(CACHE_DIR, "graph-import");
|
|
9
11
|
const BATCH_SIZE = 50;
|
|
10
12
|
|
|
11
13
|
async function executeBatch(
|
|
@@ -428,25 +430,244 @@ function warnIfOptionalFilesMissing(): void {
|
|
|
428
430
|
);
|
|
429
431
|
}
|
|
430
432
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
433
|
+
type GraphData = {
|
|
434
|
+
fileEntities: FileEntity[];
|
|
435
|
+
ruleEntities: RuleEntity[];
|
|
436
|
+
adrEntities: AdrEntity[];
|
|
437
|
+
chunkEntities: ChunkEntity[];
|
|
438
|
+
moduleEntities: ModuleEntity[];
|
|
439
|
+
projectEntities: ProjectEntity[];
|
|
440
|
+
constrains: Relation[];
|
|
441
|
+
implementsEdges: Relation[];
|
|
442
|
+
supersedes: Relation[];
|
|
443
|
+
defines: Relation[];
|
|
444
|
+
calls: CallRelation[];
|
|
445
|
+
callsSql: Relation[];
|
|
446
|
+
usesConfigKey: Relation[];
|
|
447
|
+
usesResourceKey: Relation[];
|
|
448
|
+
usesSettingKey: Relation[];
|
|
449
|
+
imports: ImportRelation[];
|
|
450
|
+
contains: Relation[];
|
|
451
|
+
containsModule: Relation[];
|
|
452
|
+
exports: Relation[];
|
|
453
|
+
includesFile: Relation[];
|
|
454
|
+
referencesProject: Relation[];
|
|
455
|
+
usesResource: Relation[];
|
|
456
|
+
usesSetting: Relation[];
|
|
457
|
+
usesConfig: Relation[];
|
|
458
|
+
transformsConfig: Relation[];
|
|
459
|
+
};
|
|
434
460
|
|
|
435
|
-
|
|
436
|
-
|
|
461
|
+
function parseGraphData(): GraphData {
|
|
462
|
+
return {
|
|
463
|
+
fileEntities: parseFiles(readEntityFile("entities.file.jsonl")),
|
|
464
|
+
ruleEntities: parseRules(readEntityFile("entities.rule.jsonl")),
|
|
465
|
+
adrEntities: parseAdrs(readEntityFile("entities.adr.jsonl")),
|
|
466
|
+
chunkEntities: parseChunks(readEntityFile("entities.chunk.jsonl")),
|
|
467
|
+
moduleEntities: parseModules(readEntityFile("entities.module.jsonl")),
|
|
468
|
+
projectEntities: parseProjects(readEntityFile("entities.project.jsonl")),
|
|
469
|
+
constrains: parseRelations("relations.constrains.jsonl", "note"),
|
|
470
|
+
implementsEdges: parseRelations("relations.implements.jsonl", "note"),
|
|
471
|
+
supersedes: parseRelations("relations.supersedes.jsonl", "reason"),
|
|
472
|
+
defines: parseSimpleRelations("relations.defines.jsonl"),
|
|
473
|
+
calls: parseCallRelations("relations.calls.jsonl"),
|
|
474
|
+
callsSql: parseRelations("relations.calls_sql.jsonl", "note"),
|
|
475
|
+
usesConfigKey: parseRelations("relations.uses_config_key.jsonl", "note"),
|
|
476
|
+
usesResourceKey: parseRelations("relations.uses_resource_key.jsonl", "note"),
|
|
477
|
+
usesSettingKey: parseRelations("relations.uses_setting_key.jsonl", "note"),
|
|
478
|
+
imports: parseImportRelations("relations.imports.jsonl"),
|
|
479
|
+
contains: parseSimpleRelations("relations.contains.jsonl"),
|
|
480
|
+
containsModule: parseSimpleRelations("relations.contains_module.jsonl"),
|
|
481
|
+
exports: parseSimpleRelations("relations.exports.jsonl"),
|
|
482
|
+
includesFile: parseSimpleRelations("relations.includes_file.jsonl"),
|
|
483
|
+
referencesProject: parseRelations("relations.references_project.jsonl", "note"),
|
|
484
|
+
usesResource: parseRelations("relations.uses_resource.jsonl", "note"),
|
|
485
|
+
usesSetting: parseRelations("relations.uses_setting.jsonl", "note"),
|
|
486
|
+
usesConfig: parseRelations("relations.uses_config.jsonl", "note"),
|
|
487
|
+
transformsConfig: parseRelations("relations.transforms_config.jsonl", "note")
|
|
488
|
+
};
|
|
489
|
+
}
|
|
437
490
|
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
491
|
+
// Mirror the MATCH (...)-CREATE skip semantics of the prepared-statement
|
|
492
|
+
// loader: an edge whose endpoint node was never created is silently dropped.
|
|
493
|
+
// COPY into a rel table instead errors on an unknown primary key, so edges
|
|
494
|
+
// must be pre-filtered against the node-id sets to stay byte-identical.
|
|
495
|
+
function filterEdges<T extends { from: string; to: string }>(
|
|
496
|
+
edges: Iterable<T>,
|
|
497
|
+
fromSet: Set<string>,
|
|
498
|
+
toSet: Set<string>
|
|
499
|
+
): Iterable<T> {
|
|
500
|
+
return (function* filteredEdges() {
|
|
501
|
+
for (const edge of edges) {
|
|
502
|
+
if (fromSet.has(edge.from) && toSet.has(edge.to)) {
|
|
503
|
+
yield edge;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
})();
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
function* ids<T extends { id: string }>(items: Iterable<T>): Iterable<string> {
|
|
510
|
+
for (const item of items) {
|
|
511
|
+
yield item.id;
|
|
442
512
|
}
|
|
443
|
-
|
|
513
|
+
}
|
|
444
514
|
|
|
445
|
-
|
|
446
|
-
const
|
|
515
|
+
function* csvRows<T>(items: Iterable<T>, toRow: (item: T) => CsvValue[]): Iterable<CsvValue[]> {
|
|
516
|
+
for (const item of items) {
|
|
517
|
+
yield toRow(item);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
447
520
|
|
|
448
|
-
|
|
449
|
-
|
|
521
|
+
async function copyTable(
|
|
522
|
+
conn: Connection,
|
|
523
|
+
table: string,
|
|
524
|
+
header: string[],
|
|
525
|
+
rows: Iterable<CsvValue[]>
|
|
526
|
+
): Promise<void> {
|
|
527
|
+
const csvPath = path.join(GRAPH_IMPORT_DIR, `${table}.csv`);
|
|
528
|
+
const rowCount = writeCsv(csvPath, header, rows);
|
|
529
|
+
if (rowCount === 0) {
|
|
530
|
+
fs.rmSync(csvPath, { force: true });
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
// ryugraph parses COPY's path as a double-quoted literal; toCopyPathLiteral
|
|
534
|
+
// normalizes separators and escapes any embedded quote so a repo/cache path
|
|
535
|
+
// containing a double quote still bulk-loads instead of falling back.
|
|
536
|
+
const csvForCopy = toCopyPathLiteral(csvPath);
|
|
537
|
+
await conn.query(`COPY ${table} FROM "${csvForCopy}" ${CSV_COPY_OPTIONS};`);
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// Bulk path: one COPY per table instead of a prepared statement per row.
|
|
541
|
+
// Only used on the reset path, where the freshly created tables are empty.
|
|
542
|
+
// Produces a byte-identical graph to rowByRowLoad (same parsed arrays, same
|
|
543
|
+
// node-then-edge order, same dangling-edge filtering).
|
|
544
|
+
async function bulkLoad(conn: Connection, data: GraphData): Promise<void> {
|
|
545
|
+
fs.rmSync(GRAPH_IMPORT_DIR, { recursive: true, force: true });
|
|
546
|
+
fs.mkdirSync(GRAPH_IMPORT_DIR, { recursive: true });
|
|
547
|
+
|
|
548
|
+
try {
|
|
549
|
+
const fileIds = new Set(ids(data.fileEntities));
|
|
550
|
+
const ruleIds = new Set(ids(data.ruleEntities));
|
|
551
|
+
const adrIds = new Set(ids(data.adrEntities));
|
|
552
|
+
const chunkIds = new Set(ids(data.chunkEntities));
|
|
553
|
+
const moduleIds = new Set(ids(data.moduleEntities));
|
|
554
|
+
const projectIds = new Set(ids(data.projectEntities));
|
|
555
|
+
|
|
556
|
+
// Nodes first — edges reference them by primary key.
|
|
557
|
+
await copyTable(
|
|
558
|
+
conn,
|
|
559
|
+
"File",
|
|
560
|
+
["id", "path", "kind", "excerpt", "checksum", "updated_at", "source_of_truth", "trust_level", "status"],
|
|
561
|
+
csvRows(data.fileEntities, (e) => [
|
|
562
|
+
e.id, e.path, e.kind, e.excerpt, e.checksum, e.updated_at, e.source_of_truth, e.trust_level, e.status
|
|
563
|
+
])
|
|
564
|
+
);
|
|
565
|
+
await copyTable(
|
|
566
|
+
conn,
|
|
567
|
+
"Rule",
|
|
568
|
+
["id", "title", "body", "scope", "priority", "updated_at", "source_of_truth", "trust_level", "status"],
|
|
569
|
+
csvRows(data.ruleEntities, (e) => [
|
|
570
|
+
e.id, e.title, e.body, e.scope, e.priority, e.updated_at, e.source_of_truth, e.trust_level, e.status
|
|
571
|
+
])
|
|
572
|
+
);
|
|
573
|
+
await copyTable(
|
|
574
|
+
conn,
|
|
575
|
+
"ADR",
|
|
576
|
+
["id", "path", "title", "body", "decision_date", "supersedes_id", "source_of_truth", "trust_level", "status"],
|
|
577
|
+
csvRows(data.adrEntities, (e) => [
|
|
578
|
+
e.id, e.path, e.title, e.body, e.decision_date, e.supersedes_id, e.source_of_truth, e.trust_level, e.status
|
|
579
|
+
])
|
|
580
|
+
);
|
|
581
|
+
await copyTable(
|
|
582
|
+
conn,
|
|
583
|
+
"Chunk",
|
|
584
|
+
[
|
|
585
|
+
"id", "file_id", "name", "kind", "signature", "body", "description", "start_line", "end_line",
|
|
586
|
+
"language", "exported", "checksum", "updated_at", "source_of_truth", "trust_level", "status"
|
|
587
|
+
],
|
|
588
|
+
csvRows(data.chunkEntities, (e) => [
|
|
589
|
+
e.id, e.file_id, e.name, e.kind, e.signature, e.body, e.description, e.start_line, e.end_line,
|
|
590
|
+
e.language, e.exported, e.checksum, e.updated_at, e.source_of_truth, e.trust_level, e.status
|
|
591
|
+
])
|
|
592
|
+
);
|
|
593
|
+
await copyTable(
|
|
594
|
+
conn,
|
|
595
|
+
"Module",
|
|
596
|
+
["id", "path", "name", "summary", "file_count", "exported_symbols", "updated_at", "source_of_truth", "trust_level", "status"],
|
|
597
|
+
csvRows(data.moduleEntities, (e) => [
|
|
598
|
+
e.id, e.path, e.name, e.summary, e.file_count, e.exported_symbols, e.updated_at, e.source_of_truth, e.trust_level, e.status
|
|
599
|
+
])
|
|
600
|
+
);
|
|
601
|
+
await copyTable(
|
|
602
|
+
conn,
|
|
603
|
+
"Project",
|
|
604
|
+
[
|
|
605
|
+
"id", "path", "name", "kind", "language", "target_framework", "summary", "file_count",
|
|
606
|
+
"updated_at", "source_of_truth", "trust_level", "status"
|
|
607
|
+
],
|
|
608
|
+
csvRows(data.projectEntities, (e) => [
|
|
609
|
+
e.id, e.path, e.name, e.kind, e.language, e.target_framework, e.summary, e.file_count,
|
|
610
|
+
e.updated_at, e.source_of_truth, e.trust_level, e.status
|
|
611
|
+
])
|
|
612
|
+
);
|
|
613
|
+
|
|
614
|
+
// Edges, filtered to existing endpoints. Header names are positional only
|
|
615
|
+
// (COPY into a rel table reads src key, dst key, then properties in schema
|
|
616
|
+
// order), but emitting real names keeps the CSVs self-describing.
|
|
617
|
+
await copyTable(conn, "CONSTRAINS", ["from", "to", "note"],
|
|
618
|
+
csvRows(filterEdges(data.constrains, ruleIds, fileIds), (e) => [e.from, e.to, e.note]));
|
|
619
|
+
await copyTable(conn, "IMPLEMENTS", ["from", "to", "note"],
|
|
620
|
+
csvRows(filterEdges(data.implementsEdges, fileIds, ruleIds), (e) => [e.from, e.to, e.note]));
|
|
621
|
+
await copyTable(conn, "SUPERSEDES", ["from", "to", "reason"],
|
|
622
|
+
csvRows(filterEdges(data.supersedes, adrIds, adrIds), (e) => [e.from, e.to, e.note]));
|
|
623
|
+
await copyTable(conn, "DEFINES", ["from", "to"],
|
|
624
|
+
csvRows(filterEdges(data.defines, fileIds, chunkIds), (e) => [e.from, e.to]));
|
|
625
|
+
await copyTable(conn, "CALLS", ["from", "to", "call_type"],
|
|
626
|
+
csvRows(filterEdges(data.calls, chunkIds, chunkIds), (e) => [e.from, e.to, e.call_type]));
|
|
627
|
+
await copyTable(conn, "IMPORTS", ["from", "to", "import_name"],
|
|
628
|
+
csvRows(filterEdges(data.imports, chunkIds, fileIds), (e) => [e.from, e.to, e.import_name]));
|
|
629
|
+
await copyTable(conn, "CALLS_SQL", ["from", "to", "note"],
|
|
630
|
+
csvRows(filterEdges(data.callsSql, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
|
|
631
|
+
await copyTable(conn, "USES_CONFIG_KEY", ["from", "to", "note"],
|
|
632
|
+
csvRows(filterEdges(data.usesConfigKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
|
|
633
|
+
await copyTable(conn, "USES_RESOURCE_KEY", ["from", "to", "note"],
|
|
634
|
+
csvRows(filterEdges(data.usesResourceKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
|
|
635
|
+
await copyTable(conn, "USES_SETTING_KEY", ["from", "to", "note"],
|
|
636
|
+
csvRows(filterEdges(data.usesSettingKey, fileIds, chunkIds), (e) => [e.from, e.to, e.note]));
|
|
637
|
+
await copyTable(conn, "CONTAINS", ["from", "to"],
|
|
638
|
+
csvRows(filterEdges(data.contains, moduleIds, fileIds), (e) => [e.from, e.to]));
|
|
639
|
+
await copyTable(conn, "CONTAINS_MODULE", ["from", "to"],
|
|
640
|
+
csvRows(filterEdges(data.containsModule, moduleIds, moduleIds), (e) => [e.from, e.to]));
|
|
641
|
+
await copyTable(conn, "EXPORTS", ["from", "to"],
|
|
642
|
+
csvRows(filterEdges(data.exports, moduleIds, chunkIds), (e) => [e.from, e.to]));
|
|
643
|
+
await copyTable(conn, "INCLUDES_FILE", ["from", "to"],
|
|
644
|
+
csvRows(filterEdges(data.includesFile, projectIds, fileIds), (e) => [e.from, e.to]));
|
|
645
|
+
await copyTable(conn, "REFERENCES_PROJECT", ["from", "to", "note"],
|
|
646
|
+
csvRows(filterEdges(data.referencesProject, projectIds, projectIds), (e) => [e.from, e.to, e.note]));
|
|
647
|
+
await copyTable(conn, "USES_RESOURCE", ["from", "to", "note"],
|
|
648
|
+
csvRows(filterEdges(data.usesResource, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
|
|
649
|
+
await copyTable(conn, "USES_SETTING", ["from", "to", "note"],
|
|
650
|
+
csvRows(filterEdges(data.usesSetting, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
|
|
651
|
+
await copyTable(conn, "USES_CONFIG", ["from", "to", "note"],
|
|
652
|
+
csvRows(filterEdges(data.usesConfig, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
|
|
653
|
+
await copyTable(conn, "TRANSFORMS_CONFIG", ["from", "to", "note"],
|
|
654
|
+
csvRows(filterEdges(data.transformsConfig, fileIds, fileIds), (e) => [e.from, e.to, e.note]));
|
|
655
|
+
} finally {
|
|
656
|
+
fs.rmSync(GRAPH_IMPORT_DIR, { recursive: true, force: true });
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
// Original loader: clears every table, then inserts each node/edge through a
|
|
661
|
+
// prepared statement in batches. Used for --no-reset and as the fallback when
|
|
662
|
+
// bulk COPY fails. MATCH-based edge inserts skip dangling endpoints silently.
|
|
663
|
+
async function rowByRowLoad(conn: Connection, data: GraphData): Promise<void> {
|
|
664
|
+
const {
|
|
665
|
+
fileEntities, ruleEntities, adrEntities, chunkEntities, moduleEntities, projectEntities,
|
|
666
|
+
constrains, implementsEdges, supersedes, defines, calls, callsSql,
|
|
667
|
+
usesConfigKey, usesResourceKey, usesSettingKey, imports,
|
|
668
|
+
contains, containsModule, exports, includesFile, referencesProject,
|
|
669
|
+
usesResource, usesSetting, usesConfig, transformsConfig
|
|
670
|
+
} = data;
|
|
450
671
|
|
|
451
672
|
// Delete all relations first, then all nodes, to avoid orphaned edges
|
|
452
673
|
await conn.query("MATCH (a:ADR)-[r:SUPERSEDES]->(b:ADR) DELETE r;");
|
|
@@ -477,32 +698,6 @@ async function main(): Promise<void> {
|
|
|
477
698
|
await conn.query("MATCH (n:Project) DELETE n;");
|
|
478
699
|
await conn.query("MATCH (n:File) DELETE n;");
|
|
479
700
|
|
|
480
|
-
const fileEntities = parseFiles(readEntityFile("entities.file.jsonl"));
|
|
481
|
-
const ruleEntities = parseRules(readEntityFile("entities.rule.jsonl"));
|
|
482
|
-
const adrEntities = parseAdrs(readEntityFile("entities.adr.jsonl"));
|
|
483
|
-
const chunkEntities = parseChunks(readEntityFile("entities.chunk.jsonl"));
|
|
484
|
-
const constrains = parseRelations("relations.constrains.jsonl", "note");
|
|
485
|
-
const implementsEdges = parseRelations("relations.implements.jsonl", "note");
|
|
486
|
-
const supersedes = parseRelations("relations.supersedes.jsonl", "reason");
|
|
487
|
-
const defines = parseSimpleRelations("relations.defines.jsonl");
|
|
488
|
-
const calls = parseCallRelations("relations.calls.jsonl");
|
|
489
|
-
const callsSql = parseRelations("relations.calls_sql.jsonl", "note");
|
|
490
|
-
const usesConfigKey = parseRelations("relations.uses_config_key.jsonl", "note");
|
|
491
|
-
const usesResourceKey = parseRelations("relations.uses_resource_key.jsonl", "note");
|
|
492
|
-
const usesSettingKey = parseRelations("relations.uses_setting_key.jsonl", "note");
|
|
493
|
-
const imports = parseImportRelations("relations.imports.jsonl");
|
|
494
|
-
const moduleEntities = parseModules(readEntityFile("entities.module.jsonl"));
|
|
495
|
-
const projectEntities = parseProjects(readEntityFile("entities.project.jsonl"));
|
|
496
|
-
const contains = parseSimpleRelations("relations.contains.jsonl");
|
|
497
|
-
const containsModule = parseSimpleRelations("relations.contains_module.jsonl");
|
|
498
|
-
const exports = parseSimpleRelations("relations.exports.jsonl");
|
|
499
|
-
const includesFile = parseSimpleRelations("relations.includes_file.jsonl");
|
|
500
|
-
const referencesProject = parseRelations("relations.references_project.jsonl", "note");
|
|
501
|
-
const usesResource = parseRelations("relations.uses_resource.jsonl", "note");
|
|
502
|
-
const usesSetting = parseRelations("relations.uses_setting.jsonl", "note");
|
|
503
|
-
const usesConfig = parseRelations("relations.uses_config.jsonl", "note");
|
|
504
|
-
const transformsConfig = parseRelations("relations.transforms_config.jsonl", "note");
|
|
505
|
-
|
|
506
701
|
const insertFile = await conn.prepare(`
|
|
507
702
|
CREATE (f:File {
|
|
508
703
|
id: $id,
|
|
@@ -725,6 +920,47 @@ async function main(): Promise<void> {
|
|
|
725
920
|
await executeBatch(conn, insertUsesSetting, usesSetting);
|
|
726
921
|
await executeBatch(conn, insertUsesConfig, usesConfig);
|
|
727
922
|
await executeBatch(conn, insertTransformsConfig, transformsConfig);
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
async function main(): Promise<void> {
|
|
926
|
+
const args = new Set(process.argv.slice(2));
|
|
927
|
+
const reset = !args.has("--no-reset");
|
|
928
|
+
|
|
929
|
+
await ensureRequiredFiles();
|
|
930
|
+
warnIfOptionalFilesMissing();
|
|
931
|
+
|
|
932
|
+
if (reset) {
|
|
933
|
+
fs.rmSync(DB_PATH, { recursive: true, force: true });
|
|
934
|
+
fs.rmSync(`${DB_PATH}.wal`, { force: true });
|
|
935
|
+
fs.rmSync(`${DB_PATH}.shm`, { force: true });
|
|
936
|
+
}
|
|
937
|
+
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
|
|
938
|
+
|
|
939
|
+
const db = new ryugraph.Database(DB_PATH);
|
|
940
|
+
const conn = new ryugraph.Connection(db);
|
|
941
|
+
|
|
942
|
+
const ontologyStatements = parseOntologyStatements(fs.readFileSync(ONTOLOGY_PATH, "utf8"));
|
|
943
|
+
await executeStatements(conn, ontologyStatements);
|
|
944
|
+
|
|
945
|
+
const data = parseGraphData();
|
|
946
|
+
|
|
947
|
+
const bulkEnabled = reset && process.env.CORTEX_GRAPH_BULK_LOAD !== "never";
|
|
948
|
+
let usedBulk = false;
|
|
949
|
+
if (bulkEnabled) {
|
|
950
|
+
try {
|
|
951
|
+
await bulkLoad(conn, data);
|
|
952
|
+
usedBulk = true;
|
|
953
|
+
console.log("[graph-load] loaded via COPY bulk import");
|
|
954
|
+
} catch (error) {
|
|
955
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
956
|
+
console.warn(
|
|
957
|
+
`[graph-load] bulk COPY load failed (${message}); falling back to row-by-row inserts`
|
|
958
|
+
);
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
if (!usedBulk) {
|
|
962
|
+
await rowByRowLoad(conn, data);
|
|
963
|
+
}
|
|
728
964
|
|
|
729
965
|
const fileCount = await rows(await conn.query("MATCH (f:File) RETURN count(*) AS count;"));
|
|
730
966
|
const ruleCount = await rows(await conn.query("MATCH (r:Rule) RETURN count(*) AS count;"));
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal LRU cache on Map insertion order: reads refresh recency, writes
|
|
3
|
+
* evict the least-recently-used entry once capacity is reached.
|
|
4
|
+
*/
|
|
5
|
+
export class LruCache<K, V> {
|
|
6
|
+
private readonly capacity: number;
|
|
7
|
+
private readonly entries = new Map<K, V>();
|
|
8
|
+
|
|
9
|
+
constructor(capacity: number) {
|
|
10
|
+
if (!Number.isInteger(capacity) || capacity < 1) {
|
|
11
|
+
throw new Error(`LruCache capacity must be a positive integer, got ${capacity}`);
|
|
12
|
+
}
|
|
13
|
+
this.capacity = capacity;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
get(key: K): V | undefined {
|
|
17
|
+
if (!this.entries.has(key)) {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
const value = this.entries.get(key) as V;
|
|
21
|
+
this.entries.delete(key);
|
|
22
|
+
this.entries.set(key, value);
|
|
23
|
+
return value;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
set(key: K, value: V): void {
|
|
27
|
+
if (this.entries.has(key)) {
|
|
28
|
+
this.entries.delete(key);
|
|
29
|
+
} else if (this.entries.size >= this.capacity) {
|
|
30
|
+
const oldest = this.entries.keys().next();
|
|
31
|
+
if (!oldest.done) {
|
|
32
|
+
this.entries.delete(oldest.value);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
this.entries.set(key, value);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
get size(): number {
|
|
39
|
+
return this.entries.size;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -251,7 +251,7 @@ export function legacyDataAccessBoost(entity: SearchEntity, queryTokens: string[
|
|
|
251
251
|
return boost;
|
|
252
252
|
}
|
|
253
253
|
|
|
254
|
-
export function cosineSimilarity(a:
|
|
254
|
+
export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
|
|
255
255
|
if (a.length === 0 || b.length === 0 || a.length !== b.length) {
|
|
256
256
|
return 0;
|
|
257
257
|
}
|
|
@@ -23,13 +23,13 @@ export function buildSearchResults(params: {
|
|
|
23
23
|
includeScores: boolean;
|
|
24
24
|
includeMatchedRules: boolean;
|
|
25
25
|
includeContent: boolean;
|
|
26
|
-
queryVector:
|
|
27
|
-
embeddingVectors: Map<string,
|
|
26
|
+
queryVector: Float32Array | null;
|
|
27
|
+
embeddingVectors: Map<string, Float32Array>;
|
|
28
28
|
topK: number;
|
|
29
29
|
minLexicalRelevance: number;
|
|
30
30
|
minVectorRelevance: number;
|
|
31
31
|
semanticScorer: (queryTokens: string[], queryPhrase: string, text: string) => number;
|
|
32
|
-
vectorScorer: (a:
|
|
32
|
+
vectorScorer: (a: Float32Array, b: Float32Array) => number;
|
|
33
33
|
recencyScorer: (isoDate: string) => number;
|
|
34
34
|
legacyDataAccessBooster: (entity: SearchEntity, queryTokens: string[], queryPhrase: string) => number;
|
|
35
35
|
}): Record<string, unknown>[] {
|
|
@@ -151,7 +151,12 @@ export type SearchEntity = {
|
|
|
151
151
|
|
|
152
152
|
export type EmbeddingIndex = {
|
|
153
153
|
model: string | null;
|
|
154
|
-
|
|
154
|
+
// Stored as Float32Array (not number[]): ~4x less memory than boxed
|
|
155
|
+
// float64 arrays, and a monomorphic typed-array cosine loop. The JSONL on
|
|
156
|
+
// disk stays canonical 6-decimal text; the float32 re-encode perturbs
|
|
157
|
+
// values by ~1e-8, far below the rounding already applied and below the
|
|
158
|
+
// 4-decimal score rounding at the API boundary.
|
|
159
|
+
vectors: Map<string, Float32Array>;
|
|
155
160
|
warning?: string;
|
|
156
161
|
};
|
|
157
162
|
|