@danielblomma/cortex-mcp 2.4.1 → 2.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/README.md +5 -0
  3. package/bin/cli/arguments.mjs +60 -0
  4. package/bin/cli/context-passthrough.mjs +129 -0
  5. package/bin/cli/daemon.mjs +157 -0
  6. package/bin/cli/enterprise.mjs +516 -0
  7. package/bin/cli/help.mjs +88 -0
  8. package/bin/cli/hooks.mjs +199 -0
  9. package/bin/cli/mcp-command.mjs +87 -0
  10. package/bin/cli/paths.mjs +14 -0
  11. package/bin/cli/process.mjs +49 -0
  12. package/bin/cli/project-commands.mjs +237 -0
  13. package/bin/cli/project-runtime.mjs +34 -0
  14. package/bin/cli/query-command.mjs +18 -0
  15. package/bin/cli/router.mjs +66 -0
  16. package/bin/cli/run-command.mjs +44 -0
  17. package/bin/cli/scaffold-ownership.mjs +936 -0
  18. package/bin/cli/scaffold.mjs +687 -0
  19. package/bin/cli/stage-command.mjs +9 -0
  20. package/bin/cli/telemetry-command.mjs +18 -0
  21. package/bin/cli/trusted-runtime.mjs +18 -0
  22. package/bin/cortex.mjs +6 -1834
  23. package/mcp-registry-submission.json +1 -1
  24. package/package.json +3 -2
  25. package/scaffold/ownership/baseline-v2.4.1.json +22 -0
  26. package/scaffold/ownership/current.json +4 -0
  27. package/scaffold/ownership/v1.json +456 -0
  28. package/scaffold/scripts/ingest-parsers.mjs +1 -387
  29. package/scaffold/scripts/ingest-worker.mjs +4 -1
  30. package/scaffold/scripts/ingest.mjs +5 -3899
  31. package/scaffold/scripts/lib/ingest/arguments.mjs +78 -0
  32. package/scaffold/scripts/lib/ingest/chunks.mjs +212 -0
  33. package/scaffold/scripts/lib/ingest/config.mjs +120 -0
  34. package/scaffold/scripts/lib/ingest/constants.mjs +222 -0
  35. package/scaffold/scripts/lib/ingest/files.mjs +387 -0
  36. package/scaffold/scripts/lib/ingest/incremental-state.mjs +131 -0
  37. package/scaffold/scripts/lib/ingest/io.mjs +78 -0
  38. package/scaffold/scripts/lib/ingest/main.mjs +76 -0
  39. package/scaffold/scripts/lib/ingest/parser-composition.mjs +140 -0
  40. package/scaffold/scripts/lib/ingest/parser-registry.mjs +387 -0
  41. package/scaffold/scripts/lib/ingest/pipeline-stages.mjs +1625 -0
  42. package/scaffold/scripts/lib/ingest/projects.mjs +272 -0
  43. package/scaffold/scripts/lib/ingest/relations.mjs +860 -0
  44. package/scaffold/scripts/lib/ingest/runtime-paths.mjs +11 -0
  45. package/scaffold/scripts/lib/ingest/workers.mjs +264 -0
@@ -1,3886 +1,11 @@
1
1
  #!/usr/bin/env node
2
- import crypto from "node:crypto";
3
- import fs from "node:fs";
4
- import os from "node:os";
5
2
  import path from "node:path";
6
3
  import { fileURLToPath } from "node:url";
7
- import { execSync } from "node:child_process";
8
- import { Worker } from "node:worker_threads";
9
- import {
10
- loadParsers,
11
- getChunkParserForExtension,
12
- parseCSharpProject,
13
- hasCSharpProjectParser,
14
- isCSharpParserAvailable,
15
- getCSharpParserRuntime,
16
- PARALLEL_SAFE_LANGUAGES
17
- } from "./ingest-parsers.mjs";
4
+ import { main } from "./lib/ingest/main.mjs";
18
5
 
19
- const __filename = fileURLToPath(import.meta.url);
20
- const __dirname = path.dirname(__filename);
21
- const REPO_ROOT = process.env.CORTEX_PROJECT_ROOT
22
- ? path.resolve(process.env.CORTEX_PROJECT_ROOT)
23
- : path.resolve(__dirname, "..", "..");
24
- const CONTEXT_DIR = path.join(REPO_ROOT, ".context");
25
- const CACHE_DIR = path.join(CONTEXT_DIR, "cache");
26
- const DB_IMPORT_DIR = path.join(CONTEXT_DIR, "db", "import");
27
-
28
- const SUPPORTED_TEXT_EXTENSIONS = new Set([
29
- ".md",
30
- ".mdx",
31
- ".txt",
32
- ".adoc",
33
- ".rst",
34
- ".yaml",
35
- ".yml",
36
- ".json",
37
- ".toml",
38
- ".csv",
39
- ".ts",
40
- ".tsx",
41
- ".mts",
42
- ".cts",
43
- ".js",
44
- ".jsx",
45
- ".mjs",
46
- ".cjs",
47
- ".py",
48
- ".go",
49
- ".java",
50
- ".cs",
51
- ".vb",
52
- ".sln",
53
- ".vbproj",
54
- ".csproj",
55
- ".fsproj",
56
- ".props",
57
- ".targets",
58
- ".config",
59
- ".resx",
60
- ".settings",
61
- ".rb",
62
- ".rs",
63
- ".php",
64
- ".swift",
65
- ".kt",
66
- ".sql",
67
- ".sh",
68
- ".bash",
69
- ".zsh",
70
- ".ps1",
71
- ".c",
72
- ".h",
73
- ".cpp",
74
- ".hpp",
75
- ".cc",
76
- ".hh",
77
- ".bas",
78
- ".cls",
79
- ".frm",
80
- ".ctl"
81
- ]);
82
-
83
- const LEGACY_DOTNET_METADATA_EXTENSIONS = new Set([
84
- ".sln",
85
- ".vbproj",
86
- ".csproj",
87
- ".fsproj",
88
- ".props",
89
- ".targets",
90
- ".config",
91
- ".resx",
92
- ".settings"
93
- ]);
94
-
95
- const PROJECT_DEFINITION_EXTENSIONS = new Set([".sln", ".vbproj", ".csproj", ".fsproj", ".vcxproj"]);
96
- const STRUCTURED_NON_CODE_CHUNK_EXTENSIONS = new Set([".config", ".resx", ".settings", ".md", ".mdx"]);
97
-
98
- const CODE_FILE_EXTENSIONS = new Set([
99
- ".ts",
100
- ".tsx",
101
- ".mts",
102
- ".cts",
103
- ".js",
104
- ".jsx",
105
- ".mjs",
106
- ".cjs",
107
- ".py",
108
- ".go",
109
- ".java",
110
- ".cs",
111
- ".vb",
112
- ".rb",
113
- ".rs",
114
- ".php",
115
- ".swift",
116
- ".kt",
117
- ".sql",
118
- ".sh",
119
- ".bash",
120
- ".zsh",
121
- ".ps1",
122
- ".c",
123
- ".h",
124
- ".cpp",
125
- ".hpp",
126
- ".cc",
127
- ".hh",
128
- ".bas",
129
- ".cls",
130
- ".frm",
131
- ".ctl"
132
- ]);
133
-
134
- const SQL_REFERENCE_SOURCE_EXTENSIONS = new Set([
135
- ".vb",
136
- ".cs",
137
- ".config",
138
- ".resx",
139
- ".settings"
140
- ]);
141
- const NAMED_RESOURCE_REFERENCE_SOURCE_EXTENSIONS = new Set([".vb", ".cs"]);
142
-
143
- const SQL_OBJECT_REFERENCE_PATTERNS = [
144
- /\b(?:SqlCommand|OleDbCommand|OdbcCommand)\s*\(\s*"([^"\r\n]{2,200})"/gi,
145
- /\bCommandText\s*=\s*"([^"\r\n]{2,500})"/gi,
146
- /\bCommandType\s*=\s*(?:CommandType\.)?StoredProcedure[\s\S]{0,240}?"([^"\r\n]{2,200})"/gi,
147
- /"([^"\r\n]{2,200})"[\s\S]{0,240}?\bCommandType\s*=\s*(?:CommandType\.)?StoredProcedure/gi
148
- ];
149
-
150
- const SQL_STRING_REFERENCE_PATTERNS = [
151
- /\bexec(?:ute)?\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
152
- /\bfrom\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
153
- /\bjoin\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
154
- /\bupdate\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
155
- /\binsert\s+into\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
156
- /\bdelete\s+from\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
157
- /\bmerge\s+into\s+([#@]?[A-Za-z0-9_[\].]+)/gi
158
- ];
159
-
160
- const SQL_RESOURCE_KEY_PATTERNS = [
161
- /\bMy\.Resources\.([A-Za-z_][A-Za-z0-9_]*)/g,
162
- /\bResources\.([A-Za-z_][A-Za-z0-9_]*)/g,
163
- /\bMy\.Settings\.([A-Za-z_][A-Za-z0-9_]*)/g,
164
- /\b(?:[A-Za-z_][A-Za-z0-9_]*\.)?Settings\.Default\.([A-Za-z_][A-Za-z0-9_]*)/g,
165
- /\bGetString\(\s*"([^"\r\n]+)"/g,
166
- /\bGetObject\(\s*"([^"\r\n]+)"/g
167
- ];
168
- const CONFIG_KEY_REFERENCE_PATTERNS = [
169
- /\bConfigurationManager\.ConnectionStrings\s*\[\s*"([^"\r\n]+)"\s*\]/g,
170
- /\bConfigurationManager\.ConnectionStrings\s*\(\s*"([^"\r\n]+)"\s*\)/g,
171
- /\bConfigurationManager\.AppSettings\s*\[\s*"([^"\r\n]+)"\s*\]/g,
172
- /\bConfigurationManager\.AppSettings\s*\(\s*"([^"\r\n]+)"\s*\)/g,
173
- /\bGetConnectionString\(\s*"([^"\r\n]+)"\s*\)/g,
174
- /\bGetAppSetting\(\s*"([^"\r\n]+)"\s*\)/g
175
- ];
176
-
177
- const SKIP_DIRECTORIES = new Set([
178
- ".git",
179
- ".idea",
180
- ".vscode",
181
- "node_modules",
182
- "bin",
183
- "obj",
184
- "dist",
185
- "build",
186
- "coverage",
187
- ".next",
188
- ".cache",
189
- ".context"
190
- ]);
191
-
192
- const MAX_FILE_BYTES = 1024 * 1024;
193
- const MAX_CONTENT_CHARS = 60000;
194
- const MAX_BODY_CHARS = 12000;
195
- const RULE_KEYWORD_LIMIT = 20;
196
- const DEFAULT_CHUNK_WINDOW_LINES = 80;
197
- const DEFAULT_CHUNK_OVERLAP_LINES = 16;
198
- const DEFAULT_CHUNK_SPLIT_MIN_LINES = 120;
199
- const DEFAULT_CHUNK_MAX_WINDOWS = 8;
200
- const IMPORT_RESOLUTION_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".json"];
201
- const IMPORT_RUNTIME_JS_EXTENSIONS = new Set([".js", ".jsx", ".mjs", ".cjs"]);
202
- const IMPORT_RUNTIME_JS_RESOLUTION_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"];
203
- const CPP_IMPORT_RESOLUTION_EXTENSIONS = [".h", ".hh", ".hpp", ".c", ".cc", ".cpp"];
204
-
205
- const STOP_WORDS = new Set([
206
- "the",
207
- "and",
208
- "for",
209
- "with",
210
- "from",
211
- "that",
212
- "this",
213
- "must",
214
- "when",
215
- "where",
216
- "into",
217
- "used",
218
- "using",
219
- "only",
220
- "true",
221
- "false",
222
- "unless",
223
- "should",
224
- "global",
225
- "active",
226
- "rule",
227
- "rules",
228
- "data",
229
- "file",
230
- "files",
231
- "code",
232
- "docs",
233
- "context",
234
- "och",
235
- "det",
236
- "att",
237
- "som",
238
- "med",
239
- "för",
240
- "utan",
241
- "eller",
242
- "inte",
243
- "ska",
244
- "skall",
245
- "måste",
246
- "kan",
247
- "vid",
248
- "alla"
249
- ]);
250
-
251
- function parseArgs(argv) {
252
- const args = new Set(argv.slice(2));
253
- if (args.has("--help") || args.has("-h")) {
254
- printHelp();
255
- process.exit(0);
256
- }
257
-
258
- return {
259
- mode: args.has("--changed") ? "changed" : "full",
260
- verbose: args.has("--verbose")
261
- };
262
- }
263
-
264
- function printHelp() {
265
- console.log("Usage: ./scripts/ingest.sh [--changed] [--verbose]");
266
- console.log("");
267
- console.log("Options:");
268
- console.log(" --changed Ingest only changed/untracked files when git is available.");
269
- console.log(" --verbose Print skipped files and additional diagnostics.");
270
- console.log(" -h, --help Show this help message.");
271
- }
272
-
273
- function ensureDirectory(directoryPath) {
274
- fs.mkdirSync(directoryPath, { recursive: true });
275
- }
276
-
277
- function isTextFile(relPath) {
278
- const ext = path.extname(relPath).toLowerCase();
279
- const base = path.basename(relPath).toLowerCase();
280
- if (SUPPORTED_TEXT_EXTENSIONS.has(ext)) {
281
- return true;
282
- }
283
-
284
- return base === "readme" || base.startsWith("readme.");
285
- }
286
-
287
- function isBinaryBuffer(buffer) {
288
- const scanLength = Math.min(buffer.length, 4000);
289
- for (let index = 0; index < scanLength; index += 1) {
290
- if (buffer[index] === 0) {
291
- return true;
292
- }
293
- }
294
-
295
- return false;
296
- }
297
-
298
- function toPosixPath(value) {
299
- return value.split(path.sep).join("/");
300
- }
301
-
302
- function normalizeToken(value) {
303
- return value.toLowerCase().replace(/[^a-z0-9]/g, "");
304
- }
305
-
306
- function tokenizeKeywords(value) {
307
- return value
308
- .toLowerCase()
309
- .split(/[^a-z0-9]+/g)
310
- .map((token) => token.trim())
311
- .filter((token) => token.length >= 3 && !STOP_WORDS.has(token));
312
- }
313
-
314
- function uniqueSorted(values) {
315
- return [...new Set(values)].sort();
316
- }
317
-
318
- function parsePositiveIntegerEnv(name, fallback) {
319
- const rawValue = process.env[name];
320
- if (!rawValue) {
321
- return fallback;
322
- }
323
- const parsed = Number(rawValue);
324
- if (!Number.isFinite(parsed) || parsed < 1) {
325
- return fallback;
326
- }
327
- return Math.floor(parsed);
328
- }
329
-
330
- function parseNonNegativeIntegerEnv(name, fallback) {
331
- const rawValue = process.env[name];
332
- if (!rawValue) {
333
- return fallback;
334
- }
335
- const parsed = Number(rawValue);
336
- if (!Number.isFinite(parsed) || parsed < 0) {
337
- return fallback;
338
- }
339
- return Math.floor(parsed);
340
- }
341
-
342
- function isTruthyEnv(value) {
343
- if (typeof value !== "string") {
344
- return false;
345
- }
346
- const normalized = value.trim().toLowerCase();
347
- return normalized !== "" && normalized !== "0" && normalized !== "false" && normalized !== "no" && normalized !== "off";
348
- }
349
-
350
- function createIngestMemoryTrace() {
351
- const enabled = isTruthyEnv(process.env.CORTEX_INGEST_TRACE_MEMORY);
352
-
353
- return {
354
- enabled,
355
- checkpoint(label, counts = {}) {
356
- if (!enabled) {
357
- return;
358
- }
359
-
360
- const memory = process.memoryUsage();
361
- process.stderr.write(
362
- `${JSON.stringify({
363
- type: "cortex.ingest.memory",
364
- label,
365
- rss_bytes: memory.rss,
366
- rss_mb: Number((memory.rss / 1024 / 1024).toFixed(2)),
367
- heap_used_bytes: memory.heapUsed,
368
- external_bytes: memory.external,
369
- counts
370
- })}\n`
371
- );
372
- }
373
- };
374
- }
375
-
376
- function parseSourcePaths(configText) {
377
- const sourcePaths = [];
378
- const lines = configText.split(/\r?\n/);
379
- let inSourcePaths = false;
380
-
381
- for (const line of lines) {
382
- if (!inSourcePaths && /^source_paths:\s*$/.test(line.trim())) {
383
- inSourcePaths = true;
384
- continue;
385
- }
386
-
387
- if (!inSourcePaths) {
388
- continue;
389
- }
390
-
391
- const entryMatch = line.match(/^\s*-\s*(.+?)\s*$/);
392
- if (entryMatch) {
393
- const unquoted = entryMatch[1].replace(/^['"]|['"]$/g, "");
394
- sourcePaths.push(unquoted);
395
- continue;
396
- }
397
-
398
- if (line.trim() !== "" && !/^\s/.test(line)) {
399
- break;
400
- }
401
- }
402
-
403
- return sourcePaths;
404
- }
405
-
406
- function parseRules(rulesText) {
407
- const lines = rulesText.split(/\r?\n/);
408
- const rules = [];
409
- let current = null;
410
-
411
- const pushCurrent = () => {
412
- if (!current || !current.id) {
413
- return;
414
- }
415
- rules.push({
416
- id: current.id,
417
- description: current.description ?? "",
418
- priority: Number.isFinite(current.priority) ? current.priority : 0,
419
- enforce: current.enforce === true
420
- });
421
- };
422
-
423
- for (const line of lines) {
424
- const idMatch = line.match(/^\s*-\s*id:\s*(.+?)\s*$/);
425
- if (idMatch) {
426
- pushCurrent();
427
- current = { id: idMatch[1].replace(/^['"]|['"]$/g, "") };
428
- continue;
429
- }
430
-
431
- if (!current) {
432
- continue;
433
- }
434
-
435
- const descriptionMatch = line.match(/^\s*description:\s*(.+?)\s*$/);
436
- if (descriptionMatch) {
437
- current.description = descriptionMatch[1].replace(/^['"]|['"]$/g, "");
438
- continue;
439
- }
440
-
441
- const priorityMatch = line.match(/^\s*priority:\s*(\d+)\s*$/);
442
- if (priorityMatch) {
443
- current.priority = Number(priorityMatch[1]);
444
- continue;
445
- }
446
-
447
- const enforceMatch = line.match(/^\s*enforce:\s*(true|false)\s*$/i);
448
- if (enforceMatch) {
449
- current.enforce = enforceMatch[1].toLowerCase() === "true";
450
- }
451
- }
452
-
453
- pushCurrent();
454
- return rules;
455
- }
456
-
457
- function walkDirectory(directoryPath, files) {
458
- const entries = fs.readdirSync(directoryPath, { withFileTypes: true });
459
- for (const entry of entries) {
460
- const absolutePath = path.join(directoryPath, entry.name);
461
- if (entry.isDirectory() && shouldSkipDirectory(absolutePath, entry.name)) {
462
- continue;
463
- }
464
-
465
- if (entry.isDirectory()) {
466
- walkDirectory(absolutePath, files);
467
- continue;
468
- }
469
-
470
- if (entry.isFile()) {
471
- files.add(absolutePath);
472
- }
473
- }
474
- }
475
-
476
- function shouldSkipDirectory(absolutePath, entryName) {
477
- if (entryName === "bin" && path.resolve(path.dirname(absolutePath)) === REPO_ROOT) {
478
- return false;
479
- }
480
- return SKIP_DIRECTORIES.has(entryName);
481
- }
482
-
483
- function normalizeSourcePrefix(sourcePath) {
484
- const source = toPosixPath(sourcePath).replace(/^\.\/+/, "").replace(/\/+$/, "");
485
- return source === "." ? "" : source;
486
- }
487
-
488
- function normalizeRelativePath(relPath) {
489
- return toPosixPath(relPath).replace(/^\.\/+/, "").replace(/\/+$/, "");
490
- }
491
-
492
- function hasSkippedDirectorySegment(relPath) {
493
- const parts = normalizeRelativePath(relPath).split("/").filter(Boolean);
494
- return parts.some((part, index) => {
495
- if (part === "bin" && index === 0) {
496
- return false;
497
- }
498
- return SKIP_DIRECTORIES.has(part);
499
- });
500
- }
501
-
502
- function hasSourcePrefix(relPath, sourcePaths) {
503
- const normalizedRelPath = normalizeRelativePath(relPath);
504
- if (!normalizedRelPath || hasSkippedDirectorySegment(normalizedRelPath)) {
505
- return false;
506
- }
507
- return sourcePaths.some((sourcePath) => {
508
- const source = normalizeSourcePrefix(sourcePath);
509
- return source === "" || normalizedRelPath === source || normalizedRelPath.startsWith(`${source}/`);
510
- });
511
- }
512
-
513
- function pushImportResolutionCandidate(candidates, seenCandidates, candidatePath) {
514
- if (!seenCandidates.has(candidatePath)) {
515
- seenCandidates.add(candidatePath);
516
- candidates.push(candidatePath);
517
- }
518
- }
519
-
520
- function isCppLikeFilePath(filePath) {
521
- return [".c", ".h", ".cc", ".cpp", ".hh", ".hpp"].includes(path.posix.extname(filePath).toLowerCase());
522
- }
523
-
524
- function resolveRelativeImportTargetId(filePath, importPath, indexedFileIds) {
525
- const isCppLike = isCppLikeFilePath(filePath);
526
- const isRelativeImport = importPath.startsWith(".");
527
- const isLocalCppInclude =
528
- isCppLike && !path.posix.isAbsolute(importPath) && !/^[A-Za-z]:[\\/]/.test(importPath);
529
-
530
- if (!isRelativeImport && !isLocalCppInclude) {
531
- return null;
532
- }
533
-
534
- const basePath = path.posix.normalize(path.posix.join(path.posix.dirname(filePath), importPath));
535
- const candidates = [];
536
- const seenCandidates = new Set();
537
- pushImportResolutionCandidate(candidates, seenCandidates, basePath);
538
-
539
- if (path.posix.extname(basePath) === "") {
540
- const extensions = isCppLike ? CPP_IMPORT_RESOLUTION_EXTENSIONS : IMPORT_RESOLUTION_EXTENSIONS;
541
- for (const extension of extensions) {
542
- pushImportResolutionCandidate(candidates, seenCandidates, `${basePath}${extension}`);
543
- }
544
- if (!isCppLike) {
545
- for (const extension of IMPORT_RESOLUTION_EXTENSIONS) {
546
- pushImportResolutionCandidate(candidates, seenCandidates, path.posix.join(basePath, `index${extension}`));
547
- }
548
- }
549
- } else if (IMPORT_RUNTIME_JS_EXTENSIONS.has(path.posix.extname(basePath))) {
550
- const extension = path.posix.extname(basePath);
551
- const stemPath = basePath.slice(0, -extension.length);
552
- for (const candidateExtension of IMPORT_RUNTIME_JS_RESOLUTION_EXTENSIONS) {
553
- pushImportResolutionCandidate(candidates, seenCandidates, `${stemPath}${candidateExtension}`);
554
- }
555
- }
556
-
557
- for (const candidate of candidates) {
558
- const targetFileId = `file:${candidate}`;
559
- if (indexedFileIds.has(targetFileId)) {
560
- return targetFileId;
561
- }
562
- }
563
-
564
- return null;
565
- }
566
-
567
- function getGitChanges() {
568
- try {
569
- const output = execSync("git status --porcelain", {
570
- cwd: REPO_ROOT,
571
- stdio: ["ignore", "pipe", "ignore"],
572
- encoding: "utf8"
573
- });
574
-
575
- const changed = new Set();
576
- const deleted = new Set();
577
-
578
- for (const line of output.split(/\r?\n/)) {
579
- if (!line) continue;
580
- const status = line.slice(0, 2);
581
- const payload = line.slice(3).trim();
582
- if (!payload) continue;
583
-
584
- if (payload.includes(" -> ")) {
585
- const [fromPath, toPath] = payload.split(" -> ");
586
- deleted.add(path.resolve(REPO_ROOT, fromPath));
587
- changed.add(path.resolve(REPO_ROOT, toPath));
588
- continue;
589
- }
590
-
591
- const absolutePath = path.resolve(REPO_ROOT, payload);
592
- if (status.includes("D")) {
593
- deleted.add(absolutePath);
594
- } else {
595
- changed.add(absolutePath);
596
- }
597
- }
598
-
599
- return {
600
- changed: [...changed],
601
- deleted: [...deleted]
602
- };
603
- } catch {
604
- return {
605
- changed: [],
606
- deleted: []
607
- };
608
- }
609
- }
610
-
611
- function collectCandidateFiles(sourcePaths, mode) {
612
- const candidates = new Set();
613
- const deletedRelPaths = new Set();
614
-
615
- if (mode === "changed") {
616
- const gitChanges = getGitChanges();
617
- if (gitChanges.changed.length > 0 || gitChanges.deleted.length > 0) {
618
- for (const absolutePath of gitChanges.changed) {
619
- if (!fs.existsSync(absolutePath)) {
620
- continue;
621
- }
622
-
623
- const stats = fs.statSync(absolutePath);
624
- if (stats.isFile()) {
625
- const relPath = toPosixPath(path.relative(REPO_ROOT, absolutePath));
626
- if (hasSourcePrefix(relPath, sourcePaths)) {
627
- candidates.add(absolutePath);
628
- }
629
- continue;
630
- }
631
-
632
- if (stats.isDirectory()) {
633
- const nestedFiles = new Set();
634
- walkDirectory(absolutePath, nestedFiles);
635
- for (const nestedPath of nestedFiles) {
636
- const nestedRelPath = toPosixPath(path.relative(REPO_ROOT, nestedPath));
637
- if (hasSourcePrefix(nestedRelPath, sourcePaths)) {
638
- candidates.add(nestedPath);
639
- }
640
- }
641
- }
642
- }
643
-
644
- for (const deletedPath of gitChanges.deleted) {
645
- const relPath = toPosixPath(path.relative(REPO_ROOT, deletedPath));
646
- if (hasSourcePrefix(relPath, sourcePaths)) {
647
- deletedRelPaths.add(relPath);
648
- }
649
- }
650
-
651
- return {
652
- candidates,
653
- incrementalMode: true,
654
- deletedRelPaths: [...deletedRelPaths]
655
- };
656
- }
657
- }
658
-
659
- for (const sourcePath of sourcePaths) {
660
- if (hasSkippedDirectorySegment(sourcePath)) {
661
- continue;
662
- }
663
- const absoluteSourcePath = path.resolve(REPO_ROOT, sourcePath);
664
- if (!fs.existsSync(absoluteSourcePath)) {
665
- continue;
666
- }
667
-
668
- const stats = fs.statSync(absoluteSourcePath);
669
- if (stats.isFile()) {
670
- candidates.add(absoluteSourcePath);
671
- continue;
672
- }
673
-
674
- if (stats.isDirectory()) {
675
- walkDirectory(absoluteSourcePath, candidates);
676
- }
677
- }
678
-
679
- return {
680
- candidates,
681
- incrementalMode: false,
682
- deletedRelPaths: []
683
- };
684
- }
685
-
686
- function detectKind(relPath) {
687
- const lower = relPath.toLowerCase();
688
- const ext = path.extname(lower);
689
- const isAdrPath =
690
- /(^|\/)(adr|adrs|decisions)(\/|$)/.test(lower) ||
691
- /(^|\/)adr[-_ ]?\d+/.test(path.basename(lower));
692
-
693
- if (isAdrPath) {
694
- return "ADR";
695
- }
696
-
697
- if (
698
- lower.startsWith("docs/") ||
699
- ext === ".md" ||
700
- ext === ".mdx" ||
701
- ext === ".txt" ||
702
- ext === ".adoc" ||
703
- ext === ".rst"
704
- ) {
705
- return "DOC";
706
- }
707
-
708
- if (LEGACY_DOTNET_METADATA_EXTENSIONS.has(ext) || !CODE_FILE_EXTENSIONS.has(ext)) {
709
- return "DOC";
710
- }
711
-
712
- return "CODE";
713
- }
714
-
715
- function trustLevelForKind(kind) {
716
- if (kind === "ADR") return 95;
717
- if (kind === "CODE") return 80;
718
- return 70;
719
- }
720
-
721
- function checksum(buffer) {
722
- return crypto.createHash("sha256").update(buffer).digest("hex");
723
- }
724
-
725
- function normalizeWhitespace(value) {
726
- return value.replace(/\s+/g, " ").trim();
727
- }
728
-
729
- function extractTitle(content, fallbackTitle) {
730
- const lines = content.split(/\r?\n/);
731
- for (const line of lines) {
732
- const match = line.match(/^#\s+(.+)\s*$/);
733
- if (match) return match[1].trim();
734
- }
735
-
736
- return fallbackTitle;
737
- }
738
-
739
- function parseDecisionDate(content, fallbackDate) {
740
- const datePatterns = [
741
- /^\s*date:\s*["']?(\d{4}-\d{2}-\d{2})["']?\s*$/im,
742
- /^\s*decision[_\s-]*date:\s*["']?(\d{4}-\d{2}-\d{2})["']?\s*$/im
743
- ];
744
-
745
- for (const pattern of datePatterns) {
746
- const match = content.match(pattern);
747
- if (match && !Number.isNaN(Date.parse(match[1]))) {
748
- return match[1];
749
- }
750
- }
751
-
752
- return fallbackDate.slice(0, 10);
753
- }
754
-
755
- function adrTokens(adrRecord) {
756
- const fileBase = path.basename(adrRecord.path).replace(path.extname(adrRecord.path), "");
757
- const tokens = new Set([
758
- normalizeToken(adrRecord.id),
759
- normalizeToken(fileBase),
760
- normalizeToken(adrRecord.title)
761
- ]);
762
-
763
- const numberMatch = fileBase.match(/(\d+)/);
764
- if (numberMatch) {
765
- tokens.add(normalizeToken(`adr-${numberMatch[1]}`));
766
- tokens.add(normalizeToken(numberMatch[1]));
767
- }
768
-
769
- return [...tokens].filter(Boolean);
770
- }
771
-
772
- function findSupersedesReferences(content) {
773
- const refs = new Set();
774
- const pattern = /(?:supersedes|ersätter)\s*[:\-]?\s*([A-Za-z0-9._/-]+)/gi;
775
- let match;
776
- while ((match = pattern.exec(content)) !== null) {
777
- refs.add(match[1]);
778
- }
779
-
780
- return [...refs];
781
- }
782
-
783
- function writeJsonl(filePath, records) {
784
- const fd = fs.openSync(filePath, "w");
785
- try {
786
- for (const record of records) {
787
- fs.writeSync(fd, `${JSON.stringify(record)}\n`, undefined, "utf8");
788
- }
789
- } finally {
790
- fs.closeSync(fd);
791
- }
792
- }
793
-
794
- function stageJsonl(filePath, records) {
795
- const stagedPath = `${filePath}.tmp-${process.pid}-${Date.now()}`;
796
- writeJsonl(stagedPath, records);
797
- return {
798
- commit() {
799
- fs.renameSync(stagedPath, filePath);
800
- },
801
- discard() {
802
- fs.rmSync(stagedPath, { force: true });
803
- }
804
- };
805
- }
806
-
807
- function countFileContentRecords(fileRecords) {
808
- let count = 0;
809
- for (const fileRecord of fileRecords) {
810
- if (Object.prototype.hasOwnProperty.call(fileRecord, "content")) {
811
- count += 1;
812
- }
813
- }
814
- return count;
815
- }
816
-
817
- function sanitizeTsvCell(value) {
818
- if (value === null || value === undefined) return "";
819
- return String(value).replace(/\t/g, " ").replace(/\r?\n/g, " ");
820
- }
821
-
822
- function writeTsv(filePath, headers, rows) {
823
- const fd = fs.openSync(filePath, "w");
824
- try {
825
- fs.writeSync(fd, `${headers.join("\t")}\n`, undefined, "utf8");
826
- for (const row of rows) {
827
- fs.writeSync(fd, `${row.map((value) => sanitizeTsvCell(value)).join("\t")}\n`, undefined, "utf8");
828
- }
829
- } finally {
830
- fs.closeSync(fd);
831
- }
832
- }
833
-
834
- function* mapRows(records, project) {
835
- for (const record of records) {
836
- yield project(record);
837
- }
838
- }
839
-
840
- function readJsonlSafe(filePath) {
841
- if (!fs.existsSync(filePath)) {
842
- return [];
843
- }
844
-
845
- return fs
846
- .readFileSync(filePath, "utf8")
847
- .split(/\r?\n/)
848
- .map((line) => line.trim())
849
- .filter(Boolean)
850
- .map((line) => {
851
- try {
852
- return JSON.parse(line);
853
- } catch {
854
- return null;
855
- }
856
- })
857
- .filter((record) => record !== null);
858
- }
859
-
860
- function relationKey(...parts) {
861
- return parts.map((part) => String(part ?? "")).join("|");
862
- }
863
-
864
- function uniqueRelations(relations) {
865
- const deduped = new Map();
866
- for (const relation of relations) {
867
- const key = relationKey(relation.from, relation.to, relation.note);
868
- if (!deduped.has(key)) {
869
- deduped.set(key, relation);
870
- }
871
- }
872
- return [...deduped.values()].sort((a, b) =>
873
- relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
874
- );
875
- }
876
-
877
- function normalizeSqlName(value) {
878
- if (!value) {
879
- return "";
880
- }
881
-
882
- return String(value)
883
- .trim()
884
- .replace(/[;"`]/g, "")
885
- .replace(/\[(.+?)\]/g, "$1")
886
- .replace(/\s+/g, "")
887
- .replace(/^\.+|\.+$/g, "")
888
- .replace(/\.\.+/g, ".")
889
- .toLowerCase();
890
- }
891
-
892
- function sqlChunkAliases(name) {
893
- const normalized = normalizeSqlName(name);
894
- if (!normalized) {
895
- return [];
896
- }
897
-
898
- const aliases = new Set([normalized]);
899
- const parts = normalized.split(".").filter(Boolean);
900
- if (parts.length > 1) {
901
- aliases.add(parts[parts.length - 1]);
902
- }
903
- return [...aliases];
904
- }
905
-
906
- function configChunkAliases(chunk) {
907
- const aliases = new Set();
908
- const rawKey = String(chunk?.configKey ?? chunk?.name ?? "");
909
- const normalizedKey = normalizeToken(rawKey);
910
- if (normalizedKey) {
911
- aliases.add(normalizedKey);
912
- }
913
- const chunkName = String(chunk?.name ?? "");
914
- const tail = chunkName.split(".").pop() ?? "";
915
- const normalizedTail = normalizeToken(tail);
916
- if (normalizedTail) {
917
- aliases.add(normalizedTail);
918
- }
919
- return [...aliases];
920
- }
921
-
922
- function namedEntryChunkAliases(chunk) {
923
- const aliases = new Set();
924
- const rawKey = String(chunk?.resourceKey ?? chunk?.configKey ?? chunk?.name ?? "");
925
- const normalizedKey = normalizeToken(rawKey);
926
- if (normalizedKey) {
927
- aliases.add(normalizedKey);
928
- }
929
- const chunkName = String(chunk?.name ?? "");
930
- const tail = chunkName.split(".").pop() ?? "";
931
- const normalizedTail = normalizeToken(tail);
932
- if (normalizedTail) {
933
- aliases.add(normalizedTail);
934
- }
935
- return [...aliases];
936
- }
937
-
938
- function buildChunkAliasIndexes(chunkRecords) {
939
- const sqlChunkIdsByAlias = new Map();
940
- const configChunkIdsByAlias = new Map();
941
- const resourceChunkIdsByAlias = new Map();
942
- const settingChunkIdsByAlias = new Map();
943
-
944
- for (const chunk of chunkRecords) {
945
- if (isWindowChunkId(chunk?.id)) {
946
- continue;
947
- }
948
-
949
- const language = String(chunk?.language ?? "").toLowerCase();
950
- if (language === "sql") {
951
- for (const alias of sqlChunkAliases(chunk.name)) {
952
- const existing = sqlChunkIdsByAlias.get(alias) ?? [];
953
- sqlChunkIdsByAlias.set(alias, [...existing, chunk.id]);
954
- }
955
- continue;
956
- }
957
-
958
- if (language === "config") {
959
- for (const alias of configChunkAliases(chunk)) {
960
- const existing = configChunkIdsByAlias.get(alias) ?? [];
961
- configChunkIdsByAlias.set(alias, [...existing, chunk.id]);
962
- }
963
- continue;
964
- }
965
-
966
- if (language === "resource") {
967
- for (const alias of namedEntryChunkAliases(chunk)) {
968
- const existing = resourceChunkIdsByAlias.get(alias) ?? [];
969
- resourceChunkIdsByAlias.set(alias, [...existing, chunk.id]);
970
- }
971
- continue;
972
- }
973
-
974
- if (language === "settings") {
975
- for (const alias of namedEntryChunkAliases(chunk)) {
976
- const existing = settingChunkIdsByAlias.get(alias) ?? [];
977
- settingChunkIdsByAlias.set(alias, [...existing, chunk.id]);
978
- }
979
- }
980
- }
981
-
982
- return {
983
- sqlChunkIdsByAlias,
984
- configChunkIdsByAlias,
985
- resourceChunkIdsByAlias,
986
- settingChunkIdsByAlias
987
- };
988
- }
989
-
990
- function extractSqlReferenceNamesFromString(text) {
991
- const refs = new Set();
992
-
993
- const normalizedName = normalizeSqlName(text);
994
- if (/^[a-z0-9_.]+$/i.test(normalizedName) && normalizedName.includes(".")) {
995
- refs.add(normalizedName);
996
- }
997
-
998
- for (const pattern of SQL_STRING_REFERENCE_PATTERNS) {
999
- let match;
1000
- while ((match = pattern.exec(text)) !== null) {
1001
- const name = normalizeSqlName(match[1]);
1002
- if (!name || name.startsWith("@") || name.startsWith("#")) {
1003
- continue;
1004
- }
1005
- refs.add(name);
1006
- }
1007
- }
1008
-
1009
- return [...refs];
1010
- }
1011
-
1012
- function parseResxSqlReferenceMap(content) {
1013
- const refsByKey = new Map();
1014
- const dataPattern = /<data\b[^>]*\bname="([^"]+)"[^>]*>([\s\S]*?)<\/data>/gi;
1015
- let match;
1016
-
1017
- while ((match = dataPattern.exec(content)) !== null) {
1018
- const key = normalizeToken(decodeXmlEntities(match[1]));
1019
- if (!key) {
1020
- continue;
1021
- }
1022
-
1023
- const valueMatch = match[2].match(/<value>([\s\S]*?)<\/value>/i);
1024
- if (!valueMatch) {
1025
- continue;
1026
- }
1027
-
1028
- const value = decodeXmlEntities(valueMatch[1]).trim();
1029
- const refs = extractSqlReferenceNamesFromString(value);
1030
- if (refs.length === 0) {
1031
- continue;
1032
- }
1033
-
1034
- const existing = refsByKey.get(key) ?? [];
1035
- refsByKey.set(key, uniqueSorted([...existing, ...refs]));
1036
- }
1037
-
1038
- return refsByKey;
1039
- }
1040
-
1041
- function parseResxKeyMap(content) {
1042
- const fileKeys = new Map();
1043
- const dataPattern = /<data\b[^>]*\bname="([^"]+)"[^>]*>/gi;
1044
- let match;
1045
-
1046
- while ((match = dataPattern.exec(content)) !== null) {
1047
- const key = normalizeToken(decodeXmlEntities(match[1]));
1048
- if (!key) {
1049
- continue;
1050
- }
1051
- fileKeys.set(key, true);
1052
- }
1053
-
1054
- return fileKeys;
1055
- }
1056
-
1057
- function parseSettingsSqlReferenceMap(content) {
1058
- const refsByKey = new Map();
1059
- const settingPattern = /<Setting\b[^>]*\bName="([^"]+)"[^>]*>([\s\S]*?)<\/Setting>/gi;
1060
- let match;
1061
-
1062
- while ((match = settingPattern.exec(content)) !== null) {
1063
- const key = normalizeToken(decodeXmlEntities(match[1]));
1064
- if (!key) {
1065
- continue;
1066
- }
1067
-
1068
- const valueMatch = match[2].match(/<Value(?:\s[^>]*)?>([\s\S]*?)<\/Value>/i);
1069
- if (!valueMatch) {
1070
- continue;
1071
- }
1072
-
1073
- const value = decodeXmlEntities(valueMatch[1]).trim();
1074
- const refs = extractSqlReferenceNamesFromString(value);
1075
- if (refs.length === 0) {
1076
- continue;
1077
- }
1078
-
1079
- const existing = refsByKey.get(key) ?? [];
1080
- refsByKey.set(key, uniqueSorted([...existing, ...refs]));
1081
- }
1082
-
1083
- return refsByKey;
1084
- }
1085
-
1086
- function parseSettingsKeyMap(content) {
1087
- const fileKeys = new Map();
1088
- const settingPattern = /<Setting\b[^>]*\bName="([^"]+)"[^>]*>/gi;
1089
- let match;
1090
-
1091
- while ((match = settingPattern.exec(content)) !== null) {
1092
- const key = normalizeToken(decodeXmlEntities(match[1]));
1093
- if (!key) {
1094
- continue;
1095
- }
1096
- fileKeys.set(key, true);
1097
- }
1098
-
1099
- return fileKeys;
1100
- }
1101
-
1102
- function parseConfigKeyMap(content) {
1103
- const fileKeys = new Map();
1104
- const addPattern = /<add\b([^>]+?)\/?>/gi;
1105
- let match;
1106
-
1107
- while ((match = addPattern.exec(content)) !== null) {
1108
- const attributes = match[1];
1109
- const nameMatch = attributes.match(/\bname="([^"]+)"/i);
1110
- const keyMatch = attributes.match(/\bkey="([^"]+)"/i);
1111
- const normalized = normalizeToken(decodeXmlEntities(nameMatch?.[1] ?? keyMatch?.[1] ?? ""));
1112
- if (!normalized) {
1113
- continue;
1114
- }
1115
- fileKeys.set(normalized, true);
1116
- }
1117
-
1118
- return fileKeys;
1119
- }
1120
-
1121
- function buildSqlResourceReferenceMap(fileRecords) {
1122
- const refsByKey = new Map();
1123
-
1124
- for (const fileRecord of fileRecords) {
1125
- const ext = path.extname(fileRecord.path).toLowerCase();
1126
- let fileRefs = null;
1127
- if (ext === ".resx") {
1128
- fileRefs = parseResxSqlReferenceMap(fileRecord.content);
1129
- } else if (ext === ".settings") {
1130
- fileRefs = parseSettingsSqlReferenceMap(fileRecord.content);
1131
- }
1132
-
1133
- if (!fileRefs) {
1134
- continue;
1135
- }
1136
-
1137
- for (const [key, refs] of fileRefs.entries()) {
1138
- const existing = refsByKey.get(key) ?? [];
1139
- refsByKey.set(key, uniqueSorted([...existing, ...refs]));
1140
- }
1141
- }
1142
-
1143
- return refsByKey;
1144
- }
1145
-
1146
- function buildNamedResourceFileMaps(fileRecords) {
1147
- const resourceFilesByKey = new Map();
1148
- const settingFilesByKey = new Map();
1149
- const configFilesByKey = new Map();
1150
-
1151
- for (const fileRecord of fileRecords) {
1152
- const ext = path.extname(fileRecord.path).toLowerCase();
1153
- const keyMap =
1154
- ext === ".resx"
1155
- ? parseResxKeyMap(fileRecord.content)
1156
- : ext === ".settings"
1157
- ? parseSettingsKeyMap(fileRecord.content)
1158
- : ext === ".config"
1159
- ? parseConfigKeyMap(fileRecord.content)
1160
- : null;
1161
-
1162
- if (!keyMap) {
1163
- continue;
1164
- }
1165
-
1166
- for (const key of keyMap.keys()) {
1167
- const targetMap =
1168
- ext === ".resx" ? resourceFilesByKey : ext === ".settings" ? settingFilesByKey : configFilesByKey;
1169
- const list = targetMap.get(key) ?? [];
1170
- list.push(fileRecord.id);
1171
- targetMap.set(key, uniqueSorted(list));
1172
- }
1173
- }
1174
-
1175
- return { resourceFilesByKey, settingFilesByKey, configFilesByKey };
1176
- }
1177
-
1178
- function extractSqlResourceKeyReferences(content) {
1179
- const keys = new Set();
1180
-
1181
- for (const pattern of SQL_RESOURCE_KEY_PATTERNS) {
1182
- let match;
1183
- while ((match = pattern.exec(content)) !== null) {
1184
- const key = normalizeToken(match[1]);
1185
- if (key) {
1186
- keys.add(key);
1187
- }
1188
- }
1189
- }
1190
-
1191
- return [...keys];
1192
- }
1193
-
1194
- function extractConfigKeyReferences(content) {
1195
- const keys = new Set();
1196
-
1197
- for (const pattern of CONFIG_KEY_REFERENCE_PATTERNS) {
1198
- let match;
1199
- while ((match = pattern.exec(content)) !== null) {
1200
- const key = normalizeToken(match[1]);
1201
- if (key) {
1202
- keys.add(key);
1203
- }
1204
- }
1205
- }
1206
-
1207
- return [...keys];
1208
- }
1209
-
1210
- function shouldExtractNamedResourceReferences(filePath) {
1211
- return NAMED_RESOURCE_REFERENCE_SOURCE_EXTENSIONS.has(path.extname(filePath).toLowerCase());
1212
- }
1213
-
1214
- function generateNamedResourceRelations(fileRecords) {
1215
- const { resourceFilesByKey, settingFilesByKey, configFilesByKey } = buildNamedResourceFileMaps(fileRecords);
1216
- const usesResourceRelations = [];
1217
- const usesSettingRelations = [];
1218
- const usesConfigRelations = [];
1219
- const resourceSeen = new Set();
1220
- const settingSeen = new Set();
1221
- const configSeen = new Set();
1222
-
1223
- for (const fileRecord of fileRecords) {
1224
- if (!shouldExtractNamedResourceReferences(fileRecord.path)) {
1225
- continue;
1226
- }
1227
-
1228
- for (const key of extractSqlResourceKeyReferences(fileRecord.content)) {
1229
- for (const targetFileId of resourceFilesByKey.get(key) ?? []) {
1230
- const relKey = relationKey(fileRecord.id, targetFileId, key);
1231
- if (!resourceSeen.has(relKey) && fileRecord.id !== targetFileId) {
1232
- resourceSeen.add(relKey);
1233
- usesResourceRelations.push({
1234
- from: fileRecord.id,
1235
- to: targetFileId,
1236
- note: key
1237
- });
1238
- }
1239
- }
1240
-
1241
- for (const targetFileId of settingFilesByKey.get(key) ?? []) {
1242
- const relKey = relationKey(fileRecord.id, targetFileId, key);
1243
- if (!settingSeen.has(relKey) && fileRecord.id !== targetFileId) {
1244
- settingSeen.add(relKey);
1245
- usesSettingRelations.push({
1246
- from: fileRecord.id,
1247
- to: targetFileId,
1248
- note: key
1249
- });
1250
- }
1251
- }
1252
- }
1253
-
1254
- for (const key of extractConfigKeyReferences(fileRecord.content)) {
1255
- for (const targetFileId of configFilesByKey.get(key) ?? []) {
1256
- const relKey = relationKey(fileRecord.id, targetFileId, key);
1257
- if (!configSeen.has(relKey) && fileRecord.id !== targetFileId) {
1258
- configSeen.add(relKey);
1259
- usesConfigRelations.push({
1260
- from: fileRecord.id,
1261
- to: targetFileId,
1262
- note: key
1263
- });
1264
- }
1265
- }
1266
- }
1267
- }
1268
-
1269
- usesResourceRelations.sort((a, b) =>
1270
- relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
1271
- );
1272
- usesSettingRelations.sort((a, b) =>
1273
- relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
1274
- );
1275
- usesConfigRelations.sort((a, b) =>
1276
- relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
1277
- );
1278
-
1279
- return { usesResourceRelations, usesSettingRelations, usesConfigRelations };
1280
- }
1281
-
1282
- function parseConfigIncludeTargets(fileRecord) {
1283
- const relPath = toPosixPath(String(fileRecord?.path ?? "").trim());
1284
- const lowerPath = relPath.toLowerCase();
1285
- if (!lowerPath.endsWith(".config")) {
1286
- return [];
1287
- }
1288
-
1289
- const content = String(fileRecord?.content ?? "");
1290
- const dir = path.posix.dirname(relPath);
1291
- const includes = [];
1292
- const sectionPattern =
1293
- /<([A-Za-z_][A-Za-z0-9_.:-]*)\b([^>]*?)\b(configSource|file)="([^"]+)"([^>]*)>/gi;
1294
- let match;
1295
-
1296
- while ((match = sectionPattern.exec(content)) !== null) {
1297
- const sectionName = String(match[1] ?? "").trim().toLowerCase();
1298
- const attributeName = String(match[3] ?? "").trim().toLowerCase();
1299
- const includePath = decodeXmlEntities(match[4] ?? "").trim().replace(/\\/g, "/");
1300
- if (!sectionName || !attributeName || !includePath) {
1301
- continue;
1302
- }
1303
- if (includePath.startsWith("/") || includePath.startsWith("~")) {
1304
- continue;
1305
- }
1306
-
1307
- const resolvedPath = path.posix.normalize(dir === "." ? includePath : `${dir}/${includePath}`);
1308
- if (!resolvedPath || resolvedPath.startsWith("../")) {
1309
- continue;
1310
- }
1311
-
1312
- includes.push({
1313
- targetPath: resolvedPath,
1314
- note: `${sectionName}:${attributeName}`
1315
- });
1316
- }
1317
-
1318
- return includes;
1319
- }
1320
-
1321
- function generateConfigIncludeRelations(fileRecords) {
1322
- const fileIdByPath = new Map(fileRecords.map((record) => [toPosixPath(record.path), record.id]));
1323
- const relations = [];
1324
- const seen = new Set();
1325
-
1326
- for (const fileRecord of fileRecords) {
1327
- for (const include of parseConfigIncludeTargets(fileRecord)) {
1328
- const targetFileId = fileIdByPath.get(include.targetPath);
1329
- if (!targetFileId || targetFileId === fileRecord.id) {
1330
- continue;
1331
- }
1332
- const key = relationKey(fileRecord.id, targetFileId, include.note);
1333
- if (seen.has(key)) {
1334
- continue;
1335
- }
1336
- seen.add(key);
1337
- relations.push({
1338
- from: fileRecord.id,
1339
- to: targetFileId,
1340
- note: include.note
1341
- });
1342
- }
1343
- }
1344
-
1345
- relations.sort((a, b) => relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note)));
1346
- return relations;
1347
- }
1348
-
1349
- function parseSectionHandlerDeclarations(content) {
1350
- const declarations = [];
1351
- const sectionPattern = /<section\b([^>]*?)\/?>/gi;
1352
- let match;
1353
-
1354
- while ((match = sectionPattern.exec(String(content ?? ""))) !== null) {
1355
- const attrs = match[1] ?? "";
1356
- const nameMatch = attrs.match(/\bname="([^"]+)"/i);
1357
- const typeMatch = attrs.match(/\btype="([^"]+)"/i);
1358
- const sectionName = normalizeToken(decodeXmlEntities(nameMatch?.[1] ?? ""));
1359
- const typeValue = decodeXmlEntities(typeMatch?.[1] ?? "").trim();
1360
- if (!sectionName || !typeValue) {
1361
- continue;
1362
- }
1363
-
1364
- const typeParts = typeValue.split(",").map((part) => part.trim()).filter(Boolean);
1365
- const fullTypeName = typeParts[0] ?? "";
1366
- const assemblyName = typeParts[1] ?? "";
1367
- const shortTypeName = fullTypeName.split(".").pop()?.split("+").pop() ?? "";
1368
- const normalizedTypeName = normalizeToken(shortTypeName);
1369
- const normalizedAssemblyName = normalizeToken(assemblyName);
1370
- if (!normalizedTypeName && !normalizedAssemblyName) {
1371
- continue;
1372
- }
1373
-
1374
- declarations.push({
1375
- sectionName,
1376
- normalizedTypeName,
1377
- normalizedAssemblyName
1378
- });
1379
- }
1380
-
1381
- return declarations;
1382
- }
1383
-
1384
- function buildProjectAssemblyFileMap(fileRecords) {
1385
- const aliasMap = new Map();
1386
-
1387
- for (const fileRecord of fileRecords) {
1388
- const ext = path.extname(fileRecord.path).toLowerCase();
1389
- if (!PROJECT_DEFINITION_EXTENSIONS.has(ext) || ext === ".sln") {
1390
- continue;
1391
- }
1392
-
1393
- const aliases = uniqueSorted([
1394
- normalizeToken(extractXmlTagValue(fileRecord.content, "AssemblyName")),
1395
- normalizeToken(extractXmlTagValue(fileRecord.content, "RootNamespace")),
1396
- normalizeToken(path.basename(fileRecord.path, ext))
1397
- ].filter(Boolean));
1398
-
1399
- for (const alias of aliases) {
1400
- const existing = aliasMap.get(alias) ?? [];
1401
- aliasMap.set(alias, uniqueSorted([...existing, fileRecord.id]));
1402
- }
1403
- }
1404
-
1405
- return aliasMap;
1406
- }
1407
-
1408
- function extractDeclaredTypeNames(fileRecord) {
1409
- const ext = path.extname(fileRecord.path).toLowerCase();
1410
- const pattern =
1411
- ext === ".vb"
1412
- ? /\b(?:Public|Friend|Private|Protected|Partial|MustInherit|NotInheritable|Shadows|Default|Overridable|Overrides|Shared|\s)*(?:Class|Module|Structure|Interface|Enum)\s+([A-Za-z_][A-Za-z0-9_]*)/gi
1413
- : ext === ".cs"
1414
- ? /\b(?:public|internal|private|protected|abstract|sealed|static|partial|\s)*(?:class|struct|interface|enum)\s+([A-Za-z_][A-Za-z0-9_]*)/gi
1415
- : null;
1416
-
1417
- if (!pattern) {
1418
- return [];
1419
- }
1420
-
1421
- const typeNames = new Set();
1422
- let match;
1423
- while ((match = pattern.exec(String(fileRecord.content ?? ""))) !== null) {
1424
- const normalized = normalizeToken(match[1] ?? "");
1425
- if (normalized) {
1426
- typeNames.add(normalized);
1427
- }
1428
- }
1429
-
1430
- return [...typeNames];
1431
- }
1432
-
1433
- function buildCodeTypeFileMap(fileRecords) {
1434
- const typeMap = new Map();
1435
-
1436
- for (const fileRecord of fileRecords) {
1437
- if (fileRecord.kind !== "CODE") {
1438
- continue;
1439
- }
1440
- for (const typeName of extractDeclaredTypeNames(fileRecord)) {
1441
- const existing = typeMap.get(typeName) ?? [];
1442
- typeMap.set(typeName, uniqueSorted([...existing, fileRecord.id]));
1443
- }
1444
- }
1445
-
1446
- return typeMap;
1447
- }
1448
-
1449
- function longestCommonPathPrefixLength(pathA, pathB) {
1450
- const partsA = toPosixPath(pathA).split("/").filter(Boolean);
1451
- const partsB = toPosixPath(pathB).split("/").filter(Boolean);
1452
- const limit = Math.min(partsA.length, partsB.length);
1453
- let count = 0;
1454
- while (count < limit && partsA[count] === partsB[count]) {
1455
- count += 1;
1456
- }
1457
- return count;
1458
- }
1459
-
1460
- function generateMachineConfigRelations(fileRecords) {
1461
- const machineConfigs = fileRecords.filter(
1462
- (record) => path.basename(record.path).toLowerCase() === "machine.config"
1463
- );
1464
- if (machineConfigs.length === 0) {
1465
- return [];
1466
- }
1467
-
1468
- const relations = [];
1469
- const seen = new Set();
1470
-
1471
- for (const fileRecord of fileRecords) {
1472
- const lowerPath = fileRecord.path.toLowerCase();
1473
- if (
1474
- !lowerPath.endsWith(".config") ||
1475
- path.basename(lowerPath) === "machine.config" ||
1476
- !/<configuration\b/i.test(String(fileRecord.content ?? "")) ||
1477
- parseConfigTransformTarget(fileRecord)
1478
- ) {
1479
- continue;
1480
- }
1481
-
1482
- const rankedTargets = machineConfigs
1483
- .filter((candidate) => candidate.id !== fileRecord.id)
1484
- .map((candidate) => ({
1485
- id: candidate.id,
1486
- score: longestCommonPathPrefixLength(fileRecord.path, candidate.path)
1487
- }))
1488
- .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
1489
-
1490
- const target = rankedTargets[0];
1491
- if (!target) {
1492
- continue;
1493
- }
1494
-
1495
- const key = relationKey(fileRecord.id, target.id, "inherits:machine");
1496
- if (seen.has(key)) {
1497
- continue;
1498
- }
1499
- seen.add(key);
1500
- relations.push({
1501
- from: fileRecord.id,
1502
- to: target.id,
1503
- note: "inherits:machine"
1504
- });
1505
- }
1506
-
1507
- return uniqueRelations(relations);
1508
- }
1509
-
1510
- function generateSectionHandlerRelations(fileRecords) {
1511
- const projectAssemblyFileMap = buildProjectAssemblyFileMap(fileRecords);
1512
- const codeTypeFileMap = buildCodeTypeFileMap(fileRecords);
1513
- const relations = [];
1514
-
1515
- for (const fileRecord of fileRecords) {
1516
- if (!fileRecord.path.toLowerCase().endsWith(".config")) {
1517
- continue;
1518
- }
1519
-
1520
- for (const declaration of parseSectionHandlerDeclarations(fileRecord.content)) {
1521
- const note = `section_handler:${declaration.sectionName}`;
1522
-
1523
- for (const targetFileId of projectAssemblyFileMap.get(declaration.normalizedAssemblyName) ?? []) {
1524
- relations.push({
1525
- from: fileRecord.id,
1526
- to: targetFileId,
1527
- note
1528
- });
1529
- }
1530
-
1531
- for (const targetFileId of codeTypeFileMap.get(declaration.normalizedTypeName) ?? []) {
1532
- relations.push({
1533
- from: fileRecord.id,
1534
- to: targetFileId,
1535
- note
1536
- });
1537
- }
1538
- }
1539
- }
1540
-
1541
- return uniqueRelations(relations.filter((relation) => relation.from !== relation.to));
1542
- }
1543
-
1544
- function generateConfigTransformKeyRelations(fileRecords, chunkRecords) {
1545
- const fileIdByPath = new Map(fileRecords.map((record) => [toPosixPath(record.path), record.id]));
1546
- const chunkFileIdById = new Map(chunkRecords.map((chunk) => [chunk.id, chunk.file_id]));
1547
- const configChunkIdsByAlias = new Map();
1548
-
1549
- for (const chunk of chunkRecords) {
1550
- if (isWindowChunkId(chunk.id) || String(chunk.language ?? "").toLowerCase() !== "config") {
1551
- continue;
1552
- }
1553
- for (const alias of configChunkAliases(chunk)) {
1554
- const existing = configChunkIdsByAlias.get(alias) ?? [];
1555
- configChunkIdsByAlias.set(alias, [...existing, chunk.id]);
1556
- }
1557
- }
1558
-
1559
- const relations = [];
1560
- for (const fileRecord of fileRecords) {
1561
- const transform = parseConfigTransformTarget(fileRecord);
1562
- if (!transform) {
1563
- continue;
1564
- }
1565
-
1566
- const targetFileId = fileIdByPath.get(transform.targetPath);
1567
- if (!targetFileId) {
1568
- continue;
1569
- }
1570
-
1571
- for (const key of parseConfigKeyMap(fileRecord.content).keys()) {
1572
- for (const targetChunkId of configChunkIdsByAlias.get(key) ?? []) {
1573
- if (chunkFileIdById.get(targetChunkId) !== targetFileId) {
1574
- continue;
1575
- }
1576
- relations.push({
1577
- from: fileRecord.id,
1578
- to: targetChunkId,
1579
- note: `${key}:${transform.environment}`
1580
- });
1581
- }
1582
- }
1583
- }
1584
-
1585
- return uniqueRelations(relations);
1586
- }
1587
-
1588
- function parseConfigTransformTarget(fileRecord) {
1589
- const relPath = toPosixPath(String(fileRecord?.path ?? "").trim());
1590
- const lowerPath = relPath.toLowerCase();
1591
- if (!lowerPath.endsWith(".config")) {
1592
- return null;
1593
- }
1594
-
1595
- const content = String(fileRecord?.content ?? "");
1596
- if (!/\bxdt:(?:transform|locator)\b/i.test(content) && !/\bxmlns:xdt=/i.test(content)) {
1597
- return null;
1598
- }
1599
-
1600
- const dir = path.posix.dirname(relPath);
1601
- const baseName = path.posix.basename(relPath, ".config");
1602
- const match = baseName.match(/^(.+)\.([^.]+)$/);
1603
- if (!match) {
1604
- return null;
1605
- }
1606
-
1607
- const baseStem = match[1]?.trim();
1608
- const environment = match[2]?.trim();
1609
- if (!baseStem || !environment) {
1610
- return null;
1611
- }
1612
-
1613
- const targetPath = dir === "." ? `${baseStem}.config` : `${dir}/${baseStem}.config`;
1614
- return {
1615
- targetPath,
1616
- environment: normalizeToken(environment)
1617
- };
1618
- }
1619
-
1620
- function generateConfigTransformRelations(fileRecords) {
1621
- const fileIdByPath = new Map(fileRecords.map((record) => [toPosixPath(record.path), record.id]));
1622
- const relations = [];
1623
- const seen = new Set();
1624
-
1625
- for (const fileRecord of fileRecords) {
1626
- const transform = parseConfigTransformTarget(fileRecord);
1627
- if (!transform) {
1628
- continue;
1629
- }
1630
-
1631
- const targetFileId = fileIdByPath.get(transform.targetPath);
1632
- if (!targetFileId || targetFileId === fileRecord.id) {
1633
- continue;
1634
- }
1635
-
1636
- const key = relationKey(fileRecord.id, targetFileId, transform.environment);
1637
- if (seen.has(key)) {
1638
- continue;
1639
- }
1640
- seen.add(key);
1641
- relations.push({
1642
- from: fileRecord.id,
1643
- to: targetFileId,
1644
- note: transform.environment
1645
- });
1646
- }
1647
-
1648
- relations.sort((a, b) => relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note)));
1649
- return relations;
1650
- }
1651
-
1652
- function shouldExtractSqlReferences(filePath) {
1653
- return SQL_REFERENCE_SOURCE_EXTENSIONS.has(path.extname(filePath).toLowerCase());
1654
- }
1655
-
1656
- function extractSqlObjectReferencesFromContent(content, filePath = "", sqlResourceReferenceMap = new Map()) {
1657
- const refs = new Set();
1658
- const ext = path.extname(filePath).toLowerCase();
1659
-
1660
- if (ext === ".resx") {
1661
- for (const values of parseResxSqlReferenceMap(content).values()) {
1662
- for (const ref of values) {
1663
- refs.add(ref);
1664
- }
1665
- }
1666
- } else if (ext === ".settings") {
1667
- for (const values of parseSettingsSqlReferenceMap(content).values()) {
1668
- for (const ref of values) {
1669
- refs.add(ref);
1670
- }
1671
- }
1672
- }
1673
-
1674
- for (const pattern of SQL_OBJECT_REFERENCE_PATTERNS) {
1675
- let match;
1676
- while ((match = pattern.exec(content)) !== null) {
1677
- for (const ref of extractSqlReferenceNamesFromString(match[1])) {
1678
- refs.add(ref);
1679
- }
1680
- }
1681
- }
1682
-
1683
- if (sqlResourceReferenceMap.size > 0) {
1684
- for (const key of extractSqlResourceKeyReferences(content)) {
1685
- for (const ref of sqlResourceReferenceMap.get(key) ?? []) {
1686
- refs.add(ref);
1687
- }
1688
- }
1689
- }
1690
-
1691
- return uniqueSorted([...refs]);
1692
- }
1693
-
1694
- function decodeXmlEntities(value) {
1695
- return String(value)
1696
- .replace(/&quot;/g, '"')
1697
- .replace(/&apos;/g, "'")
1698
- .replace(/&lt;/g, "<")
1699
- .replace(/&gt;/g, ">")
1700
- .replace(/&amp;/g, "&");
1701
- }
1702
-
1703
- function projectIdFor(filePath) {
1704
- return `project:${filePath}`;
1705
- }
1706
-
1707
- function isProjectDefinitionFile(filePath) {
1708
- return PROJECT_DEFINITION_EXTENSIONS.has(path.extname(filePath).toLowerCase());
1709
- }
1710
-
1711
- function resolveProjectRelativePath(baseFilePath, includePath) {
1712
- if (!includePath) {
1713
- return null;
1714
- }
1715
-
1716
- const normalizedInclude = toPosixPath(decodeXmlEntities(includePath).trim().replace(/\\/g, "/"));
1717
- if (!normalizedInclude) {
1718
- return null;
1719
- }
1720
-
1721
- const resolved = path.resolve(REPO_ROOT, path.dirname(baseFilePath), normalizedInclude);
1722
- const relPath = toPosixPath(path.relative(REPO_ROOT, resolved));
1723
- if (!relPath || relPath.startsWith("../")) {
1724
- return null;
1725
- }
1726
-
1727
- return relPath;
1728
- }
1729
-
1730
- function projectLanguageForExtension(ext) {
1731
- switch (ext) {
1732
- case ".vbproj":
1733
- return "vbnet";
1734
- case ".csproj":
1735
- return "csharp";
1736
- case ".fsproj":
1737
- return "fsharp";
1738
- case ".vcxproj":
1739
- return "cpp";
1740
- case ".sln":
1741
- return "solution";
1742
- default:
1743
- return "dotnet";
1744
- }
1745
- }
1746
-
1747
- function extractXmlTagValue(content, tagName) {
1748
- const match = content.match(new RegExp(`<${tagName}>([\\s\\S]*?)<\\/${tagName}>`, "i"));
1749
- return match ? decodeXmlEntities(match[1]).trim() : "";
1750
- }
1751
-
1752
- function collectXmlIncludeValues(content, elementNames) {
1753
- const values = [];
1754
- const pattern = new RegExp(
1755
- `<(?:${elementNames.join("|")})\\b[^>]*\\bInclude="([^"]+)"[^>]*\\/?>`,
1756
- "gi"
1757
- );
1758
- let match;
1759
- while ((match = pattern.exec(content)) !== null) {
1760
- values.push(decodeXmlEntities(match[1]).trim());
1761
- }
1762
- return values;
1763
- }
1764
-
1765
- function parseSolutionProject(fileRecord, indexedFileIds) {
1766
- const declaredMembers = [];
1767
- const referencesProjectRelations = [];
1768
- const includesFileRelations = [];
1769
- const fileRelationKeys = new Set();
1770
- const ext = path.extname(fileRecord.path).toLowerCase();
1771
- const fallbackName = path.basename(fileRecord.path, ext);
1772
- const projectPattern =
1773
- /^Project\([^)]*\)\s*=\s*"([^"]+)",\s*"([^"]+\.(?:vbproj|csproj|fsproj|vcxproj))",\s*"\{[^"]+\}"$/gim;
1774
-
1775
- let match;
1776
- while ((match = projectPattern.exec(fileRecord.content)) !== null) {
1777
- const memberName = match[1].trim();
1778
- const memberPath = resolveProjectRelativePath(fileRecord.path, match[2]);
1779
- if (!memberPath) {
1780
- continue;
1781
- }
1782
- declaredMembers.push({ name: memberName, path: memberPath });
1783
- const targetId = projectIdFor(memberPath);
1784
- if (indexedFileIds.has(`file:${memberPath}`)) {
1785
- referencesProjectRelations.push({
1786
- from: projectIdFor(fileRecord.path),
1787
- to: targetId,
1788
- note: `solution_member:${memberName}`
1789
- });
1790
- }
1791
- }
1792
-
1793
- for (const fileId of [`file:${fileRecord.path}`]) {
1794
- if (indexedFileIds.has(fileId) && !fileRelationKeys.has(fileId)) {
1795
- fileRelationKeys.add(fileId);
1796
- includesFileRelations.push({ from: projectIdFor(fileRecord.path), to: fileId });
1797
- }
1798
- }
1799
-
1800
- const summaryParts = [`Solution ${fallbackName}`];
1801
- if (declaredMembers.length > 0) {
1802
- summaryParts.push(`Contains ${declaredMembers.length} project references`);
1803
- }
1804
-
1805
- return {
1806
- project: {
1807
- id: projectIdFor(fileRecord.path),
1808
- path: fileRecord.path,
1809
- name: fallbackName,
1810
- kind: "solution",
1811
- language: projectLanguageForExtension(ext),
1812
- target_framework: "",
1813
- summary: `${summaryParts.join(". ")}.`,
1814
- file_count: includesFileRelations.length,
1815
- updated_at: fileRecord.updated_at,
1816
- source_of_truth: false,
1817
- trust_level: 78,
1818
- status: "active"
1819
- },
1820
- includesFileRelations,
1821
- referencesProjectRelations
1822
- };
1823
- }
1824
-
1825
- function parseDotNetProject(fileRecord, indexedFileIds) {
1826
- const ext = path.extname(fileRecord.path).toLowerCase();
1827
- const fallbackName = path.basename(fileRecord.path, ext);
1828
- const assemblyName = extractXmlTagValue(fileRecord.content, "AssemblyName");
1829
- const rootNamespace = extractXmlTagValue(fileRecord.content, "RootNamespace");
1830
- const targetFrameworkRaw =
1831
- extractXmlTagValue(fileRecord.content, "TargetFramework") ||
1832
- extractXmlTagValue(fileRecord.content, "TargetFrameworkVersion") ||
1833
- extractXmlTagValue(fileRecord.content, "TargetFrameworks");
1834
- const targetFramework = targetFrameworkRaw.split(";")[0].trim();
1835
- const includeCandidates = collectXmlIncludeValues(fileRecord.content, [
1836
- "Compile",
1837
- "Content",
1838
- "EmbeddedResource",
1839
- "None",
1840
- "Page",
1841
- "ApplicationDefinition"
1842
- ]);
1843
- const projectReferenceCandidates = collectXmlIncludeValues(fileRecord.content, ["ProjectReference"]);
1844
- const includesFileRelations = [];
1845
- const referencesProjectRelations = [];
1846
- const fileRelationKeys = new Set();
1847
-
1848
- const addFileRelation = (relPath) => {
1849
- const fileId = `file:${relPath}`;
1850
- if (!indexedFileIds.has(fileId) || fileRelationKeys.has(fileId)) {
1851
- return;
1852
- }
1853
- fileRelationKeys.add(fileId);
1854
- includesFileRelations.push({
1855
- from: projectIdFor(fileRecord.path),
1856
- to: fileId
1857
- });
1858
- };
1859
-
1860
- addFileRelation(fileRecord.path);
1861
-
1862
- for (const includePath of includeCandidates) {
1863
- const relPath = resolveProjectRelativePath(fileRecord.path, includePath);
1864
- if (!relPath) {
1865
- continue;
1866
- }
1867
- addFileRelation(relPath);
1868
- }
1869
-
1870
- for (const includePath of projectReferenceCandidates) {
1871
- const relPath = resolveProjectRelativePath(fileRecord.path, includePath);
1872
- if (!relPath) {
1873
- continue;
1874
- }
1875
- const targetFileId = `file:${relPath}`;
1876
- if (!indexedFileIds.has(targetFileId)) {
1877
- continue;
1878
- }
1879
- referencesProjectRelations.push({
1880
- from: projectIdFor(fileRecord.path),
1881
- to: projectIdFor(relPath),
1882
- note: includePath
1883
- });
1884
- }
1885
-
1886
- const summaryParts = [
1887
- `${projectLanguageForExtension(ext).toUpperCase()} project ${assemblyName || rootNamespace || fallbackName}`
1888
- ];
1889
- if (targetFramework) {
1890
- summaryParts.push(`Target framework ${targetFramework}`);
1891
- }
1892
- if (includesFileRelations.length > 1) {
1893
- summaryParts.push(`Includes ${includesFileRelations.length - 1} indexed project files`);
1894
- }
1895
- if (referencesProjectRelations.length > 0) {
1896
- summaryParts.push(`References ${referencesProjectRelations.length} projects`);
1897
- }
1898
-
1899
- return {
1900
- project: {
1901
- id: projectIdFor(fileRecord.path),
1902
- path: fileRecord.path,
1903
- name: assemblyName || rootNamespace || fallbackName,
1904
- kind: "project",
1905
- language: projectLanguageForExtension(ext),
1906
- target_framework: targetFramework,
1907
- summary: `${summaryParts.join(". ")}.`,
1908
- file_count: includesFileRelations.length,
1909
- updated_at: fileRecord.updated_at,
1910
- source_of_truth: false,
1911
- trust_level: 80,
1912
- status: "active"
1913
- },
1914
- includesFileRelations,
1915
- referencesProjectRelations
1916
- };
1917
- }
1918
-
1919
- function generateProjects(fileRecords) {
1920
- const indexedFileIds = new Set(fileRecords.map((record) => record.id));
1921
- const projectRecords = [];
1922
- const includesFileRelations = [];
1923
- const referencesProjectRelations = [];
1924
- const includeKeys = new Set();
1925
- const referenceKeys = new Set();
1926
-
1927
- for (const fileRecord of fileRecords) {
1928
- if (!isProjectDefinitionFile(fileRecord.path)) {
1929
- continue;
1930
- }
1931
-
1932
- const ext = path.extname(fileRecord.path).toLowerCase();
1933
- const parsed =
1934
- ext === ".sln"
1935
- ? parseSolutionProject(fileRecord, indexedFileIds)
1936
- : parseDotNetProject(fileRecord, indexedFileIds);
1937
-
1938
- projectRecords.push(parsed.project);
1939
-
1940
- for (const relation of parsed.includesFileRelations) {
1941
- const key = relationKey(relation.from, relation.to);
1942
- if (includeKeys.has(key)) {
1943
- continue;
1944
- }
1945
- includeKeys.add(key);
1946
- includesFileRelations.push(relation);
1947
- }
1948
-
1949
- for (const relation of parsed.referencesProjectRelations) {
1950
- const key = relationKey(relation.from, relation.to, relation.note);
1951
- if (referenceKeys.has(key)) {
1952
- continue;
1953
- }
1954
- referenceKeys.add(key);
1955
- referencesProjectRelations.push(relation);
1956
- }
1957
- }
1958
-
1959
- projectRecords.sort((a, b) => a.path.localeCompare(b.path));
1960
- includesFileRelations.sort((a, b) => relationKey(a.from, a.to).localeCompare(relationKey(b.from, b.to)));
1961
- referencesProjectRelations.sort((a, b) =>
1962
- relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
1963
- );
1964
-
1965
- return {
1966
- projects: projectRecords,
1967
- includesFileRelations,
1968
- referencesProjectRelations
1969
- };
1970
- }
1971
-
1972
- function removeChunkStateForFile(fileId, chunkRecordMap, definesRelationMap, callsRelationMap, importsRelationMap, callsSqlRelationMap) {
1973
- const removedChunkIds = new Set();
1974
-
1975
- for (const [chunkId, chunkRecord] of chunkRecordMap.entries()) {
1976
- if (chunkRecord.file_id === fileId) {
1977
- removedChunkIds.add(chunkId);
1978
- chunkRecordMap.delete(chunkId);
1979
- }
1980
- }
1981
-
1982
- if (removedChunkIds.size === 0) {
1983
- return;
1984
- }
1985
-
1986
- for (const [key, relation] of definesRelationMap.entries()) {
1987
- if (relation.from === fileId || removedChunkIds.has(relation.to)) {
1988
- definesRelationMap.delete(key);
1989
- }
1990
- }
1991
-
1992
- for (const [key, relation] of callsRelationMap.entries()) {
1993
- if (removedChunkIds.has(relation.from) || removedChunkIds.has(relation.to)) {
1994
- callsRelationMap.delete(key);
1995
- }
1996
- }
1997
-
1998
- for (const [key, relation] of importsRelationMap.entries()) {
1999
- if (removedChunkIds.has(relation.from)) {
2000
- importsRelationMap.delete(key);
2001
- }
2002
- }
2003
-
2004
- for (const [key, relation] of callsSqlRelationMap.entries()) {
2005
- if (relation.from === fileId || removedChunkIds.has(relation.to)) {
2006
- callsSqlRelationMap.delete(key);
2007
- }
2008
- }
2009
- }
2010
-
2011
- function hydrateIncrementalChunkState(fileRecords) {
2012
- const fileIdSet = new Set(fileRecords.map((record) => record.id));
2013
- const chunkRecordMap = new Map();
2014
- const definesRelationMap = new Map();
2015
- const callsRelationMap = new Map();
2016
- const importsRelationMap = new Map();
2017
- const callsSqlRelationMap = new Map();
2018
-
2019
- for (const record of readJsonlSafe(path.join(CACHE_DIR, "entities.chunk.jsonl"))) {
2020
- if (!record || typeof record !== "object") continue;
2021
- const chunkId = String(record.id ?? "");
2022
- const fileId = String(record.file_id ?? "");
2023
- if (!chunkId || !fileIdSet.has(fileId)) {
2024
- continue;
2025
- }
2026
- chunkRecordMap.set(chunkId, {
2027
- ...record,
2028
- id: chunkId,
2029
- file_id: fileId
2030
- });
2031
- }
2032
-
2033
- const chunkIdSet = new Set(chunkRecordMap.keys());
2034
-
2035
- for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.defines.jsonl"))) {
2036
- if (!record || typeof record !== "object") continue;
2037
- const from = String(record.from ?? "");
2038
- const to = String(record.to ?? "");
2039
- if (!fileIdSet.has(from) || !chunkIdSet.has(to)) {
2040
- continue;
2041
- }
2042
- definesRelationMap.set(relationKey(from, to), { from, to });
2043
- }
2044
-
2045
- for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.calls.jsonl"))) {
2046
- if (!record || typeof record !== "object") continue;
2047
- const from = String(record.from ?? "");
2048
- const to = String(record.to ?? "");
2049
- const callType = String(record.call_type ?? "direct");
2050
- if (!chunkIdSet.has(from) || !chunkIdSet.has(to)) {
2051
- continue;
2052
- }
2053
- callsRelationMap.set(relationKey(from, to, callType), {
2054
- from,
2055
- to,
2056
- call_type: callType
2057
- });
2058
- }
2059
-
2060
- for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.imports.jsonl"))) {
2061
- if (!record || typeof record !== "object") continue;
2062
- const from = String(record.from ?? "");
2063
- const to = String(record.to ?? "");
2064
- const importName = String(record.import_name ?? "");
2065
- if (!chunkIdSet.has(from) || !fileIdSet.has(to)) {
2066
- continue;
2067
- }
2068
- importsRelationMap.set(relationKey(from, to, importName), {
2069
- from,
2070
- to,
2071
- import_name: importName
2072
- });
2073
- }
2074
-
2075
- for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.calls_sql.jsonl"))) {
2076
- if (!record || typeof record !== "object") continue;
2077
- const from = String(record.from ?? "");
2078
- const to = String(record.to ?? "");
2079
- const note = String(record.note ?? "");
2080
- if (!fileIdSet.has(from) || !chunkIdSet.has(to)) {
2081
- continue;
2082
- }
2083
- callsSqlRelationMap.set(relationKey(from, to, note), {
2084
- from,
2085
- to,
2086
- note
2087
- });
2088
- }
2089
-
2090
- return {
2091
- chunkRecordMap,
2092
- definesRelationMap,
2093
- callsRelationMap,
2094
- importsRelationMap,
2095
- callsSqlRelationMap
2096
- };
2097
- }
2098
-
2099
- function normalizeRuleTokens(ruleRecord) {
2100
- const idParts = ruleRecord.id.split(/[._-]+/g);
2101
- const descriptionTokens = tokenizeKeywords(ruleRecord.body);
2102
- const rawKeywords = [...idParts, ...descriptionTokens];
2103
- const normalized = rawKeywords
2104
- .map((token) => token.toLowerCase().replace(/[^a-z0-9]/g, ""))
2105
- .filter((token) => token.length >= 3 && !STOP_WORDS.has(token));
2106
-
2107
- return uniqueSorted(normalized).slice(0, RULE_KEYWORD_LIMIT);
2108
- }
2109
-
2110
- function fileTokenSet(fileRecord) {
2111
- const tokenSource = `${fileRecord.path}\n${fileRecord.content.slice(0, 12000)}`;
2112
- return new Set(tokenizeKeywords(tokenSource));
2113
- }
2114
-
2115
- function collectRuleKeywordMatch(ruleKeywords, tokens, minimumMatches) {
2116
- const required = Math.min(minimumMatches, Math.max(1, ruleKeywords.length));
2117
- let count = 0;
2118
- const sample = [];
2119
-
2120
- for (const keyword of ruleKeywords) {
2121
- if (!tokens.has(keyword)) {
2122
- continue;
2123
- }
2124
- count += 1;
2125
- if (sample.length < 5) {
2126
- sample.push(keyword);
2127
- }
2128
- if (count >= required && sample.length >= 5) {
2129
- break;
2130
- }
2131
- }
2132
-
2133
- return { matched: count >= required, sample };
2134
- }
2135
-
2136
- function chunkIdFor(filePath, chunk) {
2137
- const startLine = Number.isFinite(chunk.startLine) ? chunk.startLine : 0;
2138
- const endLine = Number.isFinite(chunk.endLine) ? chunk.endLine : startLine;
2139
- return `chunk:${filePath}:${chunk.name}:${startLine}-${endLine}`;
2140
- }
2141
-
2142
- function generateChunkDescription(chunk) {
2143
- const parts = [chunk.kind];
2144
- if (chunk.exported) parts.push("exported");
2145
- if (chunk.async) parts.push("async");
2146
- parts.push(chunk.signature);
2147
-
2148
- if (typeof chunk.description === "string" && chunk.description.trim().length > 10) {
2149
- parts.push(normalizeWhitespace(chunk.description).slice(0, 200));
2150
- }
2151
-
2152
- // Extract leading JSDoc/comment from body
2153
- // Match leading JSDoc (/** */), block (/* */) and line (//) comments
2154
- const commentMatch = chunk.body.match(/^(?:\s*(?:\/\*[\s\S]*?\*\/|\/\/[^\n]*)[\s\n]*)+/);
2155
- if (commentMatch) {
2156
- const cleaned = commentMatch[0]
2157
- .replace(/\/\*\*|\*\/|\*|\/\//g, "")
2158
- .replace(/\s+/g, " ").trim()
2159
- .slice(0, 200);
2160
- if (cleaned.length > 10) parts.push(cleaned);
2161
- }
2162
-
2163
- return parts.join(". ") + ".";
2164
- }
2165
-
2166
- function generateModuleSummary(dir, files, exportNames, repoRoot = REPO_ROOT) {
2167
- // Check for README.md in directory
2168
- const readmePath = path.join(repoRoot, dir, "README.md");
2169
- if (fs.existsSync(readmePath)) {
2170
- try {
2171
- const content = fs.readFileSync(readmePath, "utf8");
2172
- // Skip first heading line, take first 300 chars
2173
- const lines = content.split(/\r?\n/);
2174
- const startIdx = lines.findIndex(l => !l.startsWith("#") && l.trim().length > 0);
2175
- if (startIdx >= 0) {
2176
- const excerpt = lines.slice(startIdx).join(" ").trim().slice(0, 300);
2177
- if (excerpt.length > 20) return excerpt;
2178
- }
2179
- } catch {
2180
- // fall through to auto-generated summary
2181
- }
2182
- }
2183
-
2184
- const name = path.basename(dir);
2185
- const codeFiles = files.filter(f => f.kind === "CODE");
2186
- const docFiles = files.filter(f => f.kind !== "CODE");
2187
-
2188
- const parts = [`Module ${name}`];
2189
- parts.push(`Contains ${files.length} files (${codeFiles.length} code, ${docFiles.length} docs)`);
2190
-
2191
- // Detect common file extension pattern
2192
- const exts = new Set(codeFiles.map(f => path.extname(f.path).toLowerCase()));
2193
- if (exts.size === 1) {
2194
- const ext = [...exts][0];
2195
- const extNames = {
2196
- ".ts": "TypeScript",
2197
- ".tsx": "TypeScript React",
2198
- ".mts": "TypeScript ESM",
2199
- ".cts": "TypeScript CommonJS",
2200
- ".js": "JavaScript",
2201
- ".jsx": "JavaScript React",
2202
- ".mjs": "JavaScript ESM",
2203
- ".cjs": "JavaScript CommonJS"
2204
- };
2205
- if (extNames[ext]) parts.push(`${extNames[ext]} source files`);
2206
- }
2207
-
2208
- if (exportNames.length > 0) {
2209
- parts.push(`Key exports: ${exportNames.slice(0, 5).join(", ")}`);
2210
- }
2211
-
2212
- return parts.join(". ") + ".";
2213
- }
2214
-
2215
- function generateModules(fileRecords, chunkRecords) {
2216
- const dirFiles = new Map();
2217
- const dirChunks = new Map();
2218
- const fileById = new Map(fileRecords.map(f => [f.id, f]));
2219
-
2220
- for (const file of fileRecords) {
2221
- const dir = path.dirname(file.path);
2222
- if (!dirFiles.has(dir)) dirFiles.set(dir, []);
2223
- dirFiles.get(dir).push(file);
2224
- }
2225
-
2226
- for (const chunk of chunkRecords) {
2227
- if (!chunk.exported || isWindowChunkId(chunk.id)) continue;
2228
- const file = fileById.get(chunk.file_id);
2229
- if (!file) continue;
2230
- const dir = path.dirname(file.path);
2231
- if (!dirChunks.has(dir)) dirChunks.set(dir, []);
2232
- dirChunks.get(dir).push(chunk);
2233
- }
2234
-
2235
- const modules = [];
2236
- const containsRelations = [];
2237
- const containsModuleRelations = [];
2238
- const exportsRelations = [];
2239
-
2240
- const MIN_MODULE_FILES = 2;
2241
-
2242
- for (const [dir, files] of dirFiles) {
2243
- if (files.length < MIN_MODULE_FILES) continue;
2244
-
2245
- const exports = dirChunks.get(dir) || [];
2246
- const exportNames = [...new Set(exports.slice(0, 20).map(c => c.name))];
2247
- const moduleId = `module:${dir}`;
2248
-
2249
- modules.push({
2250
- id: moduleId,
2251
- path: dir,
2252
- name: path.basename(dir),
2253
- summary: generateModuleSummary(dir, files, exportNames),
2254
- file_count: files.length,
2255
- exported_symbols: exportNames.join(", "),
2256
- updated_at: files.reduce((latest, f) => f.updated_at > latest ? f.updated_at : latest, ""),
2257
- source_of_truth: false,
2258
- trust_level: 75,
2259
- status: "active"
2260
- });
2261
-
2262
- // CONTAINS: Module -> File
2263
- for (const file of files) {
2264
- containsRelations.push({ from: moduleId, to: file.id });
2265
- }
2266
-
2267
- // EXPORTS: Module -> Chunk
2268
- for (const chunk of exports) {
2269
- exportsRelations.push({ from: moduleId, to: chunk.id });
2270
- }
2271
- }
2272
-
2273
- // CONTAINS_MODULE: parent Module -> child Module
2274
- const moduleDirs = new Set(modules.map(m => m.path));
2275
- for (const dir of moduleDirs) {
2276
- const parent = path.dirname(dir);
2277
- if (parent !== dir && moduleDirs.has(parent)) {
2278
- containsModuleRelations.push({
2279
- from: `module:${parent}`,
2280
- to: `module:${dir}`
2281
- });
2282
- }
2283
- }
2284
-
2285
- return { modules, containsRelations, containsModuleRelations, exportsRelations };
2286
- }
2287
-
2288
- function isWindowChunkId(chunkId) {
2289
- return typeof chunkId === "string" && chunkId.includes(":window:");
2290
- }
2291
-
2292
- function splitChunkIntoWindows(chunkRecord, options) {
2293
- const { windowLines, overlapLines, splitMinLines, maxWindows, chunkBody } = options;
2294
- const sourceBody = typeof chunkBody === "string" ? chunkBody : chunkRecord.body;
2295
- const lines = sourceBody.split(/\r?\n/);
2296
- const totalLines = lines.length;
2297
- if (totalLines < splitMinLines || totalLines <= windowLines) {
2298
- return [];
2299
- }
2300
-
2301
- const windows = [];
2302
- const safeOverlap = Math.max(0, Math.min(overlapLines, windowLines - 1));
2303
- let start = 0;
2304
- let windowIndex = 1;
2305
-
2306
- while (start < totalLines && windows.length < maxWindows) {
2307
- const isLastAllowedWindow = windows.length + 1 >= maxWindows;
2308
- const end = isLastAllowedWindow ? totalLines : Math.min(totalLines, start + windowLines);
2309
- const windowStartLine = chunkRecord.start_line + start;
2310
- const windowEndLine = chunkRecord.start_line + Math.max(0, end - 1);
2311
- const windowBody = lines.slice(start, end).join("\n");
2312
- const persistedBody = isLastAllowedWindow ? windowBody : windowBody.slice(0, MAX_BODY_CHARS);
2313
- windows.push({
2314
- id: `${chunkRecord.id}:window:${windowIndex}:${windowStartLine}-${windowEndLine}`,
2315
- file_id: chunkRecord.file_id,
2316
- name: `${chunkRecord.name}#window${windowIndex}`,
2317
- kind: chunkRecord.kind,
2318
- signature: `${chunkRecord.signature} [window ${windowIndex}]`,
2319
- body: persistedBody,
2320
- description: chunkRecord.description || "",
2321
- start_line: windowStartLine,
2322
- end_line: windowEndLine,
2323
- language: chunkRecord.language,
2324
- exported: chunkRecord.exported || false,
2325
- checksum: checksum(Buffer.from(windowBody)),
2326
- updated_at: chunkRecord.updated_at,
2327
- trust_level: chunkRecord.trust_level,
2328
- status: chunkRecord.status,
2329
- source_of_truth: chunkRecord.source_of_truth
2330
- });
2331
-
2332
- if (end >= totalLines) {
2333
- break;
2334
- }
2335
-
2336
- start = end - safeOverlap;
2337
- windowIndex += 1;
2338
- }
2339
-
2340
- return windows;
2341
- }
2342
-
2343
- function resolveIngestWorkerCount(taskCount) {
2344
- const raw = process.env.CORTEX_INGEST_WORKERS;
2345
- const configured = raw !== undefined ? Number.parseInt(raw, 10) : Number.NaN;
2346
- const cpuBudget = Math.max(1, (os.availableParallelism?.() ?? os.cpus().length) - 1);
2347
- const defaultWorkerLimit = taskCount >= 1000 ? 4 : 8;
2348
- const desired = Number.isFinite(configured) && configured >= 0 ? configured : Math.min(cpuBudget, defaultWorkerLimit);
2349
- if (desired <= 1) return 1;
2350
- // Worker spin-up plus per-worker WASM grammar init dominates on small or
2351
- // incremental runs; stay sequential until there is enough work to amortize.
2352
- if (taskCount < 50) return 1;
2353
- return Math.min(desired, taskCount);
2354
- }
2355
-
2356
- function createEmptyWorkerParseStream(tasks, workerCount) {
2357
- const stats = () => ({
2358
- worker_tasks: tasks.length,
2359
- worker_count: Number.isFinite(workerCount) ? workerCount : 0,
2360
- worker_tasks_assigned: 0,
2361
- worker_tasks_settled: 0,
2362
- worker_tasks_unsettled_fallback: tasks.length,
2363
- worker_results: 0,
2364
- worker_results_consumed: 0,
2365
- worker_results_retained: 0,
2366
- worker_results_retained_peak: 0,
2367
- worker_results_pending: 0,
2368
- worker_results_missing: tasks.length,
2369
- worker_waiters: 0
2370
- });
2371
- return {
2372
- hasTask: () => false,
2373
- stats,
2374
- take: async () => undefined,
2375
- drain: async () => stats()
2376
- };
2377
- }
2378
-
2379
- function startWorkerParseStream(tasks, { workerCount, verbose, workerUrl } = {}) {
2380
- if (tasks.length === 0) {
2381
- return createEmptyWorkerParseStream(tasks, workerCount);
2382
- }
2383
-
2384
- const resolvedWorkerUrl = workerUrl ?? new URL("./ingest-worker.mjs", import.meta.url);
2385
- const poolSize = Math.min(workerCount, tasks.length);
2386
- // Defensive: an undefined/0/negative/NaN workerCount yields poolSize < 1, in
2387
- // which case no workers are ever spawned and the pool would never resolve.
2388
- // Return empty so the caller parses everything inline. (Math.min(undefined, n)
2389
- // is NaN, which is why this is `!(>= 1)` rather than `< 1`.)
2390
- if (!(poolSize >= 1)) {
2391
- return createEmptyWorkerParseStream(tasks, workerCount);
2392
- }
2393
-
2394
- const taskIds = new Set(tasks.map((task) => task.id));
2395
- const results = new Map();
2396
- const missingResults = new Set();
2397
- const waiters = new Map();
2398
- const workers = [];
2399
- const inflight = new Map(); // worker -> taskId being parsed, or null when idle
2400
- let nextTask = 0;
2401
- let alive = poolSize;
2402
- let finished = false;
2403
- let resolveDone;
2404
- const done = new Promise((resolve) => {
2405
- resolveDone = resolve;
2406
- });
2407
- const state = {
2408
- assigned: 0,
2409
- settled: 0,
2410
- successful: 0,
2411
- consumed: 0,
2412
- retainedPeak: 0
2413
- };
2414
-
2415
- const stats = () => ({
2416
- worker_tasks: tasks.length,
2417
- worker_count: poolSize,
2418
- worker_tasks_assigned: state.assigned,
2419
- worker_tasks_settled: state.settled,
2420
- worker_tasks_unsettled_fallback: finished ? Math.max(0, tasks.length - state.settled) : 0,
2421
- worker_results: state.successful,
2422
- worker_results_consumed: state.consumed,
2423
- worker_results_retained: results.size,
2424
- worker_results_retained_peak: state.retainedPeak,
2425
- worker_results_pending: finished ? 0 : Math.max(0, tasks.length - state.settled),
2426
- worker_results_missing: tasks.length - state.successful,
2427
- worker_waiters: waiters.size
2428
- });
2429
-
2430
- const finish = () => {
2431
- if (finished) return;
2432
- finished = true;
2433
- for (const resolve of waiters.values()) {
2434
- resolve(undefined);
2435
- }
2436
- waiters.clear();
2437
- resolveDone();
2438
- };
2439
-
2440
- // A task is "settled" once it has a result, was skipped, or its worker
2441
- // died holding it. Finish when every task is settled, or when no worker is
2442
- // left alive to make progress (any still-queued tasks then parse inline).
2443
- const maybeFinish = () => {
2444
- if (state.settled >= tasks.length || alive <= 0) {
2445
- finish();
2446
- }
2447
- };
2448
-
2449
- const resolveWaiter = (taskId, result) => {
2450
- const resolve = waiters.get(taskId);
2451
- if (!resolve) {
2452
- return false;
2453
- }
2454
- waiters.delete(taskId);
2455
- if (result !== undefined) {
2456
- state.consumed += 1;
2457
- }
2458
- resolve(result);
2459
- return true;
2460
- };
2461
-
2462
- const settleTask = (taskId, result) => {
2463
- state.settled += 1;
2464
- if (result !== undefined) {
2465
- state.successful += 1;
2466
- if (!resolveWaiter(taskId, result)) {
2467
- results.set(taskId, result);
2468
- state.retainedPeak = Math.max(state.retainedPeak, results.size);
2469
- }
2470
- } else {
2471
- missingResults.add(taskId);
2472
- resolveWaiter(taskId, undefined);
2473
- }
2474
- maybeFinish();
2475
- };
2476
-
2477
- const assign = (worker) => {
2478
- if (nextTask >= tasks.length) {
2479
- inflight.set(worker, null);
2480
- worker.postMessage({ type: "shutdown" });
2481
- return;
2482
- }
2483
- const task = tasks[nextTask++];
2484
- state.assigned += 1;
2485
- inflight.set(worker, task.id);
2486
- worker.postMessage({
2487
- taskId: task.id,
2488
- ext: task.ext,
2489
- content: task.content,
2490
- absolutePath: task.absolutePath,
2491
- contentLimit: task.contentLimit,
2492
- filePath: task.path
2493
- });
2494
- };
2495
-
2496
- const onMessage = (worker, message) => {
2497
- if (finished) return;
2498
- inflight.set(worker, null);
2499
- if (message.ok) {
2500
- settleTask(message.taskId, message.result);
2501
- } else {
2502
- if (verbose) {
2503
- console.log(`[ingest] worker skipped ${message.taskId}: ${message.reason}`);
2504
- }
2505
- settleTask(message.taskId, undefined);
2506
- }
2507
- if (!finished) {
2508
- assign(worker);
2509
- }
2510
- };
2511
-
2512
- const onExit = (worker) => {
2513
- if (finished) return;
2514
- alive -= 1;
2515
- const taskId = inflight.get(worker);
2516
- if (taskId != null) {
2517
- // Worker exited mid-parse without posting a result (OOM, native abort,
2518
- // process.exit) and without an 'error' event. Count its in-flight task
2519
- // as settled so it falls back to inline parsing rather than leaving the
2520
- // pool waiting on a dead worker forever.
2521
- if (verbose) {
2522
- console.log(`[ingest] worker exited mid-task ${taskId}; will parse inline`);
2523
- }
2524
- inflight.set(worker, null);
2525
- settleTask(taskId, undefined);
2526
- return;
2527
- }
2528
- maybeFinish();
2529
- };
2530
-
2531
- for (let i = 0; i < poolSize; i += 1) {
2532
- const worker = new Worker(resolvedWorkerUrl);
2533
- workers.push(worker);
2534
- worker.on("message", (message) => onMessage(worker, message));
2535
- worker.on("error", (error) => {
2536
- // Uncaught exception in the worker. The 'exit' event that always
2537
- // follows does the task accounting (idempotent via inflight), so we
2538
- // only log here to avoid double-counting.
2539
- if (verbose) {
2540
- console.log(`[ingest] worker error: ${error.message}`);
2541
- }
2542
- });
2543
- worker.on("exit", () => onExit(worker));
2544
- }
2545
-
2546
- for (const worker of workers) {
2547
- assign(worker);
2548
- }
2549
-
2550
- return {
2551
- hasTask(taskId) {
2552
- return taskIds.has(taskId);
2553
- },
2554
- stats,
2555
- async take(taskId) {
2556
- if (!taskIds.has(taskId)) {
2557
- return undefined;
2558
- }
2559
- if (results.has(taskId)) {
2560
- const result = results.get(taskId);
2561
- results.delete(taskId);
2562
- state.consumed += 1;
2563
- return result;
2564
- }
2565
- if (missingResults.has(taskId) || finished) {
2566
- return undefined;
2567
- }
2568
- return new Promise((resolve) => {
2569
- waiters.set(taskId, resolve);
2570
- });
2571
- },
2572
- async drain() {
2573
- await done;
2574
- await Promise.all(workers.map((worker) => worker.terminate().catch(() => {})));
2575
- return stats();
2576
- }
2577
- };
2578
- }
2579
-
2580
- // Parse files in a worker pool, returning Map<fileId, parseResult> for the
2581
- // tasks that completed. Any task not present (worker miss, skip, or a fatal
2582
- // worker error) is left to the caller's inline parse — the same parse
2583
- // function on the same content, so output is identical either way. Never
2584
- // rejects: a fatal worker error resolves with the partial results collected
2585
- // so far.
2586
- async function parseFilesInWorkers(tasks, options = {}) {
2587
- const stream = startWorkerParseStream(tasks, options);
2588
- const results = new Map();
2589
- for (const task of tasks) {
2590
- const result = await stream.take(task.id);
2591
- if (result) {
2592
- results.set(task.id, result);
2593
- }
2594
- }
2595
-
2596
- await stream.drain();
2597
- return results;
2598
- }
2599
-
2600
- async function main() {
2601
- await loadParsers();
2602
- const { mode, verbose } = parseArgs(process.argv);
2603
- const memoryTrace = createIngestMemoryTrace();
2604
- const configPath = path.join(CONTEXT_DIR, "config.yaml");
2605
- const rulesPath = path.join(CONTEXT_DIR, "rules.yaml");
2606
-
2607
- if (!fs.existsSync(configPath)) {
2608
- throw new Error(`Missing config: ${configPath}`);
2609
- }
2610
- if (!fs.existsSync(rulesPath)) {
2611
- throw new Error(`Missing rules: ${rulesPath}`);
2612
- }
2613
-
2614
- ensureDirectory(CACHE_DIR);
2615
- ensureDirectory(DB_IMPORT_DIR);
2616
-
2617
- const configText = fs.readFileSync(configPath, "utf8");
2618
- const sourcePaths = parseSourcePaths(configText);
2619
- if (sourcePaths.length === 0) {
2620
- throw new Error("No source_paths found in .context/config.yaml");
2621
- }
2622
-
2623
- const rules = parseRules(fs.readFileSync(rulesPath, "utf8"));
2624
- memoryTrace.checkpoint("scan:start", {
2625
- mode,
2626
- source_paths: sourcePaths.length,
2627
- rules: rules.length
2628
- });
2629
- const { candidates, incrementalMode, deletedRelPaths } = collectCandidateFiles(sourcePaths, mode);
2630
- const chunkWindowLines = parsePositiveIntegerEnv(
2631
- "CORTEX_CHUNK_WINDOW_LINES",
2632
- DEFAULT_CHUNK_WINDOW_LINES
2633
- );
2634
- const chunkOverlapLines = Math.max(
2635
- 0,
2636
- Math.min(
2637
- chunkWindowLines - 1,
2638
- parseNonNegativeIntegerEnv("CORTEX_CHUNK_OVERLAP_LINES", DEFAULT_CHUNK_OVERLAP_LINES)
2639
- )
2640
- );
2641
- const chunkSplitMinLines = Math.max(
2642
- chunkWindowLines + 1,
2643
- parsePositiveIntegerEnv("CORTEX_CHUNK_SPLIT_MIN_LINES", DEFAULT_CHUNK_SPLIT_MIN_LINES)
2644
- );
2645
- const chunkMaxWindows = parsePositiveIntegerEnv(
2646
- "CORTEX_CHUNK_MAX_WINDOWS",
2647
- DEFAULT_CHUNK_MAX_WINDOWS
2648
- );
2649
-
2650
- const fileRecordMap = new Map();
2651
- const adrRecordMap = new Map();
2652
- const skipped = {
2653
- unsupported: 0,
2654
- tooLarge: 0,
2655
- binary: 0
2656
- };
2657
-
2658
- if (incrementalMode) {
2659
- const existingFiles = readJsonlSafe(path.join(CACHE_DIR, "entities.file.jsonl"));
2660
- for (const record of existingFiles) {
2661
- if (!record || typeof record !== "object") continue;
2662
- const filePath = toPosixPath(String(record.path ?? ""));
2663
- if (!filePath || !hasSourcePrefix(filePath, sourcePaths)) {
2664
- continue;
2665
- }
2666
- const absolutePath = path.resolve(REPO_ROOT, filePath);
2667
- if (!fs.existsSync(absolutePath)) {
2668
- continue;
2669
- }
2670
- fileRecordMap.set(String(record.id ?? `file:${filePath}`), {
2671
- ...record,
2672
- id: String(record.id ?? `file:${filePath}`),
2673
- path: filePath,
2674
- kind: String(record.kind ?? detectKind(filePath)),
2675
- content: String(record.content ?? "")
2676
- });
2677
- }
2678
-
2679
- const existingAdrs = readJsonlSafe(path.join(CACHE_DIR, "entities.adr.jsonl"));
2680
- for (const adr of existingAdrs) {
2681
- if (!adr || typeof adr !== "object") continue;
2682
- const adrPath = toPosixPath(String(adr.path ?? ""));
2683
- if (!adrPath || !hasSourcePrefix(adrPath, sourcePaths)) {
2684
- continue;
2685
- }
2686
- if (!fs.existsSync(path.resolve(REPO_ROOT, adrPath))) {
2687
- continue;
2688
- }
2689
- adrRecordMap.set(String(adr.id ?? ""), {
2690
- ...adr,
2691
- id: String(adr.id ?? ""),
2692
- path: adrPath
2693
- });
2694
- }
2695
- }
2696
-
2697
- for (const relPath of deletedRelPaths) {
2698
- fileRecordMap.delete(`file:${relPath}`);
2699
- const relPrefix = relPath.endsWith("/") ? relPath : `${relPath}/`;
2700
- for (const [fileId, fileRecord] of fileRecordMap.entries()) {
2701
- if (String(fileRecord.path ?? "").startsWith(relPrefix)) {
2702
- fileRecordMap.delete(fileId);
2703
- }
2704
- }
2705
-
2706
- for (const [adrId, adrRecord] of adrRecordMap.entries()) {
2707
- if (adrRecord.path === relPath || String(adrRecord.path ?? "").startsWith(relPrefix)) {
2708
- adrRecordMap.delete(adrId);
2709
- }
2710
- }
2711
- }
2712
-
2713
- for (const absolutePath of [...candidates].sort()) {
2714
- const relPath = toPosixPath(path.relative(REPO_ROOT, absolutePath));
2715
- if (!isTextFile(relPath)) {
2716
- skipped.unsupported += 1;
2717
- if (verbose) console.log(`[ingest] skip unsupported: ${relPath}`);
2718
- continue;
2719
- }
2720
-
2721
- const stats = fs.statSync(absolutePath);
2722
- if (stats.size > MAX_FILE_BYTES) {
2723
- skipped.tooLarge += 1;
2724
- if (verbose) console.log(`[ingest] skip large: ${relPath}`);
2725
- continue;
2726
- }
2727
-
2728
- const buffer = fs.readFileSync(absolutePath);
2729
- if (isBinaryBuffer(buffer)) {
2730
- skipped.binary += 1;
2731
- if (verbose) console.log(`[ingest] skip binary: ${relPath}`);
2732
- continue;
2733
- }
2734
-
2735
- const content = buffer.toString("utf8");
2736
- const kind = detectKind(relPath);
2737
- const id = `file:${relPath}`;
2738
- const updatedAt = stats.mtime.toISOString();
2739
- const sourceOfTruth = kind === "ADR";
2740
- const trustLevel = trustLevelForKind(kind);
2741
-
2742
- const fileRecord = {
2743
- id,
2744
- path: relPath,
2745
- kind,
2746
- checksum: checksum(buffer),
2747
- updated_at: updatedAt,
2748
- source_of_truth: sourceOfTruth,
2749
- trust_level: trustLevel,
2750
- status: "active",
2751
- size_bytes: stats.size,
2752
- excerpt: normalizeWhitespace(content).slice(0, 500),
2753
- content: content.slice(0, MAX_CONTENT_CHARS)
2754
- };
2755
- fileRecordMap.set(fileRecord.id, fileRecord);
2756
-
2757
- if (kind === "ADR") {
2758
- const title = extractTitle(content, path.basename(relPath, path.extname(relPath)));
2759
- const adrRecord = {
2760
- id: `adr:${path.basename(relPath, path.extname(relPath)).toLowerCase()}`,
2761
- path: relPath,
2762
- title,
2763
- body: content.slice(0, MAX_BODY_CHARS),
2764
- decision_date: parseDecisionDate(content, updatedAt),
2765
- supersedes_id: "",
2766
- source_of_truth: true,
2767
- trust_level: 95,
2768
- status: "active"
2769
- };
2770
- adrRecordMap.set(adrRecord.id, adrRecord);
2771
- } else {
2772
- for (const [adrId, adrRecord] of adrRecordMap.entries()) {
2773
- if (adrRecord.path === relPath) {
2774
- adrRecordMap.delete(adrId);
2775
- }
2776
- }
2777
- }
2778
- }
2779
-
2780
- const fileRecords = [...fileRecordMap.values()].sort((a, b) => a.path.localeCompare(b.path));
2781
- const adrRecords = [...adrRecordMap.values()].sort((a, b) => a.path.localeCompare(b.path));
2782
- memoryTrace.checkpoint("scan:file_records", {
2783
- candidates: candidates.size,
2784
- incremental_mode: incrementalMode,
2785
- deleted_paths: deletedRelPaths.length,
2786
- files: fileRecords.length,
2787
- adrs: adrRecords.length,
2788
- skipped_unsupported: skipped.unsupported,
2789
- skipped_too_large: skipped.tooLarge,
2790
- skipped_binary: skipped.binary
2791
- });
2792
- const csharpFileCount = fileRecords.filter((record) => path.extname(record.path).toLowerCase() === ".cs").length;
2793
- const csharpRuntime = csharpFileCount > 0 ? getCSharpParserRuntime() : null;
2794
- const indexedFileIds = new Set(fileRecords.map((record) => record.id));
2795
- const changedFileIds = new Set(
2796
- [...candidates].map((absolutePath) => `file:${toPosixPath(path.relative(REPO_ROOT, absolutePath))}`)
2797
- );
2798
-
2799
- const {
2800
- chunkRecordMap,
2801
- definesRelationMap,
2802
- callsRelationMap,
2803
- importsRelationMap,
2804
- callsSqlRelationMap
2805
- } = incrementalMode
2806
- ? hydrateIncrementalChunkState(fileRecords)
2807
- : {
2808
- chunkRecordMap: new Map(),
2809
- definesRelationMap: new Map(),
2810
- callsRelationMap: new Map(),
2811
- importsRelationMap: new Map(),
2812
- callsSqlRelationMap: new Map()
2813
- };
2814
- memoryTrace.checkpoint("hydration:complete", {
2815
- incremental_mode: incrementalMode,
2816
- cached_chunks: chunkRecordMap.size,
2817
- cached_defines_relations: definesRelationMap.size,
2818
- cached_calls_relations: callsRelationMap.size,
2819
- cached_imports_relations: importsRelationMap.size,
2820
- cached_calls_sql_relations: callsSqlRelationMap.size
2821
- });
2822
-
2823
- const cachedChunkFileIds = new Set(
2824
- [...chunkRecordMap.values()].map((record) => String(record.file_id ?? "")).filter(Boolean)
2825
- );
2826
- const cachedSqlReferenceFileIds = new Set(
2827
- [...callsSqlRelationMap.values()].map((record) => String(record.from ?? "")).filter(Boolean)
2828
- );
2829
- const usesConfigKeyRelationMap = new Map();
2830
- const usesResourceKeyRelationMap = new Map();
2831
- const usesSettingKeyRelationMap = new Map();
2832
-
2833
- // Extract chunks from changed or uncached code files
2834
- let windowedChunkCount = 0;
2835
- let {
2836
- sqlChunkIdsByAlias,
2837
- configChunkIdsByAlias,
2838
- resourceChunkIdsByAlias,
2839
- settingChunkIdsByAlias
2840
- } = buildChunkAliasIndexes([...chunkRecordMap.values()]);
2841
- const deferredSqlCallEdges = [];
2842
-
2843
- // C# project-wide batch parse: when Roslyn is available and batching
2844
- // isn't disabled, compile all .cs files together via CSharpCompilation
2845
- // to enable SemanticModel-resolved calls (e.g. "System.IO.File.ReadAllText"
2846
- // instead of bare "ReadAllText"). Falls back silently to per-file parse
2847
- // if batch isn't usable.
2848
- const csharpBatchCache = new Map();
2849
- if (
2850
- hasCSharpProjectParser() &&
2851
- isCSharpParserAvailable() &&
2852
- process.env.CORTEX_CSHARP_BATCH !== "never"
2853
- ) {
2854
- const csharpFilesForBatch = fileRecords.filter((r) => {
2855
- if (r.kind !== "CODE") return false;
2856
- if (path.extname(r.path).toLowerCase() !== ".cs") return false;
2857
- return !incrementalMode || changedFileIds.has(r.id) || !cachedChunkFileIds.has(r.id);
2858
- });
2859
- if (csharpFilesForBatch.length > 0) {
2860
- const allCsharpInputs = fileRecords
2861
- .filter((r) => r.kind === "CODE" && path.extname(r.path).toLowerCase() === ".cs")
2862
- .map((r) => ({ path: r.path, content: r.content }));
2863
- try {
2864
- const batchResult = parseCSharpProject(allCsharpInputs);
2865
- for (const [filePath, result] of batchResult) {
2866
- csharpBatchCache.set(filePath, result);
2867
- }
2868
- } catch (error) {
2869
- if (verbose) {
2870
- console.log(`[ingest] C# batch parse failed, falling back per-file: ${error.message}`);
2871
- }
2872
- }
2873
- }
2874
- }
2875
-
2876
- // Determine which files need parsing (single pass over the gates, shared by
2877
- // the worker dispatch and the merge loop below so the two cannot diverge).
2878
- const parseEligible = new Map();
2879
- for (const fileRecord of fileRecords) {
2880
- const ext = path.extname(fileRecord.path).toLowerCase();
2881
- const parser = getChunkParserForExtension(ext);
2882
- const isStructuredNonCodeChunk = STRUCTURED_NON_CODE_CHUNK_EXTENSIONS.has(ext);
2883
- if (fileRecord.kind !== "CODE" && !isStructuredNonCodeChunk) continue;
2884
- if (!parser) continue;
2885
- if (typeof parser.isAvailable === "function" && !(await parser.isAvailable())) continue;
2886
-
2887
- const shouldParseFile =
2888
- !incrementalMode || changedFileIds.has(fileRecord.id) || !cachedChunkFileIds.has(fileRecord.id);
2889
- if (!shouldParseFile) {
2890
- continue;
2891
- }
2892
- parseEligible.set(fileRecord.id, { parser, ext });
2893
- }
2894
- memoryTrace.checkpoint("parse:eligible", {
2895
- files: fileRecords.length,
2896
- parse_eligible: parseEligible.size,
2897
- csharp_batch_cached: csharpBatchCache.size
2898
- });
2899
-
2900
- // Parse parallel-safe files (tree-sitter/JS, no subprocess, no cross-file
2901
- // state) in a worker pool. C# batch-cached files and any worker miss fall
2902
- // through to inline parsing in the loop, so the result is byte-identical to
2903
- // the sequential path regardless of where a file actually parsed.
2904
- const workerTasks = fileRecords
2905
- .filter((fileRecord) => {
2906
- const eligible = parseEligible.get(fileRecord.id);
2907
- if (!eligible) return false;
2908
- if (!PARALLEL_SAFE_LANGUAGES.has(eligible.parser.language)) return false;
2909
- if (eligible.parser.language === "csharp" && csharpBatchCache.has(fileRecord.path)) return false;
2910
- return true;
2911
- })
2912
- .map((fileRecord) => ({
2913
- id: fileRecord.id,
2914
- ext: parseEligible.get(fileRecord.id).ext,
2915
- absolutePath: path.resolve(REPO_ROOT, fileRecord.path),
2916
- contentLimit: MAX_CONTENT_CHARS,
2917
- path: fileRecord.path
2918
- }));
2919
- const workerCount = resolveIngestWorkerCount(workerTasks.length);
2920
- memoryTrace.checkpoint("parse:workers_start", {
2921
- worker_tasks: workerTasks.length,
2922
- worker_count: workerCount
2923
- });
2924
- const workerStream =
2925
- workerCount > 1 ? startWorkerParseStream(workerTasks, { workerCount, verbose }) : null;
2926
- if (!workerStream) {
2927
- memoryTrace.checkpoint("parse:workers_complete", {
2928
- worker_tasks: workerTasks.length,
2929
- worker_count: workerCount,
2930
- worker_results: 0,
2931
- worker_results_consumed: 0,
2932
- worker_results_retained: 0,
2933
- worker_results_retained_peak: 0,
2934
- worker_results_pending: 0,
2935
- worker_results_missing: workerTasks.length
2936
- });
2937
- }
2938
-
2939
- for (const fileRecord of fileRecords) {
2940
- const eligible = parseEligible.get(fileRecord.id);
2941
- if (!eligible) continue;
2942
- const { parser } = eligible;
2943
-
2944
- removeChunkStateForFile(
2945
- fileRecord.id,
2946
- chunkRecordMap,
2947
- definesRelationMap,
2948
- callsRelationMap,
2949
- importsRelationMap,
2950
- callsSqlRelationMap
2951
- );
2952
-
2953
- try {
2954
- let parseResult;
2955
- if (parser.language === "csharp" && csharpBatchCache.has(fileRecord.path)) {
2956
- parseResult = csharpBatchCache.get(fileRecord.path);
2957
- } else if (workerStream?.hasTask(fileRecord.id)) {
2958
- parseResult = await workerStream.take(fileRecord.id);
2959
- if (!parseResult) {
2960
- parseResult = await parser.parse(fileRecord.content, fileRecord.path, parser.language);
2961
- }
2962
- } else {
2963
- parseResult = await parser.parse(fileRecord.content, fileRecord.path, parser.language);
2964
- }
2965
-
2966
- if (parseResult.errors.length > 0 && verbose) {
2967
- console.log(`[ingest] parse errors in ${fileRecord.path}:`, parseResult.errors[0].message);
2968
- }
2969
-
2970
- const parsedChunks = [];
2971
- const chunkIdsByName = new Map();
2972
-
2973
- for (const chunk of parseResult.chunks) {
2974
- const chunkId = chunkIdFor(fileRecord.path, chunk);
2975
- parsedChunks.push({ chunk, chunkId });
2976
- if (!chunkIdsByName.has(chunk.name)) {
2977
- chunkIdsByName.set(chunk.name, []);
2978
- }
2979
- chunkIdsByName.get(chunk.name).push(chunkId);
2980
- if (parser.language === "sql") {
2981
- for (const alias of sqlChunkAliases(chunk.name)) {
2982
- if (!sqlChunkIdsByAlias.has(alias)) {
2983
- sqlChunkIdsByAlias.set(alias, []);
2984
- }
2985
- sqlChunkIdsByAlias.get(alias).push(chunkId);
2986
- }
2987
- deferredSqlCallEdges.push({
2988
- chunkId,
2989
- calls: Array.isArray(chunk.calls) ? chunk.calls : []
2990
- });
2991
- } else if (parser.language === "config") {
2992
- for (const alias of configChunkAliases(chunk)) {
2993
- if (!configChunkIdsByAlias.has(alias)) {
2994
- configChunkIdsByAlias.set(alias, []);
2995
- }
2996
- configChunkIdsByAlias.get(alias).push(chunkId);
2997
- }
2998
- deferredSqlCallEdges.push({
2999
- chunkId,
3000
- calls: Array.isArray(chunk.calls) ? chunk.calls : []
3001
- });
3002
- } else if (parser.language === "resource") {
3003
- for (const alias of namedEntryChunkAliases(chunk)) {
3004
- if (!resourceChunkIdsByAlias.has(alias)) {
3005
- resourceChunkIdsByAlias.set(alias, []);
3006
- }
3007
- resourceChunkIdsByAlias.get(alias).push(chunkId);
3008
- }
3009
- deferredSqlCallEdges.push({
3010
- chunkId,
3011
- calls: Array.isArray(chunk.calls) ? chunk.calls : []
3012
- });
3013
- } else if (parser.language === "settings") {
3014
- for (const alias of namedEntryChunkAliases(chunk)) {
3015
- if (!settingChunkIdsByAlias.has(alias)) {
3016
- settingChunkIdsByAlias.set(alias, []);
3017
- }
3018
- settingChunkIdsByAlias.get(alias).push(chunkId);
3019
- }
3020
- deferredSqlCallEdges.push({
3021
- chunkId,
3022
- calls: Array.isArray(chunk.calls) ? chunk.calls : []
3023
- });
3024
- }
3025
-
3026
- const chunkRecord = {
3027
- id: chunkId,
3028
- file_id: fileRecord.id,
3029
- name: chunk.name,
3030
- kind: chunk.kind,
3031
- signature: chunk.signature,
3032
- body: chunk.body.slice(0, MAX_BODY_CHARS), // Limit chunk body size
3033
- description: generateChunkDescription(chunk),
3034
- start_line: chunk.startLine,
3035
- end_line: chunk.endLine,
3036
- language: chunk.language,
3037
- exported: Boolean(chunk.exported),
3038
- checksum: checksum(Buffer.from(chunk.body)),
3039
- updated_at: fileRecord.updated_at,
3040
- trust_level: fileRecord.trust_level,
3041
- status:
3042
- typeof fileRecord.status === "string" && fileRecord.status.trim().length > 0
3043
- ? fileRecord.status
3044
- : "active",
3045
- source_of_truth: Boolean(fileRecord.source_of_truth)
3046
- };
3047
- chunkRecordMap.set(chunkId, chunkRecord);
3048
-
3049
- // DEFINES relation: File -> Chunk
3050
- definesRelationMap.set(relationKey(fileRecord.id, chunkId), {
3051
- from: fileRecord.id,
3052
- to: chunkId
3053
- });
3054
-
3055
- const windows = splitChunkIntoWindows(chunkRecord, {
3056
- windowLines: chunkWindowLines,
3057
- overlapLines: chunkOverlapLines,
3058
- splitMinLines: chunkSplitMinLines,
3059
- maxWindows: chunkMaxWindows,
3060
- chunkBody: chunk.body
3061
- });
3062
- if (windows.length > 0) {
3063
- windowedChunkCount += windows.length;
3064
- for (const windowChunk of windows) {
3065
- chunkRecordMap.set(windowChunk.id, windowChunk);
3066
- definesRelationMap.set(relationKey(fileRecord.id, windowChunk.id), {
3067
- from: fileRecord.id,
3068
- to: windowChunk.id
3069
- });
3070
- }
3071
- }
3072
-
3073
- // IMPORTS relations: Chunk -> File
3074
- for (const importPath of chunk.imports || []) {
3075
- const targetFileId = resolveRelativeImportTargetId(fileRecord.path, importPath, indexedFileIds);
3076
- if (!targetFileId) {
3077
- continue;
3078
- }
3079
-
3080
- importsRelationMap.set(relationKey(chunkId, targetFileId, importPath), {
3081
- from: chunkId,
3082
- to: targetFileId,
3083
- import_name: importPath
3084
- });
3085
- }
3086
- }
3087
-
3088
- const seenCallEdges = new Set();
3089
- for (const { chunk, chunkId } of parsedChunks) {
3090
- // CALLS relations: Chunk -> Chunk (within same file)
3091
- for (const calledName of chunk.calls || []) {
3092
- const targetChunkIds = chunkIdsByName.get(calledName) || [];
3093
- for (const targetChunkId of targetChunkIds) {
3094
- const callKey = `${chunkId}|${targetChunkId}|direct`;
3095
- if (seenCallEdges.has(callKey)) {
3096
- continue;
3097
- }
3098
- seenCallEdges.add(callKey);
3099
- callsRelationMap.set(relationKey(chunkId, targetChunkId, "direct"), {
3100
- from: chunkId,
3101
- to: targetChunkId,
3102
- call_type: "direct"
3103
- });
3104
- }
3105
- }
3106
- }
3107
- } catch (error) {
3108
- if (verbose) {
3109
- console.log(`[ingest] failed to parse ${fileRecord.path}: ${error instanceof Error ? error.message : String(error)}`);
3110
- }
3111
- }
3112
- }
3113
- const finalWorkerStats = workerStream ? await workerStream.drain() : null;
3114
- if (finalWorkerStats) {
3115
- memoryTrace.checkpoint("parse:workers_complete", finalWorkerStats);
3116
- if (verbose) {
3117
- console.log(
3118
- `[ingest] parsed ${finalWorkerStats.worker_results}/${workerTasks.length} files across ${workerCount} workers`
3119
- );
3120
- }
3121
- }
3122
- memoryTrace.checkpoint("parse:merge_complete", {
3123
- chunk_map: chunkRecordMap.size,
3124
- defines_relations: definesRelationMap.size,
3125
- calls_relations: callsRelationMap.size,
3126
- imports_relations: importsRelationMap.size,
3127
- calls_sql_relations: callsSqlRelationMap.size,
3128
- deferred_sql_edges: deferredSqlCallEdges.length,
3129
- windowed_chunks: windowedChunkCount,
3130
- worker_results_retained: finalWorkerStats?.worker_results_retained ?? 0,
3131
- worker_results_retained_peak: finalWorkerStats?.worker_results_retained_peak ?? 0,
3132
- worker_results_pending: finalWorkerStats?.worker_results_pending ?? 0
3133
- });
3134
-
3135
- const chunkRecords = [...chunkRecordMap.values()].sort((a, b) => String(a.id).localeCompare(String(b.id)));
3136
- ({
3137
- sqlChunkIdsByAlias,
3138
- configChunkIdsByAlias,
3139
- resourceChunkIdsByAlias,
3140
- settingChunkIdsByAlias
3141
- } = buildChunkAliasIndexes(chunkRecords));
3142
-
3143
- // Filter CALLS relations to only valid targets (chunks that actually exist)
3144
- const chunkIdSet = new Set(chunkRecords.map(c => c.id));
3145
- const validDefinesRelations = [...definesRelationMap.values()].filter(
3146
- (rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
3147
- );
3148
- const totalCallsRelations = callsRelationMap.size;
3149
- for (const edge of deferredSqlCallEdges) {
3150
- for (const calledName of edge.calls) {
3151
- for (const alias of sqlChunkAliases(calledName)) {
3152
- const targetChunkIds = sqlChunkIdsByAlias.get(alias) || [];
3153
- for (const targetChunkId of targetChunkIds) {
3154
- if (targetChunkId === edge.chunkId) {
3155
- continue;
3156
- }
3157
- callsRelationMap.set(relationKey(edge.chunkId, targetChunkId, "sql_reference"), {
3158
- from: edge.chunkId,
3159
- to: targetChunkId,
3160
- call_type: "sql_reference"
3161
- });
3162
- }
3163
- }
3164
- }
3165
- }
3166
- const validCallsRelations = [...callsRelationMap.values()].filter(
3167
- (rel) => chunkIdSet.has(rel.from) && chunkIdSet.has(rel.to)
3168
- );
3169
- const validImportsRelations = [...importsRelationMap.values()].filter(
3170
- (rel) => chunkIdSet.has(rel.from) && indexedFileIds.has(rel.to)
3171
- );
3172
- const sqlDefinitionsChanged =
3173
- incrementalMode &&
3174
- fileRecords.some(
3175
- (fileRecord) =>
3176
- changedFileIds.has(fileRecord.id) && path.extname(fileRecord.path).toLowerCase() === ".sql"
3177
- );
3178
- const sqlResourceReferenceMap = buildSqlResourceReferenceMap(fileRecords);
3179
- for (const fileRecord of fileRecords) {
3180
- if (!shouldExtractSqlReferences(fileRecord.path)) {
3181
- continue;
3182
- }
3183
-
3184
- const shouldAnalyzeFile =
3185
- !incrementalMode ||
3186
- sqlDefinitionsChanged ||
3187
- changedFileIds.has(fileRecord.id) ||
3188
- !cachedSqlReferenceFileIds.has(fileRecord.id);
3189
- if (!shouldAnalyzeFile) {
3190
- continue;
3191
- }
3192
-
3193
- for (const [key, relation] of callsSqlRelationMap.entries()) {
3194
- if (relation.from === fileRecord.id) {
3195
- callsSqlRelationMap.delete(key);
3196
- }
3197
- }
3198
-
3199
- for (const refName of extractSqlObjectReferencesFromContent(
3200
- fileRecord.content,
3201
- fileRecord.path,
3202
- sqlResourceReferenceMap
3203
- )) {
3204
- for (const alias of sqlChunkAliases(refName)) {
3205
- const targetChunkIds = sqlChunkIdsByAlias.get(alias) || [];
3206
- for (const targetChunkId of targetChunkIds) {
3207
- callsSqlRelationMap.set(relationKey(fileRecord.id, targetChunkId, refName), {
3208
- from: fileRecord.id,
3209
- to: targetChunkId,
3210
- note: refName
3211
- });
3212
- }
3213
- }
3214
- }
3215
- }
3216
- const validCallsSqlRelations = [...callsSqlRelationMap.values()].filter(
3217
- (rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
3218
- );
3219
- for (const fileRecord of fileRecords) {
3220
- if (!shouldExtractNamedResourceReferences(fileRecord.path)) {
3221
- continue;
3222
- }
3223
-
3224
- for (const key of extractSqlResourceKeyReferences(fileRecord.content)) {
3225
- for (const targetChunkId of resourceChunkIdsByAlias.get(key) ?? []) {
3226
- usesResourceKeyRelationMap.set(relationKey(fileRecord.id, targetChunkId, key), {
3227
- from: fileRecord.id,
3228
- to: targetChunkId,
3229
- note: key
3230
- });
3231
- }
3232
- for (const targetChunkId of settingChunkIdsByAlias.get(key) ?? []) {
3233
- usesSettingKeyRelationMap.set(relationKey(fileRecord.id, targetChunkId, key), {
3234
- from: fileRecord.id,
3235
- to: targetChunkId,
3236
- note: key
3237
- });
3238
- }
3239
- }
3240
-
3241
- for (const key of extractConfigKeyReferences(fileRecord.content)) {
3242
- for (const targetChunkId of configChunkIdsByAlias.get(key) ?? []) {
3243
- usesConfigKeyRelationMap.set(relationKey(fileRecord.id, targetChunkId, key), {
3244
- from: fileRecord.id,
3245
- to: targetChunkId,
3246
- note: key
3247
- });
3248
- }
3249
- }
3250
- }
3251
- for (const relation of generateConfigTransformKeyRelations(fileRecords, chunkRecords)) {
3252
- usesConfigKeyRelationMap.set(relationKey(relation.from, relation.to, relation.note), relation);
3253
- }
3254
- const validUsesConfigKeyRelations = [...usesConfigKeyRelationMap.values()].filter(
3255
- (rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
3256
- );
3257
- const validUsesResourceKeyRelations = [...usesResourceKeyRelationMap.values()].filter(
3258
- (rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
3259
- );
3260
- const validUsesSettingKeyRelations = [...usesSettingKeyRelationMap.values()].filter(
3261
- (rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
3262
- );
3263
- memoryTrace.checkpoint("materialize:chunks_relations", {
3264
- chunks: chunkRecords.length,
3265
- chunk_ids: chunkIdSet.size,
3266
- relations_defines: validDefinesRelations.length,
3267
- relations_calls: validCallsRelations.length,
3268
- relations_imports: validImportsRelations.length,
3269
- relations_calls_sql: validCallsSqlRelations.length,
3270
- relations_uses_config_key: validUsesConfigKeyRelations.length,
3271
- relations_uses_resource_key: validUsesResourceKeyRelations.length,
3272
- relations_uses_setting_key: validUsesSettingKeyRelations.length
3273
- });
3274
-
3275
- if (verbose && chunkRecords.length > 0) {
3276
- console.log(`[ingest] extracted ${chunkRecords.length} chunks from ${fileRecords.filter(f => f.kind === "CODE").length} code files`);
3277
- if (windowedChunkCount > 0) {
3278
- console.log(
3279
- `[ingest] overlap windows added=${windowedChunkCount} (window_lines=${chunkWindowLines}, overlap_lines=${chunkOverlapLines}, max_windows=${chunkMaxWindows})`
3280
- );
3281
- }
3282
- console.log(`[ingest] ${validCallsRelations.length} call relations (${totalCallsRelations - validCallsRelations.length} filtered)`);
3283
- if (validCallsSqlRelations.length > 0) {
3284
- console.log(`[ingest] sql call links=${validCallsSqlRelations.length}`);
3285
- }
3286
- if (validUsesConfigKeyRelations.length > 0) {
3287
- console.log(`[ingest] uses_config_key=${validUsesConfigKeyRelations.length}`);
3288
- }
3289
- if (validUsesResourceKeyRelations.length > 0 || validUsesSettingKeyRelations.length > 0) {
3290
- console.log(
3291
- `[ingest] uses_resource_key=${validUsesResourceKeyRelations.length} uses_setting_key=${validUsesSettingKeyRelations.length}`
3292
- );
3293
- }
3294
- }
3295
-
3296
- const csharpChunkCount = chunkRecords.filter((record) => record.language === "csharp").length;
3297
- const parserHealth = {};
3298
- if (csharpFileCount > 0) {
3299
- parserHealth.csharp = {
3300
- files: csharpFileCount,
3301
- available: Boolean(csharpRuntime?.available),
3302
- reason: csharpRuntime?.available ? null : (csharpRuntime?.reason ?? "C# parser unavailable"),
3303
- chunks: csharpChunkCount,
3304
- };
3305
-
3306
- if (!csharpRuntime?.available) {
3307
- console.log(`[ingest] warning csharp parser unavailable: ${parserHealth.csharp.reason}`);
3308
- } else if (csharpChunkCount === 0) {
3309
- console.log("[ingest] warning csharp parser produced 0 chunks across C# files");
3310
- }
3311
- }
3312
-
3313
- // Generate Module entities and relations
3314
- const moduleResult = generateModules(fileRecords, chunkRecords);
3315
- const moduleRecords = moduleResult.modules;
3316
- const moduleContainsRelations = moduleResult.containsRelations;
3317
- const moduleContainsModuleRelations = moduleResult.containsModuleRelations;
3318
- const moduleExportsRelations = moduleResult.exportsRelations;
3319
- const projectResult = generateProjects(fileRecords);
3320
- const projectRecords = projectResult.projects;
3321
- const projectIncludesFileRelations = projectResult.includesFileRelations;
3322
- const projectReferencesProjectRelations = projectResult.referencesProjectRelations;
3323
- const namedResourceRelationResult = generateNamedResourceRelations(fileRecords);
3324
- const usesResourceRelations = namedResourceRelationResult.usesResourceRelations;
3325
- const usesSettingRelations = namedResourceRelationResult.usesSettingRelations;
3326
- const configIncludeRelations = generateConfigIncludeRelations(fileRecords);
3327
- const machineConfigRelations = generateMachineConfigRelations(fileRecords);
3328
- const sectionHandlerRelations = generateSectionHandlerRelations(fileRecords);
3329
- const usesConfigRelations = uniqueRelations([
3330
- ...namedResourceRelationResult.usesConfigRelations,
3331
- ...configIncludeRelations,
3332
- ...machineConfigRelations,
3333
- ...sectionHandlerRelations
3334
- ]);
3335
- const configTransformRelations = generateConfigTransformRelations(fileRecords);
3336
- memoryTrace.checkpoint("materialize:modules_projects_relations", {
3337
- modules: moduleRecords.length,
3338
- projects: projectRecords.length,
3339
- relations_contains: moduleContainsRelations.length,
3340
- relations_contains_module: moduleContainsModuleRelations.length,
3341
- relations_exports: moduleExportsRelations.length,
3342
- relations_includes_file: projectIncludesFileRelations.length,
3343
- relations_references_project: projectReferencesProjectRelations.length,
3344
- relations_uses_resource: usesResourceRelations.length,
3345
- relations_uses_setting: usesSettingRelations.length,
3346
- relations_uses_config: usesConfigRelations.length,
3347
- relations_transforms_config: configTransformRelations.length
3348
- });
3349
-
3350
- if (verbose && moduleRecords.length > 0) {
3351
- console.log(`[ingest] modules=${moduleRecords.length} contains=${moduleContainsRelations.length} contains_module=${moduleContainsModuleRelations.length} exports=${moduleExportsRelations.length}`);
3352
- }
3353
- if (verbose && projectRecords.length > 0) {
3354
- console.log(
3355
- `[ingest] projects=${projectRecords.length} includes_file=${projectIncludesFileRelations.length} references_project=${projectReferencesProjectRelations.length}`
3356
- );
3357
- }
3358
- if (
3359
- verbose &&
3360
- (
3361
- usesResourceRelations.length > 0 ||
3362
- usesSettingRelations.length > 0 ||
3363
- usesConfigRelations.length > 0 ||
3364
- configTransformRelations.length > 0
3365
- )
3366
- ) {
3367
- console.log(
3368
- `[ingest] uses_resource=${usesResourceRelations.length} uses_setting=${usesSettingRelations.length} uses_config=${usesConfigRelations.length} transforms_config=${configTransformRelations.length}`
3369
- );
3370
- }
3371
-
3372
- const ruleRecords = rules.map((rule) => ({
3373
- id: rule.id,
3374
- title: rule.id,
3375
- body: rule.description,
3376
- scope: "global",
3377
- updated_at: new Date().toISOString(),
3378
- source_of_truth: true,
3379
- trust_level: 95,
3380
- status: rule.enforce ? "active" : "draft",
3381
- priority: rule.priority
3382
- }));
3383
-
3384
- const adrTokenIndex = new Map();
3385
- for (const adrRecord of adrRecords) {
3386
- for (const token of adrTokens(adrRecord)) {
3387
- if (!adrTokenIndex.has(token)) {
3388
- adrTokenIndex.set(token, adrRecord.id);
3389
- }
3390
- }
3391
- }
3392
-
3393
- const supersedesRelations = [];
3394
- for (const adrRecord of adrRecords) {
3395
- const refs = findSupersedesReferences(adrRecord.body);
3396
- for (const ref of refs) {
3397
- const target = adrTokenIndex.get(normalizeToken(ref));
3398
- if (!target || target === adrRecord.id) {
3399
- continue;
3400
- }
3401
- adrRecord.supersedes_id = target;
3402
- supersedesRelations.push({
3403
- from: adrRecord.id,
3404
- to: target,
3405
- reason: `Supersedes ${ref}`
3406
- });
3407
- }
3408
- }
3409
-
3410
- const constrainsRelations = [];
3411
- const implementsRelations = [];
3412
- const stagedDocumentCache = stageJsonl(path.join(CACHE_DIR, "documents.jsonl"), fileRecords);
3413
- const stagedFileEntityCache = stageJsonl(path.join(CACHE_DIR, "entities.file.jsonl"), fileRecords);
3414
- memoryTrace.checkpoint("writes:file_cache_staged", {
3415
- files: fileRecords.length,
3416
- file_content_records: countFileContentRecords(fileRecords)
3417
- });
3418
- memoryTrace.checkpoint("tokens:rule_matching_start", {
3419
- files: fileRecords.length,
3420
- rules: ruleRecords.length
3421
- });
3422
-
3423
- const ruleMatchers = ruleRecords.map((ruleRecord) => ({
3424
- ruleRecord,
3425
- needle: ruleRecord.id.toLowerCase(),
3426
- keywords: normalizeRuleTokens(ruleRecord)
3427
- }));
3428
- const constrainsByKey = new Map();
3429
- const implementsByKey = new Map();
3430
- let fileTokenSetsProcessed = 0;
3431
- let fileContentRecordsReleased = 0;
3432
-
3433
- for (const fileRecord of fileRecords) {
3434
- const lower = String(fileRecord.content ?? "").toLowerCase();
3435
- const tokens = fileTokenSet(fileRecord);
3436
- fileTokenSetsProcessed += 1;
3437
-
3438
- for (const { ruleRecord, needle, keywords } of ruleMatchers) {
3439
- const explicitMention = lower.includes(needle);
3440
- const minimumMatches = fileRecord.kind === "CODE" ? 1 : 2;
3441
- const keywordResult = explicitMention
3442
- ? { matched: false, sample: [] }
3443
- : collectRuleKeywordMatch(keywords, tokens, minimumMatches);
3444
-
3445
- if (!explicitMention && !keywordResult.matched) {
3446
- continue;
3447
- }
3448
-
3449
- const constrainsKey = `${ruleRecord.id}|${fileRecord.id}`;
3450
- if (!constrainsByKey.has(constrainsKey)) {
3451
- constrainsByKey.set(constrainsKey, {
3452
- from: ruleRecord.id,
3453
- to: fileRecord.id,
3454
- note: explicitMention
3455
- ? `Mentions ${ruleRecord.id}`
3456
- : `Keyword match ${keywordResult.sample.join(", ")}`
3457
- });
3458
- }
3459
-
3460
- if (fileRecord.kind === "CODE") {
3461
- const implementsKey = `${fileRecord.id}|${ruleRecord.id}`;
3462
- if (!implementsByKey.has(implementsKey)) {
3463
- implementsByKey.set(implementsKey, {
3464
- from: fileRecord.id,
3465
- to: ruleRecord.id,
3466
- note: explicitMention
3467
- ? `Code references ${ruleRecord.id}`
3468
- : `Code keywords ${keywordResult.sample.join(", ")}`
3469
- });
3470
- }
3471
- }
3472
- }
3473
-
3474
- if (Object.prototype.hasOwnProperty.call(fileRecord, "content")) {
3475
- delete fileRecord.content;
3476
- fileContentRecordsReleased += 1;
3477
- }
3478
- }
3479
-
3480
- for (const { ruleRecord } of ruleMatchers) {
3481
- for (const fileRecord of fileRecords) {
3482
- const constrainsKey = `${ruleRecord.id}|${fileRecord.id}`;
3483
- const constrainsRelation = constrainsByKey.get(constrainsKey);
3484
- if (constrainsRelation) {
3485
- constrainsRelations.push(constrainsRelation);
3486
- constrainsByKey.delete(constrainsKey);
3487
- }
3488
- if (fileRecord.kind === "CODE") {
3489
- const implementsKey = `${fileRecord.id}|${ruleRecord.id}`;
3490
- const implementsRelation = implementsByKey.get(implementsKey);
3491
- if (implementsRelation) {
3492
- implementsRelations.push(implementsRelation);
3493
- implementsByKey.delete(implementsKey);
3494
- }
3495
- }
3496
- }
3497
- }
3498
-
3499
- memoryTrace.checkpoint("tokens:rule_matching_complete", {
3500
- file_token_sets: fileTokenSetsProcessed,
3501
- file_token_sets_retained: 0,
3502
- file_content_records_released: fileContentRecordsReleased,
3503
- file_content_records_retained: countFileContentRecords(fileRecords),
3504
- rules: ruleRecords.length,
3505
- relations_constrains: constrainsRelations.length,
3506
- relations_implements: implementsRelations.length,
3507
- relations_supersedes: supersedesRelations.length
3508
- });
3509
-
3510
- memoryTrace.checkpoint("writes:cache_start", {
3511
- files: fileRecords.length,
3512
- adrs: adrRecords.length,
3513
- rules: ruleRecords.length,
3514
- chunks: chunkRecords.length
3515
- });
3516
- stagedDocumentCache.commit();
3517
- stagedFileEntityCache.commit();
3518
- writeJsonl(path.join(CACHE_DIR, "entities.adr.jsonl"), adrRecords);
3519
- writeJsonl(path.join(CACHE_DIR, "entities.rule.jsonl"), ruleRecords);
3520
- writeJsonl(path.join(CACHE_DIR, "entities.chunk.jsonl"), chunkRecords);
3521
- writeJsonl(path.join(CACHE_DIR, "relations.supersedes.jsonl"), supersedesRelations);
3522
- writeJsonl(path.join(CACHE_DIR, "relations.constrains.jsonl"), constrainsRelations);
3523
- writeJsonl(path.join(CACHE_DIR, "relations.implements.jsonl"), implementsRelations);
3524
- writeJsonl(path.join(CACHE_DIR, "relations.defines.jsonl"), validDefinesRelations);
3525
- writeJsonl(path.join(CACHE_DIR, "relations.calls.jsonl"), validCallsRelations);
3526
- writeJsonl(path.join(CACHE_DIR, "relations.imports.jsonl"), validImportsRelations);
3527
- writeJsonl(path.join(CACHE_DIR, "relations.calls_sql.jsonl"), validCallsSqlRelations);
3528
- writeJsonl(path.join(CACHE_DIR, "relations.uses_config_key.jsonl"), validUsesConfigKeyRelations);
3529
- writeJsonl(path.join(CACHE_DIR, "relations.uses_resource_key.jsonl"), validUsesResourceKeyRelations);
3530
- writeJsonl(path.join(CACHE_DIR, "relations.uses_setting_key.jsonl"), validUsesSettingKeyRelations);
3531
- writeJsonl(path.join(CACHE_DIR, "entities.module.jsonl"), moduleRecords);
3532
- writeJsonl(path.join(CACHE_DIR, "relations.contains.jsonl"), moduleContainsRelations);
3533
- writeJsonl(path.join(CACHE_DIR, "relations.contains_module.jsonl"), moduleContainsModuleRelations);
3534
- writeJsonl(path.join(CACHE_DIR, "relations.exports.jsonl"), moduleExportsRelations);
3535
- writeJsonl(path.join(CACHE_DIR, "entities.project.jsonl"), projectRecords);
3536
- writeJsonl(path.join(CACHE_DIR, "relations.includes_file.jsonl"), projectIncludesFileRelations);
3537
- writeJsonl(path.join(CACHE_DIR, "relations.uses_resource.jsonl"), usesResourceRelations);
3538
- writeJsonl(path.join(CACHE_DIR, "relations.uses_setting.jsonl"), usesSettingRelations);
3539
- writeJsonl(path.join(CACHE_DIR, "relations.uses_config.jsonl"), usesConfigRelations);
3540
- writeJsonl(path.join(CACHE_DIR, "relations.transforms_config.jsonl"), configTransformRelations);
3541
- writeJsonl(
3542
- path.join(CACHE_DIR, "relations.references_project.jsonl"),
3543
- projectReferencesProjectRelations
3544
- );
3545
- memoryTrace.checkpoint("writes:cache_complete", {
3546
- jsonl_files: 26,
3547
- files: fileRecords.length,
3548
- chunks: chunkRecords.length
3549
- });
3550
-
3551
- memoryTrace.checkpoint("writes:db_start", {
3552
- tsv_files: 21
3553
- });
3554
- writeTsv(
3555
- path.join(DB_IMPORT_DIR, "file_nodes.tsv"),
3556
- [
3557
- "id",
3558
- "path",
3559
- "kind",
3560
- "excerpt",
3561
- "checksum",
3562
- "updated_at",
3563
- "source_of_truth",
3564
- "trust_level",
3565
- "status"
3566
- ],
3567
- mapRows(fileRecords, (record) => [
3568
- record.id,
3569
- record.path,
3570
- record.kind,
3571
- record.excerpt,
3572
- record.checksum,
3573
- record.updated_at,
3574
- record.source_of_truth,
3575
- record.trust_level,
3576
- record.status
3577
- ])
3578
- );
3579
-
3580
- writeTsv(
3581
- path.join(DB_IMPORT_DIR, "rule_nodes.tsv"),
3582
- [
3583
- "id",
3584
- "title",
3585
- "body",
3586
- "scope",
3587
- "priority",
3588
- "updated_at",
3589
- "source_of_truth",
3590
- "trust_level",
3591
- "status"
3592
- ],
3593
- mapRows(ruleRecords, (record) => [
3594
- record.id,
3595
- record.title,
3596
- record.body,
3597
- record.scope,
3598
- record.priority,
3599
- record.updated_at,
3600
- record.source_of_truth,
3601
- record.trust_level,
3602
- record.status
3603
- ])
3604
- );
3605
-
3606
- writeTsv(
3607
- path.join(DB_IMPORT_DIR, "adr_nodes.tsv"),
3608
- [
3609
- "id",
3610
- "path",
3611
- "title",
3612
- "body",
3613
- "decision_date",
3614
- "supersedes_id",
3615
- "source_of_truth",
3616
- "trust_level",
3617
- "status"
3618
- ],
3619
- mapRows(adrRecords, (record) => [
3620
- record.id,
3621
- record.path,
3622
- record.title,
3623
- record.body,
3624
- record.decision_date,
3625
- record.supersedes_id,
3626
- record.source_of_truth,
3627
- record.trust_level,
3628
- record.status
3629
- ])
3630
- );
3631
-
3632
- writeTsv(
3633
- path.join(DB_IMPORT_DIR, "constrains_rel.tsv"),
3634
- ["from", "to", "note"],
3635
- mapRows(constrainsRelations, (record) => [record.from, record.to, record.note])
3636
- );
3637
-
3638
- writeTsv(
3639
- path.join(DB_IMPORT_DIR, "implements_rel.tsv"),
3640
- ["from", "to", "note"],
3641
- mapRows(implementsRelations, (record) => [record.from, record.to, record.note])
3642
- );
3643
-
3644
- writeTsv(
3645
- path.join(DB_IMPORT_DIR, "supersedes_rel.tsv"),
3646
- ["from", "to", "reason"],
3647
- mapRows(supersedesRelations, (record) => [record.from, record.to, record.reason])
3648
- );
3649
-
3650
- writeTsv(
3651
- path.join(DB_IMPORT_DIR, "chunk_nodes.tsv"),
3652
- [
3653
- "id",
3654
- "file_id",
3655
- "name",
3656
- "kind",
3657
- "signature",
3658
- "body",
3659
- "start_line",
3660
- "end_line",
3661
- "language",
3662
- "checksum",
3663
- "updated_at",
3664
- "trust_level"
3665
- ],
3666
- mapRows(chunkRecords, (record) => [
3667
- record.id,
3668
- record.file_id,
3669
- record.name,
3670
- record.kind,
3671
- record.signature,
3672
- record.body,
3673
- record.start_line,
3674
- record.end_line,
3675
- record.language,
3676
- record.checksum,
3677
- record.updated_at,
3678
- record.trust_level
3679
- ])
3680
- );
3681
-
3682
- writeTsv(
3683
- path.join(DB_IMPORT_DIR, "defines_rel.tsv"),
3684
- ["from", "to"],
3685
- mapRows(validDefinesRelations, (record) => [record.from, record.to])
3686
- );
3687
-
3688
- writeTsv(
3689
- path.join(DB_IMPORT_DIR, "calls_rel.tsv"),
3690
- ["from", "to", "call_type"],
3691
- mapRows(validCallsRelations, (record) => [record.from, record.to, record.call_type])
3692
- );
3693
-
3694
- writeTsv(
3695
- path.join(DB_IMPORT_DIR, "imports_rel.tsv"),
3696
- ["from", "to", "import_name"],
3697
- mapRows(validImportsRelations, (record) => [record.from, record.to, record.import_name])
3698
- );
3699
-
3700
- writeTsv(
3701
- path.join(DB_IMPORT_DIR, "calls_sql_rel.tsv"),
3702
- ["from", "to", "note"],
3703
- mapRows(validCallsSqlRelations, (record) => [record.from, record.to, record.note])
3704
- );
3705
-
3706
- writeTsv(
3707
- path.join(DB_IMPORT_DIR, "uses_config_key_rel.tsv"),
3708
- ["from", "to", "note"],
3709
- mapRows(validUsesConfigKeyRelations, (record) => [record.from, record.to, record.note])
3710
- );
3711
-
3712
- writeTsv(
3713
- path.join(DB_IMPORT_DIR, "uses_resource_key_rel.tsv"),
3714
- ["from", "to", "note"],
3715
- mapRows(validUsesResourceKeyRelations, (record) => [record.from, record.to, record.note])
3716
- );
3717
-
3718
- writeTsv(
3719
- path.join(DB_IMPORT_DIR, "uses_setting_key_rel.tsv"),
3720
- ["from", "to", "note"],
3721
- mapRows(validUsesSettingKeyRelations, (record) => [record.from, record.to, record.note])
3722
- );
3723
-
3724
- writeTsv(
3725
- path.join(DB_IMPORT_DIR, "project_nodes.tsv"),
3726
- [
3727
- "id",
3728
- "path",
3729
- "name",
3730
- "kind",
3731
- "language",
3732
- "target_framework",
3733
- "summary",
3734
- "file_count",
3735
- "updated_at",
3736
- "source_of_truth",
3737
- "trust_level",
3738
- "status"
3739
- ],
3740
- mapRows(projectRecords, (record) => [
3741
- record.id,
3742
- record.path,
3743
- record.name,
3744
- record.kind,
3745
- record.language,
3746
- record.target_framework,
3747
- record.summary,
3748
- record.file_count,
3749
- record.updated_at,
3750
- record.source_of_truth,
3751
- record.trust_level,
3752
- record.status
3753
- ])
3754
- );
3755
-
3756
- writeTsv(
3757
- path.join(DB_IMPORT_DIR, "includes_file_rel.tsv"),
3758
- ["from", "to"],
3759
- mapRows(projectIncludesFileRelations, (record) => [record.from, record.to])
3760
- );
3761
-
3762
- writeTsv(
3763
- path.join(DB_IMPORT_DIR, "references_project_rel.tsv"),
3764
- ["from", "to", "note"],
3765
- mapRows(projectReferencesProjectRelations, (record) => [record.from, record.to, record.note])
3766
- );
3767
-
3768
- writeTsv(
3769
- path.join(DB_IMPORT_DIR, "uses_resource_rel.tsv"),
3770
- ["from", "to", "note"],
3771
- mapRows(usesResourceRelations, (record) => [record.from, record.to, record.note])
3772
- );
3773
-
3774
- writeTsv(
3775
- path.join(DB_IMPORT_DIR, "uses_setting_rel.tsv"),
3776
- ["from", "to", "note"],
3777
- mapRows(usesSettingRelations, (record) => [record.from, record.to, record.note])
3778
- );
3779
-
3780
- writeTsv(
3781
- path.join(DB_IMPORT_DIR, "uses_config_rel.tsv"),
3782
- ["from", "to", "note"],
3783
- mapRows(usesConfigRelations, (record) => [record.from, record.to, record.note])
3784
- );
3785
-
3786
- writeTsv(
3787
- path.join(DB_IMPORT_DIR, "transforms_config_rel.tsv"),
3788
- ["from", "to", "note"],
3789
- mapRows(configTransformRelations, (record) => [record.from, record.to, record.note])
3790
- );
3791
- memoryTrace.checkpoint("writes:db_complete", {
3792
- tsv_files: 21
3793
- });
3794
-
3795
- const manifest = {
3796
- generated_at: new Date().toISOString(),
3797
- mode,
3798
- source_paths: sourcePaths,
3799
- counts: {
3800
- files: fileRecords.length,
3801
- adrs: adrRecords.length,
3802
- rules: ruleRecords.length,
3803
- chunks: chunkRecords.length,
3804
- relations_constrains: constrainsRelations.length,
3805
- relations_implements: implementsRelations.length,
3806
- relations_supersedes: supersedesRelations.length,
3807
- relations_defines: validDefinesRelations.length,
3808
- relations_calls: validCallsRelations.length,
3809
- relations_imports: validImportsRelations.length,
3810
- relations_calls_sql: validCallsSqlRelations.length,
3811
- relations_uses_config_key: validUsesConfigKeyRelations.length,
3812
- relations_uses_resource_key: validUsesResourceKeyRelations.length,
3813
- relations_uses_setting_key: validUsesSettingKeyRelations.length,
3814
- modules: moduleRecords.length,
3815
- relations_contains: moduleContainsRelations.length,
3816
- relations_contains_module: moduleContainsModuleRelations.length,
3817
- relations_exports: moduleExportsRelations.length,
3818
- projects: projectRecords.length,
3819
- relations_includes_file: projectIncludesFileRelations.length,
3820
- relations_references_project: projectReferencesProjectRelations.length,
3821
- relations_uses_resource: usesResourceRelations.length,
3822
- relations_uses_setting: usesSettingRelations.length,
3823
- relations_uses_config: usesConfigRelations.length,
3824
- relations_transforms_config: configTransformRelations.length
3825
- },
3826
- skipped,
3827
- parser_health: parserHealth,
3828
- incremental_mode: incrementalMode,
3829
- changed_candidates: candidates.size,
3830
- deleted_paths: deletedRelPaths.length
3831
- };
3832
-
3833
- fs.writeFileSync(path.join(CACHE_DIR, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
3834
- memoryTrace.checkpoint("writes:manifest_complete", {
3835
- files: manifest.counts.files,
3836
- chunks: manifest.counts.chunks,
3837
- total_relations:
3838
- manifest.counts.relations_constrains +
3839
- manifest.counts.relations_implements +
3840
- manifest.counts.relations_supersedes +
3841
- manifest.counts.relations_defines +
3842
- manifest.counts.relations_calls +
3843
- manifest.counts.relations_imports +
3844
- manifest.counts.relations_calls_sql +
3845
- manifest.counts.relations_uses_config_key +
3846
- manifest.counts.relations_uses_resource_key +
3847
- manifest.counts.relations_uses_setting_key +
3848
- manifest.counts.relations_contains +
3849
- manifest.counts.relations_contains_module +
3850
- manifest.counts.relations_exports +
3851
- manifest.counts.relations_includes_file +
3852
- manifest.counts.relations_references_project +
3853
- manifest.counts.relations_uses_resource +
3854
- manifest.counts.relations_uses_setting +
3855
- manifest.counts.relations_uses_config +
3856
- manifest.counts.relations_transforms_config
3857
- });
3858
-
3859
- console.log(`[ingest] mode=${mode}`);
3860
- if (incrementalMode) {
3861
- console.log(
3862
- `[ingest] incremental changed_candidates=${manifest.changed_candidates} deleted_paths=${manifest.deleted_paths}`
3863
- );
3864
- } else if (mode === "changed") {
3865
- console.log("[ingest] incremental diff unavailable; processed full source set");
3866
- }
3867
- console.log(`[ingest] files=${manifest.counts.files} adrs=${manifest.counts.adrs} rules=${manifest.counts.rules} chunks=${manifest.counts.chunks}`);
3868
- console.log(
3869
- `[ingest] rels constrains=${manifest.counts.relations_constrains} implements=${manifest.counts.relations_implements} supersedes=${manifest.counts.relations_supersedes}`
3870
- );
3871
- console.log(
3872
- `[ingest] rels defines=${manifest.counts.relations_defines} calls=${manifest.counts.relations_calls} imports=${manifest.counts.relations_imports} calls_sql=${manifest.counts.relations_calls_sql} uses_config_key=${manifest.counts.relations_uses_config_key} uses_resource_key=${manifest.counts.relations_uses_resource_key} uses_setting_key=${manifest.counts.relations_uses_setting_key}`
3873
- );
3874
- console.log(
3875
- `[ingest] rels contains=${manifest.counts.relations_contains} contains_module=${manifest.counts.relations_contains_module} exports=${manifest.counts.relations_exports} includes_file=${manifest.counts.relations_includes_file} references_project=${manifest.counts.relations_references_project} uses_resource=${manifest.counts.relations_uses_resource} uses_setting=${manifest.counts.relations_uses_setting} uses_config=${manifest.counts.relations_uses_config} transforms_config=${manifest.counts.relations_transforms_config}`
3876
- );
3877
- console.log(
3878
- `[ingest] skipped unsupported=${skipped.unsupported} too_large=${skipped.tooLarge} binary=${skipped.binary}`
3879
- );
3880
- console.log(`[ingest] wrote cache + db import files under .context/`);
3881
- }
3882
-
3883
- const isMainModule = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(__filename);
6
+ const isMainModule =
7
+ process.argv[1] &&
8
+ path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url));
3884
9
  if (isMainModule) {
3885
10
  main().catch((error) => {
3886
11
  console.error(error instanceof Error ? error.message : String(error));
@@ -3888,23 +13,4 @@ if (isMainModule) {
3888
13
  });
3889
14
  }
3890
15
 
3891
- export {
3892
- buildChunkAliasIndexes,
3893
- buildSqlResourceReferenceMap,
3894
- detectKind,
3895
- extractSqlObjectReferencesFromContent,
3896
- generateChunkDescription,
3897
- generateConfigIncludeRelations,
3898
- generateConfigTransformKeyRelations,
3899
- generateMachineConfigRelations,
3900
- generateConfigTransformRelations,
3901
- generateModuleSummary,
3902
- generateModules,
3903
- generateNamedResourceRelations,
3904
- generateProjects,
3905
- generateSectionHandlerRelations,
3906
- getChunkParserForExtension,
3907
- parseFilesInWorkers,
3908
- resolveIngestWorkerCount,
3909
- resolveRelativeImportTargetId
3910
- };
16
+ export * from "./lib/ingest/main.mjs";