@danielblomma/cortex-mcp 2.4.0 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +226 -0
- package/README.md +24 -3
- package/bin/cli/arguments.mjs +60 -0
- package/bin/cli/context-passthrough.mjs +129 -0
- package/bin/cli/daemon.mjs +157 -0
- package/bin/cli/enterprise.mjs +516 -0
- package/bin/cli/help.mjs +88 -0
- package/bin/cli/hooks.mjs +199 -0
- package/bin/cli/mcp-command.mjs +87 -0
- package/bin/cli/paths.mjs +14 -0
- package/bin/cli/process.mjs +49 -0
- package/bin/cli/project-commands.mjs +237 -0
- package/bin/cli/project-runtime.mjs +34 -0
- package/bin/cli/query-command.mjs +18 -0
- package/bin/cli/router.mjs +66 -0
- package/bin/cli/run-command.mjs +44 -0
- package/bin/cli/scaffold-ownership.mjs +936 -0
- package/bin/cli/scaffold.mjs +687 -0
- package/bin/cli/stage-command.mjs +9 -0
- package/bin/cli/telemetry-command.mjs +18 -0
- package/bin/cli/trusted-runtime.mjs +18 -0
- package/bin/cortex.mjs +12 -1607
- package/bin/daemon-control.mjs +162 -0
- package/mcp-registry-submission.json +1 -1
- package/package.json +8 -3
- package/scaffold/mcp/dist/.cortex-build-hash +1 -0
- package/scaffold/mcp/dist/cli/enterprise-setup.js +134 -0
- package/scaffold/mcp/dist/cli/govern.js +937 -0
- package/scaffold/mcp/dist/cli/query.js +409 -0
- package/scaffold/mcp/dist/cli/run.js +295 -0
- package/scaffold/mcp/dist/cli/stage.js +308 -0
- package/scaffold/mcp/dist/cli/telemetry-test.js +141 -0
- package/scaffold/mcp/dist/cli/ungoverned-detector.js +133 -0
- package/scaffold/mcp/dist/contextEntities.js +282 -0
- package/scaffold/mcp/dist/core/audit/query.js +72 -0
- package/scaffold/mcp/dist/core/audit/writer.js +28 -0
- package/scaffold/mcp/dist/core/config.js +235 -0
- package/scaffold/mcp/dist/core/enterprise-host-identity.js +193 -0
- package/scaffold/mcp/dist/core/enterprise-identity.js +18 -0
- package/scaffold/mcp/dist/core/enterprise-rotation.js +158 -0
- package/scaffold/mcp/dist/core/govern-paths.js +21 -0
- package/scaffold/mcp/dist/core/index.js +14 -0
- package/scaffold/mcp/dist/core/license.js +278 -0
- package/scaffold/mcp/dist/core/policy/enforce.js +66 -0
- package/scaffold/mcp/dist/core/policy/injection.js +172 -0
- package/scaffold/mcp/dist/core/policy/store.js +179 -0
- package/scaffold/mcp/dist/core/rbac/check.js +30 -0
- package/scaffold/mcp/dist/core/secure-endpoint.js +25 -0
- package/scaffold/mcp/dist/core/telemetry/collector.js +285 -0
- package/scaffold/mcp/dist/core/telemetry/state-dir.js +31 -0
- package/scaffold/mcp/dist/core/validators/builtins.js +632 -0
- package/scaffold/mcp/dist/core/validators/config.js +44 -0
- package/scaffold/mcp/dist/core/validators/engine.js +120 -0
- package/scaffold/mcp/dist/core/validators/evaluators/code_comments.js +236 -0
- package/scaffold/mcp/dist/core/validators/evaluators/regex.js +116 -0
- package/scaffold/mcp/dist/core/workflow/artifact-io.js +103 -0
- package/scaffold/mcp/dist/core/workflow/capabilities.js +88 -0
- package/scaffold/mcp/dist/core/workflow/default-workflows.js +102 -0
- package/scaffold/mcp/dist/core/workflow/enforcement.js +164 -0
- package/scaffold/mcp/dist/core/workflow/envelope.js +132 -0
- package/scaffold/mcp/dist/core/workflow/index.js +11 -0
- package/scaffold/mcp/dist/core/workflow/mcp-tools.js +175 -0
- package/scaffold/mcp/dist/core/workflow/resolution.js +69 -0
- package/scaffold/mcp/dist/core/workflow/run-lifecycle.js +123 -0
- package/scaffold/mcp/dist/core/workflow/schemas.js +141 -0
- package/scaffold/mcp/dist/core/workflow/synced-capability-registry.js +60 -0
- package/scaffold/mcp/dist/core/workflow/synced-registry.js +60 -0
- package/scaffold/mcp/dist/daemon/capability-sync-checker.js +242 -0
- package/scaffold/mcp/dist/daemon/client.js +130 -0
- package/scaffold/mcp/dist/daemon/egress-proxy.js +288 -0
- package/scaffold/mcp/dist/daemon/global-host-events.js +222 -0
- package/scaffold/mcp/dist/daemon/heartbeat-pusher.js +116 -0
- package/scaffold/mcp/dist/daemon/heartbeat-tracker.js +165 -0
- package/scaffold/mcp/dist/daemon/host-events-pusher.js +249 -0
- package/scaffold/mcp/dist/daemon/main.js +449 -0
- package/scaffold/mcp/dist/daemon/paths.js +36 -0
- package/scaffold/mcp/dist/daemon/project-service-registry.js +78 -0
- package/scaffold/mcp/dist/daemon/protocol.js +8 -0
- package/scaffold/mcp/dist/daemon/server.js +180 -0
- package/scaffold/mcp/dist/daemon/skill-sync-checker.js +557 -0
- package/scaffold/mcp/dist/daemon/sync-checker.js +165 -0
- package/scaffold/mcp/dist/daemon/ungoverned-scanner.js +136 -0
- package/scaffold/mcp/dist/daemon/workflow-sync-checker.js +249 -0
- package/scaffold/mcp/dist/defaults.js +6 -0
- package/scaffold/mcp/dist/embed.js +627 -0
- package/scaffold/mcp/dist/embedScheduler.js +479 -0
- package/scaffold/mcp/dist/embeddings.js +167 -0
- package/scaffold/mcp/dist/enterprise/audit/push.js +66 -0
- package/scaffold/mcp/dist/enterprise/index.js +327 -0
- package/scaffold/mcp/dist/enterprise/model/deploy.js +27 -0
- package/scaffold/mcp/dist/enterprise/policy/sync.js +129 -0
- package/scaffold/mcp/dist/enterprise/privacy/boundary.js +184 -0
- package/scaffold/mcp/dist/enterprise/reviews/changed-files.js +33 -0
- package/scaffold/mcp/dist/enterprise/reviews/pattern-context.js +202 -0
- package/scaffold/mcp/dist/enterprise/reviews/policy-selection.js +46 -0
- package/scaffold/mcp/dist/enterprise/reviews/push.js +102 -0
- package/scaffold/mcp/dist/enterprise/reviews/trust-state.js +186 -0
- package/scaffold/mcp/dist/enterprise/telemetry/sync.js +57 -0
- package/scaffold/mcp/dist/enterprise/tools/enterprise.js +826 -0
- package/scaffold/mcp/dist/enterprise/tools/harness.js +40 -0
- package/scaffold/mcp/dist/enterprise/tools/walk.js +73 -0
- package/scaffold/mcp/dist/enterprise/violations/push.js +77 -0
- package/scaffold/mcp/dist/enterprise/workflow/push.js +44 -0
- package/scaffold/mcp/dist/enterprise/workflow/state.js +329 -0
- package/scaffold/mcp/dist/frontmatter.js +33 -0
- package/scaffold/mcp/dist/graph.js +769 -0
- package/scaffold/mcp/dist/graphCsv.js +55 -0
- package/scaffold/mcp/dist/graphMetrics.js +8 -0
- package/scaffold/mcp/dist/hooks/permission-request.js +89 -0
- package/scaffold/mcp/dist/hooks/post-tool-use.js +105 -0
- package/scaffold/mcp/dist/hooks/pre-compact.js +29 -0
- package/scaffold/mcp/dist/hooks/pre-tool-use.js +78 -0
- package/scaffold/mcp/dist/hooks/session-end.js +43 -0
- package/scaffold/mcp/dist/hooks/session-start.js +41 -0
- package/scaffold/mcp/dist/hooks/shared.js +194 -0
- package/scaffold/mcp/dist/hooks/stop.js +28 -0
- package/scaffold/mcp/dist/hooks/user-prompt-submit.js +33 -0
- package/scaffold/mcp/dist/impactPresentation.js +137 -0
- package/scaffold/mcp/dist/impactRanking.js +191 -0
- package/scaffold/mcp/dist/impactResponse.js +30 -0
- package/scaffold/mcp/dist/impactResults.js +105 -0
- package/scaffold/mcp/dist/impactSeed.js +20 -0
- package/scaffold/mcp/dist/impactTraversal.js +64 -0
- package/scaffold/mcp/dist/jsonl.js +77 -0
- package/scaffold/mcp/dist/loadGraph.js +759 -0
- package/scaffold/mcp/dist/lruCache.js +38 -0
- package/scaffold/mcp/dist/paths.js +97 -0
- package/scaffold/mcp/dist/patternEvidence.js +272 -0
- package/scaffold/mcp/dist/plugin.js +81 -0
- package/scaffold/mcp/dist/presets.js +78 -0
- package/scaffold/mcp/dist/relatedResponse.js +18 -0
- package/scaffold/mcp/dist/relatedTraversal.js +78 -0
- package/scaffold/mcp/dist/rules.js +23 -0
- package/scaffold/mcp/dist/search.js +212 -0
- package/scaffold/mcp/dist/searchCore.js +457 -0
- package/scaffold/mcp/dist/searchResults.js +230 -0
- package/scaffold/mcp/dist/server.js +317 -0
- package/scaffold/mcp/dist/types.js +1 -0
- package/scaffold/mcp/package-lock.json +321 -200
- package/scaffold/mcp/package.json +15 -6
- package/scaffold/mcp/src/cli/enterprise-setup.ts +82 -8
- package/scaffold/mcp/src/cli/govern.ts +137 -52
- package/scaffold/mcp/src/cli/run.ts +53 -19
- package/scaffold/mcp/src/cli/stage.ts +8 -3
- package/scaffold/mcp/src/core/config.ts +9 -2
- package/scaffold/mcp/src/core/enterprise-host-identity.ts +259 -0
- package/scaffold/mcp/src/core/enterprise-identity.ts +20 -0
- package/scaffold/mcp/src/core/enterprise-rotation.ts +185 -0
- package/scaffold/mcp/src/core/govern-paths.ts +30 -0
- package/scaffold/mcp/src/core/license.ts +186 -17
- package/scaffold/mcp/src/core/secure-endpoint.ts +23 -0
- package/scaffold/mcp/src/core/workflow/enforcement.ts +8 -1
- package/scaffold/mcp/src/core/workflow/mcp-tools.ts +3 -0
- package/scaffold/mcp/src/core/workflow/resolution.ts +9 -1
- package/scaffold/mcp/src/core/workflow/synced-capability-registry.ts +15 -0
- package/scaffold/mcp/src/core/workflow/synced-registry.ts +15 -0
- package/scaffold/mcp/src/daemon/capability-sync-checker.ts +38 -2
- package/scaffold/mcp/src/daemon/egress-proxy.ts +14 -8
- package/scaffold/mcp/src/daemon/global-host-events.ts +288 -0
- package/scaffold/mcp/src/daemon/heartbeat-pusher.ts +4 -0
- package/scaffold/mcp/src/daemon/heartbeat-tracker.ts +2 -0
- package/scaffold/mcp/src/daemon/host-events-pusher.ts +5 -0
- package/scaffold/mcp/src/daemon/main.ts +99 -25
- package/scaffold/mcp/src/daemon/project-service-registry.ts +107 -0
- package/scaffold/mcp/src/daemon/skill-sync-checker.ts +368 -31
- package/scaffold/mcp/src/daemon/sync-checker.ts +4 -0
- package/scaffold/mcp/src/daemon/ungoverned-scanner.ts +57 -25
- package/scaffold/mcp/src/daemon/workflow-sync-checker.ts +41 -2
- package/scaffold/mcp/src/enterprise/audit/push.ts +8 -0
- package/scaffold/mcp/src/enterprise/policy/sync.ts +12 -0
- package/scaffold/mcp/src/enterprise/reviews/push.ts +9 -0
- package/scaffold/mcp/src/enterprise/reviews/trust-state.ts +3 -1
- package/scaffold/mcp/src/enterprise/telemetry/sync.ts +9 -0
- package/scaffold/mcp/src/enterprise/violations/push.ts +9 -0
- package/scaffold/mcp/src/enterprise/workflow/push.ts +7 -0
- package/scaffold/mcp/src/plugin.ts +20 -0
- package/scaffold/mcp/tests/copilot-shim.test.mjs +19 -1
- package/scaffold/mcp/tests/egress-proxy.test.mjs +37 -0
- package/scaffold/mcp/tests/enterprise-identity-sync.test.mjs +401 -0
- package/scaffold/mcp/tests/enterprise-setup.test.mjs +189 -0
- package/scaffold/mcp/tests/global-host-events.test.mjs +81 -0
- package/scaffold/mcp/tests/govern-install.test.mjs +95 -2
- package/scaffold/mcp/tests/govern-repair.test.mjs +20 -3
- package/scaffold/mcp/tests/heartbeat-tracker.test.mjs +26 -0
- package/scaffold/mcp/tests/license.test.mjs +367 -0
- package/scaffold/mcp/tests/project-service-registry.test.mjs +172 -0
- package/scaffold/mcp/tests/review-trust-contract.test.mjs +18 -0
- package/scaffold/mcp/tests/secure-enterprise-endpoint.test.mjs +46 -0
- package/scaffold/mcp/tests/skill-sync-checker.test.mjs +446 -2
- package/scaffold/mcp/tests/ungoverned-scanner.test.mjs +104 -22
- package/scaffold/mcp/tests/workflow-cli.test.mjs +14 -1
- package/scaffold/mcp/tests/workflow-synced-capabilities.test.mjs +37 -0
- package/scaffold/mcp/tests/workflow-synced-registry.test.mjs +40 -1
- package/scaffold/ownership/baseline-v2.4.1.json +22 -0
- package/scaffold/ownership/current.json +4 -0
- package/scaffold/ownership/v1.json +456 -0
- package/scaffold/scripts/ingest-parsers.mjs +1 -387
- package/scaffold/scripts/ingest-worker.mjs +4 -1
- package/scaffold/scripts/ingest.mjs +5 -3899
- package/scaffold/scripts/lib/ingest/arguments.mjs +78 -0
- package/scaffold/scripts/lib/ingest/chunks.mjs +212 -0
- package/scaffold/scripts/lib/ingest/config.mjs +120 -0
- package/scaffold/scripts/lib/ingest/constants.mjs +222 -0
- package/scaffold/scripts/lib/ingest/files.mjs +387 -0
- package/scaffold/scripts/lib/ingest/incremental-state.mjs +131 -0
- package/scaffold/scripts/lib/ingest/io.mjs +78 -0
- package/scaffold/scripts/lib/ingest/main.mjs +76 -0
- package/scaffold/scripts/lib/ingest/parser-composition.mjs +140 -0
- package/scaffold/scripts/lib/ingest/parser-registry.mjs +387 -0
- package/scaffold/scripts/lib/ingest/pipeline-stages.mjs +1625 -0
- package/scaffold/scripts/lib/ingest/projects.mjs +272 -0
- package/scaffold/scripts/lib/ingest/relations.mjs +860 -0
- package/scaffold/scripts/lib/ingest/runtime-paths.mjs +11 -0
- package/scaffold/scripts/lib/ingest/workers.mjs +264 -0
|
@@ -1,3886 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import crypto from "node:crypto";
|
|
3
|
-
import fs from "node:fs";
|
|
4
|
-
import os from "node:os";
|
|
5
2
|
import path from "node:path";
|
|
6
3
|
import { fileURLToPath } from "node:url";
|
|
7
|
-
import {
|
|
8
|
-
import { Worker } from "node:worker_threads";
|
|
9
|
-
import {
|
|
10
|
-
loadParsers,
|
|
11
|
-
getChunkParserForExtension,
|
|
12
|
-
parseCSharpProject,
|
|
13
|
-
hasCSharpProjectParser,
|
|
14
|
-
isCSharpParserAvailable,
|
|
15
|
-
getCSharpParserRuntime,
|
|
16
|
-
PARALLEL_SAFE_LANGUAGES
|
|
17
|
-
} from "./ingest-parsers.mjs";
|
|
4
|
+
import { main } from "./lib/ingest/main.mjs";
|
|
18
5
|
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
? path.resolve(process.env.CORTEX_PROJECT_ROOT)
|
|
23
|
-
: path.resolve(__dirname, "..", "..");
|
|
24
|
-
const CONTEXT_DIR = path.join(REPO_ROOT, ".context");
|
|
25
|
-
const CACHE_DIR = path.join(CONTEXT_DIR, "cache");
|
|
26
|
-
const DB_IMPORT_DIR = path.join(CONTEXT_DIR, "db", "import");
|
|
27
|
-
|
|
28
|
-
const SUPPORTED_TEXT_EXTENSIONS = new Set([
|
|
29
|
-
".md",
|
|
30
|
-
".mdx",
|
|
31
|
-
".txt",
|
|
32
|
-
".adoc",
|
|
33
|
-
".rst",
|
|
34
|
-
".yaml",
|
|
35
|
-
".yml",
|
|
36
|
-
".json",
|
|
37
|
-
".toml",
|
|
38
|
-
".csv",
|
|
39
|
-
".ts",
|
|
40
|
-
".tsx",
|
|
41
|
-
".mts",
|
|
42
|
-
".cts",
|
|
43
|
-
".js",
|
|
44
|
-
".jsx",
|
|
45
|
-
".mjs",
|
|
46
|
-
".cjs",
|
|
47
|
-
".py",
|
|
48
|
-
".go",
|
|
49
|
-
".java",
|
|
50
|
-
".cs",
|
|
51
|
-
".vb",
|
|
52
|
-
".sln",
|
|
53
|
-
".vbproj",
|
|
54
|
-
".csproj",
|
|
55
|
-
".fsproj",
|
|
56
|
-
".props",
|
|
57
|
-
".targets",
|
|
58
|
-
".config",
|
|
59
|
-
".resx",
|
|
60
|
-
".settings",
|
|
61
|
-
".rb",
|
|
62
|
-
".rs",
|
|
63
|
-
".php",
|
|
64
|
-
".swift",
|
|
65
|
-
".kt",
|
|
66
|
-
".sql",
|
|
67
|
-
".sh",
|
|
68
|
-
".bash",
|
|
69
|
-
".zsh",
|
|
70
|
-
".ps1",
|
|
71
|
-
".c",
|
|
72
|
-
".h",
|
|
73
|
-
".cpp",
|
|
74
|
-
".hpp",
|
|
75
|
-
".cc",
|
|
76
|
-
".hh",
|
|
77
|
-
".bas",
|
|
78
|
-
".cls",
|
|
79
|
-
".frm",
|
|
80
|
-
".ctl"
|
|
81
|
-
]);
|
|
82
|
-
|
|
83
|
-
const LEGACY_DOTNET_METADATA_EXTENSIONS = new Set([
|
|
84
|
-
".sln",
|
|
85
|
-
".vbproj",
|
|
86
|
-
".csproj",
|
|
87
|
-
".fsproj",
|
|
88
|
-
".props",
|
|
89
|
-
".targets",
|
|
90
|
-
".config",
|
|
91
|
-
".resx",
|
|
92
|
-
".settings"
|
|
93
|
-
]);
|
|
94
|
-
|
|
95
|
-
const PROJECT_DEFINITION_EXTENSIONS = new Set([".sln", ".vbproj", ".csproj", ".fsproj", ".vcxproj"]);
|
|
96
|
-
const STRUCTURED_NON_CODE_CHUNK_EXTENSIONS = new Set([".config", ".resx", ".settings", ".md", ".mdx"]);
|
|
97
|
-
|
|
98
|
-
const CODE_FILE_EXTENSIONS = new Set([
|
|
99
|
-
".ts",
|
|
100
|
-
".tsx",
|
|
101
|
-
".mts",
|
|
102
|
-
".cts",
|
|
103
|
-
".js",
|
|
104
|
-
".jsx",
|
|
105
|
-
".mjs",
|
|
106
|
-
".cjs",
|
|
107
|
-
".py",
|
|
108
|
-
".go",
|
|
109
|
-
".java",
|
|
110
|
-
".cs",
|
|
111
|
-
".vb",
|
|
112
|
-
".rb",
|
|
113
|
-
".rs",
|
|
114
|
-
".php",
|
|
115
|
-
".swift",
|
|
116
|
-
".kt",
|
|
117
|
-
".sql",
|
|
118
|
-
".sh",
|
|
119
|
-
".bash",
|
|
120
|
-
".zsh",
|
|
121
|
-
".ps1",
|
|
122
|
-
".c",
|
|
123
|
-
".h",
|
|
124
|
-
".cpp",
|
|
125
|
-
".hpp",
|
|
126
|
-
".cc",
|
|
127
|
-
".hh",
|
|
128
|
-
".bas",
|
|
129
|
-
".cls",
|
|
130
|
-
".frm",
|
|
131
|
-
".ctl"
|
|
132
|
-
]);
|
|
133
|
-
|
|
134
|
-
const SQL_REFERENCE_SOURCE_EXTENSIONS = new Set([
|
|
135
|
-
".vb",
|
|
136
|
-
".cs",
|
|
137
|
-
".config",
|
|
138
|
-
".resx",
|
|
139
|
-
".settings"
|
|
140
|
-
]);
|
|
141
|
-
const NAMED_RESOURCE_REFERENCE_SOURCE_EXTENSIONS = new Set([".vb", ".cs"]);
|
|
142
|
-
|
|
143
|
-
const SQL_OBJECT_REFERENCE_PATTERNS = [
|
|
144
|
-
/\b(?:SqlCommand|OleDbCommand|OdbcCommand)\s*\(\s*"([^"\r\n]{2,200})"/gi,
|
|
145
|
-
/\bCommandText\s*=\s*"([^"\r\n]{2,500})"/gi,
|
|
146
|
-
/\bCommandType\s*=\s*(?:CommandType\.)?StoredProcedure[\s\S]{0,240}?"([^"\r\n]{2,200})"/gi,
|
|
147
|
-
/"([^"\r\n]{2,200})"[\s\S]{0,240}?\bCommandType\s*=\s*(?:CommandType\.)?StoredProcedure/gi
|
|
148
|
-
];
|
|
149
|
-
|
|
150
|
-
const SQL_STRING_REFERENCE_PATTERNS = [
|
|
151
|
-
/\bexec(?:ute)?\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
|
|
152
|
-
/\bfrom\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
|
|
153
|
-
/\bjoin\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
|
|
154
|
-
/\bupdate\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
|
|
155
|
-
/\binsert\s+into\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
|
|
156
|
-
/\bdelete\s+from\s+([#@]?[A-Za-z0-9_[\].]+)/gi,
|
|
157
|
-
/\bmerge\s+into\s+([#@]?[A-Za-z0-9_[\].]+)/gi
|
|
158
|
-
];
|
|
159
|
-
|
|
160
|
-
const SQL_RESOURCE_KEY_PATTERNS = [
|
|
161
|
-
/\bMy\.Resources\.([A-Za-z_][A-Za-z0-9_]*)/g,
|
|
162
|
-
/\bResources\.([A-Za-z_][A-Za-z0-9_]*)/g,
|
|
163
|
-
/\bMy\.Settings\.([A-Za-z_][A-Za-z0-9_]*)/g,
|
|
164
|
-
/\b(?:[A-Za-z_][A-Za-z0-9_]*\.)?Settings\.Default\.([A-Za-z_][A-Za-z0-9_]*)/g,
|
|
165
|
-
/\bGetString\(\s*"([^"\r\n]+)"/g,
|
|
166
|
-
/\bGetObject\(\s*"([^"\r\n]+)"/g
|
|
167
|
-
];
|
|
168
|
-
const CONFIG_KEY_REFERENCE_PATTERNS = [
|
|
169
|
-
/\bConfigurationManager\.ConnectionStrings\s*\[\s*"([^"\r\n]+)"\s*\]/g,
|
|
170
|
-
/\bConfigurationManager\.ConnectionStrings\s*\(\s*"([^"\r\n]+)"\s*\)/g,
|
|
171
|
-
/\bConfigurationManager\.AppSettings\s*\[\s*"([^"\r\n]+)"\s*\]/g,
|
|
172
|
-
/\bConfigurationManager\.AppSettings\s*\(\s*"([^"\r\n]+)"\s*\)/g,
|
|
173
|
-
/\bGetConnectionString\(\s*"([^"\r\n]+)"\s*\)/g,
|
|
174
|
-
/\bGetAppSetting\(\s*"([^"\r\n]+)"\s*\)/g
|
|
175
|
-
];
|
|
176
|
-
|
|
177
|
-
const SKIP_DIRECTORIES = new Set([
|
|
178
|
-
".git",
|
|
179
|
-
".idea",
|
|
180
|
-
".vscode",
|
|
181
|
-
"node_modules",
|
|
182
|
-
"bin",
|
|
183
|
-
"obj",
|
|
184
|
-
"dist",
|
|
185
|
-
"build",
|
|
186
|
-
"coverage",
|
|
187
|
-
".next",
|
|
188
|
-
".cache",
|
|
189
|
-
".context"
|
|
190
|
-
]);
|
|
191
|
-
|
|
192
|
-
const MAX_FILE_BYTES = 1024 * 1024;
|
|
193
|
-
const MAX_CONTENT_CHARS = 60000;
|
|
194
|
-
const MAX_BODY_CHARS = 12000;
|
|
195
|
-
const RULE_KEYWORD_LIMIT = 20;
|
|
196
|
-
const DEFAULT_CHUNK_WINDOW_LINES = 80;
|
|
197
|
-
const DEFAULT_CHUNK_OVERLAP_LINES = 16;
|
|
198
|
-
const DEFAULT_CHUNK_SPLIT_MIN_LINES = 120;
|
|
199
|
-
const DEFAULT_CHUNK_MAX_WINDOWS = 8;
|
|
200
|
-
const IMPORT_RESOLUTION_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".json"];
|
|
201
|
-
const IMPORT_RUNTIME_JS_EXTENSIONS = new Set([".js", ".jsx", ".mjs", ".cjs"]);
|
|
202
|
-
const IMPORT_RUNTIME_JS_RESOLUTION_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"];
|
|
203
|
-
const CPP_IMPORT_RESOLUTION_EXTENSIONS = [".h", ".hh", ".hpp", ".c", ".cc", ".cpp"];
|
|
204
|
-
|
|
205
|
-
const STOP_WORDS = new Set([
|
|
206
|
-
"the",
|
|
207
|
-
"and",
|
|
208
|
-
"for",
|
|
209
|
-
"with",
|
|
210
|
-
"from",
|
|
211
|
-
"that",
|
|
212
|
-
"this",
|
|
213
|
-
"must",
|
|
214
|
-
"when",
|
|
215
|
-
"where",
|
|
216
|
-
"into",
|
|
217
|
-
"used",
|
|
218
|
-
"using",
|
|
219
|
-
"only",
|
|
220
|
-
"true",
|
|
221
|
-
"false",
|
|
222
|
-
"unless",
|
|
223
|
-
"should",
|
|
224
|
-
"global",
|
|
225
|
-
"active",
|
|
226
|
-
"rule",
|
|
227
|
-
"rules",
|
|
228
|
-
"data",
|
|
229
|
-
"file",
|
|
230
|
-
"files",
|
|
231
|
-
"code",
|
|
232
|
-
"docs",
|
|
233
|
-
"context",
|
|
234
|
-
"och",
|
|
235
|
-
"det",
|
|
236
|
-
"att",
|
|
237
|
-
"som",
|
|
238
|
-
"med",
|
|
239
|
-
"för",
|
|
240
|
-
"utan",
|
|
241
|
-
"eller",
|
|
242
|
-
"inte",
|
|
243
|
-
"ska",
|
|
244
|
-
"skall",
|
|
245
|
-
"måste",
|
|
246
|
-
"kan",
|
|
247
|
-
"vid",
|
|
248
|
-
"alla"
|
|
249
|
-
]);
|
|
250
|
-
|
|
251
|
-
function parseArgs(argv) {
|
|
252
|
-
const args = new Set(argv.slice(2));
|
|
253
|
-
if (args.has("--help") || args.has("-h")) {
|
|
254
|
-
printHelp();
|
|
255
|
-
process.exit(0);
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
return {
|
|
259
|
-
mode: args.has("--changed") ? "changed" : "full",
|
|
260
|
-
verbose: args.has("--verbose")
|
|
261
|
-
};
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
function printHelp() {
|
|
265
|
-
console.log("Usage: ./scripts/ingest.sh [--changed] [--verbose]");
|
|
266
|
-
console.log("");
|
|
267
|
-
console.log("Options:");
|
|
268
|
-
console.log(" --changed Ingest only changed/untracked files when git is available.");
|
|
269
|
-
console.log(" --verbose Print skipped files and additional diagnostics.");
|
|
270
|
-
console.log(" -h, --help Show this help message.");
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
function ensureDirectory(directoryPath) {
|
|
274
|
-
fs.mkdirSync(directoryPath, { recursive: true });
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
function isTextFile(relPath) {
|
|
278
|
-
const ext = path.extname(relPath).toLowerCase();
|
|
279
|
-
const base = path.basename(relPath).toLowerCase();
|
|
280
|
-
if (SUPPORTED_TEXT_EXTENSIONS.has(ext)) {
|
|
281
|
-
return true;
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
return base === "readme" || base.startsWith("readme.");
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
function isBinaryBuffer(buffer) {
|
|
288
|
-
const scanLength = Math.min(buffer.length, 4000);
|
|
289
|
-
for (let index = 0; index < scanLength; index += 1) {
|
|
290
|
-
if (buffer[index] === 0) {
|
|
291
|
-
return true;
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
return false;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
function toPosixPath(value) {
|
|
299
|
-
return value.split(path.sep).join("/");
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
function normalizeToken(value) {
|
|
303
|
-
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
function tokenizeKeywords(value) {
|
|
307
|
-
return value
|
|
308
|
-
.toLowerCase()
|
|
309
|
-
.split(/[^a-z0-9]+/g)
|
|
310
|
-
.map((token) => token.trim())
|
|
311
|
-
.filter((token) => token.length >= 3 && !STOP_WORDS.has(token));
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
function uniqueSorted(values) {
|
|
315
|
-
return [...new Set(values)].sort();
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
function parsePositiveIntegerEnv(name, fallback) {
|
|
319
|
-
const rawValue = process.env[name];
|
|
320
|
-
if (!rawValue) {
|
|
321
|
-
return fallback;
|
|
322
|
-
}
|
|
323
|
-
const parsed = Number(rawValue);
|
|
324
|
-
if (!Number.isFinite(parsed) || parsed < 1) {
|
|
325
|
-
return fallback;
|
|
326
|
-
}
|
|
327
|
-
return Math.floor(parsed);
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
function parseNonNegativeIntegerEnv(name, fallback) {
|
|
331
|
-
const rawValue = process.env[name];
|
|
332
|
-
if (!rawValue) {
|
|
333
|
-
return fallback;
|
|
334
|
-
}
|
|
335
|
-
const parsed = Number(rawValue);
|
|
336
|
-
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
337
|
-
return fallback;
|
|
338
|
-
}
|
|
339
|
-
return Math.floor(parsed);
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
function isTruthyEnv(value) {
|
|
343
|
-
if (typeof value !== "string") {
|
|
344
|
-
return false;
|
|
345
|
-
}
|
|
346
|
-
const normalized = value.trim().toLowerCase();
|
|
347
|
-
return normalized !== "" && normalized !== "0" && normalized !== "false" && normalized !== "no" && normalized !== "off";
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
function createIngestMemoryTrace() {
|
|
351
|
-
const enabled = isTruthyEnv(process.env.CORTEX_INGEST_TRACE_MEMORY);
|
|
352
|
-
|
|
353
|
-
return {
|
|
354
|
-
enabled,
|
|
355
|
-
checkpoint(label, counts = {}) {
|
|
356
|
-
if (!enabled) {
|
|
357
|
-
return;
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
const memory = process.memoryUsage();
|
|
361
|
-
process.stderr.write(
|
|
362
|
-
`${JSON.stringify({
|
|
363
|
-
type: "cortex.ingest.memory",
|
|
364
|
-
label,
|
|
365
|
-
rss_bytes: memory.rss,
|
|
366
|
-
rss_mb: Number((memory.rss / 1024 / 1024).toFixed(2)),
|
|
367
|
-
heap_used_bytes: memory.heapUsed,
|
|
368
|
-
external_bytes: memory.external,
|
|
369
|
-
counts
|
|
370
|
-
})}\n`
|
|
371
|
-
);
|
|
372
|
-
}
|
|
373
|
-
};
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
function parseSourcePaths(configText) {
|
|
377
|
-
const sourcePaths = [];
|
|
378
|
-
const lines = configText.split(/\r?\n/);
|
|
379
|
-
let inSourcePaths = false;
|
|
380
|
-
|
|
381
|
-
for (const line of lines) {
|
|
382
|
-
if (!inSourcePaths && /^source_paths:\s*$/.test(line.trim())) {
|
|
383
|
-
inSourcePaths = true;
|
|
384
|
-
continue;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
if (!inSourcePaths) {
|
|
388
|
-
continue;
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
const entryMatch = line.match(/^\s*-\s*(.+?)\s*$/);
|
|
392
|
-
if (entryMatch) {
|
|
393
|
-
const unquoted = entryMatch[1].replace(/^['"]|['"]$/g, "");
|
|
394
|
-
sourcePaths.push(unquoted);
|
|
395
|
-
continue;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
if (line.trim() !== "" && !/^\s/.test(line)) {
|
|
399
|
-
break;
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
return sourcePaths;
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
function parseRules(rulesText) {
|
|
407
|
-
const lines = rulesText.split(/\r?\n/);
|
|
408
|
-
const rules = [];
|
|
409
|
-
let current = null;
|
|
410
|
-
|
|
411
|
-
const pushCurrent = () => {
|
|
412
|
-
if (!current || !current.id) {
|
|
413
|
-
return;
|
|
414
|
-
}
|
|
415
|
-
rules.push({
|
|
416
|
-
id: current.id,
|
|
417
|
-
description: current.description ?? "",
|
|
418
|
-
priority: Number.isFinite(current.priority) ? current.priority : 0,
|
|
419
|
-
enforce: current.enforce === true
|
|
420
|
-
});
|
|
421
|
-
};
|
|
422
|
-
|
|
423
|
-
for (const line of lines) {
|
|
424
|
-
const idMatch = line.match(/^\s*-\s*id:\s*(.+?)\s*$/);
|
|
425
|
-
if (idMatch) {
|
|
426
|
-
pushCurrent();
|
|
427
|
-
current = { id: idMatch[1].replace(/^['"]|['"]$/g, "") };
|
|
428
|
-
continue;
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
if (!current) {
|
|
432
|
-
continue;
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
const descriptionMatch = line.match(/^\s*description:\s*(.+?)\s*$/);
|
|
436
|
-
if (descriptionMatch) {
|
|
437
|
-
current.description = descriptionMatch[1].replace(/^['"]|['"]$/g, "");
|
|
438
|
-
continue;
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
const priorityMatch = line.match(/^\s*priority:\s*(\d+)\s*$/);
|
|
442
|
-
if (priorityMatch) {
|
|
443
|
-
current.priority = Number(priorityMatch[1]);
|
|
444
|
-
continue;
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
const enforceMatch = line.match(/^\s*enforce:\s*(true|false)\s*$/i);
|
|
448
|
-
if (enforceMatch) {
|
|
449
|
-
current.enforce = enforceMatch[1].toLowerCase() === "true";
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
pushCurrent();
|
|
454
|
-
return rules;
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
function walkDirectory(directoryPath, files) {
|
|
458
|
-
const entries = fs.readdirSync(directoryPath, { withFileTypes: true });
|
|
459
|
-
for (const entry of entries) {
|
|
460
|
-
const absolutePath = path.join(directoryPath, entry.name);
|
|
461
|
-
if (entry.isDirectory() && shouldSkipDirectory(absolutePath, entry.name)) {
|
|
462
|
-
continue;
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
if (entry.isDirectory()) {
|
|
466
|
-
walkDirectory(absolutePath, files);
|
|
467
|
-
continue;
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
if (entry.isFile()) {
|
|
471
|
-
files.add(absolutePath);
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
function shouldSkipDirectory(absolutePath, entryName) {
|
|
477
|
-
if (entryName === "bin" && path.resolve(path.dirname(absolutePath)) === REPO_ROOT) {
|
|
478
|
-
return false;
|
|
479
|
-
}
|
|
480
|
-
return SKIP_DIRECTORIES.has(entryName);
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
function normalizeSourcePrefix(sourcePath) {
|
|
484
|
-
const source = toPosixPath(sourcePath).replace(/^\.\/+/, "").replace(/\/+$/, "");
|
|
485
|
-
return source === "." ? "" : source;
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
function normalizeRelativePath(relPath) {
|
|
489
|
-
return toPosixPath(relPath).replace(/^\.\/+/, "").replace(/\/+$/, "");
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
function hasSkippedDirectorySegment(relPath) {
|
|
493
|
-
const parts = normalizeRelativePath(relPath).split("/").filter(Boolean);
|
|
494
|
-
return parts.some((part, index) => {
|
|
495
|
-
if (part === "bin" && index === 0) {
|
|
496
|
-
return false;
|
|
497
|
-
}
|
|
498
|
-
return SKIP_DIRECTORIES.has(part);
|
|
499
|
-
});
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
function hasSourcePrefix(relPath, sourcePaths) {
|
|
503
|
-
const normalizedRelPath = normalizeRelativePath(relPath);
|
|
504
|
-
if (!normalizedRelPath || hasSkippedDirectorySegment(normalizedRelPath)) {
|
|
505
|
-
return false;
|
|
506
|
-
}
|
|
507
|
-
return sourcePaths.some((sourcePath) => {
|
|
508
|
-
const source = normalizeSourcePrefix(sourcePath);
|
|
509
|
-
return source === "" || normalizedRelPath === source || normalizedRelPath.startsWith(`${source}/`);
|
|
510
|
-
});
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
function pushImportResolutionCandidate(candidates, seenCandidates, candidatePath) {
|
|
514
|
-
if (!seenCandidates.has(candidatePath)) {
|
|
515
|
-
seenCandidates.add(candidatePath);
|
|
516
|
-
candidates.push(candidatePath);
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
function isCppLikeFilePath(filePath) {
|
|
521
|
-
return [".c", ".h", ".cc", ".cpp", ".hh", ".hpp"].includes(path.posix.extname(filePath).toLowerCase());
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
function resolveRelativeImportTargetId(filePath, importPath, indexedFileIds) {
|
|
525
|
-
const isCppLike = isCppLikeFilePath(filePath);
|
|
526
|
-
const isRelativeImport = importPath.startsWith(".");
|
|
527
|
-
const isLocalCppInclude =
|
|
528
|
-
isCppLike && !path.posix.isAbsolute(importPath) && !/^[A-Za-z]:[\\/]/.test(importPath);
|
|
529
|
-
|
|
530
|
-
if (!isRelativeImport && !isLocalCppInclude) {
|
|
531
|
-
return null;
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
const basePath = path.posix.normalize(path.posix.join(path.posix.dirname(filePath), importPath));
|
|
535
|
-
const candidates = [];
|
|
536
|
-
const seenCandidates = new Set();
|
|
537
|
-
pushImportResolutionCandidate(candidates, seenCandidates, basePath);
|
|
538
|
-
|
|
539
|
-
if (path.posix.extname(basePath) === "") {
|
|
540
|
-
const extensions = isCppLike ? CPP_IMPORT_RESOLUTION_EXTENSIONS : IMPORT_RESOLUTION_EXTENSIONS;
|
|
541
|
-
for (const extension of extensions) {
|
|
542
|
-
pushImportResolutionCandidate(candidates, seenCandidates, `${basePath}${extension}`);
|
|
543
|
-
}
|
|
544
|
-
if (!isCppLike) {
|
|
545
|
-
for (const extension of IMPORT_RESOLUTION_EXTENSIONS) {
|
|
546
|
-
pushImportResolutionCandidate(candidates, seenCandidates, path.posix.join(basePath, `index${extension}`));
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
} else if (IMPORT_RUNTIME_JS_EXTENSIONS.has(path.posix.extname(basePath))) {
|
|
550
|
-
const extension = path.posix.extname(basePath);
|
|
551
|
-
const stemPath = basePath.slice(0, -extension.length);
|
|
552
|
-
for (const candidateExtension of IMPORT_RUNTIME_JS_RESOLUTION_EXTENSIONS) {
|
|
553
|
-
pushImportResolutionCandidate(candidates, seenCandidates, `${stemPath}${candidateExtension}`);
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
for (const candidate of candidates) {
|
|
558
|
-
const targetFileId = `file:${candidate}`;
|
|
559
|
-
if (indexedFileIds.has(targetFileId)) {
|
|
560
|
-
return targetFileId;
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
return null;
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
function getGitChanges() {
|
|
568
|
-
try {
|
|
569
|
-
const output = execSync("git status --porcelain", {
|
|
570
|
-
cwd: REPO_ROOT,
|
|
571
|
-
stdio: ["ignore", "pipe", "ignore"],
|
|
572
|
-
encoding: "utf8"
|
|
573
|
-
});
|
|
574
|
-
|
|
575
|
-
const changed = new Set();
|
|
576
|
-
const deleted = new Set();
|
|
577
|
-
|
|
578
|
-
for (const line of output.split(/\r?\n/)) {
|
|
579
|
-
if (!line) continue;
|
|
580
|
-
const status = line.slice(0, 2);
|
|
581
|
-
const payload = line.slice(3).trim();
|
|
582
|
-
if (!payload) continue;
|
|
583
|
-
|
|
584
|
-
if (payload.includes(" -> ")) {
|
|
585
|
-
const [fromPath, toPath] = payload.split(" -> ");
|
|
586
|
-
deleted.add(path.resolve(REPO_ROOT, fromPath));
|
|
587
|
-
changed.add(path.resolve(REPO_ROOT, toPath));
|
|
588
|
-
continue;
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
const absolutePath = path.resolve(REPO_ROOT, payload);
|
|
592
|
-
if (status.includes("D")) {
|
|
593
|
-
deleted.add(absolutePath);
|
|
594
|
-
} else {
|
|
595
|
-
changed.add(absolutePath);
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
return {
|
|
600
|
-
changed: [...changed],
|
|
601
|
-
deleted: [...deleted]
|
|
602
|
-
};
|
|
603
|
-
} catch {
|
|
604
|
-
return {
|
|
605
|
-
changed: [],
|
|
606
|
-
deleted: []
|
|
607
|
-
};
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
function collectCandidateFiles(sourcePaths, mode) {
|
|
612
|
-
const candidates = new Set();
|
|
613
|
-
const deletedRelPaths = new Set();
|
|
614
|
-
|
|
615
|
-
if (mode === "changed") {
|
|
616
|
-
const gitChanges = getGitChanges();
|
|
617
|
-
if (gitChanges.changed.length > 0 || gitChanges.deleted.length > 0) {
|
|
618
|
-
for (const absolutePath of gitChanges.changed) {
|
|
619
|
-
if (!fs.existsSync(absolutePath)) {
|
|
620
|
-
continue;
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
const stats = fs.statSync(absolutePath);
|
|
624
|
-
if (stats.isFile()) {
|
|
625
|
-
const relPath = toPosixPath(path.relative(REPO_ROOT, absolutePath));
|
|
626
|
-
if (hasSourcePrefix(relPath, sourcePaths)) {
|
|
627
|
-
candidates.add(absolutePath);
|
|
628
|
-
}
|
|
629
|
-
continue;
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
if (stats.isDirectory()) {
|
|
633
|
-
const nestedFiles = new Set();
|
|
634
|
-
walkDirectory(absolutePath, nestedFiles);
|
|
635
|
-
for (const nestedPath of nestedFiles) {
|
|
636
|
-
const nestedRelPath = toPosixPath(path.relative(REPO_ROOT, nestedPath));
|
|
637
|
-
if (hasSourcePrefix(nestedRelPath, sourcePaths)) {
|
|
638
|
-
candidates.add(nestedPath);
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
for (const deletedPath of gitChanges.deleted) {
|
|
645
|
-
const relPath = toPosixPath(path.relative(REPO_ROOT, deletedPath));
|
|
646
|
-
if (hasSourcePrefix(relPath, sourcePaths)) {
|
|
647
|
-
deletedRelPaths.add(relPath);
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
return {
|
|
652
|
-
candidates,
|
|
653
|
-
incrementalMode: true,
|
|
654
|
-
deletedRelPaths: [...deletedRelPaths]
|
|
655
|
-
};
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
for (const sourcePath of sourcePaths) {
|
|
660
|
-
if (hasSkippedDirectorySegment(sourcePath)) {
|
|
661
|
-
continue;
|
|
662
|
-
}
|
|
663
|
-
const absoluteSourcePath = path.resolve(REPO_ROOT, sourcePath);
|
|
664
|
-
if (!fs.existsSync(absoluteSourcePath)) {
|
|
665
|
-
continue;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
const stats = fs.statSync(absoluteSourcePath);
|
|
669
|
-
if (stats.isFile()) {
|
|
670
|
-
candidates.add(absoluteSourcePath);
|
|
671
|
-
continue;
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
if (stats.isDirectory()) {
|
|
675
|
-
walkDirectory(absoluteSourcePath, candidates);
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
return {
|
|
680
|
-
candidates,
|
|
681
|
-
incrementalMode: false,
|
|
682
|
-
deletedRelPaths: []
|
|
683
|
-
};
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
function detectKind(relPath) {
|
|
687
|
-
const lower = relPath.toLowerCase();
|
|
688
|
-
const ext = path.extname(lower);
|
|
689
|
-
const isAdrPath =
|
|
690
|
-
/(^|\/)(adr|adrs|decisions)(\/|$)/.test(lower) ||
|
|
691
|
-
/(^|\/)adr[-_ ]?\d+/.test(path.basename(lower));
|
|
692
|
-
|
|
693
|
-
if (isAdrPath) {
|
|
694
|
-
return "ADR";
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
if (
|
|
698
|
-
lower.startsWith("docs/") ||
|
|
699
|
-
ext === ".md" ||
|
|
700
|
-
ext === ".mdx" ||
|
|
701
|
-
ext === ".txt" ||
|
|
702
|
-
ext === ".adoc" ||
|
|
703
|
-
ext === ".rst"
|
|
704
|
-
) {
|
|
705
|
-
return "DOC";
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
if (LEGACY_DOTNET_METADATA_EXTENSIONS.has(ext) || !CODE_FILE_EXTENSIONS.has(ext)) {
|
|
709
|
-
return "DOC";
|
|
710
|
-
}
|
|
711
|
-
|
|
712
|
-
return "CODE";
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
function trustLevelForKind(kind) {
|
|
716
|
-
if (kind === "ADR") return 95;
|
|
717
|
-
if (kind === "CODE") return 80;
|
|
718
|
-
return 70;
|
|
719
|
-
}
|
|
720
|
-
|
|
721
|
-
function checksum(buffer) {
|
|
722
|
-
return crypto.createHash("sha256").update(buffer).digest("hex");
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
function normalizeWhitespace(value) {
|
|
726
|
-
return value.replace(/\s+/g, " ").trim();
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
function extractTitle(content, fallbackTitle) {
|
|
730
|
-
const lines = content.split(/\r?\n/);
|
|
731
|
-
for (const line of lines) {
|
|
732
|
-
const match = line.match(/^#\s+(.+)\s*$/);
|
|
733
|
-
if (match) return match[1].trim();
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
return fallbackTitle;
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
function parseDecisionDate(content, fallbackDate) {
|
|
740
|
-
const datePatterns = [
|
|
741
|
-
/^\s*date:\s*["']?(\d{4}-\d{2}-\d{2})["']?\s*$/im,
|
|
742
|
-
/^\s*decision[_\s-]*date:\s*["']?(\d{4}-\d{2}-\d{2})["']?\s*$/im
|
|
743
|
-
];
|
|
744
|
-
|
|
745
|
-
for (const pattern of datePatterns) {
|
|
746
|
-
const match = content.match(pattern);
|
|
747
|
-
if (match && !Number.isNaN(Date.parse(match[1]))) {
|
|
748
|
-
return match[1];
|
|
749
|
-
}
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
return fallbackDate.slice(0, 10);
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
function adrTokens(adrRecord) {
|
|
756
|
-
const fileBase = path.basename(adrRecord.path).replace(path.extname(adrRecord.path), "");
|
|
757
|
-
const tokens = new Set([
|
|
758
|
-
normalizeToken(adrRecord.id),
|
|
759
|
-
normalizeToken(fileBase),
|
|
760
|
-
normalizeToken(adrRecord.title)
|
|
761
|
-
]);
|
|
762
|
-
|
|
763
|
-
const numberMatch = fileBase.match(/(\d+)/);
|
|
764
|
-
if (numberMatch) {
|
|
765
|
-
tokens.add(normalizeToken(`adr-${numberMatch[1]}`));
|
|
766
|
-
tokens.add(normalizeToken(numberMatch[1]));
|
|
767
|
-
}
|
|
768
|
-
|
|
769
|
-
return [...tokens].filter(Boolean);
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
function findSupersedesReferences(content) {
|
|
773
|
-
const refs = new Set();
|
|
774
|
-
const pattern = /(?:supersedes|ersätter)\s*[:\-]?\s*([A-Za-z0-9._/-]+)/gi;
|
|
775
|
-
let match;
|
|
776
|
-
while ((match = pattern.exec(content)) !== null) {
|
|
777
|
-
refs.add(match[1]);
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
return [...refs];
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
function writeJsonl(filePath, records) {
|
|
784
|
-
const fd = fs.openSync(filePath, "w");
|
|
785
|
-
try {
|
|
786
|
-
for (const record of records) {
|
|
787
|
-
fs.writeSync(fd, `${JSON.stringify(record)}\n`, undefined, "utf8");
|
|
788
|
-
}
|
|
789
|
-
} finally {
|
|
790
|
-
fs.closeSync(fd);
|
|
791
|
-
}
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
function stageJsonl(filePath, records) {
|
|
795
|
-
const stagedPath = `${filePath}.tmp-${process.pid}-${Date.now()}`;
|
|
796
|
-
writeJsonl(stagedPath, records);
|
|
797
|
-
return {
|
|
798
|
-
commit() {
|
|
799
|
-
fs.renameSync(stagedPath, filePath);
|
|
800
|
-
},
|
|
801
|
-
discard() {
|
|
802
|
-
fs.rmSync(stagedPath, { force: true });
|
|
803
|
-
}
|
|
804
|
-
};
|
|
805
|
-
}
|
|
806
|
-
|
|
807
|
-
function countFileContentRecords(fileRecords) {
|
|
808
|
-
let count = 0;
|
|
809
|
-
for (const fileRecord of fileRecords) {
|
|
810
|
-
if (Object.prototype.hasOwnProperty.call(fileRecord, "content")) {
|
|
811
|
-
count += 1;
|
|
812
|
-
}
|
|
813
|
-
}
|
|
814
|
-
return count;
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
function sanitizeTsvCell(value) {
|
|
818
|
-
if (value === null || value === undefined) return "";
|
|
819
|
-
return String(value).replace(/\t/g, " ").replace(/\r?\n/g, " ");
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
function writeTsv(filePath, headers, rows) {
|
|
823
|
-
const fd = fs.openSync(filePath, "w");
|
|
824
|
-
try {
|
|
825
|
-
fs.writeSync(fd, `${headers.join("\t")}\n`, undefined, "utf8");
|
|
826
|
-
for (const row of rows) {
|
|
827
|
-
fs.writeSync(fd, `${row.map((value) => sanitizeTsvCell(value)).join("\t")}\n`, undefined, "utf8");
|
|
828
|
-
}
|
|
829
|
-
} finally {
|
|
830
|
-
fs.closeSync(fd);
|
|
831
|
-
}
|
|
832
|
-
}
|
|
833
|
-
|
|
834
|
-
function* mapRows(records, project) {
|
|
835
|
-
for (const record of records) {
|
|
836
|
-
yield project(record);
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
function readJsonlSafe(filePath) {
|
|
841
|
-
if (!fs.existsSync(filePath)) {
|
|
842
|
-
return [];
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
return fs
|
|
846
|
-
.readFileSync(filePath, "utf8")
|
|
847
|
-
.split(/\r?\n/)
|
|
848
|
-
.map((line) => line.trim())
|
|
849
|
-
.filter(Boolean)
|
|
850
|
-
.map((line) => {
|
|
851
|
-
try {
|
|
852
|
-
return JSON.parse(line);
|
|
853
|
-
} catch {
|
|
854
|
-
return null;
|
|
855
|
-
}
|
|
856
|
-
})
|
|
857
|
-
.filter((record) => record !== null);
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
function relationKey(...parts) {
|
|
861
|
-
return parts.map((part) => String(part ?? "")).join("|");
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
function uniqueRelations(relations) {
|
|
865
|
-
const deduped = new Map();
|
|
866
|
-
for (const relation of relations) {
|
|
867
|
-
const key = relationKey(relation.from, relation.to, relation.note);
|
|
868
|
-
if (!deduped.has(key)) {
|
|
869
|
-
deduped.set(key, relation);
|
|
870
|
-
}
|
|
871
|
-
}
|
|
872
|
-
return [...deduped.values()].sort((a, b) =>
|
|
873
|
-
relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
|
|
874
|
-
);
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
function normalizeSqlName(value) {
|
|
878
|
-
if (!value) {
|
|
879
|
-
return "";
|
|
880
|
-
}
|
|
881
|
-
|
|
882
|
-
return String(value)
|
|
883
|
-
.trim()
|
|
884
|
-
.replace(/[;"`]/g, "")
|
|
885
|
-
.replace(/\[(.+?)\]/g, "$1")
|
|
886
|
-
.replace(/\s+/g, "")
|
|
887
|
-
.replace(/^\.+|\.+$/g, "")
|
|
888
|
-
.replace(/\.\.+/g, ".")
|
|
889
|
-
.toLowerCase();
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
function sqlChunkAliases(name) {
|
|
893
|
-
const normalized = normalizeSqlName(name);
|
|
894
|
-
if (!normalized) {
|
|
895
|
-
return [];
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
const aliases = new Set([normalized]);
|
|
899
|
-
const parts = normalized.split(".").filter(Boolean);
|
|
900
|
-
if (parts.length > 1) {
|
|
901
|
-
aliases.add(parts[parts.length - 1]);
|
|
902
|
-
}
|
|
903
|
-
return [...aliases];
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
function configChunkAliases(chunk) {
|
|
907
|
-
const aliases = new Set();
|
|
908
|
-
const rawKey = String(chunk?.configKey ?? chunk?.name ?? "");
|
|
909
|
-
const normalizedKey = normalizeToken(rawKey);
|
|
910
|
-
if (normalizedKey) {
|
|
911
|
-
aliases.add(normalizedKey);
|
|
912
|
-
}
|
|
913
|
-
const chunkName = String(chunk?.name ?? "");
|
|
914
|
-
const tail = chunkName.split(".").pop() ?? "";
|
|
915
|
-
const normalizedTail = normalizeToken(tail);
|
|
916
|
-
if (normalizedTail) {
|
|
917
|
-
aliases.add(normalizedTail);
|
|
918
|
-
}
|
|
919
|
-
return [...aliases];
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
function namedEntryChunkAliases(chunk) {
|
|
923
|
-
const aliases = new Set();
|
|
924
|
-
const rawKey = String(chunk?.resourceKey ?? chunk?.configKey ?? chunk?.name ?? "");
|
|
925
|
-
const normalizedKey = normalizeToken(rawKey);
|
|
926
|
-
if (normalizedKey) {
|
|
927
|
-
aliases.add(normalizedKey);
|
|
928
|
-
}
|
|
929
|
-
const chunkName = String(chunk?.name ?? "");
|
|
930
|
-
const tail = chunkName.split(".").pop() ?? "";
|
|
931
|
-
const normalizedTail = normalizeToken(tail);
|
|
932
|
-
if (normalizedTail) {
|
|
933
|
-
aliases.add(normalizedTail);
|
|
934
|
-
}
|
|
935
|
-
return [...aliases];
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
function buildChunkAliasIndexes(chunkRecords) {
|
|
939
|
-
const sqlChunkIdsByAlias = new Map();
|
|
940
|
-
const configChunkIdsByAlias = new Map();
|
|
941
|
-
const resourceChunkIdsByAlias = new Map();
|
|
942
|
-
const settingChunkIdsByAlias = new Map();
|
|
943
|
-
|
|
944
|
-
for (const chunk of chunkRecords) {
|
|
945
|
-
if (isWindowChunkId(chunk?.id)) {
|
|
946
|
-
continue;
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
const language = String(chunk?.language ?? "").toLowerCase();
|
|
950
|
-
if (language === "sql") {
|
|
951
|
-
for (const alias of sqlChunkAliases(chunk.name)) {
|
|
952
|
-
const existing = sqlChunkIdsByAlias.get(alias) ?? [];
|
|
953
|
-
sqlChunkIdsByAlias.set(alias, [...existing, chunk.id]);
|
|
954
|
-
}
|
|
955
|
-
continue;
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
if (language === "config") {
|
|
959
|
-
for (const alias of configChunkAliases(chunk)) {
|
|
960
|
-
const existing = configChunkIdsByAlias.get(alias) ?? [];
|
|
961
|
-
configChunkIdsByAlias.set(alias, [...existing, chunk.id]);
|
|
962
|
-
}
|
|
963
|
-
continue;
|
|
964
|
-
}
|
|
965
|
-
|
|
966
|
-
if (language === "resource") {
|
|
967
|
-
for (const alias of namedEntryChunkAliases(chunk)) {
|
|
968
|
-
const existing = resourceChunkIdsByAlias.get(alias) ?? [];
|
|
969
|
-
resourceChunkIdsByAlias.set(alias, [...existing, chunk.id]);
|
|
970
|
-
}
|
|
971
|
-
continue;
|
|
972
|
-
}
|
|
973
|
-
|
|
974
|
-
if (language === "settings") {
|
|
975
|
-
for (const alias of namedEntryChunkAliases(chunk)) {
|
|
976
|
-
const existing = settingChunkIdsByAlias.get(alias) ?? [];
|
|
977
|
-
settingChunkIdsByAlias.set(alias, [...existing, chunk.id]);
|
|
978
|
-
}
|
|
979
|
-
}
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
return {
|
|
983
|
-
sqlChunkIdsByAlias,
|
|
984
|
-
configChunkIdsByAlias,
|
|
985
|
-
resourceChunkIdsByAlias,
|
|
986
|
-
settingChunkIdsByAlias
|
|
987
|
-
};
|
|
988
|
-
}
|
|
989
|
-
|
|
990
|
-
function extractSqlReferenceNamesFromString(text) {
|
|
991
|
-
const refs = new Set();
|
|
992
|
-
|
|
993
|
-
const normalizedName = normalizeSqlName(text);
|
|
994
|
-
if (/^[a-z0-9_.]+$/i.test(normalizedName) && normalizedName.includes(".")) {
|
|
995
|
-
refs.add(normalizedName);
|
|
996
|
-
}
|
|
997
|
-
|
|
998
|
-
for (const pattern of SQL_STRING_REFERENCE_PATTERNS) {
|
|
999
|
-
let match;
|
|
1000
|
-
while ((match = pattern.exec(text)) !== null) {
|
|
1001
|
-
const name = normalizeSqlName(match[1]);
|
|
1002
|
-
if (!name || name.startsWith("@") || name.startsWith("#")) {
|
|
1003
|
-
continue;
|
|
1004
|
-
}
|
|
1005
|
-
refs.add(name);
|
|
1006
|
-
}
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
return [...refs];
|
|
1010
|
-
}
|
|
1011
|
-
|
|
1012
|
-
function parseResxSqlReferenceMap(content) {
|
|
1013
|
-
const refsByKey = new Map();
|
|
1014
|
-
const dataPattern = /<data\b[^>]*\bname="([^"]+)"[^>]*>([\s\S]*?)<\/data>/gi;
|
|
1015
|
-
let match;
|
|
1016
|
-
|
|
1017
|
-
while ((match = dataPattern.exec(content)) !== null) {
|
|
1018
|
-
const key = normalizeToken(decodeXmlEntities(match[1]));
|
|
1019
|
-
if (!key) {
|
|
1020
|
-
continue;
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
const valueMatch = match[2].match(/<value>([\s\S]*?)<\/value>/i);
|
|
1024
|
-
if (!valueMatch) {
|
|
1025
|
-
continue;
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
const value = decodeXmlEntities(valueMatch[1]).trim();
|
|
1029
|
-
const refs = extractSqlReferenceNamesFromString(value);
|
|
1030
|
-
if (refs.length === 0) {
|
|
1031
|
-
continue;
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
const existing = refsByKey.get(key) ?? [];
|
|
1035
|
-
refsByKey.set(key, uniqueSorted([...existing, ...refs]));
|
|
1036
|
-
}
|
|
1037
|
-
|
|
1038
|
-
return refsByKey;
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
function parseResxKeyMap(content) {
|
|
1042
|
-
const fileKeys = new Map();
|
|
1043
|
-
const dataPattern = /<data\b[^>]*\bname="([^"]+)"[^>]*>/gi;
|
|
1044
|
-
let match;
|
|
1045
|
-
|
|
1046
|
-
while ((match = dataPattern.exec(content)) !== null) {
|
|
1047
|
-
const key = normalizeToken(decodeXmlEntities(match[1]));
|
|
1048
|
-
if (!key) {
|
|
1049
|
-
continue;
|
|
1050
|
-
}
|
|
1051
|
-
fileKeys.set(key, true);
|
|
1052
|
-
}
|
|
1053
|
-
|
|
1054
|
-
return fileKeys;
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
|
-
function parseSettingsSqlReferenceMap(content) {
|
|
1058
|
-
const refsByKey = new Map();
|
|
1059
|
-
const settingPattern = /<Setting\b[^>]*\bName="([^"]+)"[^>]*>([\s\S]*?)<\/Setting>/gi;
|
|
1060
|
-
let match;
|
|
1061
|
-
|
|
1062
|
-
while ((match = settingPattern.exec(content)) !== null) {
|
|
1063
|
-
const key = normalizeToken(decodeXmlEntities(match[1]));
|
|
1064
|
-
if (!key) {
|
|
1065
|
-
continue;
|
|
1066
|
-
}
|
|
1067
|
-
|
|
1068
|
-
const valueMatch = match[2].match(/<Value(?:\s[^>]*)?>([\s\S]*?)<\/Value>/i);
|
|
1069
|
-
if (!valueMatch) {
|
|
1070
|
-
continue;
|
|
1071
|
-
}
|
|
1072
|
-
|
|
1073
|
-
const value = decodeXmlEntities(valueMatch[1]).trim();
|
|
1074
|
-
const refs = extractSqlReferenceNamesFromString(value);
|
|
1075
|
-
if (refs.length === 0) {
|
|
1076
|
-
continue;
|
|
1077
|
-
}
|
|
1078
|
-
|
|
1079
|
-
const existing = refsByKey.get(key) ?? [];
|
|
1080
|
-
refsByKey.set(key, uniqueSorted([...existing, ...refs]));
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
return refsByKey;
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
function parseSettingsKeyMap(content) {
|
|
1087
|
-
const fileKeys = new Map();
|
|
1088
|
-
const settingPattern = /<Setting\b[^>]*\bName="([^"]+)"[^>]*>/gi;
|
|
1089
|
-
let match;
|
|
1090
|
-
|
|
1091
|
-
while ((match = settingPattern.exec(content)) !== null) {
|
|
1092
|
-
const key = normalizeToken(decodeXmlEntities(match[1]));
|
|
1093
|
-
if (!key) {
|
|
1094
|
-
continue;
|
|
1095
|
-
}
|
|
1096
|
-
fileKeys.set(key, true);
|
|
1097
|
-
}
|
|
1098
|
-
|
|
1099
|
-
return fileKeys;
|
|
1100
|
-
}
|
|
1101
|
-
|
|
1102
|
-
function parseConfigKeyMap(content) {
|
|
1103
|
-
const fileKeys = new Map();
|
|
1104
|
-
const addPattern = /<add\b([^>]+?)\/?>/gi;
|
|
1105
|
-
let match;
|
|
1106
|
-
|
|
1107
|
-
while ((match = addPattern.exec(content)) !== null) {
|
|
1108
|
-
const attributes = match[1];
|
|
1109
|
-
const nameMatch = attributes.match(/\bname="([^"]+)"/i);
|
|
1110
|
-
const keyMatch = attributes.match(/\bkey="([^"]+)"/i);
|
|
1111
|
-
const normalized = normalizeToken(decodeXmlEntities(nameMatch?.[1] ?? keyMatch?.[1] ?? ""));
|
|
1112
|
-
if (!normalized) {
|
|
1113
|
-
continue;
|
|
1114
|
-
}
|
|
1115
|
-
fileKeys.set(normalized, true);
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
return fileKeys;
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
function buildSqlResourceReferenceMap(fileRecords) {
|
|
1122
|
-
const refsByKey = new Map();
|
|
1123
|
-
|
|
1124
|
-
for (const fileRecord of fileRecords) {
|
|
1125
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
1126
|
-
let fileRefs = null;
|
|
1127
|
-
if (ext === ".resx") {
|
|
1128
|
-
fileRefs = parseResxSqlReferenceMap(fileRecord.content);
|
|
1129
|
-
} else if (ext === ".settings") {
|
|
1130
|
-
fileRefs = parseSettingsSqlReferenceMap(fileRecord.content);
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
if (!fileRefs) {
|
|
1134
|
-
continue;
|
|
1135
|
-
}
|
|
1136
|
-
|
|
1137
|
-
for (const [key, refs] of fileRefs.entries()) {
|
|
1138
|
-
const existing = refsByKey.get(key) ?? [];
|
|
1139
|
-
refsByKey.set(key, uniqueSorted([...existing, ...refs]));
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
|
|
1143
|
-
return refsByKey;
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
function buildNamedResourceFileMaps(fileRecords) {
|
|
1147
|
-
const resourceFilesByKey = new Map();
|
|
1148
|
-
const settingFilesByKey = new Map();
|
|
1149
|
-
const configFilesByKey = new Map();
|
|
1150
|
-
|
|
1151
|
-
for (const fileRecord of fileRecords) {
|
|
1152
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
1153
|
-
const keyMap =
|
|
1154
|
-
ext === ".resx"
|
|
1155
|
-
? parseResxKeyMap(fileRecord.content)
|
|
1156
|
-
: ext === ".settings"
|
|
1157
|
-
? parseSettingsKeyMap(fileRecord.content)
|
|
1158
|
-
: ext === ".config"
|
|
1159
|
-
? parseConfigKeyMap(fileRecord.content)
|
|
1160
|
-
: null;
|
|
1161
|
-
|
|
1162
|
-
if (!keyMap) {
|
|
1163
|
-
continue;
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
for (const key of keyMap.keys()) {
|
|
1167
|
-
const targetMap =
|
|
1168
|
-
ext === ".resx" ? resourceFilesByKey : ext === ".settings" ? settingFilesByKey : configFilesByKey;
|
|
1169
|
-
const list = targetMap.get(key) ?? [];
|
|
1170
|
-
list.push(fileRecord.id);
|
|
1171
|
-
targetMap.set(key, uniqueSorted(list));
|
|
1172
|
-
}
|
|
1173
|
-
}
|
|
1174
|
-
|
|
1175
|
-
return { resourceFilesByKey, settingFilesByKey, configFilesByKey };
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
function extractSqlResourceKeyReferences(content) {
|
|
1179
|
-
const keys = new Set();
|
|
1180
|
-
|
|
1181
|
-
for (const pattern of SQL_RESOURCE_KEY_PATTERNS) {
|
|
1182
|
-
let match;
|
|
1183
|
-
while ((match = pattern.exec(content)) !== null) {
|
|
1184
|
-
const key = normalizeToken(match[1]);
|
|
1185
|
-
if (key) {
|
|
1186
|
-
keys.add(key);
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
}
|
|
1190
|
-
|
|
1191
|
-
return [...keys];
|
|
1192
|
-
}
|
|
1193
|
-
|
|
1194
|
-
function extractConfigKeyReferences(content) {
|
|
1195
|
-
const keys = new Set();
|
|
1196
|
-
|
|
1197
|
-
for (const pattern of CONFIG_KEY_REFERENCE_PATTERNS) {
|
|
1198
|
-
let match;
|
|
1199
|
-
while ((match = pattern.exec(content)) !== null) {
|
|
1200
|
-
const key = normalizeToken(match[1]);
|
|
1201
|
-
if (key) {
|
|
1202
|
-
keys.add(key);
|
|
1203
|
-
}
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
return [...keys];
|
|
1208
|
-
}
|
|
1209
|
-
|
|
1210
|
-
function shouldExtractNamedResourceReferences(filePath) {
|
|
1211
|
-
return NAMED_RESOURCE_REFERENCE_SOURCE_EXTENSIONS.has(path.extname(filePath).toLowerCase());
|
|
1212
|
-
}
|
|
1213
|
-
|
|
1214
|
-
function generateNamedResourceRelations(fileRecords) {
|
|
1215
|
-
const { resourceFilesByKey, settingFilesByKey, configFilesByKey } = buildNamedResourceFileMaps(fileRecords);
|
|
1216
|
-
const usesResourceRelations = [];
|
|
1217
|
-
const usesSettingRelations = [];
|
|
1218
|
-
const usesConfigRelations = [];
|
|
1219
|
-
const resourceSeen = new Set();
|
|
1220
|
-
const settingSeen = new Set();
|
|
1221
|
-
const configSeen = new Set();
|
|
1222
|
-
|
|
1223
|
-
for (const fileRecord of fileRecords) {
|
|
1224
|
-
if (!shouldExtractNamedResourceReferences(fileRecord.path)) {
|
|
1225
|
-
continue;
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
for (const key of extractSqlResourceKeyReferences(fileRecord.content)) {
|
|
1229
|
-
for (const targetFileId of resourceFilesByKey.get(key) ?? []) {
|
|
1230
|
-
const relKey = relationKey(fileRecord.id, targetFileId, key);
|
|
1231
|
-
if (!resourceSeen.has(relKey) && fileRecord.id !== targetFileId) {
|
|
1232
|
-
resourceSeen.add(relKey);
|
|
1233
|
-
usesResourceRelations.push({
|
|
1234
|
-
from: fileRecord.id,
|
|
1235
|
-
to: targetFileId,
|
|
1236
|
-
note: key
|
|
1237
|
-
});
|
|
1238
|
-
}
|
|
1239
|
-
}
|
|
1240
|
-
|
|
1241
|
-
for (const targetFileId of settingFilesByKey.get(key) ?? []) {
|
|
1242
|
-
const relKey = relationKey(fileRecord.id, targetFileId, key);
|
|
1243
|
-
if (!settingSeen.has(relKey) && fileRecord.id !== targetFileId) {
|
|
1244
|
-
settingSeen.add(relKey);
|
|
1245
|
-
usesSettingRelations.push({
|
|
1246
|
-
from: fileRecord.id,
|
|
1247
|
-
to: targetFileId,
|
|
1248
|
-
note: key
|
|
1249
|
-
});
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
for (const key of extractConfigKeyReferences(fileRecord.content)) {
|
|
1255
|
-
for (const targetFileId of configFilesByKey.get(key) ?? []) {
|
|
1256
|
-
const relKey = relationKey(fileRecord.id, targetFileId, key);
|
|
1257
|
-
if (!configSeen.has(relKey) && fileRecord.id !== targetFileId) {
|
|
1258
|
-
configSeen.add(relKey);
|
|
1259
|
-
usesConfigRelations.push({
|
|
1260
|
-
from: fileRecord.id,
|
|
1261
|
-
to: targetFileId,
|
|
1262
|
-
note: key
|
|
1263
|
-
});
|
|
1264
|
-
}
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
|
|
1269
|
-
usesResourceRelations.sort((a, b) =>
|
|
1270
|
-
relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
|
|
1271
|
-
);
|
|
1272
|
-
usesSettingRelations.sort((a, b) =>
|
|
1273
|
-
relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
|
|
1274
|
-
);
|
|
1275
|
-
usesConfigRelations.sort((a, b) =>
|
|
1276
|
-
relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
|
|
1277
|
-
);
|
|
1278
|
-
|
|
1279
|
-
return { usesResourceRelations, usesSettingRelations, usesConfigRelations };
|
|
1280
|
-
}
|
|
1281
|
-
|
|
1282
|
-
function parseConfigIncludeTargets(fileRecord) {
|
|
1283
|
-
const relPath = toPosixPath(String(fileRecord?.path ?? "").trim());
|
|
1284
|
-
const lowerPath = relPath.toLowerCase();
|
|
1285
|
-
if (!lowerPath.endsWith(".config")) {
|
|
1286
|
-
return [];
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
const content = String(fileRecord?.content ?? "");
|
|
1290
|
-
const dir = path.posix.dirname(relPath);
|
|
1291
|
-
const includes = [];
|
|
1292
|
-
const sectionPattern =
|
|
1293
|
-
/<([A-Za-z_][A-Za-z0-9_.:-]*)\b([^>]*?)\b(configSource|file)="([^"]+)"([^>]*)>/gi;
|
|
1294
|
-
let match;
|
|
1295
|
-
|
|
1296
|
-
while ((match = sectionPattern.exec(content)) !== null) {
|
|
1297
|
-
const sectionName = String(match[1] ?? "").trim().toLowerCase();
|
|
1298
|
-
const attributeName = String(match[3] ?? "").trim().toLowerCase();
|
|
1299
|
-
const includePath = decodeXmlEntities(match[4] ?? "").trim().replace(/\\/g, "/");
|
|
1300
|
-
if (!sectionName || !attributeName || !includePath) {
|
|
1301
|
-
continue;
|
|
1302
|
-
}
|
|
1303
|
-
if (includePath.startsWith("/") || includePath.startsWith("~")) {
|
|
1304
|
-
continue;
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1307
|
-
const resolvedPath = path.posix.normalize(dir === "." ? includePath : `${dir}/${includePath}`);
|
|
1308
|
-
if (!resolvedPath || resolvedPath.startsWith("../")) {
|
|
1309
|
-
continue;
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
includes.push({
|
|
1313
|
-
targetPath: resolvedPath,
|
|
1314
|
-
note: `${sectionName}:${attributeName}`
|
|
1315
|
-
});
|
|
1316
|
-
}
|
|
1317
|
-
|
|
1318
|
-
return includes;
|
|
1319
|
-
}
|
|
1320
|
-
|
|
1321
|
-
function generateConfigIncludeRelations(fileRecords) {
|
|
1322
|
-
const fileIdByPath = new Map(fileRecords.map((record) => [toPosixPath(record.path), record.id]));
|
|
1323
|
-
const relations = [];
|
|
1324
|
-
const seen = new Set();
|
|
1325
|
-
|
|
1326
|
-
for (const fileRecord of fileRecords) {
|
|
1327
|
-
for (const include of parseConfigIncludeTargets(fileRecord)) {
|
|
1328
|
-
const targetFileId = fileIdByPath.get(include.targetPath);
|
|
1329
|
-
if (!targetFileId || targetFileId === fileRecord.id) {
|
|
1330
|
-
continue;
|
|
1331
|
-
}
|
|
1332
|
-
const key = relationKey(fileRecord.id, targetFileId, include.note);
|
|
1333
|
-
if (seen.has(key)) {
|
|
1334
|
-
continue;
|
|
1335
|
-
}
|
|
1336
|
-
seen.add(key);
|
|
1337
|
-
relations.push({
|
|
1338
|
-
from: fileRecord.id,
|
|
1339
|
-
to: targetFileId,
|
|
1340
|
-
note: include.note
|
|
1341
|
-
});
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
|
|
1345
|
-
relations.sort((a, b) => relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note)));
|
|
1346
|
-
return relations;
|
|
1347
|
-
}
|
|
1348
|
-
|
|
1349
|
-
function parseSectionHandlerDeclarations(content) {
|
|
1350
|
-
const declarations = [];
|
|
1351
|
-
const sectionPattern = /<section\b([^>]*?)\/?>/gi;
|
|
1352
|
-
let match;
|
|
1353
|
-
|
|
1354
|
-
while ((match = sectionPattern.exec(String(content ?? ""))) !== null) {
|
|
1355
|
-
const attrs = match[1] ?? "";
|
|
1356
|
-
const nameMatch = attrs.match(/\bname="([^"]+)"/i);
|
|
1357
|
-
const typeMatch = attrs.match(/\btype="([^"]+)"/i);
|
|
1358
|
-
const sectionName = normalizeToken(decodeXmlEntities(nameMatch?.[1] ?? ""));
|
|
1359
|
-
const typeValue = decodeXmlEntities(typeMatch?.[1] ?? "").trim();
|
|
1360
|
-
if (!sectionName || !typeValue) {
|
|
1361
|
-
continue;
|
|
1362
|
-
}
|
|
1363
|
-
|
|
1364
|
-
const typeParts = typeValue.split(",").map((part) => part.trim()).filter(Boolean);
|
|
1365
|
-
const fullTypeName = typeParts[0] ?? "";
|
|
1366
|
-
const assemblyName = typeParts[1] ?? "";
|
|
1367
|
-
const shortTypeName = fullTypeName.split(".").pop()?.split("+").pop() ?? "";
|
|
1368
|
-
const normalizedTypeName = normalizeToken(shortTypeName);
|
|
1369
|
-
const normalizedAssemblyName = normalizeToken(assemblyName);
|
|
1370
|
-
if (!normalizedTypeName && !normalizedAssemblyName) {
|
|
1371
|
-
continue;
|
|
1372
|
-
}
|
|
1373
|
-
|
|
1374
|
-
declarations.push({
|
|
1375
|
-
sectionName,
|
|
1376
|
-
normalizedTypeName,
|
|
1377
|
-
normalizedAssemblyName
|
|
1378
|
-
});
|
|
1379
|
-
}
|
|
1380
|
-
|
|
1381
|
-
return declarations;
|
|
1382
|
-
}
|
|
1383
|
-
|
|
1384
|
-
function buildProjectAssemblyFileMap(fileRecords) {
|
|
1385
|
-
const aliasMap = new Map();
|
|
1386
|
-
|
|
1387
|
-
for (const fileRecord of fileRecords) {
|
|
1388
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
1389
|
-
if (!PROJECT_DEFINITION_EXTENSIONS.has(ext) || ext === ".sln") {
|
|
1390
|
-
continue;
|
|
1391
|
-
}
|
|
1392
|
-
|
|
1393
|
-
const aliases = uniqueSorted([
|
|
1394
|
-
normalizeToken(extractXmlTagValue(fileRecord.content, "AssemblyName")),
|
|
1395
|
-
normalizeToken(extractXmlTagValue(fileRecord.content, "RootNamespace")),
|
|
1396
|
-
normalizeToken(path.basename(fileRecord.path, ext))
|
|
1397
|
-
].filter(Boolean));
|
|
1398
|
-
|
|
1399
|
-
for (const alias of aliases) {
|
|
1400
|
-
const existing = aliasMap.get(alias) ?? [];
|
|
1401
|
-
aliasMap.set(alias, uniqueSorted([...existing, fileRecord.id]));
|
|
1402
|
-
}
|
|
1403
|
-
}
|
|
1404
|
-
|
|
1405
|
-
return aliasMap;
|
|
1406
|
-
}
|
|
1407
|
-
|
|
1408
|
-
function extractDeclaredTypeNames(fileRecord) {
|
|
1409
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
1410
|
-
const pattern =
|
|
1411
|
-
ext === ".vb"
|
|
1412
|
-
? /\b(?:Public|Friend|Private|Protected|Partial|MustInherit|NotInheritable|Shadows|Default|Overridable|Overrides|Shared|\s)*(?:Class|Module|Structure|Interface|Enum)\s+([A-Za-z_][A-Za-z0-9_]*)/gi
|
|
1413
|
-
: ext === ".cs"
|
|
1414
|
-
? /\b(?:public|internal|private|protected|abstract|sealed|static|partial|\s)*(?:class|struct|interface|enum)\s+([A-Za-z_][A-Za-z0-9_]*)/gi
|
|
1415
|
-
: null;
|
|
1416
|
-
|
|
1417
|
-
if (!pattern) {
|
|
1418
|
-
return [];
|
|
1419
|
-
}
|
|
1420
|
-
|
|
1421
|
-
const typeNames = new Set();
|
|
1422
|
-
let match;
|
|
1423
|
-
while ((match = pattern.exec(String(fileRecord.content ?? ""))) !== null) {
|
|
1424
|
-
const normalized = normalizeToken(match[1] ?? "");
|
|
1425
|
-
if (normalized) {
|
|
1426
|
-
typeNames.add(normalized);
|
|
1427
|
-
}
|
|
1428
|
-
}
|
|
1429
|
-
|
|
1430
|
-
return [...typeNames];
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
function buildCodeTypeFileMap(fileRecords) {
|
|
1434
|
-
const typeMap = new Map();
|
|
1435
|
-
|
|
1436
|
-
for (const fileRecord of fileRecords) {
|
|
1437
|
-
if (fileRecord.kind !== "CODE") {
|
|
1438
|
-
continue;
|
|
1439
|
-
}
|
|
1440
|
-
for (const typeName of extractDeclaredTypeNames(fileRecord)) {
|
|
1441
|
-
const existing = typeMap.get(typeName) ?? [];
|
|
1442
|
-
typeMap.set(typeName, uniqueSorted([...existing, fileRecord.id]));
|
|
1443
|
-
}
|
|
1444
|
-
}
|
|
1445
|
-
|
|
1446
|
-
return typeMap;
|
|
1447
|
-
}
|
|
1448
|
-
|
|
1449
|
-
function longestCommonPathPrefixLength(pathA, pathB) {
|
|
1450
|
-
const partsA = toPosixPath(pathA).split("/").filter(Boolean);
|
|
1451
|
-
const partsB = toPosixPath(pathB).split("/").filter(Boolean);
|
|
1452
|
-
const limit = Math.min(partsA.length, partsB.length);
|
|
1453
|
-
let count = 0;
|
|
1454
|
-
while (count < limit && partsA[count] === partsB[count]) {
|
|
1455
|
-
count += 1;
|
|
1456
|
-
}
|
|
1457
|
-
return count;
|
|
1458
|
-
}
|
|
1459
|
-
|
|
1460
|
-
function generateMachineConfigRelations(fileRecords) {
|
|
1461
|
-
const machineConfigs = fileRecords.filter(
|
|
1462
|
-
(record) => path.basename(record.path).toLowerCase() === "machine.config"
|
|
1463
|
-
);
|
|
1464
|
-
if (machineConfigs.length === 0) {
|
|
1465
|
-
return [];
|
|
1466
|
-
}
|
|
1467
|
-
|
|
1468
|
-
const relations = [];
|
|
1469
|
-
const seen = new Set();
|
|
1470
|
-
|
|
1471
|
-
for (const fileRecord of fileRecords) {
|
|
1472
|
-
const lowerPath = fileRecord.path.toLowerCase();
|
|
1473
|
-
if (
|
|
1474
|
-
!lowerPath.endsWith(".config") ||
|
|
1475
|
-
path.basename(lowerPath) === "machine.config" ||
|
|
1476
|
-
!/<configuration\b/i.test(String(fileRecord.content ?? "")) ||
|
|
1477
|
-
parseConfigTransformTarget(fileRecord)
|
|
1478
|
-
) {
|
|
1479
|
-
continue;
|
|
1480
|
-
}
|
|
1481
|
-
|
|
1482
|
-
const rankedTargets = machineConfigs
|
|
1483
|
-
.filter((candidate) => candidate.id !== fileRecord.id)
|
|
1484
|
-
.map((candidate) => ({
|
|
1485
|
-
id: candidate.id,
|
|
1486
|
-
score: longestCommonPathPrefixLength(fileRecord.path, candidate.path)
|
|
1487
|
-
}))
|
|
1488
|
-
.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
1489
|
-
|
|
1490
|
-
const target = rankedTargets[0];
|
|
1491
|
-
if (!target) {
|
|
1492
|
-
continue;
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
const key = relationKey(fileRecord.id, target.id, "inherits:machine");
|
|
1496
|
-
if (seen.has(key)) {
|
|
1497
|
-
continue;
|
|
1498
|
-
}
|
|
1499
|
-
seen.add(key);
|
|
1500
|
-
relations.push({
|
|
1501
|
-
from: fileRecord.id,
|
|
1502
|
-
to: target.id,
|
|
1503
|
-
note: "inherits:machine"
|
|
1504
|
-
});
|
|
1505
|
-
}
|
|
1506
|
-
|
|
1507
|
-
return uniqueRelations(relations);
|
|
1508
|
-
}
|
|
1509
|
-
|
|
1510
|
-
function generateSectionHandlerRelations(fileRecords) {
|
|
1511
|
-
const projectAssemblyFileMap = buildProjectAssemblyFileMap(fileRecords);
|
|
1512
|
-
const codeTypeFileMap = buildCodeTypeFileMap(fileRecords);
|
|
1513
|
-
const relations = [];
|
|
1514
|
-
|
|
1515
|
-
for (const fileRecord of fileRecords) {
|
|
1516
|
-
if (!fileRecord.path.toLowerCase().endsWith(".config")) {
|
|
1517
|
-
continue;
|
|
1518
|
-
}
|
|
1519
|
-
|
|
1520
|
-
for (const declaration of parseSectionHandlerDeclarations(fileRecord.content)) {
|
|
1521
|
-
const note = `section_handler:${declaration.sectionName}`;
|
|
1522
|
-
|
|
1523
|
-
for (const targetFileId of projectAssemblyFileMap.get(declaration.normalizedAssemblyName) ?? []) {
|
|
1524
|
-
relations.push({
|
|
1525
|
-
from: fileRecord.id,
|
|
1526
|
-
to: targetFileId,
|
|
1527
|
-
note
|
|
1528
|
-
});
|
|
1529
|
-
}
|
|
1530
|
-
|
|
1531
|
-
for (const targetFileId of codeTypeFileMap.get(declaration.normalizedTypeName) ?? []) {
|
|
1532
|
-
relations.push({
|
|
1533
|
-
from: fileRecord.id,
|
|
1534
|
-
to: targetFileId,
|
|
1535
|
-
note
|
|
1536
|
-
});
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
}
|
|
1540
|
-
|
|
1541
|
-
return uniqueRelations(relations.filter((relation) => relation.from !== relation.to));
|
|
1542
|
-
}
|
|
1543
|
-
|
|
1544
|
-
function generateConfigTransformKeyRelations(fileRecords, chunkRecords) {
|
|
1545
|
-
const fileIdByPath = new Map(fileRecords.map((record) => [toPosixPath(record.path), record.id]));
|
|
1546
|
-
const chunkFileIdById = new Map(chunkRecords.map((chunk) => [chunk.id, chunk.file_id]));
|
|
1547
|
-
const configChunkIdsByAlias = new Map();
|
|
1548
|
-
|
|
1549
|
-
for (const chunk of chunkRecords) {
|
|
1550
|
-
if (isWindowChunkId(chunk.id) || String(chunk.language ?? "").toLowerCase() !== "config") {
|
|
1551
|
-
continue;
|
|
1552
|
-
}
|
|
1553
|
-
for (const alias of configChunkAliases(chunk)) {
|
|
1554
|
-
const existing = configChunkIdsByAlias.get(alias) ?? [];
|
|
1555
|
-
configChunkIdsByAlias.set(alias, [...existing, chunk.id]);
|
|
1556
|
-
}
|
|
1557
|
-
}
|
|
1558
|
-
|
|
1559
|
-
const relations = [];
|
|
1560
|
-
for (const fileRecord of fileRecords) {
|
|
1561
|
-
const transform = parseConfigTransformTarget(fileRecord);
|
|
1562
|
-
if (!transform) {
|
|
1563
|
-
continue;
|
|
1564
|
-
}
|
|
1565
|
-
|
|
1566
|
-
const targetFileId = fileIdByPath.get(transform.targetPath);
|
|
1567
|
-
if (!targetFileId) {
|
|
1568
|
-
continue;
|
|
1569
|
-
}
|
|
1570
|
-
|
|
1571
|
-
for (const key of parseConfigKeyMap(fileRecord.content).keys()) {
|
|
1572
|
-
for (const targetChunkId of configChunkIdsByAlias.get(key) ?? []) {
|
|
1573
|
-
if (chunkFileIdById.get(targetChunkId) !== targetFileId) {
|
|
1574
|
-
continue;
|
|
1575
|
-
}
|
|
1576
|
-
relations.push({
|
|
1577
|
-
from: fileRecord.id,
|
|
1578
|
-
to: targetChunkId,
|
|
1579
|
-
note: `${key}:${transform.environment}`
|
|
1580
|
-
});
|
|
1581
|
-
}
|
|
1582
|
-
}
|
|
1583
|
-
}
|
|
1584
|
-
|
|
1585
|
-
return uniqueRelations(relations);
|
|
1586
|
-
}
|
|
1587
|
-
|
|
1588
|
-
function parseConfigTransformTarget(fileRecord) {
|
|
1589
|
-
const relPath = toPosixPath(String(fileRecord?.path ?? "").trim());
|
|
1590
|
-
const lowerPath = relPath.toLowerCase();
|
|
1591
|
-
if (!lowerPath.endsWith(".config")) {
|
|
1592
|
-
return null;
|
|
1593
|
-
}
|
|
1594
|
-
|
|
1595
|
-
const content = String(fileRecord?.content ?? "");
|
|
1596
|
-
if (!/\bxdt:(?:transform|locator)\b/i.test(content) && !/\bxmlns:xdt=/i.test(content)) {
|
|
1597
|
-
return null;
|
|
1598
|
-
}
|
|
1599
|
-
|
|
1600
|
-
const dir = path.posix.dirname(relPath);
|
|
1601
|
-
const baseName = path.posix.basename(relPath, ".config");
|
|
1602
|
-
const match = baseName.match(/^(.+)\.([^.]+)$/);
|
|
1603
|
-
if (!match) {
|
|
1604
|
-
return null;
|
|
1605
|
-
}
|
|
1606
|
-
|
|
1607
|
-
const baseStem = match[1]?.trim();
|
|
1608
|
-
const environment = match[2]?.trim();
|
|
1609
|
-
if (!baseStem || !environment) {
|
|
1610
|
-
return null;
|
|
1611
|
-
}
|
|
1612
|
-
|
|
1613
|
-
const targetPath = dir === "." ? `${baseStem}.config` : `${dir}/${baseStem}.config`;
|
|
1614
|
-
return {
|
|
1615
|
-
targetPath,
|
|
1616
|
-
environment: normalizeToken(environment)
|
|
1617
|
-
};
|
|
1618
|
-
}
|
|
1619
|
-
|
|
1620
|
-
function generateConfigTransformRelations(fileRecords) {
|
|
1621
|
-
const fileIdByPath = new Map(fileRecords.map((record) => [toPosixPath(record.path), record.id]));
|
|
1622
|
-
const relations = [];
|
|
1623
|
-
const seen = new Set();
|
|
1624
|
-
|
|
1625
|
-
for (const fileRecord of fileRecords) {
|
|
1626
|
-
const transform = parseConfigTransformTarget(fileRecord);
|
|
1627
|
-
if (!transform) {
|
|
1628
|
-
continue;
|
|
1629
|
-
}
|
|
1630
|
-
|
|
1631
|
-
const targetFileId = fileIdByPath.get(transform.targetPath);
|
|
1632
|
-
if (!targetFileId || targetFileId === fileRecord.id) {
|
|
1633
|
-
continue;
|
|
1634
|
-
}
|
|
1635
|
-
|
|
1636
|
-
const key = relationKey(fileRecord.id, targetFileId, transform.environment);
|
|
1637
|
-
if (seen.has(key)) {
|
|
1638
|
-
continue;
|
|
1639
|
-
}
|
|
1640
|
-
seen.add(key);
|
|
1641
|
-
relations.push({
|
|
1642
|
-
from: fileRecord.id,
|
|
1643
|
-
to: targetFileId,
|
|
1644
|
-
note: transform.environment
|
|
1645
|
-
});
|
|
1646
|
-
}
|
|
1647
|
-
|
|
1648
|
-
relations.sort((a, b) => relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note)));
|
|
1649
|
-
return relations;
|
|
1650
|
-
}
|
|
1651
|
-
|
|
1652
|
-
function shouldExtractSqlReferences(filePath) {
|
|
1653
|
-
return SQL_REFERENCE_SOURCE_EXTENSIONS.has(path.extname(filePath).toLowerCase());
|
|
1654
|
-
}
|
|
1655
|
-
|
|
1656
|
-
function extractSqlObjectReferencesFromContent(content, filePath = "", sqlResourceReferenceMap = new Map()) {
|
|
1657
|
-
const refs = new Set();
|
|
1658
|
-
const ext = path.extname(filePath).toLowerCase();
|
|
1659
|
-
|
|
1660
|
-
if (ext === ".resx") {
|
|
1661
|
-
for (const values of parseResxSqlReferenceMap(content).values()) {
|
|
1662
|
-
for (const ref of values) {
|
|
1663
|
-
refs.add(ref);
|
|
1664
|
-
}
|
|
1665
|
-
}
|
|
1666
|
-
} else if (ext === ".settings") {
|
|
1667
|
-
for (const values of parseSettingsSqlReferenceMap(content).values()) {
|
|
1668
|
-
for (const ref of values) {
|
|
1669
|
-
refs.add(ref);
|
|
1670
|
-
}
|
|
1671
|
-
}
|
|
1672
|
-
}
|
|
1673
|
-
|
|
1674
|
-
for (const pattern of SQL_OBJECT_REFERENCE_PATTERNS) {
|
|
1675
|
-
let match;
|
|
1676
|
-
while ((match = pattern.exec(content)) !== null) {
|
|
1677
|
-
for (const ref of extractSqlReferenceNamesFromString(match[1])) {
|
|
1678
|
-
refs.add(ref);
|
|
1679
|
-
}
|
|
1680
|
-
}
|
|
1681
|
-
}
|
|
1682
|
-
|
|
1683
|
-
if (sqlResourceReferenceMap.size > 0) {
|
|
1684
|
-
for (const key of extractSqlResourceKeyReferences(content)) {
|
|
1685
|
-
for (const ref of sqlResourceReferenceMap.get(key) ?? []) {
|
|
1686
|
-
refs.add(ref);
|
|
1687
|
-
}
|
|
1688
|
-
}
|
|
1689
|
-
}
|
|
1690
|
-
|
|
1691
|
-
return uniqueSorted([...refs]);
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
function decodeXmlEntities(value) {
|
|
1695
|
-
return String(value)
|
|
1696
|
-
.replace(/"/g, '"')
|
|
1697
|
-
.replace(/'/g, "'")
|
|
1698
|
-
.replace(/</g, "<")
|
|
1699
|
-
.replace(/>/g, ">")
|
|
1700
|
-
.replace(/&/g, "&");
|
|
1701
|
-
}
|
|
1702
|
-
|
|
1703
|
-
function projectIdFor(filePath) {
|
|
1704
|
-
return `project:${filePath}`;
|
|
1705
|
-
}
|
|
1706
|
-
|
|
1707
|
-
function isProjectDefinitionFile(filePath) {
|
|
1708
|
-
return PROJECT_DEFINITION_EXTENSIONS.has(path.extname(filePath).toLowerCase());
|
|
1709
|
-
}
|
|
1710
|
-
|
|
1711
|
-
function resolveProjectRelativePath(baseFilePath, includePath) {
|
|
1712
|
-
if (!includePath) {
|
|
1713
|
-
return null;
|
|
1714
|
-
}
|
|
1715
|
-
|
|
1716
|
-
const normalizedInclude = toPosixPath(decodeXmlEntities(includePath).trim().replace(/\\/g, "/"));
|
|
1717
|
-
if (!normalizedInclude) {
|
|
1718
|
-
return null;
|
|
1719
|
-
}
|
|
1720
|
-
|
|
1721
|
-
const resolved = path.resolve(REPO_ROOT, path.dirname(baseFilePath), normalizedInclude);
|
|
1722
|
-
const relPath = toPosixPath(path.relative(REPO_ROOT, resolved));
|
|
1723
|
-
if (!relPath || relPath.startsWith("../")) {
|
|
1724
|
-
return null;
|
|
1725
|
-
}
|
|
1726
|
-
|
|
1727
|
-
return relPath;
|
|
1728
|
-
}
|
|
1729
|
-
|
|
1730
|
-
function projectLanguageForExtension(ext) {
|
|
1731
|
-
switch (ext) {
|
|
1732
|
-
case ".vbproj":
|
|
1733
|
-
return "vbnet";
|
|
1734
|
-
case ".csproj":
|
|
1735
|
-
return "csharp";
|
|
1736
|
-
case ".fsproj":
|
|
1737
|
-
return "fsharp";
|
|
1738
|
-
case ".vcxproj":
|
|
1739
|
-
return "cpp";
|
|
1740
|
-
case ".sln":
|
|
1741
|
-
return "solution";
|
|
1742
|
-
default:
|
|
1743
|
-
return "dotnet";
|
|
1744
|
-
}
|
|
1745
|
-
}
|
|
1746
|
-
|
|
1747
|
-
function extractXmlTagValue(content, tagName) {
|
|
1748
|
-
const match = content.match(new RegExp(`<${tagName}>([\\s\\S]*?)<\\/${tagName}>`, "i"));
|
|
1749
|
-
return match ? decodeXmlEntities(match[1]).trim() : "";
|
|
1750
|
-
}
|
|
1751
|
-
|
|
1752
|
-
function collectXmlIncludeValues(content, elementNames) {
|
|
1753
|
-
const values = [];
|
|
1754
|
-
const pattern = new RegExp(
|
|
1755
|
-
`<(?:${elementNames.join("|")})\\b[^>]*\\bInclude="([^"]+)"[^>]*\\/?>`,
|
|
1756
|
-
"gi"
|
|
1757
|
-
);
|
|
1758
|
-
let match;
|
|
1759
|
-
while ((match = pattern.exec(content)) !== null) {
|
|
1760
|
-
values.push(decodeXmlEntities(match[1]).trim());
|
|
1761
|
-
}
|
|
1762
|
-
return values;
|
|
1763
|
-
}
|
|
1764
|
-
|
|
1765
|
-
function parseSolutionProject(fileRecord, indexedFileIds) {
|
|
1766
|
-
const declaredMembers = [];
|
|
1767
|
-
const referencesProjectRelations = [];
|
|
1768
|
-
const includesFileRelations = [];
|
|
1769
|
-
const fileRelationKeys = new Set();
|
|
1770
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
1771
|
-
const fallbackName = path.basename(fileRecord.path, ext);
|
|
1772
|
-
const projectPattern =
|
|
1773
|
-
/^Project\([^)]*\)\s*=\s*"([^"]+)",\s*"([^"]+\.(?:vbproj|csproj|fsproj|vcxproj))",\s*"\{[^"]+\}"$/gim;
|
|
1774
|
-
|
|
1775
|
-
let match;
|
|
1776
|
-
while ((match = projectPattern.exec(fileRecord.content)) !== null) {
|
|
1777
|
-
const memberName = match[1].trim();
|
|
1778
|
-
const memberPath = resolveProjectRelativePath(fileRecord.path, match[2]);
|
|
1779
|
-
if (!memberPath) {
|
|
1780
|
-
continue;
|
|
1781
|
-
}
|
|
1782
|
-
declaredMembers.push({ name: memberName, path: memberPath });
|
|
1783
|
-
const targetId = projectIdFor(memberPath);
|
|
1784
|
-
if (indexedFileIds.has(`file:${memberPath}`)) {
|
|
1785
|
-
referencesProjectRelations.push({
|
|
1786
|
-
from: projectIdFor(fileRecord.path),
|
|
1787
|
-
to: targetId,
|
|
1788
|
-
note: `solution_member:${memberName}`
|
|
1789
|
-
});
|
|
1790
|
-
}
|
|
1791
|
-
}
|
|
1792
|
-
|
|
1793
|
-
for (const fileId of [`file:${fileRecord.path}`]) {
|
|
1794
|
-
if (indexedFileIds.has(fileId) && !fileRelationKeys.has(fileId)) {
|
|
1795
|
-
fileRelationKeys.add(fileId);
|
|
1796
|
-
includesFileRelations.push({ from: projectIdFor(fileRecord.path), to: fileId });
|
|
1797
|
-
}
|
|
1798
|
-
}
|
|
1799
|
-
|
|
1800
|
-
const summaryParts = [`Solution ${fallbackName}`];
|
|
1801
|
-
if (declaredMembers.length > 0) {
|
|
1802
|
-
summaryParts.push(`Contains ${declaredMembers.length} project references`);
|
|
1803
|
-
}
|
|
1804
|
-
|
|
1805
|
-
return {
|
|
1806
|
-
project: {
|
|
1807
|
-
id: projectIdFor(fileRecord.path),
|
|
1808
|
-
path: fileRecord.path,
|
|
1809
|
-
name: fallbackName,
|
|
1810
|
-
kind: "solution",
|
|
1811
|
-
language: projectLanguageForExtension(ext),
|
|
1812
|
-
target_framework: "",
|
|
1813
|
-
summary: `${summaryParts.join(". ")}.`,
|
|
1814
|
-
file_count: includesFileRelations.length,
|
|
1815
|
-
updated_at: fileRecord.updated_at,
|
|
1816
|
-
source_of_truth: false,
|
|
1817
|
-
trust_level: 78,
|
|
1818
|
-
status: "active"
|
|
1819
|
-
},
|
|
1820
|
-
includesFileRelations,
|
|
1821
|
-
referencesProjectRelations
|
|
1822
|
-
};
|
|
1823
|
-
}
|
|
1824
|
-
|
|
1825
|
-
function parseDotNetProject(fileRecord, indexedFileIds) {
|
|
1826
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
1827
|
-
const fallbackName = path.basename(fileRecord.path, ext);
|
|
1828
|
-
const assemblyName = extractXmlTagValue(fileRecord.content, "AssemblyName");
|
|
1829
|
-
const rootNamespace = extractXmlTagValue(fileRecord.content, "RootNamespace");
|
|
1830
|
-
const targetFrameworkRaw =
|
|
1831
|
-
extractXmlTagValue(fileRecord.content, "TargetFramework") ||
|
|
1832
|
-
extractXmlTagValue(fileRecord.content, "TargetFrameworkVersion") ||
|
|
1833
|
-
extractXmlTagValue(fileRecord.content, "TargetFrameworks");
|
|
1834
|
-
const targetFramework = targetFrameworkRaw.split(";")[0].trim();
|
|
1835
|
-
const includeCandidates = collectXmlIncludeValues(fileRecord.content, [
|
|
1836
|
-
"Compile",
|
|
1837
|
-
"Content",
|
|
1838
|
-
"EmbeddedResource",
|
|
1839
|
-
"None",
|
|
1840
|
-
"Page",
|
|
1841
|
-
"ApplicationDefinition"
|
|
1842
|
-
]);
|
|
1843
|
-
const projectReferenceCandidates = collectXmlIncludeValues(fileRecord.content, ["ProjectReference"]);
|
|
1844
|
-
const includesFileRelations = [];
|
|
1845
|
-
const referencesProjectRelations = [];
|
|
1846
|
-
const fileRelationKeys = new Set();
|
|
1847
|
-
|
|
1848
|
-
const addFileRelation = (relPath) => {
|
|
1849
|
-
const fileId = `file:${relPath}`;
|
|
1850
|
-
if (!indexedFileIds.has(fileId) || fileRelationKeys.has(fileId)) {
|
|
1851
|
-
return;
|
|
1852
|
-
}
|
|
1853
|
-
fileRelationKeys.add(fileId);
|
|
1854
|
-
includesFileRelations.push({
|
|
1855
|
-
from: projectIdFor(fileRecord.path),
|
|
1856
|
-
to: fileId
|
|
1857
|
-
});
|
|
1858
|
-
};
|
|
1859
|
-
|
|
1860
|
-
addFileRelation(fileRecord.path);
|
|
1861
|
-
|
|
1862
|
-
for (const includePath of includeCandidates) {
|
|
1863
|
-
const relPath = resolveProjectRelativePath(fileRecord.path, includePath);
|
|
1864
|
-
if (!relPath) {
|
|
1865
|
-
continue;
|
|
1866
|
-
}
|
|
1867
|
-
addFileRelation(relPath);
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
for (const includePath of projectReferenceCandidates) {
|
|
1871
|
-
const relPath = resolveProjectRelativePath(fileRecord.path, includePath);
|
|
1872
|
-
if (!relPath) {
|
|
1873
|
-
continue;
|
|
1874
|
-
}
|
|
1875
|
-
const targetFileId = `file:${relPath}`;
|
|
1876
|
-
if (!indexedFileIds.has(targetFileId)) {
|
|
1877
|
-
continue;
|
|
1878
|
-
}
|
|
1879
|
-
referencesProjectRelations.push({
|
|
1880
|
-
from: projectIdFor(fileRecord.path),
|
|
1881
|
-
to: projectIdFor(relPath),
|
|
1882
|
-
note: includePath
|
|
1883
|
-
});
|
|
1884
|
-
}
|
|
1885
|
-
|
|
1886
|
-
const summaryParts = [
|
|
1887
|
-
`${projectLanguageForExtension(ext).toUpperCase()} project ${assemblyName || rootNamespace || fallbackName}`
|
|
1888
|
-
];
|
|
1889
|
-
if (targetFramework) {
|
|
1890
|
-
summaryParts.push(`Target framework ${targetFramework}`);
|
|
1891
|
-
}
|
|
1892
|
-
if (includesFileRelations.length > 1) {
|
|
1893
|
-
summaryParts.push(`Includes ${includesFileRelations.length - 1} indexed project files`);
|
|
1894
|
-
}
|
|
1895
|
-
if (referencesProjectRelations.length > 0) {
|
|
1896
|
-
summaryParts.push(`References ${referencesProjectRelations.length} projects`);
|
|
1897
|
-
}
|
|
1898
|
-
|
|
1899
|
-
return {
|
|
1900
|
-
project: {
|
|
1901
|
-
id: projectIdFor(fileRecord.path),
|
|
1902
|
-
path: fileRecord.path,
|
|
1903
|
-
name: assemblyName || rootNamespace || fallbackName,
|
|
1904
|
-
kind: "project",
|
|
1905
|
-
language: projectLanguageForExtension(ext),
|
|
1906
|
-
target_framework: targetFramework,
|
|
1907
|
-
summary: `${summaryParts.join(". ")}.`,
|
|
1908
|
-
file_count: includesFileRelations.length,
|
|
1909
|
-
updated_at: fileRecord.updated_at,
|
|
1910
|
-
source_of_truth: false,
|
|
1911
|
-
trust_level: 80,
|
|
1912
|
-
status: "active"
|
|
1913
|
-
},
|
|
1914
|
-
includesFileRelations,
|
|
1915
|
-
referencesProjectRelations
|
|
1916
|
-
};
|
|
1917
|
-
}
|
|
1918
|
-
|
|
1919
|
-
function generateProjects(fileRecords) {
|
|
1920
|
-
const indexedFileIds = new Set(fileRecords.map((record) => record.id));
|
|
1921
|
-
const projectRecords = [];
|
|
1922
|
-
const includesFileRelations = [];
|
|
1923
|
-
const referencesProjectRelations = [];
|
|
1924
|
-
const includeKeys = new Set();
|
|
1925
|
-
const referenceKeys = new Set();
|
|
1926
|
-
|
|
1927
|
-
for (const fileRecord of fileRecords) {
|
|
1928
|
-
if (!isProjectDefinitionFile(fileRecord.path)) {
|
|
1929
|
-
continue;
|
|
1930
|
-
}
|
|
1931
|
-
|
|
1932
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
1933
|
-
const parsed =
|
|
1934
|
-
ext === ".sln"
|
|
1935
|
-
? parseSolutionProject(fileRecord, indexedFileIds)
|
|
1936
|
-
: parseDotNetProject(fileRecord, indexedFileIds);
|
|
1937
|
-
|
|
1938
|
-
projectRecords.push(parsed.project);
|
|
1939
|
-
|
|
1940
|
-
for (const relation of parsed.includesFileRelations) {
|
|
1941
|
-
const key = relationKey(relation.from, relation.to);
|
|
1942
|
-
if (includeKeys.has(key)) {
|
|
1943
|
-
continue;
|
|
1944
|
-
}
|
|
1945
|
-
includeKeys.add(key);
|
|
1946
|
-
includesFileRelations.push(relation);
|
|
1947
|
-
}
|
|
1948
|
-
|
|
1949
|
-
for (const relation of parsed.referencesProjectRelations) {
|
|
1950
|
-
const key = relationKey(relation.from, relation.to, relation.note);
|
|
1951
|
-
if (referenceKeys.has(key)) {
|
|
1952
|
-
continue;
|
|
1953
|
-
}
|
|
1954
|
-
referenceKeys.add(key);
|
|
1955
|
-
referencesProjectRelations.push(relation);
|
|
1956
|
-
}
|
|
1957
|
-
}
|
|
1958
|
-
|
|
1959
|
-
projectRecords.sort((a, b) => a.path.localeCompare(b.path));
|
|
1960
|
-
includesFileRelations.sort((a, b) => relationKey(a.from, a.to).localeCompare(relationKey(b.from, b.to)));
|
|
1961
|
-
referencesProjectRelations.sort((a, b) =>
|
|
1962
|
-
relationKey(a.from, a.to, a.note).localeCompare(relationKey(b.from, b.to, b.note))
|
|
1963
|
-
);
|
|
1964
|
-
|
|
1965
|
-
return {
|
|
1966
|
-
projects: projectRecords,
|
|
1967
|
-
includesFileRelations,
|
|
1968
|
-
referencesProjectRelations
|
|
1969
|
-
};
|
|
1970
|
-
}
|
|
1971
|
-
|
|
1972
|
-
function removeChunkStateForFile(fileId, chunkRecordMap, definesRelationMap, callsRelationMap, importsRelationMap, callsSqlRelationMap) {
|
|
1973
|
-
const removedChunkIds = new Set();
|
|
1974
|
-
|
|
1975
|
-
for (const [chunkId, chunkRecord] of chunkRecordMap.entries()) {
|
|
1976
|
-
if (chunkRecord.file_id === fileId) {
|
|
1977
|
-
removedChunkIds.add(chunkId);
|
|
1978
|
-
chunkRecordMap.delete(chunkId);
|
|
1979
|
-
}
|
|
1980
|
-
}
|
|
1981
|
-
|
|
1982
|
-
if (removedChunkIds.size === 0) {
|
|
1983
|
-
return;
|
|
1984
|
-
}
|
|
1985
|
-
|
|
1986
|
-
for (const [key, relation] of definesRelationMap.entries()) {
|
|
1987
|
-
if (relation.from === fileId || removedChunkIds.has(relation.to)) {
|
|
1988
|
-
definesRelationMap.delete(key);
|
|
1989
|
-
}
|
|
1990
|
-
}
|
|
1991
|
-
|
|
1992
|
-
for (const [key, relation] of callsRelationMap.entries()) {
|
|
1993
|
-
if (removedChunkIds.has(relation.from) || removedChunkIds.has(relation.to)) {
|
|
1994
|
-
callsRelationMap.delete(key);
|
|
1995
|
-
}
|
|
1996
|
-
}
|
|
1997
|
-
|
|
1998
|
-
for (const [key, relation] of importsRelationMap.entries()) {
|
|
1999
|
-
if (removedChunkIds.has(relation.from)) {
|
|
2000
|
-
importsRelationMap.delete(key);
|
|
2001
|
-
}
|
|
2002
|
-
}
|
|
2003
|
-
|
|
2004
|
-
for (const [key, relation] of callsSqlRelationMap.entries()) {
|
|
2005
|
-
if (relation.from === fileId || removedChunkIds.has(relation.to)) {
|
|
2006
|
-
callsSqlRelationMap.delete(key);
|
|
2007
|
-
}
|
|
2008
|
-
}
|
|
2009
|
-
}
|
|
2010
|
-
|
|
2011
|
-
function hydrateIncrementalChunkState(fileRecords) {
|
|
2012
|
-
const fileIdSet = new Set(fileRecords.map((record) => record.id));
|
|
2013
|
-
const chunkRecordMap = new Map();
|
|
2014
|
-
const definesRelationMap = new Map();
|
|
2015
|
-
const callsRelationMap = new Map();
|
|
2016
|
-
const importsRelationMap = new Map();
|
|
2017
|
-
const callsSqlRelationMap = new Map();
|
|
2018
|
-
|
|
2019
|
-
for (const record of readJsonlSafe(path.join(CACHE_DIR, "entities.chunk.jsonl"))) {
|
|
2020
|
-
if (!record || typeof record !== "object") continue;
|
|
2021
|
-
const chunkId = String(record.id ?? "");
|
|
2022
|
-
const fileId = String(record.file_id ?? "");
|
|
2023
|
-
if (!chunkId || !fileIdSet.has(fileId)) {
|
|
2024
|
-
continue;
|
|
2025
|
-
}
|
|
2026
|
-
chunkRecordMap.set(chunkId, {
|
|
2027
|
-
...record,
|
|
2028
|
-
id: chunkId,
|
|
2029
|
-
file_id: fileId
|
|
2030
|
-
});
|
|
2031
|
-
}
|
|
2032
|
-
|
|
2033
|
-
const chunkIdSet = new Set(chunkRecordMap.keys());
|
|
2034
|
-
|
|
2035
|
-
for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.defines.jsonl"))) {
|
|
2036
|
-
if (!record || typeof record !== "object") continue;
|
|
2037
|
-
const from = String(record.from ?? "");
|
|
2038
|
-
const to = String(record.to ?? "");
|
|
2039
|
-
if (!fileIdSet.has(from) || !chunkIdSet.has(to)) {
|
|
2040
|
-
continue;
|
|
2041
|
-
}
|
|
2042
|
-
definesRelationMap.set(relationKey(from, to), { from, to });
|
|
2043
|
-
}
|
|
2044
|
-
|
|
2045
|
-
for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.calls.jsonl"))) {
|
|
2046
|
-
if (!record || typeof record !== "object") continue;
|
|
2047
|
-
const from = String(record.from ?? "");
|
|
2048
|
-
const to = String(record.to ?? "");
|
|
2049
|
-
const callType = String(record.call_type ?? "direct");
|
|
2050
|
-
if (!chunkIdSet.has(from) || !chunkIdSet.has(to)) {
|
|
2051
|
-
continue;
|
|
2052
|
-
}
|
|
2053
|
-
callsRelationMap.set(relationKey(from, to, callType), {
|
|
2054
|
-
from,
|
|
2055
|
-
to,
|
|
2056
|
-
call_type: callType
|
|
2057
|
-
});
|
|
2058
|
-
}
|
|
2059
|
-
|
|
2060
|
-
for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.imports.jsonl"))) {
|
|
2061
|
-
if (!record || typeof record !== "object") continue;
|
|
2062
|
-
const from = String(record.from ?? "");
|
|
2063
|
-
const to = String(record.to ?? "");
|
|
2064
|
-
const importName = String(record.import_name ?? "");
|
|
2065
|
-
if (!chunkIdSet.has(from) || !fileIdSet.has(to)) {
|
|
2066
|
-
continue;
|
|
2067
|
-
}
|
|
2068
|
-
importsRelationMap.set(relationKey(from, to, importName), {
|
|
2069
|
-
from,
|
|
2070
|
-
to,
|
|
2071
|
-
import_name: importName
|
|
2072
|
-
});
|
|
2073
|
-
}
|
|
2074
|
-
|
|
2075
|
-
for (const record of readJsonlSafe(path.join(CACHE_DIR, "relations.calls_sql.jsonl"))) {
|
|
2076
|
-
if (!record || typeof record !== "object") continue;
|
|
2077
|
-
const from = String(record.from ?? "");
|
|
2078
|
-
const to = String(record.to ?? "");
|
|
2079
|
-
const note = String(record.note ?? "");
|
|
2080
|
-
if (!fileIdSet.has(from) || !chunkIdSet.has(to)) {
|
|
2081
|
-
continue;
|
|
2082
|
-
}
|
|
2083
|
-
callsSqlRelationMap.set(relationKey(from, to, note), {
|
|
2084
|
-
from,
|
|
2085
|
-
to,
|
|
2086
|
-
note
|
|
2087
|
-
});
|
|
2088
|
-
}
|
|
2089
|
-
|
|
2090
|
-
return {
|
|
2091
|
-
chunkRecordMap,
|
|
2092
|
-
definesRelationMap,
|
|
2093
|
-
callsRelationMap,
|
|
2094
|
-
importsRelationMap,
|
|
2095
|
-
callsSqlRelationMap
|
|
2096
|
-
};
|
|
2097
|
-
}
|
|
2098
|
-
|
|
2099
|
-
function normalizeRuleTokens(ruleRecord) {
|
|
2100
|
-
const idParts = ruleRecord.id.split(/[._-]+/g);
|
|
2101
|
-
const descriptionTokens = tokenizeKeywords(ruleRecord.body);
|
|
2102
|
-
const rawKeywords = [...idParts, ...descriptionTokens];
|
|
2103
|
-
const normalized = rawKeywords
|
|
2104
|
-
.map((token) => token.toLowerCase().replace(/[^a-z0-9]/g, ""))
|
|
2105
|
-
.filter((token) => token.length >= 3 && !STOP_WORDS.has(token));
|
|
2106
|
-
|
|
2107
|
-
return uniqueSorted(normalized).slice(0, RULE_KEYWORD_LIMIT);
|
|
2108
|
-
}
|
|
2109
|
-
|
|
2110
|
-
function fileTokenSet(fileRecord) {
|
|
2111
|
-
const tokenSource = `${fileRecord.path}\n${fileRecord.content.slice(0, 12000)}`;
|
|
2112
|
-
return new Set(tokenizeKeywords(tokenSource));
|
|
2113
|
-
}
|
|
2114
|
-
|
|
2115
|
-
function collectRuleKeywordMatch(ruleKeywords, tokens, minimumMatches) {
|
|
2116
|
-
const required = Math.min(minimumMatches, Math.max(1, ruleKeywords.length));
|
|
2117
|
-
let count = 0;
|
|
2118
|
-
const sample = [];
|
|
2119
|
-
|
|
2120
|
-
for (const keyword of ruleKeywords) {
|
|
2121
|
-
if (!tokens.has(keyword)) {
|
|
2122
|
-
continue;
|
|
2123
|
-
}
|
|
2124
|
-
count += 1;
|
|
2125
|
-
if (sample.length < 5) {
|
|
2126
|
-
sample.push(keyword);
|
|
2127
|
-
}
|
|
2128
|
-
if (count >= required && sample.length >= 5) {
|
|
2129
|
-
break;
|
|
2130
|
-
}
|
|
2131
|
-
}
|
|
2132
|
-
|
|
2133
|
-
return { matched: count >= required, sample };
|
|
2134
|
-
}
|
|
2135
|
-
|
|
2136
|
-
function chunkIdFor(filePath, chunk) {
|
|
2137
|
-
const startLine = Number.isFinite(chunk.startLine) ? chunk.startLine : 0;
|
|
2138
|
-
const endLine = Number.isFinite(chunk.endLine) ? chunk.endLine : startLine;
|
|
2139
|
-
return `chunk:${filePath}:${chunk.name}:${startLine}-${endLine}`;
|
|
2140
|
-
}
|
|
2141
|
-
|
|
2142
|
-
function generateChunkDescription(chunk) {
|
|
2143
|
-
const parts = [chunk.kind];
|
|
2144
|
-
if (chunk.exported) parts.push("exported");
|
|
2145
|
-
if (chunk.async) parts.push("async");
|
|
2146
|
-
parts.push(chunk.signature);
|
|
2147
|
-
|
|
2148
|
-
if (typeof chunk.description === "string" && chunk.description.trim().length > 10) {
|
|
2149
|
-
parts.push(normalizeWhitespace(chunk.description).slice(0, 200));
|
|
2150
|
-
}
|
|
2151
|
-
|
|
2152
|
-
// Extract leading JSDoc/comment from body
|
|
2153
|
-
// Match leading JSDoc (/** */), block (/* */) and line (//) comments
|
|
2154
|
-
const commentMatch = chunk.body.match(/^(?:\s*(?:\/\*[\s\S]*?\*\/|\/\/[^\n]*)[\s\n]*)+/);
|
|
2155
|
-
if (commentMatch) {
|
|
2156
|
-
const cleaned = commentMatch[0]
|
|
2157
|
-
.replace(/\/\*\*|\*\/|\*|\/\//g, "")
|
|
2158
|
-
.replace(/\s+/g, " ").trim()
|
|
2159
|
-
.slice(0, 200);
|
|
2160
|
-
if (cleaned.length > 10) parts.push(cleaned);
|
|
2161
|
-
}
|
|
2162
|
-
|
|
2163
|
-
return parts.join(". ") + ".";
|
|
2164
|
-
}
|
|
2165
|
-
|
|
2166
|
-
function generateModuleSummary(dir, files, exportNames, repoRoot = REPO_ROOT) {
|
|
2167
|
-
// Check for README.md in directory
|
|
2168
|
-
const readmePath = path.join(repoRoot, dir, "README.md");
|
|
2169
|
-
if (fs.existsSync(readmePath)) {
|
|
2170
|
-
try {
|
|
2171
|
-
const content = fs.readFileSync(readmePath, "utf8");
|
|
2172
|
-
// Skip first heading line, take first 300 chars
|
|
2173
|
-
const lines = content.split(/\r?\n/);
|
|
2174
|
-
const startIdx = lines.findIndex(l => !l.startsWith("#") && l.trim().length > 0);
|
|
2175
|
-
if (startIdx >= 0) {
|
|
2176
|
-
const excerpt = lines.slice(startIdx).join(" ").trim().slice(0, 300);
|
|
2177
|
-
if (excerpt.length > 20) return excerpt;
|
|
2178
|
-
}
|
|
2179
|
-
} catch {
|
|
2180
|
-
// fall through to auto-generated summary
|
|
2181
|
-
}
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2184
|
-
const name = path.basename(dir);
|
|
2185
|
-
const codeFiles = files.filter(f => f.kind === "CODE");
|
|
2186
|
-
const docFiles = files.filter(f => f.kind !== "CODE");
|
|
2187
|
-
|
|
2188
|
-
const parts = [`Module ${name}`];
|
|
2189
|
-
parts.push(`Contains ${files.length} files (${codeFiles.length} code, ${docFiles.length} docs)`);
|
|
2190
|
-
|
|
2191
|
-
// Detect common file extension pattern
|
|
2192
|
-
const exts = new Set(codeFiles.map(f => path.extname(f.path).toLowerCase()));
|
|
2193
|
-
if (exts.size === 1) {
|
|
2194
|
-
const ext = [...exts][0];
|
|
2195
|
-
const extNames = {
|
|
2196
|
-
".ts": "TypeScript",
|
|
2197
|
-
".tsx": "TypeScript React",
|
|
2198
|
-
".mts": "TypeScript ESM",
|
|
2199
|
-
".cts": "TypeScript CommonJS",
|
|
2200
|
-
".js": "JavaScript",
|
|
2201
|
-
".jsx": "JavaScript React",
|
|
2202
|
-
".mjs": "JavaScript ESM",
|
|
2203
|
-
".cjs": "JavaScript CommonJS"
|
|
2204
|
-
};
|
|
2205
|
-
if (extNames[ext]) parts.push(`${extNames[ext]} source files`);
|
|
2206
|
-
}
|
|
2207
|
-
|
|
2208
|
-
if (exportNames.length > 0) {
|
|
2209
|
-
parts.push(`Key exports: ${exportNames.slice(0, 5).join(", ")}`);
|
|
2210
|
-
}
|
|
2211
|
-
|
|
2212
|
-
return parts.join(". ") + ".";
|
|
2213
|
-
}
|
|
2214
|
-
|
|
2215
|
-
function generateModules(fileRecords, chunkRecords) {
|
|
2216
|
-
const dirFiles = new Map();
|
|
2217
|
-
const dirChunks = new Map();
|
|
2218
|
-
const fileById = new Map(fileRecords.map(f => [f.id, f]));
|
|
2219
|
-
|
|
2220
|
-
for (const file of fileRecords) {
|
|
2221
|
-
const dir = path.dirname(file.path);
|
|
2222
|
-
if (!dirFiles.has(dir)) dirFiles.set(dir, []);
|
|
2223
|
-
dirFiles.get(dir).push(file);
|
|
2224
|
-
}
|
|
2225
|
-
|
|
2226
|
-
for (const chunk of chunkRecords) {
|
|
2227
|
-
if (!chunk.exported || isWindowChunkId(chunk.id)) continue;
|
|
2228
|
-
const file = fileById.get(chunk.file_id);
|
|
2229
|
-
if (!file) continue;
|
|
2230
|
-
const dir = path.dirname(file.path);
|
|
2231
|
-
if (!dirChunks.has(dir)) dirChunks.set(dir, []);
|
|
2232
|
-
dirChunks.get(dir).push(chunk);
|
|
2233
|
-
}
|
|
2234
|
-
|
|
2235
|
-
const modules = [];
|
|
2236
|
-
const containsRelations = [];
|
|
2237
|
-
const containsModuleRelations = [];
|
|
2238
|
-
const exportsRelations = [];
|
|
2239
|
-
|
|
2240
|
-
const MIN_MODULE_FILES = 2;
|
|
2241
|
-
|
|
2242
|
-
for (const [dir, files] of dirFiles) {
|
|
2243
|
-
if (files.length < MIN_MODULE_FILES) continue;
|
|
2244
|
-
|
|
2245
|
-
const exports = dirChunks.get(dir) || [];
|
|
2246
|
-
const exportNames = [...new Set(exports.slice(0, 20).map(c => c.name))];
|
|
2247
|
-
const moduleId = `module:${dir}`;
|
|
2248
|
-
|
|
2249
|
-
modules.push({
|
|
2250
|
-
id: moduleId,
|
|
2251
|
-
path: dir,
|
|
2252
|
-
name: path.basename(dir),
|
|
2253
|
-
summary: generateModuleSummary(dir, files, exportNames),
|
|
2254
|
-
file_count: files.length,
|
|
2255
|
-
exported_symbols: exportNames.join(", "),
|
|
2256
|
-
updated_at: files.reduce((latest, f) => f.updated_at > latest ? f.updated_at : latest, ""),
|
|
2257
|
-
source_of_truth: false,
|
|
2258
|
-
trust_level: 75,
|
|
2259
|
-
status: "active"
|
|
2260
|
-
});
|
|
2261
|
-
|
|
2262
|
-
// CONTAINS: Module -> File
|
|
2263
|
-
for (const file of files) {
|
|
2264
|
-
containsRelations.push({ from: moduleId, to: file.id });
|
|
2265
|
-
}
|
|
2266
|
-
|
|
2267
|
-
// EXPORTS: Module -> Chunk
|
|
2268
|
-
for (const chunk of exports) {
|
|
2269
|
-
exportsRelations.push({ from: moduleId, to: chunk.id });
|
|
2270
|
-
}
|
|
2271
|
-
}
|
|
2272
|
-
|
|
2273
|
-
// CONTAINS_MODULE: parent Module -> child Module
|
|
2274
|
-
const moduleDirs = new Set(modules.map(m => m.path));
|
|
2275
|
-
for (const dir of moduleDirs) {
|
|
2276
|
-
const parent = path.dirname(dir);
|
|
2277
|
-
if (parent !== dir && moduleDirs.has(parent)) {
|
|
2278
|
-
containsModuleRelations.push({
|
|
2279
|
-
from: `module:${parent}`,
|
|
2280
|
-
to: `module:${dir}`
|
|
2281
|
-
});
|
|
2282
|
-
}
|
|
2283
|
-
}
|
|
2284
|
-
|
|
2285
|
-
return { modules, containsRelations, containsModuleRelations, exportsRelations };
|
|
2286
|
-
}
|
|
2287
|
-
|
|
2288
|
-
function isWindowChunkId(chunkId) {
|
|
2289
|
-
return typeof chunkId === "string" && chunkId.includes(":window:");
|
|
2290
|
-
}
|
|
2291
|
-
|
|
2292
|
-
function splitChunkIntoWindows(chunkRecord, options) {
|
|
2293
|
-
const { windowLines, overlapLines, splitMinLines, maxWindows, chunkBody } = options;
|
|
2294
|
-
const sourceBody = typeof chunkBody === "string" ? chunkBody : chunkRecord.body;
|
|
2295
|
-
const lines = sourceBody.split(/\r?\n/);
|
|
2296
|
-
const totalLines = lines.length;
|
|
2297
|
-
if (totalLines < splitMinLines || totalLines <= windowLines) {
|
|
2298
|
-
return [];
|
|
2299
|
-
}
|
|
2300
|
-
|
|
2301
|
-
const windows = [];
|
|
2302
|
-
const safeOverlap = Math.max(0, Math.min(overlapLines, windowLines - 1));
|
|
2303
|
-
let start = 0;
|
|
2304
|
-
let windowIndex = 1;
|
|
2305
|
-
|
|
2306
|
-
while (start < totalLines && windows.length < maxWindows) {
|
|
2307
|
-
const isLastAllowedWindow = windows.length + 1 >= maxWindows;
|
|
2308
|
-
const end = isLastAllowedWindow ? totalLines : Math.min(totalLines, start + windowLines);
|
|
2309
|
-
const windowStartLine = chunkRecord.start_line + start;
|
|
2310
|
-
const windowEndLine = chunkRecord.start_line + Math.max(0, end - 1);
|
|
2311
|
-
const windowBody = lines.slice(start, end).join("\n");
|
|
2312
|
-
const persistedBody = isLastAllowedWindow ? windowBody : windowBody.slice(0, MAX_BODY_CHARS);
|
|
2313
|
-
windows.push({
|
|
2314
|
-
id: `${chunkRecord.id}:window:${windowIndex}:${windowStartLine}-${windowEndLine}`,
|
|
2315
|
-
file_id: chunkRecord.file_id,
|
|
2316
|
-
name: `${chunkRecord.name}#window${windowIndex}`,
|
|
2317
|
-
kind: chunkRecord.kind,
|
|
2318
|
-
signature: `${chunkRecord.signature} [window ${windowIndex}]`,
|
|
2319
|
-
body: persistedBody,
|
|
2320
|
-
description: chunkRecord.description || "",
|
|
2321
|
-
start_line: windowStartLine,
|
|
2322
|
-
end_line: windowEndLine,
|
|
2323
|
-
language: chunkRecord.language,
|
|
2324
|
-
exported: chunkRecord.exported || false,
|
|
2325
|
-
checksum: checksum(Buffer.from(windowBody)),
|
|
2326
|
-
updated_at: chunkRecord.updated_at,
|
|
2327
|
-
trust_level: chunkRecord.trust_level,
|
|
2328
|
-
status: chunkRecord.status,
|
|
2329
|
-
source_of_truth: chunkRecord.source_of_truth
|
|
2330
|
-
});
|
|
2331
|
-
|
|
2332
|
-
if (end >= totalLines) {
|
|
2333
|
-
break;
|
|
2334
|
-
}
|
|
2335
|
-
|
|
2336
|
-
start = end - safeOverlap;
|
|
2337
|
-
windowIndex += 1;
|
|
2338
|
-
}
|
|
2339
|
-
|
|
2340
|
-
return windows;
|
|
2341
|
-
}
|
|
2342
|
-
|
|
2343
|
-
function resolveIngestWorkerCount(taskCount) {
|
|
2344
|
-
const raw = process.env.CORTEX_INGEST_WORKERS;
|
|
2345
|
-
const configured = raw !== undefined ? Number.parseInt(raw, 10) : Number.NaN;
|
|
2346
|
-
const cpuBudget = Math.max(1, (os.availableParallelism?.() ?? os.cpus().length) - 1);
|
|
2347
|
-
const defaultWorkerLimit = taskCount >= 1000 ? 4 : 8;
|
|
2348
|
-
const desired = Number.isFinite(configured) && configured >= 0 ? configured : Math.min(cpuBudget, defaultWorkerLimit);
|
|
2349
|
-
if (desired <= 1) return 1;
|
|
2350
|
-
// Worker spin-up plus per-worker WASM grammar init dominates on small or
|
|
2351
|
-
// incremental runs; stay sequential until there is enough work to amortize.
|
|
2352
|
-
if (taskCount < 50) return 1;
|
|
2353
|
-
return Math.min(desired, taskCount);
|
|
2354
|
-
}
|
|
2355
|
-
|
|
2356
|
-
function createEmptyWorkerParseStream(tasks, workerCount) {
|
|
2357
|
-
const stats = () => ({
|
|
2358
|
-
worker_tasks: tasks.length,
|
|
2359
|
-
worker_count: Number.isFinite(workerCount) ? workerCount : 0,
|
|
2360
|
-
worker_tasks_assigned: 0,
|
|
2361
|
-
worker_tasks_settled: 0,
|
|
2362
|
-
worker_tasks_unsettled_fallback: tasks.length,
|
|
2363
|
-
worker_results: 0,
|
|
2364
|
-
worker_results_consumed: 0,
|
|
2365
|
-
worker_results_retained: 0,
|
|
2366
|
-
worker_results_retained_peak: 0,
|
|
2367
|
-
worker_results_pending: 0,
|
|
2368
|
-
worker_results_missing: tasks.length,
|
|
2369
|
-
worker_waiters: 0
|
|
2370
|
-
});
|
|
2371
|
-
return {
|
|
2372
|
-
hasTask: () => false,
|
|
2373
|
-
stats,
|
|
2374
|
-
take: async () => undefined,
|
|
2375
|
-
drain: async () => stats()
|
|
2376
|
-
};
|
|
2377
|
-
}
|
|
2378
|
-
|
|
2379
|
-
function startWorkerParseStream(tasks, { workerCount, verbose, workerUrl } = {}) {
|
|
2380
|
-
if (tasks.length === 0) {
|
|
2381
|
-
return createEmptyWorkerParseStream(tasks, workerCount);
|
|
2382
|
-
}
|
|
2383
|
-
|
|
2384
|
-
const resolvedWorkerUrl = workerUrl ?? new URL("./ingest-worker.mjs", import.meta.url);
|
|
2385
|
-
const poolSize = Math.min(workerCount, tasks.length);
|
|
2386
|
-
// Defensive: an undefined/0/negative/NaN workerCount yields poolSize < 1, in
|
|
2387
|
-
// which case no workers are ever spawned and the pool would never resolve.
|
|
2388
|
-
// Return empty so the caller parses everything inline. (Math.min(undefined, n)
|
|
2389
|
-
// is NaN, which is why this is `!(>= 1)` rather than `< 1`.)
|
|
2390
|
-
if (!(poolSize >= 1)) {
|
|
2391
|
-
return createEmptyWorkerParseStream(tasks, workerCount);
|
|
2392
|
-
}
|
|
2393
|
-
|
|
2394
|
-
const taskIds = new Set(tasks.map((task) => task.id));
|
|
2395
|
-
const results = new Map();
|
|
2396
|
-
const missingResults = new Set();
|
|
2397
|
-
const waiters = new Map();
|
|
2398
|
-
const workers = [];
|
|
2399
|
-
const inflight = new Map(); // worker -> taskId being parsed, or null when idle
|
|
2400
|
-
let nextTask = 0;
|
|
2401
|
-
let alive = poolSize;
|
|
2402
|
-
let finished = false;
|
|
2403
|
-
let resolveDone;
|
|
2404
|
-
const done = new Promise((resolve) => {
|
|
2405
|
-
resolveDone = resolve;
|
|
2406
|
-
});
|
|
2407
|
-
const state = {
|
|
2408
|
-
assigned: 0,
|
|
2409
|
-
settled: 0,
|
|
2410
|
-
successful: 0,
|
|
2411
|
-
consumed: 0,
|
|
2412
|
-
retainedPeak: 0
|
|
2413
|
-
};
|
|
2414
|
-
|
|
2415
|
-
const stats = () => ({
|
|
2416
|
-
worker_tasks: tasks.length,
|
|
2417
|
-
worker_count: poolSize,
|
|
2418
|
-
worker_tasks_assigned: state.assigned,
|
|
2419
|
-
worker_tasks_settled: state.settled,
|
|
2420
|
-
worker_tasks_unsettled_fallback: finished ? Math.max(0, tasks.length - state.settled) : 0,
|
|
2421
|
-
worker_results: state.successful,
|
|
2422
|
-
worker_results_consumed: state.consumed,
|
|
2423
|
-
worker_results_retained: results.size,
|
|
2424
|
-
worker_results_retained_peak: state.retainedPeak,
|
|
2425
|
-
worker_results_pending: finished ? 0 : Math.max(0, tasks.length - state.settled),
|
|
2426
|
-
worker_results_missing: tasks.length - state.successful,
|
|
2427
|
-
worker_waiters: waiters.size
|
|
2428
|
-
});
|
|
2429
|
-
|
|
2430
|
-
const finish = () => {
|
|
2431
|
-
if (finished) return;
|
|
2432
|
-
finished = true;
|
|
2433
|
-
for (const resolve of waiters.values()) {
|
|
2434
|
-
resolve(undefined);
|
|
2435
|
-
}
|
|
2436
|
-
waiters.clear();
|
|
2437
|
-
resolveDone();
|
|
2438
|
-
};
|
|
2439
|
-
|
|
2440
|
-
// A task is "settled" once it has a result, was skipped, or its worker
|
|
2441
|
-
// died holding it. Finish when every task is settled, or when no worker is
|
|
2442
|
-
// left alive to make progress (any still-queued tasks then parse inline).
|
|
2443
|
-
const maybeFinish = () => {
|
|
2444
|
-
if (state.settled >= tasks.length || alive <= 0) {
|
|
2445
|
-
finish();
|
|
2446
|
-
}
|
|
2447
|
-
};
|
|
2448
|
-
|
|
2449
|
-
const resolveWaiter = (taskId, result) => {
|
|
2450
|
-
const resolve = waiters.get(taskId);
|
|
2451
|
-
if (!resolve) {
|
|
2452
|
-
return false;
|
|
2453
|
-
}
|
|
2454
|
-
waiters.delete(taskId);
|
|
2455
|
-
if (result !== undefined) {
|
|
2456
|
-
state.consumed += 1;
|
|
2457
|
-
}
|
|
2458
|
-
resolve(result);
|
|
2459
|
-
return true;
|
|
2460
|
-
};
|
|
2461
|
-
|
|
2462
|
-
const settleTask = (taskId, result) => {
|
|
2463
|
-
state.settled += 1;
|
|
2464
|
-
if (result !== undefined) {
|
|
2465
|
-
state.successful += 1;
|
|
2466
|
-
if (!resolveWaiter(taskId, result)) {
|
|
2467
|
-
results.set(taskId, result);
|
|
2468
|
-
state.retainedPeak = Math.max(state.retainedPeak, results.size);
|
|
2469
|
-
}
|
|
2470
|
-
} else {
|
|
2471
|
-
missingResults.add(taskId);
|
|
2472
|
-
resolveWaiter(taskId, undefined);
|
|
2473
|
-
}
|
|
2474
|
-
maybeFinish();
|
|
2475
|
-
};
|
|
2476
|
-
|
|
2477
|
-
const assign = (worker) => {
|
|
2478
|
-
if (nextTask >= tasks.length) {
|
|
2479
|
-
inflight.set(worker, null);
|
|
2480
|
-
worker.postMessage({ type: "shutdown" });
|
|
2481
|
-
return;
|
|
2482
|
-
}
|
|
2483
|
-
const task = tasks[nextTask++];
|
|
2484
|
-
state.assigned += 1;
|
|
2485
|
-
inflight.set(worker, task.id);
|
|
2486
|
-
worker.postMessage({
|
|
2487
|
-
taskId: task.id,
|
|
2488
|
-
ext: task.ext,
|
|
2489
|
-
content: task.content,
|
|
2490
|
-
absolutePath: task.absolutePath,
|
|
2491
|
-
contentLimit: task.contentLimit,
|
|
2492
|
-
filePath: task.path
|
|
2493
|
-
});
|
|
2494
|
-
};
|
|
2495
|
-
|
|
2496
|
-
const onMessage = (worker, message) => {
|
|
2497
|
-
if (finished) return;
|
|
2498
|
-
inflight.set(worker, null);
|
|
2499
|
-
if (message.ok) {
|
|
2500
|
-
settleTask(message.taskId, message.result);
|
|
2501
|
-
} else {
|
|
2502
|
-
if (verbose) {
|
|
2503
|
-
console.log(`[ingest] worker skipped ${message.taskId}: ${message.reason}`);
|
|
2504
|
-
}
|
|
2505
|
-
settleTask(message.taskId, undefined);
|
|
2506
|
-
}
|
|
2507
|
-
if (!finished) {
|
|
2508
|
-
assign(worker);
|
|
2509
|
-
}
|
|
2510
|
-
};
|
|
2511
|
-
|
|
2512
|
-
const onExit = (worker) => {
|
|
2513
|
-
if (finished) return;
|
|
2514
|
-
alive -= 1;
|
|
2515
|
-
const taskId = inflight.get(worker);
|
|
2516
|
-
if (taskId != null) {
|
|
2517
|
-
// Worker exited mid-parse without posting a result (OOM, native abort,
|
|
2518
|
-
// process.exit) and without an 'error' event. Count its in-flight task
|
|
2519
|
-
// as settled so it falls back to inline parsing rather than leaving the
|
|
2520
|
-
// pool waiting on a dead worker forever.
|
|
2521
|
-
if (verbose) {
|
|
2522
|
-
console.log(`[ingest] worker exited mid-task ${taskId}; will parse inline`);
|
|
2523
|
-
}
|
|
2524
|
-
inflight.set(worker, null);
|
|
2525
|
-
settleTask(taskId, undefined);
|
|
2526
|
-
return;
|
|
2527
|
-
}
|
|
2528
|
-
maybeFinish();
|
|
2529
|
-
};
|
|
2530
|
-
|
|
2531
|
-
for (let i = 0; i < poolSize; i += 1) {
|
|
2532
|
-
const worker = new Worker(resolvedWorkerUrl);
|
|
2533
|
-
workers.push(worker);
|
|
2534
|
-
worker.on("message", (message) => onMessage(worker, message));
|
|
2535
|
-
worker.on("error", (error) => {
|
|
2536
|
-
// Uncaught exception in the worker. The 'exit' event that always
|
|
2537
|
-
// follows does the task accounting (idempotent via inflight), so we
|
|
2538
|
-
// only log here to avoid double-counting.
|
|
2539
|
-
if (verbose) {
|
|
2540
|
-
console.log(`[ingest] worker error: ${error.message}`);
|
|
2541
|
-
}
|
|
2542
|
-
});
|
|
2543
|
-
worker.on("exit", () => onExit(worker));
|
|
2544
|
-
}
|
|
2545
|
-
|
|
2546
|
-
for (const worker of workers) {
|
|
2547
|
-
assign(worker);
|
|
2548
|
-
}
|
|
2549
|
-
|
|
2550
|
-
return {
|
|
2551
|
-
hasTask(taskId) {
|
|
2552
|
-
return taskIds.has(taskId);
|
|
2553
|
-
},
|
|
2554
|
-
stats,
|
|
2555
|
-
async take(taskId) {
|
|
2556
|
-
if (!taskIds.has(taskId)) {
|
|
2557
|
-
return undefined;
|
|
2558
|
-
}
|
|
2559
|
-
if (results.has(taskId)) {
|
|
2560
|
-
const result = results.get(taskId);
|
|
2561
|
-
results.delete(taskId);
|
|
2562
|
-
state.consumed += 1;
|
|
2563
|
-
return result;
|
|
2564
|
-
}
|
|
2565
|
-
if (missingResults.has(taskId) || finished) {
|
|
2566
|
-
return undefined;
|
|
2567
|
-
}
|
|
2568
|
-
return new Promise((resolve) => {
|
|
2569
|
-
waiters.set(taskId, resolve);
|
|
2570
|
-
});
|
|
2571
|
-
},
|
|
2572
|
-
async drain() {
|
|
2573
|
-
await done;
|
|
2574
|
-
await Promise.all(workers.map((worker) => worker.terminate().catch(() => {})));
|
|
2575
|
-
return stats();
|
|
2576
|
-
}
|
|
2577
|
-
};
|
|
2578
|
-
}
|
|
2579
|
-
|
|
2580
|
-
// Parse files in a worker pool, returning Map<fileId, parseResult> for the
|
|
2581
|
-
// tasks that completed. Any task not present (worker miss, skip, or a fatal
|
|
2582
|
-
// worker error) is left to the caller's inline parse — the same parse
|
|
2583
|
-
// function on the same content, so output is identical either way. Never
|
|
2584
|
-
// rejects: a fatal worker error resolves with the partial results collected
|
|
2585
|
-
// so far.
|
|
2586
|
-
async function parseFilesInWorkers(tasks, options = {}) {
|
|
2587
|
-
const stream = startWorkerParseStream(tasks, options);
|
|
2588
|
-
const results = new Map();
|
|
2589
|
-
for (const task of tasks) {
|
|
2590
|
-
const result = await stream.take(task.id);
|
|
2591
|
-
if (result) {
|
|
2592
|
-
results.set(task.id, result);
|
|
2593
|
-
}
|
|
2594
|
-
}
|
|
2595
|
-
|
|
2596
|
-
await stream.drain();
|
|
2597
|
-
return results;
|
|
2598
|
-
}
|
|
2599
|
-
|
|
2600
|
-
async function main() {
|
|
2601
|
-
await loadParsers();
|
|
2602
|
-
const { mode, verbose } = parseArgs(process.argv);
|
|
2603
|
-
const memoryTrace = createIngestMemoryTrace();
|
|
2604
|
-
const configPath = path.join(CONTEXT_DIR, "config.yaml");
|
|
2605
|
-
const rulesPath = path.join(CONTEXT_DIR, "rules.yaml");
|
|
2606
|
-
|
|
2607
|
-
if (!fs.existsSync(configPath)) {
|
|
2608
|
-
throw new Error(`Missing config: ${configPath}`);
|
|
2609
|
-
}
|
|
2610
|
-
if (!fs.existsSync(rulesPath)) {
|
|
2611
|
-
throw new Error(`Missing rules: ${rulesPath}`);
|
|
2612
|
-
}
|
|
2613
|
-
|
|
2614
|
-
ensureDirectory(CACHE_DIR);
|
|
2615
|
-
ensureDirectory(DB_IMPORT_DIR);
|
|
2616
|
-
|
|
2617
|
-
const configText = fs.readFileSync(configPath, "utf8");
|
|
2618
|
-
const sourcePaths = parseSourcePaths(configText);
|
|
2619
|
-
if (sourcePaths.length === 0) {
|
|
2620
|
-
throw new Error("No source_paths found in .context/config.yaml");
|
|
2621
|
-
}
|
|
2622
|
-
|
|
2623
|
-
const rules = parseRules(fs.readFileSync(rulesPath, "utf8"));
|
|
2624
|
-
memoryTrace.checkpoint("scan:start", {
|
|
2625
|
-
mode,
|
|
2626
|
-
source_paths: sourcePaths.length,
|
|
2627
|
-
rules: rules.length
|
|
2628
|
-
});
|
|
2629
|
-
const { candidates, incrementalMode, deletedRelPaths } = collectCandidateFiles(sourcePaths, mode);
|
|
2630
|
-
const chunkWindowLines = parsePositiveIntegerEnv(
|
|
2631
|
-
"CORTEX_CHUNK_WINDOW_LINES",
|
|
2632
|
-
DEFAULT_CHUNK_WINDOW_LINES
|
|
2633
|
-
);
|
|
2634
|
-
const chunkOverlapLines = Math.max(
|
|
2635
|
-
0,
|
|
2636
|
-
Math.min(
|
|
2637
|
-
chunkWindowLines - 1,
|
|
2638
|
-
parseNonNegativeIntegerEnv("CORTEX_CHUNK_OVERLAP_LINES", DEFAULT_CHUNK_OVERLAP_LINES)
|
|
2639
|
-
)
|
|
2640
|
-
);
|
|
2641
|
-
const chunkSplitMinLines = Math.max(
|
|
2642
|
-
chunkWindowLines + 1,
|
|
2643
|
-
parsePositiveIntegerEnv("CORTEX_CHUNK_SPLIT_MIN_LINES", DEFAULT_CHUNK_SPLIT_MIN_LINES)
|
|
2644
|
-
);
|
|
2645
|
-
const chunkMaxWindows = parsePositiveIntegerEnv(
|
|
2646
|
-
"CORTEX_CHUNK_MAX_WINDOWS",
|
|
2647
|
-
DEFAULT_CHUNK_MAX_WINDOWS
|
|
2648
|
-
);
|
|
2649
|
-
|
|
2650
|
-
const fileRecordMap = new Map();
|
|
2651
|
-
const adrRecordMap = new Map();
|
|
2652
|
-
const skipped = {
|
|
2653
|
-
unsupported: 0,
|
|
2654
|
-
tooLarge: 0,
|
|
2655
|
-
binary: 0
|
|
2656
|
-
};
|
|
2657
|
-
|
|
2658
|
-
if (incrementalMode) {
|
|
2659
|
-
const existingFiles = readJsonlSafe(path.join(CACHE_DIR, "entities.file.jsonl"));
|
|
2660
|
-
for (const record of existingFiles) {
|
|
2661
|
-
if (!record || typeof record !== "object") continue;
|
|
2662
|
-
const filePath = toPosixPath(String(record.path ?? ""));
|
|
2663
|
-
if (!filePath || !hasSourcePrefix(filePath, sourcePaths)) {
|
|
2664
|
-
continue;
|
|
2665
|
-
}
|
|
2666
|
-
const absolutePath = path.resolve(REPO_ROOT, filePath);
|
|
2667
|
-
if (!fs.existsSync(absolutePath)) {
|
|
2668
|
-
continue;
|
|
2669
|
-
}
|
|
2670
|
-
fileRecordMap.set(String(record.id ?? `file:${filePath}`), {
|
|
2671
|
-
...record,
|
|
2672
|
-
id: String(record.id ?? `file:${filePath}`),
|
|
2673
|
-
path: filePath,
|
|
2674
|
-
kind: String(record.kind ?? detectKind(filePath)),
|
|
2675
|
-
content: String(record.content ?? "")
|
|
2676
|
-
});
|
|
2677
|
-
}
|
|
2678
|
-
|
|
2679
|
-
const existingAdrs = readJsonlSafe(path.join(CACHE_DIR, "entities.adr.jsonl"));
|
|
2680
|
-
for (const adr of existingAdrs) {
|
|
2681
|
-
if (!adr || typeof adr !== "object") continue;
|
|
2682
|
-
const adrPath = toPosixPath(String(adr.path ?? ""));
|
|
2683
|
-
if (!adrPath || !hasSourcePrefix(adrPath, sourcePaths)) {
|
|
2684
|
-
continue;
|
|
2685
|
-
}
|
|
2686
|
-
if (!fs.existsSync(path.resolve(REPO_ROOT, adrPath))) {
|
|
2687
|
-
continue;
|
|
2688
|
-
}
|
|
2689
|
-
adrRecordMap.set(String(adr.id ?? ""), {
|
|
2690
|
-
...adr,
|
|
2691
|
-
id: String(adr.id ?? ""),
|
|
2692
|
-
path: adrPath
|
|
2693
|
-
});
|
|
2694
|
-
}
|
|
2695
|
-
}
|
|
2696
|
-
|
|
2697
|
-
for (const relPath of deletedRelPaths) {
|
|
2698
|
-
fileRecordMap.delete(`file:${relPath}`);
|
|
2699
|
-
const relPrefix = relPath.endsWith("/") ? relPath : `${relPath}/`;
|
|
2700
|
-
for (const [fileId, fileRecord] of fileRecordMap.entries()) {
|
|
2701
|
-
if (String(fileRecord.path ?? "").startsWith(relPrefix)) {
|
|
2702
|
-
fileRecordMap.delete(fileId);
|
|
2703
|
-
}
|
|
2704
|
-
}
|
|
2705
|
-
|
|
2706
|
-
for (const [adrId, adrRecord] of adrRecordMap.entries()) {
|
|
2707
|
-
if (adrRecord.path === relPath || String(adrRecord.path ?? "").startsWith(relPrefix)) {
|
|
2708
|
-
adrRecordMap.delete(adrId);
|
|
2709
|
-
}
|
|
2710
|
-
}
|
|
2711
|
-
}
|
|
2712
|
-
|
|
2713
|
-
for (const absolutePath of [...candidates].sort()) {
|
|
2714
|
-
const relPath = toPosixPath(path.relative(REPO_ROOT, absolutePath));
|
|
2715
|
-
if (!isTextFile(relPath)) {
|
|
2716
|
-
skipped.unsupported += 1;
|
|
2717
|
-
if (verbose) console.log(`[ingest] skip unsupported: ${relPath}`);
|
|
2718
|
-
continue;
|
|
2719
|
-
}
|
|
2720
|
-
|
|
2721
|
-
const stats = fs.statSync(absolutePath);
|
|
2722
|
-
if (stats.size > MAX_FILE_BYTES) {
|
|
2723
|
-
skipped.tooLarge += 1;
|
|
2724
|
-
if (verbose) console.log(`[ingest] skip large: ${relPath}`);
|
|
2725
|
-
continue;
|
|
2726
|
-
}
|
|
2727
|
-
|
|
2728
|
-
const buffer = fs.readFileSync(absolutePath);
|
|
2729
|
-
if (isBinaryBuffer(buffer)) {
|
|
2730
|
-
skipped.binary += 1;
|
|
2731
|
-
if (verbose) console.log(`[ingest] skip binary: ${relPath}`);
|
|
2732
|
-
continue;
|
|
2733
|
-
}
|
|
2734
|
-
|
|
2735
|
-
const content = buffer.toString("utf8");
|
|
2736
|
-
const kind = detectKind(relPath);
|
|
2737
|
-
const id = `file:${relPath}`;
|
|
2738
|
-
const updatedAt = stats.mtime.toISOString();
|
|
2739
|
-
const sourceOfTruth = kind === "ADR";
|
|
2740
|
-
const trustLevel = trustLevelForKind(kind);
|
|
2741
|
-
|
|
2742
|
-
const fileRecord = {
|
|
2743
|
-
id,
|
|
2744
|
-
path: relPath,
|
|
2745
|
-
kind,
|
|
2746
|
-
checksum: checksum(buffer),
|
|
2747
|
-
updated_at: updatedAt,
|
|
2748
|
-
source_of_truth: sourceOfTruth,
|
|
2749
|
-
trust_level: trustLevel,
|
|
2750
|
-
status: "active",
|
|
2751
|
-
size_bytes: stats.size,
|
|
2752
|
-
excerpt: normalizeWhitespace(content).slice(0, 500),
|
|
2753
|
-
content: content.slice(0, MAX_CONTENT_CHARS)
|
|
2754
|
-
};
|
|
2755
|
-
fileRecordMap.set(fileRecord.id, fileRecord);
|
|
2756
|
-
|
|
2757
|
-
if (kind === "ADR") {
|
|
2758
|
-
const title = extractTitle(content, path.basename(relPath, path.extname(relPath)));
|
|
2759
|
-
const adrRecord = {
|
|
2760
|
-
id: `adr:${path.basename(relPath, path.extname(relPath)).toLowerCase()}`,
|
|
2761
|
-
path: relPath,
|
|
2762
|
-
title,
|
|
2763
|
-
body: content.slice(0, MAX_BODY_CHARS),
|
|
2764
|
-
decision_date: parseDecisionDate(content, updatedAt),
|
|
2765
|
-
supersedes_id: "",
|
|
2766
|
-
source_of_truth: true,
|
|
2767
|
-
trust_level: 95,
|
|
2768
|
-
status: "active"
|
|
2769
|
-
};
|
|
2770
|
-
adrRecordMap.set(adrRecord.id, adrRecord);
|
|
2771
|
-
} else {
|
|
2772
|
-
for (const [adrId, adrRecord] of adrRecordMap.entries()) {
|
|
2773
|
-
if (adrRecord.path === relPath) {
|
|
2774
|
-
adrRecordMap.delete(adrId);
|
|
2775
|
-
}
|
|
2776
|
-
}
|
|
2777
|
-
}
|
|
2778
|
-
}
|
|
2779
|
-
|
|
2780
|
-
const fileRecords = [...fileRecordMap.values()].sort((a, b) => a.path.localeCompare(b.path));
|
|
2781
|
-
const adrRecords = [...adrRecordMap.values()].sort((a, b) => a.path.localeCompare(b.path));
|
|
2782
|
-
memoryTrace.checkpoint("scan:file_records", {
|
|
2783
|
-
candidates: candidates.size,
|
|
2784
|
-
incremental_mode: incrementalMode,
|
|
2785
|
-
deleted_paths: deletedRelPaths.length,
|
|
2786
|
-
files: fileRecords.length,
|
|
2787
|
-
adrs: adrRecords.length,
|
|
2788
|
-
skipped_unsupported: skipped.unsupported,
|
|
2789
|
-
skipped_too_large: skipped.tooLarge,
|
|
2790
|
-
skipped_binary: skipped.binary
|
|
2791
|
-
});
|
|
2792
|
-
const csharpFileCount = fileRecords.filter((record) => path.extname(record.path).toLowerCase() === ".cs").length;
|
|
2793
|
-
const csharpRuntime = csharpFileCount > 0 ? getCSharpParserRuntime() : null;
|
|
2794
|
-
const indexedFileIds = new Set(fileRecords.map((record) => record.id));
|
|
2795
|
-
const changedFileIds = new Set(
|
|
2796
|
-
[...candidates].map((absolutePath) => `file:${toPosixPath(path.relative(REPO_ROOT, absolutePath))}`)
|
|
2797
|
-
);
|
|
2798
|
-
|
|
2799
|
-
const {
|
|
2800
|
-
chunkRecordMap,
|
|
2801
|
-
definesRelationMap,
|
|
2802
|
-
callsRelationMap,
|
|
2803
|
-
importsRelationMap,
|
|
2804
|
-
callsSqlRelationMap
|
|
2805
|
-
} = incrementalMode
|
|
2806
|
-
? hydrateIncrementalChunkState(fileRecords)
|
|
2807
|
-
: {
|
|
2808
|
-
chunkRecordMap: new Map(),
|
|
2809
|
-
definesRelationMap: new Map(),
|
|
2810
|
-
callsRelationMap: new Map(),
|
|
2811
|
-
importsRelationMap: new Map(),
|
|
2812
|
-
callsSqlRelationMap: new Map()
|
|
2813
|
-
};
|
|
2814
|
-
memoryTrace.checkpoint("hydration:complete", {
|
|
2815
|
-
incremental_mode: incrementalMode,
|
|
2816
|
-
cached_chunks: chunkRecordMap.size,
|
|
2817
|
-
cached_defines_relations: definesRelationMap.size,
|
|
2818
|
-
cached_calls_relations: callsRelationMap.size,
|
|
2819
|
-
cached_imports_relations: importsRelationMap.size,
|
|
2820
|
-
cached_calls_sql_relations: callsSqlRelationMap.size
|
|
2821
|
-
});
|
|
2822
|
-
|
|
2823
|
-
const cachedChunkFileIds = new Set(
|
|
2824
|
-
[...chunkRecordMap.values()].map((record) => String(record.file_id ?? "")).filter(Boolean)
|
|
2825
|
-
);
|
|
2826
|
-
const cachedSqlReferenceFileIds = new Set(
|
|
2827
|
-
[...callsSqlRelationMap.values()].map((record) => String(record.from ?? "")).filter(Boolean)
|
|
2828
|
-
);
|
|
2829
|
-
const usesConfigKeyRelationMap = new Map();
|
|
2830
|
-
const usesResourceKeyRelationMap = new Map();
|
|
2831
|
-
const usesSettingKeyRelationMap = new Map();
|
|
2832
|
-
|
|
2833
|
-
// Extract chunks from changed or uncached code files
|
|
2834
|
-
let windowedChunkCount = 0;
|
|
2835
|
-
let {
|
|
2836
|
-
sqlChunkIdsByAlias,
|
|
2837
|
-
configChunkIdsByAlias,
|
|
2838
|
-
resourceChunkIdsByAlias,
|
|
2839
|
-
settingChunkIdsByAlias
|
|
2840
|
-
} = buildChunkAliasIndexes([...chunkRecordMap.values()]);
|
|
2841
|
-
const deferredSqlCallEdges = [];
|
|
2842
|
-
|
|
2843
|
-
// C# project-wide batch parse: when Roslyn is available and batching
|
|
2844
|
-
// isn't disabled, compile all .cs files together via CSharpCompilation
|
|
2845
|
-
// to enable SemanticModel-resolved calls (e.g. "System.IO.File.ReadAllText"
|
|
2846
|
-
// instead of bare "ReadAllText"). Falls back silently to per-file parse
|
|
2847
|
-
// if batch isn't usable.
|
|
2848
|
-
const csharpBatchCache = new Map();
|
|
2849
|
-
if (
|
|
2850
|
-
hasCSharpProjectParser() &&
|
|
2851
|
-
isCSharpParserAvailable() &&
|
|
2852
|
-
process.env.CORTEX_CSHARP_BATCH !== "never"
|
|
2853
|
-
) {
|
|
2854
|
-
const csharpFilesForBatch = fileRecords.filter((r) => {
|
|
2855
|
-
if (r.kind !== "CODE") return false;
|
|
2856
|
-
if (path.extname(r.path).toLowerCase() !== ".cs") return false;
|
|
2857
|
-
return !incrementalMode || changedFileIds.has(r.id) || !cachedChunkFileIds.has(r.id);
|
|
2858
|
-
});
|
|
2859
|
-
if (csharpFilesForBatch.length > 0) {
|
|
2860
|
-
const allCsharpInputs = fileRecords
|
|
2861
|
-
.filter((r) => r.kind === "CODE" && path.extname(r.path).toLowerCase() === ".cs")
|
|
2862
|
-
.map((r) => ({ path: r.path, content: r.content }));
|
|
2863
|
-
try {
|
|
2864
|
-
const batchResult = parseCSharpProject(allCsharpInputs);
|
|
2865
|
-
for (const [filePath, result] of batchResult) {
|
|
2866
|
-
csharpBatchCache.set(filePath, result);
|
|
2867
|
-
}
|
|
2868
|
-
} catch (error) {
|
|
2869
|
-
if (verbose) {
|
|
2870
|
-
console.log(`[ingest] C# batch parse failed, falling back per-file: ${error.message}`);
|
|
2871
|
-
}
|
|
2872
|
-
}
|
|
2873
|
-
}
|
|
2874
|
-
}
|
|
2875
|
-
|
|
2876
|
-
// Determine which files need parsing (single pass over the gates, shared by
|
|
2877
|
-
// the worker dispatch and the merge loop below so the two cannot diverge).
|
|
2878
|
-
const parseEligible = new Map();
|
|
2879
|
-
for (const fileRecord of fileRecords) {
|
|
2880
|
-
const ext = path.extname(fileRecord.path).toLowerCase();
|
|
2881
|
-
const parser = getChunkParserForExtension(ext);
|
|
2882
|
-
const isStructuredNonCodeChunk = STRUCTURED_NON_CODE_CHUNK_EXTENSIONS.has(ext);
|
|
2883
|
-
if (fileRecord.kind !== "CODE" && !isStructuredNonCodeChunk) continue;
|
|
2884
|
-
if (!parser) continue;
|
|
2885
|
-
if (typeof parser.isAvailable === "function" && !(await parser.isAvailable())) continue;
|
|
2886
|
-
|
|
2887
|
-
const shouldParseFile =
|
|
2888
|
-
!incrementalMode || changedFileIds.has(fileRecord.id) || !cachedChunkFileIds.has(fileRecord.id);
|
|
2889
|
-
if (!shouldParseFile) {
|
|
2890
|
-
continue;
|
|
2891
|
-
}
|
|
2892
|
-
parseEligible.set(fileRecord.id, { parser, ext });
|
|
2893
|
-
}
|
|
2894
|
-
memoryTrace.checkpoint("parse:eligible", {
|
|
2895
|
-
files: fileRecords.length,
|
|
2896
|
-
parse_eligible: parseEligible.size,
|
|
2897
|
-
csharp_batch_cached: csharpBatchCache.size
|
|
2898
|
-
});
|
|
2899
|
-
|
|
2900
|
-
// Parse parallel-safe files (tree-sitter/JS, no subprocess, no cross-file
|
|
2901
|
-
// state) in a worker pool. C# batch-cached files and any worker miss fall
|
|
2902
|
-
// through to inline parsing in the loop, so the result is byte-identical to
|
|
2903
|
-
// the sequential path regardless of where a file actually parsed.
|
|
2904
|
-
const workerTasks = fileRecords
|
|
2905
|
-
.filter((fileRecord) => {
|
|
2906
|
-
const eligible = parseEligible.get(fileRecord.id);
|
|
2907
|
-
if (!eligible) return false;
|
|
2908
|
-
if (!PARALLEL_SAFE_LANGUAGES.has(eligible.parser.language)) return false;
|
|
2909
|
-
if (eligible.parser.language === "csharp" && csharpBatchCache.has(fileRecord.path)) return false;
|
|
2910
|
-
return true;
|
|
2911
|
-
})
|
|
2912
|
-
.map((fileRecord) => ({
|
|
2913
|
-
id: fileRecord.id,
|
|
2914
|
-
ext: parseEligible.get(fileRecord.id).ext,
|
|
2915
|
-
absolutePath: path.resolve(REPO_ROOT, fileRecord.path),
|
|
2916
|
-
contentLimit: MAX_CONTENT_CHARS,
|
|
2917
|
-
path: fileRecord.path
|
|
2918
|
-
}));
|
|
2919
|
-
const workerCount = resolveIngestWorkerCount(workerTasks.length);
|
|
2920
|
-
memoryTrace.checkpoint("parse:workers_start", {
|
|
2921
|
-
worker_tasks: workerTasks.length,
|
|
2922
|
-
worker_count: workerCount
|
|
2923
|
-
});
|
|
2924
|
-
const workerStream =
|
|
2925
|
-
workerCount > 1 ? startWorkerParseStream(workerTasks, { workerCount, verbose }) : null;
|
|
2926
|
-
if (!workerStream) {
|
|
2927
|
-
memoryTrace.checkpoint("parse:workers_complete", {
|
|
2928
|
-
worker_tasks: workerTasks.length,
|
|
2929
|
-
worker_count: workerCount,
|
|
2930
|
-
worker_results: 0,
|
|
2931
|
-
worker_results_consumed: 0,
|
|
2932
|
-
worker_results_retained: 0,
|
|
2933
|
-
worker_results_retained_peak: 0,
|
|
2934
|
-
worker_results_pending: 0,
|
|
2935
|
-
worker_results_missing: workerTasks.length
|
|
2936
|
-
});
|
|
2937
|
-
}
|
|
2938
|
-
|
|
2939
|
-
for (const fileRecord of fileRecords) {
|
|
2940
|
-
const eligible = parseEligible.get(fileRecord.id);
|
|
2941
|
-
if (!eligible) continue;
|
|
2942
|
-
const { parser } = eligible;
|
|
2943
|
-
|
|
2944
|
-
removeChunkStateForFile(
|
|
2945
|
-
fileRecord.id,
|
|
2946
|
-
chunkRecordMap,
|
|
2947
|
-
definesRelationMap,
|
|
2948
|
-
callsRelationMap,
|
|
2949
|
-
importsRelationMap,
|
|
2950
|
-
callsSqlRelationMap
|
|
2951
|
-
);
|
|
2952
|
-
|
|
2953
|
-
try {
|
|
2954
|
-
let parseResult;
|
|
2955
|
-
if (parser.language === "csharp" && csharpBatchCache.has(fileRecord.path)) {
|
|
2956
|
-
parseResult = csharpBatchCache.get(fileRecord.path);
|
|
2957
|
-
} else if (workerStream?.hasTask(fileRecord.id)) {
|
|
2958
|
-
parseResult = await workerStream.take(fileRecord.id);
|
|
2959
|
-
if (!parseResult) {
|
|
2960
|
-
parseResult = await parser.parse(fileRecord.content, fileRecord.path, parser.language);
|
|
2961
|
-
}
|
|
2962
|
-
} else {
|
|
2963
|
-
parseResult = await parser.parse(fileRecord.content, fileRecord.path, parser.language);
|
|
2964
|
-
}
|
|
2965
|
-
|
|
2966
|
-
if (parseResult.errors.length > 0 && verbose) {
|
|
2967
|
-
console.log(`[ingest] parse errors in ${fileRecord.path}:`, parseResult.errors[0].message);
|
|
2968
|
-
}
|
|
2969
|
-
|
|
2970
|
-
const parsedChunks = [];
|
|
2971
|
-
const chunkIdsByName = new Map();
|
|
2972
|
-
|
|
2973
|
-
for (const chunk of parseResult.chunks) {
|
|
2974
|
-
const chunkId = chunkIdFor(fileRecord.path, chunk);
|
|
2975
|
-
parsedChunks.push({ chunk, chunkId });
|
|
2976
|
-
if (!chunkIdsByName.has(chunk.name)) {
|
|
2977
|
-
chunkIdsByName.set(chunk.name, []);
|
|
2978
|
-
}
|
|
2979
|
-
chunkIdsByName.get(chunk.name).push(chunkId);
|
|
2980
|
-
if (parser.language === "sql") {
|
|
2981
|
-
for (const alias of sqlChunkAliases(chunk.name)) {
|
|
2982
|
-
if (!sqlChunkIdsByAlias.has(alias)) {
|
|
2983
|
-
sqlChunkIdsByAlias.set(alias, []);
|
|
2984
|
-
}
|
|
2985
|
-
sqlChunkIdsByAlias.get(alias).push(chunkId);
|
|
2986
|
-
}
|
|
2987
|
-
deferredSqlCallEdges.push({
|
|
2988
|
-
chunkId,
|
|
2989
|
-
calls: Array.isArray(chunk.calls) ? chunk.calls : []
|
|
2990
|
-
});
|
|
2991
|
-
} else if (parser.language === "config") {
|
|
2992
|
-
for (const alias of configChunkAliases(chunk)) {
|
|
2993
|
-
if (!configChunkIdsByAlias.has(alias)) {
|
|
2994
|
-
configChunkIdsByAlias.set(alias, []);
|
|
2995
|
-
}
|
|
2996
|
-
configChunkIdsByAlias.get(alias).push(chunkId);
|
|
2997
|
-
}
|
|
2998
|
-
deferredSqlCallEdges.push({
|
|
2999
|
-
chunkId,
|
|
3000
|
-
calls: Array.isArray(chunk.calls) ? chunk.calls : []
|
|
3001
|
-
});
|
|
3002
|
-
} else if (parser.language === "resource") {
|
|
3003
|
-
for (const alias of namedEntryChunkAliases(chunk)) {
|
|
3004
|
-
if (!resourceChunkIdsByAlias.has(alias)) {
|
|
3005
|
-
resourceChunkIdsByAlias.set(alias, []);
|
|
3006
|
-
}
|
|
3007
|
-
resourceChunkIdsByAlias.get(alias).push(chunkId);
|
|
3008
|
-
}
|
|
3009
|
-
deferredSqlCallEdges.push({
|
|
3010
|
-
chunkId,
|
|
3011
|
-
calls: Array.isArray(chunk.calls) ? chunk.calls : []
|
|
3012
|
-
});
|
|
3013
|
-
} else if (parser.language === "settings") {
|
|
3014
|
-
for (const alias of namedEntryChunkAliases(chunk)) {
|
|
3015
|
-
if (!settingChunkIdsByAlias.has(alias)) {
|
|
3016
|
-
settingChunkIdsByAlias.set(alias, []);
|
|
3017
|
-
}
|
|
3018
|
-
settingChunkIdsByAlias.get(alias).push(chunkId);
|
|
3019
|
-
}
|
|
3020
|
-
deferredSqlCallEdges.push({
|
|
3021
|
-
chunkId,
|
|
3022
|
-
calls: Array.isArray(chunk.calls) ? chunk.calls : []
|
|
3023
|
-
});
|
|
3024
|
-
}
|
|
3025
|
-
|
|
3026
|
-
const chunkRecord = {
|
|
3027
|
-
id: chunkId,
|
|
3028
|
-
file_id: fileRecord.id,
|
|
3029
|
-
name: chunk.name,
|
|
3030
|
-
kind: chunk.kind,
|
|
3031
|
-
signature: chunk.signature,
|
|
3032
|
-
body: chunk.body.slice(0, MAX_BODY_CHARS), // Limit chunk body size
|
|
3033
|
-
description: generateChunkDescription(chunk),
|
|
3034
|
-
start_line: chunk.startLine,
|
|
3035
|
-
end_line: chunk.endLine,
|
|
3036
|
-
language: chunk.language,
|
|
3037
|
-
exported: Boolean(chunk.exported),
|
|
3038
|
-
checksum: checksum(Buffer.from(chunk.body)),
|
|
3039
|
-
updated_at: fileRecord.updated_at,
|
|
3040
|
-
trust_level: fileRecord.trust_level,
|
|
3041
|
-
status:
|
|
3042
|
-
typeof fileRecord.status === "string" && fileRecord.status.trim().length > 0
|
|
3043
|
-
? fileRecord.status
|
|
3044
|
-
: "active",
|
|
3045
|
-
source_of_truth: Boolean(fileRecord.source_of_truth)
|
|
3046
|
-
};
|
|
3047
|
-
chunkRecordMap.set(chunkId, chunkRecord);
|
|
3048
|
-
|
|
3049
|
-
// DEFINES relation: File -> Chunk
|
|
3050
|
-
definesRelationMap.set(relationKey(fileRecord.id, chunkId), {
|
|
3051
|
-
from: fileRecord.id,
|
|
3052
|
-
to: chunkId
|
|
3053
|
-
});
|
|
3054
|
-
|
|
3055
|
-
const windows = splitChunkIntoWindows(chunkRecord, {
|
|
3056
|
-
windowLines: chunkWindowLines,
|
|
3057
|
-
overlapLines: chunkOverlapLines,
|
|
3058
|
-
splitMinLines: chunkSplitMinLines,
|
|
3059
|
-
maxWindows: chunkMaxWindows,
|
|
3060
|
-
chunkBody: chunk.body
|
|
3061
|
-
});
|
|
3062
|
-
if (windows.length > 0) {
|
|
3063
|
-
windowedChunkCount += windows.length;
|
|
3064
|
-
for (const windowChunk of windows) {
|
|
3065
|
-
chunkRecordMap.set(windowChunk.id, windowChunk);
|
|
3066
|
-
definesRelationMap.set(relationKey(fileRecord.id, windowChunk.id), {
|
|
3067
|
-
from: fileRecord.id,
|
|
3068
|
-
to: windowChunk.id
|
|
3069
|
-
});
|
|
3070
|
-
}
|
|
3071
|
-
}
|
|
3072
|
-
|
|
3073
|
-
// IMPORTS relations: Chunk -> File
|
|
3074
|
-
for (const importPath of chunk.imports || []) {
|
|
3075
|
-
const targetFileId = resolveRelativeImportTargetId(fileRecord.path, importPath, indexedFileIds);
|
|
3076
|
-
if (!targetFileId) {
|
|
3077
|
-
continue;
|
|
3078
|
-
}
|
|
3079
|
-
|
|
3080
|
-
importsRelationMap.set(relationKey(chunkId, targetFileId, importPath), {
|
|
3081
|
-
from: chunkId,
|
|
3082
|
-
to: targetFileId,
|
|
3083
|
-
import_name: importPath
|
|
3084
|
-
});
|
|
3085
|
-
}
|
|
3086
|
-
}
|
|
3087
|
-
|
|
3088
|
-
const seenCallEdges = new Set();
|
|
3089
|
-
for (const { chunk, chunkId } of parsedChunks) {
|
|
3090
|
-
// CALLS relations: Chunk -> Chunk (within same file)
|
|
3091
|
-
for (const calledName of chunk.calls || []) {
|
|
3092
|
-
const targetChunkIds = chunkIdsByName.get(calledName) || [];
|
|
3093
|
-
for (const targetChunkId of targetChunkIds) {
|
|
3094
|
-
const callKey = `${chunkId}|${targetChunkId}|direct`;
|
|
3095
|
-
if (seenCallEdges.has(callKey)) {
|
|
3096
|
-
continue;
|
|
3097
|
-
}
|
|
3098
|
-
seenCallEdges.add(callKey);
|
|
3099
|
-
callsRelationMap.set(relationKey(chunkId, targetChunkId, "direct"), {
|
|
3100
|
-
from: chunkId,
|
|
3101
|
-
to: targetChunkId,
|
|
3102
|
-
call_type: "direct"
|
|
3103
|
-
});
|
|
3104
|
-
}
|
|
3105
|
-
}
|
|
3106
|
-
}
|
|
3107
|
-
} catch (error) {
|
|
3108
|
-
if (verbose) {
|
|
3109
|
-
console.log(`[ingest] failed to parse ${fileRecord.path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3110
|
-
}
|
|
3111
|
-
}
|
|
3112
|
-
}
|
|
3113
|
-
const finalWorkerStats = workerStream ? await workerStream.drain() : null;
|
|
3114
|
-
if (finalWorkerStats) {
|
|
3115
|
-
memoryTrace.checkpoint("parse:workers_complete", finalWorkerStats);
|
|
3116
|
-
if (verbose) {
|
|
3117
|
-
console.log(
|
|
3118
|
-
`[ingest] parsed ${finalWorkerStats.worker_results}/${workerTasks.length} files across ${workerCount} workers`
|
|
3119
|
-
);
|
|
3120
|
-
}
|
|
3121
|
-
}
|
|
3122
|
-
memoryTrace.checkpoint("parse:merge_complete", {
|
|
3123
|
-
chunk_map: chunkRecordMap.size,
|
|
3124
|
-
defines_relations: definesRelationMap.size,
|
|
3125
|
-
calls_relations: callsRelationMap.size,
|
|
3126
|
-
imports_relations: importsRelationMap.size,
|
|
3127
|
-
calls_sql_relations: callsSqlRelationMap.size,
|
|
3128
|
-
deferred_sql_edges: deferredSqlCallEdges.length,
|
|
3129
|
-
windowed_chunks: windowedChunkCount,
|
|
3130
|
-
worker_results_retained: finalWorkerStats?.worker_results_retained ?? 0,
|
|
3131
|
-
worker_results_retained_peak: finalWorkerStats?.worker_results_retained_peak ?? 0,
|
|
3132
|
-
worker_results_pending: finalWorkerStats?.worker_results_pending ?? 0
|
|
3133
|
-
});
|
|
3134
|
-
|
|
3135
|
-
const chunkRecords = [...chunkRecordMap.values()].sort((a, b) => String(a.id).localeCompare(String(b.id)));
|
|
3136
|
-
({
|
|
3137
|
-
sqlChunkIdsByAlias,
|
|
3138
|
-
configChunkIdsByAlias,
|
|
3139
|
-
resourceChunkIdsByAlias,
|
|
3140
|
-
settingChunkIdsByAlias
|
|
3141
|
-
} = buildChunkAliasIndexes(chunkRecords));
|
|
3142
|
-
|
|
3143
|
-
// Filter CALLS relations to only valid targets (chunks that actually exist)
|
|
3144
|
-
const chunkIdSet = new Set(chunkRecords.map(c => c.id));
|
|
3145
|
-
const validDefinesRelations = [...definesRelationMap.values()].filter(
|
|
3146
|
-
(rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
|
|
3147
|
-
);
|
|
3148
|
-
const totalCallsRelations = callsRelationMap.size;
|
|
3149
|
-
for (const edge of deferredSqlCallEdges) {
|
|
3150
|
-
for (const calledName of edge.calls) {
|
|
3151
|
-
for (const alias of sqlChunkAliases(calledName)) {
|
|
3152
|
-
const targetChunkIds = sqlChunkIdsByAlias.get(alias) || [];
|
|
3153
|
-
for (const targetChunkId of targetChunkIds) {
|
|
3154
|
-
if (targetChunkId === edge.chunkId) {
|
|
3155
|
-
continue;
|
|
3156
|
-
}
|
|
3157
|
-
callsRelationMap.set(relationKey(edge.chunkId, targetChunkId, "sql_reference"), {
|
|
3158
|
-
from: edge.chunkId,
|
|
3159
|
-
to: targetChunkId,
|
|
3160
|
-
call_type: "sql_reference"
|
|
3161
|
-
});
|
|
3162
|
-
}
|
|
3163
|
-
}
|
|
3164
|
-
}
|
|
3165
|
-
}
|
|
3166
|
-
const validCallsRelations = [...callsRelationMap.values()].filter(
|
|
3167
|
-
(rel) => chunkIdSet.has(rel.from) && chunkIdSet.has(rel.to)
|
|
3168
|
-
);
|
|
3169
|
-
const validImportsRelations = [...importsRelationMap.values()].filter(
|
|
3170
|
-
(rel) => chunkIdSet.has(rel.from) && indexedFileIds.has(rel.to)
|
|
3171
|
-
);
|
|
3172
|
-
const sqlDefinitionsChanged =
|
|
3173
|
-
incrementalMode &&
|
|
3174
|
-
fileRecords.some(
|
|
3175
|
-
(fileRecord) =>
|
|
3176
|
-
changedFileIds.has(fileRecord.id) && path.extname(fileRecord.path).toLowerCase() === ".sql"
|
|
3177
|
-
);
|
|
3178
|
-
const sqlResourceReferenceMap = buildSqlResourceReferenceMap(fileRecords);
|
|
3179
|
-
for (const fileRecord of fileRecords) {
|
|
3180
|
-
if (!shouldExtractSqlReferences(fileRecord.path)) {
|
|
3181
|
-
continue;
|
|
3182
|
-
}
|
|
3183
|
-
|
|
3184
|
-
const shouldAnalyzeFile =
|
|
3185
|
-
!incrementalMode ||
|
|
3186
|
-
sqlDefinitionsChanged ||
|
|
3187
|
-
changedFileIds.has(fileRecord.id) ||
|
|
3188
|
-
!cachedSqlReferenceFileIds.has(fileRecord.id);
|
|
3189
|
-
if (!shouldAnalyzeFile) {
|
|
3190
|
-
continue;
|
|
3191
|
-
}
|
|
3192
|
-
|
|
3193
|
-
for (const [key, relation] of callsSqlRelationMap.entries()) {
|
|
3194
|
-
if (relation.from === fileRecord.id) {
|
|
3195
|
-
callsSqlRelationMap.delete(key);
|
|
3196
|
-
}
|
|
3197
|
-
}
|
|
3198
|
-
|
|
3199
|
-
for (const refName of extractSqlObjectReferencesFromContent(
|
|
3200
|
-
fileRecord.content,
|
|
3201
|
-
fileRecord.path,
|
|
3202
|
-
sqlResourceReferenceMap
|
|
3203
|
-
)) {
|
|
3204
|
-
for (const alias of sqlChunkAliases(refName)) {
|
|
3205
|
-
const targetChunkIds = sqlChunkIdsByAlias.get(alias) || [];
|
|
3206
|
-
for (const targetChunkId of targetChunkIds) {
|
|
3207
|
-
callsSqlRelationMap.set(relationKey(fileRecord.id, targetChunkId, refName), {
|
|
3208
|
-
from: fileRecord.id,
|
|
3209
|
-
to: targetChunkId,
|
|
3210
|
-
note: refName
|
|
3211
|
-
});
|
|
3212
|
-
}
|
|
3213
|
-
}
|
|
3214
|
-
}
|
|
3215
|
-
}
|
|
3216
|
-
const validCallsSqlRelations = [...callsSqlRelationMap.values()].filter(
|
|
3217
|
-
(rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
|
|
3218
|
-
);
|
|
3219
|
-
for (const fileRecord of fileRecords) {
|
|
3220
|
-
if (!shouldExtractNamedResourceReferences(fileRecord.path)) {
|
|
3221
|
-
continue;
|
|
3222
|
-
}
|
|
3223
|
-
|
|
3224
|
-
for (const key of extractSqlResourceKeyReferences(fileRecord.content)) {
|
|
3225
|
-
for (const targetChunkId of resourceChunkIdsByAlias.get(key) ?? []) {
|
|
3226
|
-
usesResourceKeyRelationMap.set(relationKey(fileRecord.id, targetChunkId, key), {
|
|
3227
|
-
from: fileRecord.id,
|
|
3228
|
-
to: targetChunkId,
|
|
3229
|
-
note: key
|
|
3230
|
-
});
|
|
3231
|
-
}
|
|
3232
|
-
for (const targetChunkId of settingChunkIdsByAlias.get(key) ?? []) {
|
|
3233
|
-
usesSettingKeyRelationMap.set(relationKey(fileRecord.id, targetChunkId, key), {
|
|
3234
|
-
from: fileRecord.id,
|
|
3235
|
-
to: targetChunkId,
|
|
3236
|
-
note: key
|
|
3237
|
-
});
|
|
3238
|
-
}
|
|
3239
|
-
}
|
|
3240
|
-
|
|
3241
|
-
for (const key of extractConfigKeyReferences(fileRecord.content)) {
|
|
3242
|
-
for (const targetChunkId of configChunkIdsByAlias.get(key) ?? []) {
|
|
3243
|
-
usesConfigKeyRelationMap.set(relationKey(fileRecord.id, targetChunkId, key), {
|
|
3244
|
-
from: fileRecord.id,
|
|
3245
|
-
to: targetChunkId,
|
|
3246
|
-
note: key
|
|
3247
|
-
});
|
|
3248
|
-
}
|
|
3249
|
-
}
|
|
3250
|
-
}
|
|
3251
|
-
for (const relation of generateConfigTransformKeyRelations(fileRecords, chunkRecords)) {
|
|
3252
|
-
usesConfigKeyRelationMap.set(relationKey(relation.from, relation.to, relation.note), relation);
|
|
3253
|
-
}
|
|
3254
|
-
const validUsesConfigKeyRelations = [...usesConfigKeyRelationMap.values()].filter(
|
|
3255
|
-
(rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
|
|
3256
|
-
);
|
|
3257
|
-
const validUsesResourceKeyRelations = [...usesResourceKeyRelationMap.values()].filter(
|
|
3258
|
-
(rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
|
|
3259
|
-
);
|
|
3260
|
-
const validUsesSettingKeyRelations = [...usesSettingKeyRelationMap.values()].filter(
|
|
3261
|
-
(rel) => indexedFileIds.has(rel.from) && chunkIdSet.has(rel.to)
|
|
3262
|
-
);
|
|
3263
|
-
memoryTrace.checkpoint("materialize:chunks_relations", {
|
|
3264
|
-
chunks: chunkRecords.length,
|
|
3265
|
-
chunk_ids: chunkIdSet.size,
|
|
3266
|
-
relations_defines: validDefinesRelations.length,
|
|
3267
|
-
relations_calls: validCallsRelations.length,
|
|
3268
|
-
relations_imports: validImportsRelations.length,
|
|
3269
|
-
relations_calls_sql: validCallsSqlRelations.length,
|
|
3270
|
-
relations_uses_config_key: validUsesConfigKeyRelations.length,
|
|
3271
|
-
relations_uses_resource_key: validUsesResourceKeyRelations.length,
|
|
3272
|
-
relations_uses_setting_key: validUsesSettingKeyRelations.length
|
|
3273
|
-
});
|
|
3274
|
-
|
|
3275
|
-
if (verbose && chunkRecords.length > 0) {
|
|
3276
|
-
console.log(`[ingest] extracted ${chunkRecords.length} chunks from ${fileRecords.filter(f => f.kind === "CODE").length} code files`);
|
|
3277
|
-
if (windowedChunkCount > 0) {
|
|
3278
|
-
console.log(
|
|
3279
|
-
`[ingest] overlap windows added=${windowedChunkCount} (window_lines=${chunkWindowLines}, overlap_lines=${chunkOverlapLines}, max_windows=${chunkMaxWindows})`
|
|
3280
|
-
);
|
|
3281
|
-
}
|
|
3282
|
-
console.log(`[ingest] ${validCallsRelations.length} call relations (${totalCallsRelations - validCallsRelations.length} filtered)`);
|
|
3283
|
-
if (validCallsSqlRelations.length > 0) {
|
|
3284
|
-
console.log(`[ingest] sql call links=${validCallsSqlRelations.length}`);
|
|
3285
|
-
}
|
|
3286
|
-
if (validUsesConfigKeyRelations.length > 0) {
|
|
3287
|
-
console.log(`[ingest] uses_config_key=${validUsesConfigKeyRelations.length}`);
|
|
3288
|
-
}
|
|
3289
|
-
if (validUsesResourceKeyRelations.length > 0 || validUsesSettingKeyRelations.length > 0) {
|
|
3290
|
-
console.log(
|
|
3291
|
-
`[ingest] uses_resource_key=${validUsesResourceKeyRelations.length} uses_setting_key=${validUsesSettingKeyRelations.length}`
|
|
3292
|
-
);
|
|
3293
|
-
}
|
|
3294
|
-
}
|
|
3295
|
-
|
|
3296
|
-
const csharpChunkCount = chunkRecords.filter((record) => record.language === "csharp").length;
|
|
3297
|
-
const parserHealth = {};
|
|
3298
|
-
if (csharpFileCount > 0) {
|
|
3299
|
-
parserHealth.csharp = {
|
|
3300
|
-
files: csharpFileCount,
|
|
3301
|
-
available: Boolean(csharpRuntime?.available),
|
|
3302
|
-
reason: csharpRuntime?.available ? null : (csharpRuntime?.reason ?? "C# parser unavailable"),
|
|
3303
|
-
chunks: csharpChunkCount,
|
|
3304
|
-
};
|
|
3305
|
-
|
|
3306
|
-
if (!csharpRuntime?.available) {
|
|
3307
|
-
console.log(`[ingest] warning csharp parser unavailable: ${parserHealth.csharp.reason}`);
|
|
3308
|
-
} else if (csharpChunkCount === 0) {
|
|
3309
|
-
console.log("[ingest] warning csharp parser produced 0 chunks across C# files");
|
|
3310
|
-
}
|
|
3311
|
-
}
|
|
3312
|
-
|
|
3313
|
-
// Generate Module entities and relations
|
|
3314
|
-
const moduleResult = generateModules(fileRecords, chunkRecords);
|
|
3315
|
-
const moduleRecords = moduleResult.modules;
|
|
3316
|
-
const moduleContainsRelations = moduleResult.containsRelations;
|
|
3317
|
-
const moduleContainsModuleRelations = moduleResult.containsModuleRelations;
|
|
3318
|
-
const moduleExportsRelations = moduleResult.exportsRelations;
|
|
3319
|
-
const projectResult = generateProjects(fileRecords);
|
|
3320
|
-
const projectRecords = projectResult.projects;
|
|
3321
|
-
const projectIncludesFileRelations = projectResult.includesFileRelations;
|
|
3322
|
-
const projectReferencesProjectRelations = projectResult.referencesProjectRelations;
|
|
3323
|
-
const namedResourceRelationResult = generateNamedResourceRelations(fileRecords);
|
|
3324
|
-
const usesResourceRelations = namedResourceRelationResult.usesResourceRelations;
|
|
3325
|
-
const usesSettingRelations = namedResourceRelationResult.usesSettingRelations;
|
|
3326
|
-
const configIncludeRelations = generateConfigIncludeRelations(fileRecords);
|
|
3327
|
-
const machineConfigRelations = generateMachineConfigRelations(fileRecords);
|
|
3328
|
-
const sectionHandlerRelations = generateSectionHandlerRelations(fileRecords);
|
|
3329
|
-
const usesConfigRelations = uniqueRelations([
|
|
3330
|
-
...namedResourceRelationResult.usesConfigRelations,
|
|
3331
|
-
...configIncludeRelations,
|
|
3332
|
-
...machineConfigRelations,
|
|
3333
|
-
...sectionHandlerRelations
|
|
3334
|
-
]);
|
|
3335
|
-
const configTransformRelations = generateConfigTransformRelations(fileRecords);
|
|
3336
|
-
memoryTrace.checkpoint("materialize:modules_projects_relations", {
|
|
3337
|
-
modules: moduleRecords.length,
|
|
3338
|
-
projects: projectRecords.length,
|
|
3339
|
-
relations_contains: moduleContainsRelations.length,
|
|
3340
|
-
relations_contains_module: moduleContainsModuleRelations.length,
|
|
3341
|
-
relations_exports: moduleExportsRelations.length,
|
|
3342
|
-
relations_includes_file: projectIncludesFileRelations.length,
|
|
3343
|
-
relations_references_project: projectReferencesProjectRelations.length,
|
|
3344
|
-
relations_uses_resource: usesResourceRelations.length,
|
|
3345
|
-
relations_uses_setting: usesSettingRelations.length,
|
|
3346
|
-
relations_uses_config: usesConfigRelations.length,
|
|
3347
|
-
relations_transforms_config: configTransformRelations.length
|
|
3348
|
-
});
|
|
3349
|
-
|
|
3350
|
-
if (verbose && moduleRecords.length > 0) {
|
|
3351
|
-
console.log(`[ingest] modules=${moduleRecords.length} contains=${moduleContainsRelations.length} contains_module=${moduleContainsModuleRelations.length} exports=${moduleExportsRelations.length}`);
|
|
3352
|
-
}
|
|
3353
|
-
if (verbose && projectRecords.length > 0) {
|
|
3354
|
-
console.log(
|
|
3355
|
-
`[ingest] projects=${projectRecords.length} includes_file=${projectIncludesFileRelations.length} references_project=${projectReferencesProjectRelations.length}`
|
|
3356
|
-
);
|
|
3357
|
-
}
|
|
3358
|
-
if (
|
|
3359
|
-
verbose &&
|
|
3360
|
-
(
|
|
3361
|
-
usesResourceRelations.length > 0 ||
|
|
3362
|
-
usesSettingRelations.length > 0 ||
|
|
3363
|
-
usesConfigRelations.length > 0 ||
|
|
3364
|
-
configTransformRelations.length > 0
|
|
3365
|
-
)
|
|
3366
|
-
) {
|
|
3367
|
-
console.log(
|
|
3368
|
-
`[ingest] uses_resource=${usesResourceRelations.length} uses_setting=${usesSettingRelations.length} uses_config=${usesConfigRelations.length} transforms_config=${configTransformRelations.length}`
|
|
3369
|
-
);
|
|
3370
|
-
}
|
|
3371
|
-
|
|
3372
|
-
const ruleRecords = rules.map((rule) => ({
|
|
3373
|
-
id: rule.id,
|
|
3374
|
-
title: rule.id,
|
|
3375
|
-
body: rule.description,
|
|
3376
|
-
scope: "global",
|
|
3377
|
-
updated_at: new Date().toISOString(),
|
|
3378
|
-
source_of_truth: true,
|
|
3379
|
-
trust_level: 95,
|
|
3380
|
-
status: rule.enforce ? "active" : "draft",
|
|
3381
|
-
priority: rule.priority
|
|
3382
|
-
}));
|
|
3383
|
-
|
|
3384
|
-
const adrTokenIndex = new Map();
|
|
3385
|
-
for (const adrRecord of adrRecords) {
|
|
3386
|
-
for (const token of adrTokens(adrRecord)) {
|
|
3387
|
-
if (!adrTokenIndex.has(token)) {
|
|
3388
|
-
adrTokenIndex.set(token, adrRecord.id);
|
|
3389
|
-
}
|
|
3390
|
-
}
|
|
3391
|
-
}
|
|
3392
|
-
|
|
3393
|
-
const supersedesRelations = [];
|
|
3394
|
-
for (const adrRecord of adrRecords) {
|
|
3395
|
-
const refs = findSupersedesReferences(adrRecord.body);
|
|
3396
|
-
for (const ref of refs) {
|
|
3397
|
-
const target = adrTokenIndex.get(normalizeToken(ref));
|
|
3398
|
-
if (!target || target === adrRecord.id) {
|
|
3399
|
-
continue;
|
|
3400
|
-
}
|
|
3401
|
-
adrRecord.supersedes_id = target;
|
|
3402
|
-
supersedesRelations.push({
|
|
3403
|
-
from: adrRecord.id,
|
|
3404
|
-
to: target,
|
|
3405
|
-
reason: `Supersedes ${ref}`
|
|
3406
|
-
});
|
|
3407
|
-
}
|
|
3408
|
-
}
|
|
3409
|
-
|
|
3410
|
-
const constrainsRelations = [];
|
|
3411
|
-
const implementsRelations = [];
|
|
3412
|
-
const stagedDocumentCache = stageJsonl(path.join(CACHE_DIR, "documents.jsonl"), fileRecords);
|
|
3413
|
-
const stagedFileEntityCache = stageJsonl(path.join(CACHE_DIR, "entities.file.jsonl"), fileRecords);
|
|
3414
|
-
memoryTrace.checkpoint("writes:file_cache_staged", {
|
|
3415
|
-
files: fileRecords.length,
|
|
3416
|
-
file_content_records: countFileContentRecords(fileRecords)
|
|
3417
|
-
});
|
|
3418
|
-
memoryTrace.checkpoint("tokens:rule_matching_start", {
|
|
3419
|
-
files: fileRecords.length,
|
|
3420
|
-
rules: ruleRecords.length
|
|
3421
|
-
});
|
|
3422
|
-
|
|
3423
|
-
const ruleMatchers = ruleRecords.map((ruleRecord) => ({
|
|
3424
|
-
ruleRecord,
|
|
3425
|
-
needle: ruleRecord.id.toLowerCase(),
|
|
3426
|
-
keywords: normalizeRuleTokens(ruleRecord)
|
|
3427
|
-
}));
|
|
3428
|
-
const constrainsByKey = new Map();
|
|
3429
|
-
const implementsByKey = new Map();
|
|
3430
|
-
let fileTokenSetsProcessed = 0;
|
|
3431
|
-
let fileContentRecordsReleased = 0;
|
|
3432
|
-
|
|
3433
|
-
for (const fileRecord of fileRecords) {
|
|
3434
|
-
const lower = String(fileRecord.content ?? "").toLowerCase();
|
|
3435
|
-
const tokens = fileTokenSet(fileRecord);
|
|
3436
|
-
fileTokenSetsProcessed += 1;
|
|
3437
|
-
|
|
3438
|
-
for (const { ruleRecord, needle, keywords } of ruleMatchers) {
|
|
3439
|
-
const explicitMention = lower.includes(needle);
|
|
3440
|
-
const minimumMatches = fileRecord.kind === "CODE" ? 1 : 2;
|
|
3441
|
-
const keywordResult = explicitMention
|
|
3442
|
-
? { matched: false, sample: [] }
|
|
3443
|
-
: collectRuleKeywordMatch(keywords, tokens, minimumMatches);
|
|
3444
|
-
|
|
3445
|
-
if (!explicitMention && !keywordResult.matched) {
|
|
3446
|
-
continue;
|
|
3447
|
-
}
|
|
3448
|
-
|
|
3449
|
-
const constrainsKey = `${ruleRecord.id}|${fileRecord.id}`;
|
|
3450
|
-
if (!constrainsByKey.has(constrainsKey)) {
|
|
3451
|
-
constrainsByKey.set(constrainsKey, {
|
|
3452
|
-
from: ruleRecord.id,
|
|
3453
|
-
to: fileRecord.id,
|
|
3454
|
-
note: explicitMention
|
|
3455
|
-
? `Mentions ${ruleRecord.id}`
|
|
3456
|
-
: `Keyword match ${keywordResult.sample.join(", ")}`
|
|
3457
|
-
});
|
|
3458
|
-
}
|
|
3459
|
-
|
|
3460
|
-
if (fileRecord.kind === "CODE") {
|
|
3461
|
-
const implementsKey = `${fileRecord.id}|${ruleRecord.id}`;
|
|
3462
|
-
if (!implementsByKey.has(implementsKey)) {
|
|
3463
|
-
implementsByKey.set(implementsKey, {
|
|
3464
|
-
from: fileRecord.id,
|
|
3465
|
-
to: ruleRecord.id,
|
|
3466
|
-
note: explicitMention
|
|
3467
|
-
? `Code references ${ruleRecord.id}`
|
|
3468
|
-
: `Code keywords ${keywordResult.sample.join(", ")}`
|
|
3469
|
-
});
|
|
3470
|
-
}
|
|
3471
|
-
}
|
|
3472
|
-
}
|
|
3473
|
-
|
|
3474
|
-
if (Object.prototype.hasOwnProperty.call(fileRecord, "content")) {
|
|
3475
|
-
delete fileRecord.content;
|
|
3476
|
-
fileContentRecordsReleased += 1;
|
|
3477
|
-
}
|
|
3478
|
-
}
|
|
3479
|
-
|
|
3480
|
-
for (const { ruleRecord } of ruleMatchers) {
|
|
3481
|
-
for (const fileRecord of fileRecords) {
|
|
3482
|
-
const constrainsKey = `${ruleRecord.id}|${fileRecord.id}`;
|
|
3483
|
-
const constrainsRelation = constrainsByKey.get(constrainsKey);
|
|
3484
|
-
if (constrainsRelation) {
|
|
3485
|
-
constrainsRelations.push(constrainsRelation);
|
|
3486
|
-
constrainsByKey.delete(constrainsKey);
|
|
3487
|
-
}
|
|
3488
|
-
if (fileRecord.kind === "CODE") {
|
|
3489
|
-
const implementsKey = `${fileRecord.id}|${ruleRecord.id}`;
|
|
3490
|
-
const implementsRelation = implementsByKey.get(implementsKey);
|
|
3491
|
-
if (implementsRelation) {
|
|
3492
|
-
implementsRelations.push(implementsRelation);
|
|
3493
|
-
implementsByKey.delete(implementsKey);
|
|
3494
|
-
}
|
|
3495
|
-
}
|
|
3496
|
-
}
|
|
3497
|
-
}
|
|
3498
|
-
|
|
3499
|
-
memoryTrace.checkpoint("tokens:rule_matching_complete", {
|
|
3500
|
-
file_token_sets: fileTokenSetsProcessed,
|
|
3501
|
-
file_token_sets_retained: 0,
|
|
3502
|
-
file_content_records_released: fileContentRecordsReleased,
|
|
3503
|
-
file_content_records_retained: countFileContentRecords(fileRecords),
|
|
3504
|
-
rules: ruleRecords.length,
|
|
3505
|
-
relations_constrains: constrainsRelations.length,
|
|
3506
|
-
relations_implements: implementsRelations.length,
|
|
3507
|
-
relations_supersedes: supersedesRelations.length
|
|
3508
|
-
});
|
|
3509
|
-
|
|
3510
|
-
memoryTrace.checkpoint("writes:cache_start", {
|
|
3511
|
-
files: fileRecords.length,
|
|
3512
|
-
adrs: adrRecords.length,
|
|
3513
|
-
rules: ruleRecords.length,
|
|
3514
|
-
chunks: chunkRecords.length
|
|
3515
|
-
});
|
|
3516
|
-
stagedDocumentCache.commit();
|
|
3517
|
-
stagedFileEntityCache.commit();
|
|
3518
|
-
writeJsonl(path.join(CACHE_DIR, "entities.adr.jsonl"), adrRecords);
|
|
3519
|
-
writeJsonl(path.join(CACHE_DIR, "entities.rule.jsonl"), ruleRecords);
|
|
3520
|
-
writeJsonl(path.join(CACHE_DIR, "entities.chunk.jsonl"), chunkRecords);
|
|
3521
|
-
writeJsonl(path.join(CACHE_DIR, "relations.supersedes.jsonl"), supersedesRelations);
|
|
3522
|
-
writeJsonl(path.join(CACHE_DIR, "relations.constrains.jsonl"), constrainsRelations);
|
|
3523
|
-
writeJsonl(path.join(CACHE_DIR, "relations.implements.jsonl"), implementsRelations);
|
|
3524
|
-
writeJsonl(path.join(CACHE_DIR, "relations.defines.jsonl"), validDefinesRelations);
|
|
3525
|
-
writeJsonl(path.join(CACHE_DIR, "relations.calls.jsonl"), validCallsRelations);
|
|
3526
|
-
writeJsonl(path.join(CACHE_DIR, "relations.imports.jsonl"), validImportsRelations);
|
|
3527
|
-
writeJsonl(path.join(CACHE_DIR, "relations.calls_sql.jsonl"), validCallsSqlRelations);
|
|
3528
|
-
writeJsonl(path.join(CACHE_DIR, "relations.uses_config_key.jsonl"), validUsesConfigKeyRelations);
|
|
3529
|
-
writeJsonl(path.join(CACHE_DIR, "relations.uses_resource_key.jsonl"), validUsesResourceKeyRelations);
|
|
3530
|
-
writeJsonl(path.join(CACHE_DIR, "relations.uses_setting_key.jsonl"), validUsesSettingKeyRelations);
|
|
3531
|
-
writeJsonl(path.join(CACHE_DIR, "entities.module.jsonl"), moduleRecords);
|
|
3532
|
-
writeJsonl(path.join(CACHE_DIR, "relations.contains.jsonl"), moduleContainsRelations);
|
|
3533
|
-
writeJsonl(path.join(CACHE_DIR, "relations.contains_module.jsonl"), moduleContainsModuleRelations);
|
|
3534
|
-
writeJsonl(path.join(CACHE_DIR, "relations.exports.jsonl"), moduleExportsRelations);
|
|
3535
|
-
writeJsonl(path.join(CACHE_DIR, "entities.project.jsonl"), projectRecords);
|
|
3536
|
-
writeJsonl(path.join(CACHE_DIR, "relations.includes_file.jsonl"), projectIncludesFileRelations);
|
|
3537
|
-
writeJsonl(path.join(CACHE_DIR, "relations.uses_resource.jsonl"), usesResourceRelations);
|
|
3538
|
-
writeJsonl(path.join(CACHE_DIR, "relations.uses_setting.jsonl"), usesSettingRelations);
|
|
3539
|
-
writeJsonl(path.join(CACHE_DIR, "relations.uses_config.jsonl"), usesConfigRelations);
|
|
3540
|
-
writeJsonl(path.join(CACHE_DIR, "relations.transforms_config.jsonl"), configTransformRelations);
|
|
3541
|
-
writeJsonl(
|
|
3542
|
-
path.join(CACHE_DIR, "relations.references_project.jsonl"),
|
|
3543
|
-
projectReferencesProjectRelations
|
|
3544
|
-
);
|
|
3545
|
-
memoryTrace.checkpoint("writes:cache_complete", {
|
|
3546
|
-
jsonl_files: 26,
|
|
3547
|
-
files: fileRecords.length,
|
|
3548
|
-
chunks: chunkRecords.length
|
|
3549
|
-
});
|
|
3550
|
-
|
|
3551
|
-
memoryTrace.checkpoint("writes:db_start", {
|
|
3552
|
-
tsv_files: 21
|
|
3553
|
-
});
|
|
3554
|
-
writeTsv(
|
|
3555
|
-
path.join(DB_IMPORT_DIR, "file_nodes.tsv"),
|
|
3556
|
-
[
|
|
3557
|
-
"id",
|
|
3558
|
-
"path",
|
|
3559
|
-
"kind",
|
|
3560
|
-
"excerpt",
|
|
3561
|
-
"checksum",
|
|
3562
|
-
"updated_at",
|
|
3563
|
-
"source_of_truth",
|
|
3564
|
-
"trust_level",
|
|
3565
|
-
"status"
|
|
3566
|
-
],
|
|
3567
|
-
mapRows(fileRecords, (record) => [
|
|
3568
|
-
record.id,
|
|
3569
|
-
record.path,
|
|
3570
|
-
record.kind,
|
|
3571
|
-
record.excerpt,
|
|
3572
|
-
record.checksum,
|
|
3573
|
-
record.updated_at,
|
|
3574
|
-
record.source_of_truth,
|
|
3575
|
-
record.trust_level,
|
|
3576
|
-
record.status
|
|
3577
|
-
])
|
|
3578
|
-
);
|
|
3579
|
-
|
|
3580
|
-
writeTsv(
|
|
3581
|
-
path.join(DB_IMPORT_DIR, "rule_nodes.tsv"),
|
|
3582
|
-
[
|
|
3583
|
-
"id",
|
|
3584
|
-
"title",
|
|
3585
|
-
"body",
|
|
3586
|
-
"scope",
|
|
3587
|
-
"priority",
|
|
3588
|
-
"updated_at",
|
|
3589
|
-
"source_of_truth",
|
|
3590
|
-
"trust_level",
|
|
3591
|
-
"status"
|
|
3592
|
-
],
|
|
3593
|
-
mapRows(ruleRecords, (record) => [
|
|
3594
|
-
record.id,
|
|
3595
|
-
record.title,
|
|
3596
|
-
record.body,
|
|
3597
|
-
record.scope,
|
|
3598
|
-
record.priority,
|
|
3599
|
-
record.updated_at,
|
|
3600
|
-
record.source_of_truth,
|
|
3601
|
-
record.trust_level,
|
|
3602
|
-
record.status
|
|
3603
|
-
])
|
|
3604
|
-
);
|
|
3605
|
-
|
|
3606
|
-
writeTsv(
|
|
3607
|
-
path.join(DB_IMPORT_DIR, "adr_nodes.tsv"),
|
|
3608
|
-
[
|
|
3609
|
-
"id",
|
|
3610
|
-
"path",
|
|
3611
|
-
"title",
|
|
3612
|
-
"body",
|
|
3613
|
-
"decision_date",
|
|
3614
|
-
"supersedes_id",
|
|
3615
|
-
"source_of_truth",
|
|
3616
|
-
"trust_level",
|
|
3617
|
-
"status"
|
|
3618
|
-
],
|
|
3619
|
-
mapRows(adrRecords, (record) => [
|
|
3620
|
-
record.id,
|
|
3621
|
-
record.path,
|
|
3622
|
-
record.title,
|
|
3623
|
-
record.body,
|
|
3624
|
-
record.decision_date,
|
|
3625
|
-
record.supersedes_id,
|
|
3626
|
-
record.source_of_truth,
|
|
3627
|
-
record.trust_level,
|
|
3628
|
-
record.status
|
|
3629
|
-
])
|
|
3630
|
-
);
|
|
3631
|
-
|
|
3632
|
-
writeTsv(
|
|
3633
|
-
path.join(DB_IMPORT_DIR, "constrains_rel.tsv"),
|
|
3634
|
-
["from", "to", "note"],
|
|
3635
|
-
mapRows(constrainsRelations, (record) => [record.from, record.to, record.note])
|
|
3636
|
-
);
|
|
3637
|
-
|
|
3638
|
-
writeTsv(
|
|
3639
|
-
path.join(DB_IMPORT_DIR, "implements_rel.tsv"),
|
|
3640
|
-
["from", "to", "note"],
|
|
3641
|
-
mapRows(implementsRelations, (record) => [record.from, record.to, record.note])
|
|
3642
|
-
);
|
|
3643
|
-
|
|
3644
|
-
writeTsv(
|
|
3645
|
-
path.join(DB_IMPORT_DIR, "supersedes_rel.tsv"),
|
|
3646
|
-
["from", "to", "reason"],
|
|
3647
|
-
mapRows(supersedesRelations, (record) => [record.from, record.to, record.reason])
|
|
3648
|
-
);
|
|
3649
|
-
|
|
3650
|
-
writeTsv(
|
|
3651
|
-
path.join(DB_IMPORT_DIR, "chunk_nodes.tsv"),
|
|
3652
|
-
[
|
|
3653
|
-
"id",
|
|
3654
|
-
"file_id",
|
|
3655
|
-
"name",
|
|
3656
|
-
"kind",
|
|
3657
|
-
"signature",
|
|
3658
|
-
"body",
|
|
3659
|
-
"start_line",
|
|
3660
|
-
"end_line",
|
|
3661
|
-
"language",
|
|
3662
|
-
"checksum",
|
|
3663
|
-
"updated_at",
|
|
3664
|
-
"trust_level"
|
|
3665
|
-
],
|
|
3666
|
-
mapRows(chunkRecords, (record) => [
|
|
3667
|
-
record.id,
|
|
3668
|
-
record.file_id,
|
|
3669
|
-
record.name,
|
|
3670
|
-
record.kind,
|
|
3671
|
-
record.signature,
|
|
3672
|
-
record.body,
|
|
3673
|
-
record.start_line,
|
|
3674
|
-
record.end_line,
|
|
3675
|
-
record.language,
|
|
3676
|
-
record.checksum,
|
|
3677
|
-
record.updated_at,
|
|
3678
|
-
record.trust_level
|
|
3679
|
-
])
|
|
3680
|
-
);
|
|
3681
|
-
|
|
3682
|
-
writeTsv(
|
|
3683
|
-
path.join(DB_IMPORT_DIR, "defines_rel.tsv"),
|
|
3684
|
-
["from", "to"],
|
|
3685
|
-
mapRows(validDefinesRelations, (record) => [record.from, record.to])
|
|
3686
|
-
);
|
|
3687
|
-
|
|
3688
|
-
writeTsv(
|
|
3689
|
-
path.join(DB_IMPORT_DIR, "calls_rel.tsv"),
|
|
3690
|
-
["from", "to", "call_type"],
|
|
3691
|
-
mapRows(validCallsRelations, (record) => [record.from, record.to, record.call_type])
|
|
3692
|
-
);
|
|
3693
|
-
|
|
3694
|
-
writeTsv(
|
|
3695
|
-
path.join(DB_IMPORT_DIR, "imports_rel.tsv"),
|
|
3696
|
-
["from", "to", "import_name"],
|
|
3697
|
-
mapRows(validImportsRelations, (record) => [record.from, record.to, record.import_name])
|
|
3698
|
-
);
|
|
3699
|
-
|
|
3700
|
-
writeTsv(
|
|
3701
|
-
path.join(DB_IMPORT_DIR, "calls_sql_rel.tsv"),
|
|
3702
|
-
["from", "to", "note"],
|
|
3703
|
-
mapRows(validCallsSqlRelations, (record) => [record.from, record.to, record.note])
|
|
3704
|
-
);
|
|
3705
|
-
|
|
3706
|
-
writeTsv(
|
|
3707
|
-
path.join(DB_IMPORT_DIR, "uses_config_key_rel.tsv"),
|
|
3708
|
-
["from", "to", "note"],
|
|
3709
|
-
mapRows(validUsesConfigKeyRelations, (record) => [record.from, record.to, record.note])
|
|
3710
|
-
);
|
|
3711
|
-
|
|
3712
|
-
writeTsv(
|
|
3713
|
-
path.join(DB_IMPORT_DIR, "uses_resource_key_rel.tsv"),
|
|
3714
|
-
["from", "to", "note"],
|
|
3715
|
-
mapRows(validUsesResourceKeyRelations, (record) => [record.from, record.to, record.note])
|
|
3716
|
-
);
|
|
3717
|
-
|
|
3718
|
-
writeTsv(
|
|
3719
|
-
path.join(DB_IMPORT_DIR, "uses_setting_key_rel.tsv"),
|
|
3720
|
-
["from", "to", "note"],
|
|
3721
|
-
mapRows(validUsesSettingKeyRelations, (record) => [record.from, record.to, record.note])
|
|
3722
|
-
);
|
|
3723
|
-
|
|
3724
|
-
writeTsv(
|
|
3725
|
-
path.join(DB_IMPORT_DIR, "project_nodes.tsv"),
|
|
3726
|
-
[
|
|
3727
|
-
"id",
|
|
3728
|
-
"path",
|
|
3729
|
-
"name",
|
|
3730
|
-
"kind",
|
|
3731
|
-
"language",
|
|
3732
|
-
"target_framework",
|
|
3733
|
-
"summary",
|
|
3734
|
-
"file_count",
|
|
3735
|
-
"updated_at",
|
|
3736
|
-
"source_of_truth",
|
|
3737
|
-
"trust_level",
|
|
3738
|
-
"status"
|
|
3739
|
-
],
|
|
3740
|
-
mapRows(projectRecords, (record) => [
|
|
3741
|
-
record.id,
|
|
3742
|
-
record.path,
|
|
3743
|
-
record.name,
|
|
3744
|
-
record.kind,
|
|
3745
|
-
record.language,
|
|
3746
|
-
record.target_framework,
|
|
3747
|
-
record.summary,
|
|
3748
|
-
record.file_count,
|
|
3749
|
-
record.updated_at,
|
|
3750
|
-
record.source_of_truth,
|
|
3751
|
-
record.trust_level,
|
|
3752
|
-
record.status
|
|
3753
|
-
])
|
|
3754
|
-
);
|
|
3755
|
-
|
|
3756
|
-
writeTsv(
|
|
3757
|
-
path.join(DB_IMPORT_DIR, "includes_file_rel.tsv"),
|
|
3758
|
-
["from", "to"],
|
|
3759
|
-
mapRows(projectIncludesFileRelations, (record) => [record.from, record.to])
|
|
3760
|
-
);
|
|
3761
|
-
|
|
3762
|
-
writeTsv(
|
|
3763
|
-
path.join(DB_IMPORT_DIR, "references_project_rel.tsv"),
|
|
3764
|
-
["from", "to", "note"],
|
|
3765
|
-
mapRows(projectReferencesProjectRelations, (record) => [record.from, record.to, record.note])
|
|
3766
|
-
);
|
|
3767
|
-
|
|
3768
|
-
writeTsv(
|
|
3769
|
-
path.join(DB_IMPORT_DIR, "uses_resource_rel.tsv"),
|
|
3770
|
-
["from", "to", "note"],
|
|
3771
|
-
mapRows(usesResourceRelations, (record) => [record.from, record.to, record.note])
|
|
3772
|
-
);
|
|
3773
|
-
|
|
3774
|
-
writeTsv(
|
|
3775
|
-
path.join(DB_IMPORT_DIR, "uses_setting_rel.tsv"),
|
|
3776
|
-
["from", "to", "note"],
|
|
3777
|
-
mapRows(usesSettingRelations, (record) => [record.from, record.to, record.note])
|
|
3778
|
-
);
|
|
3779
|
-
|
|
3780
|
-
writeTsv(
|
|
3781
|
-
path.join(DB_IMPORT_DIR, "uses_config_rel.tsv"),
|
|
3782
|
-
["from", "to", "note"],
|
|
3783
|
-
mapRows(usesConfigRelations, (record) => [record.from, record.to, record.note])
|
|
3784
|
-
);
|
|
3785
|
-
|
|
3786
|
-
writeTsv(
|
|
3787
|
-
path.join(DB_IMPORT_DIR, "transforms_config_rel.tsv"),
|
|
3788
|
-
["from", "to", "note"],
|
|
3789
|
-
mapRows(configTransformRelations, (record) => [record.from, record.to, record.note])
|
|
3790
|
-
);
|
|
3791
|
-
memoryTrace.checkpoint("writes:db_complete", {
|
|
3792
|
-
tsv_files: 21
|
|
3793
|
-
});
|
|
3794
|
-
|
|
3795
|
-
const manifest = {
|
|
3796
|
-
generated_at: new Date().toISOString(),
|
|
3797
|
-
mode,
|
|
3798
|
-
source_paths: sourcePaths,
|
|
3799
|
-
counts: {
|
|
3800
|
-
files: fileRecords.length,
|
|
3801
|
-
adrs: adrRecords.length,
|
|
3802
|
-
rules: ruleRecords.length,
|
|
3803
|
-
chunks: chunkRecords.length,
|
|
3804
|
-
relations_constrains: constrainsRelations.length,
|
|
3805
|
-
relations_implements: implementsRelations.length,
|
|
3806
|
-
relations_supersedes: supersedesRelations.length,
|
|
3807
|
-
relations_defines: validDefinesRelations.length,
|
|
3808
|
-
relations_calls: validCallsRelations.length,
|
|
3809
|
-
relations_imports: validImportsRelations.length,
|
|
3810
|
-
relations_calls_sql: validCallsSqlRelations.length,
|
|
3811
|
-
relations_uses_config_key: validUsesConfigKeyRelations.length,
|
|
3812
|
-
relations_uses_resource_key: validUsesResourceKeyRelations.length,
|
|
3813
|
-
relations_uses_setting_key: validUsesSettingKeyRelations.length,
|
|
3814
|
-
modules: moduleRecords.length,
|
|
3815
|
-
relations_contains: moduleContainsRelations.length,
|
|
3816
|
-
relations_contains_module: moduleContainsModuleRelations.length,
|
|
3817
|
-
relations_exports: moduleExportsRelations.length,
|
|
3818
|
-
projects: projectRecords.length,
|
|
3819
|
-
relations_includes_file: projectIncludesFileRelations.length,
|
|
3820
|
-
relations_references_project: projectReferencesProjectRelations.length,
|
|
3821
|
-
relations_uses_resource: usesResourceRelations.length,
|
|
3822
|
-
relations_uses_setting: usesSettingRelations.length,
|
|
3823
|
-
relations_uses_config: usesConfigRelations.length,
|
|
3824
|
-
relations_transforms_config: configTransformRelations.length
|
|
3825
|
-
},
|
|
3826
|
-
skipped,
|
|
3827
|
-
parser_health: parserHealth,
|
|
3828
|
-
incremental_mode: incrementalMode,
|
|
3829
|
-
changed_candidates: candidates.size,
|
|
3830
|
-
deleted_paths: deletedRelPaths.length
|
|
3831
|
-
};
|
|
3832
|
-
|
|
3833
|
-
fs.writeFileSync(path.join(CACHE_DIR, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
3834
|
-
memoryTrace.checkpoint("writes:manifest_complete", {
|
|
3835
|
-
files: manifest.counts.files,
|
|
3836
|
-
chunks: manifest.counts.chunks,
|
|
3837
|
-
total_relations:
|
|
3838
|
-
manifest.counts.relations_constrains +
|
|
3839
|
-
manifest.counts.relations_implements +
|
|
3840
|
-
manifest.counts.relations_supersedes +
|
|
3841
|
-
manifest.counts.relations_defines +
|
|
3842
|
-
manifest.counts.relations_calls +
|
|
3843
|
-
manifest.counts.relations_imports +
|
|
3844
|
-
manifest.counts.relations_calls_sql +
|
|
3845
|
-
manifest.counts.relations_uses_config_key +
|
|
3846
|
-
manifest.counts.relations_uses_resource_key +
|
|
3847
|
-
manifest.counts.relations_uses_setting_key +
|
|
3848
|
-
manifest.counts.relations_contains +
|
|
3849
|
-
manifest.counts.relations_contains_module +
|
|
3850
|
-
manifest.counts.relations_exports +
|
|
3851
|
-
manifest.counts.relations_includes_file +
|
|
3852
|
-
manifest.counts.relations_references_project +
|
|
3853
|
-
manifest.counts.relations_uses_resource +
|
|
3854
|
-
manifest.counts.relations_uses_setting +
|
|
3855
|
-
manifest.counts.relations_uses_config +
|
|
3856
|
-
manifest.counts.relations_transforms_config
|
|
3857
|
-
});
|
|
3858
|
-
|
|
3859
|
-
console.log(`[ingest] mode=${mode}`);
|
|
3860
|
-
if (incrementalMode) {
|
|
3861
|
-
console.log(
|
|
3862
|
-
`[ingest] incremental changed_candidates=${manifest.changed_candidates} deleted_paths=${manifest.deleted_paths}`
|
|
3863
|
-
);
|
|
3864
|
-
} else if (mode === "changed") {
|
|
3865
|
-
console.log("[ingest] incremental diff unavailable; processed full source set");
|
|
3866
|
-
}
|
|
3867
|
-
console.log(`[ingest] files=${manifest.counts.files} adrs=${manifest.counts.adrs} rules=${manifest.counts.rules} chunks=${manifest.counts.chunks}`);
|
|
3868
|
-
console.log(
|
|
3869
|
-
`[ingest] rels constrains=${manifest.counts.relations_constrains} implements=${manifest.counts.relations_implements} supersedes=${manifest.counts.relations_supersedes}`
|
|
3870
|
-
);
|
|
3871
|
-
console.log(
|
|
3872
|
-
`[ingest] rels defines=${manifest.counts.relations_defines} calls=${manifest.counts.relations_calls} imports=${manifest.counts.relations_imports} calls_sql=${manifest.counts.relations_calls_sql} uses_config_key=${manifest.counts.relations_uses_config_key} uses_resource_key=${manifest.counts.relations_uses_resource_key} uses_setting_key=${manifest.counts.relations_uses_setting_key}`
|
|
3873
|
-
);
|
|
3874
|
-
console.log(
|
|
3875
|
-
`[ingest] rels contains=${manifest.counts.relations_contains} contains_module=${manifest.counts.relations_contains_module} exports=${manifest.counts.relations_exports} includes_file=${manifest.counts.relations_includes_file} references_project=${manifest.counts.relations_references_project} uses_resource=${manifest.counts.relations_uses_resource} uses_setting=${manifest.counts.relations_uses_setting} uses_config=${manifest.counts.relations_uses_config} transforms_config=${manifest.counts.relations_transforms_config}`
|
|
3876
|
-
);
|
|
3877
|
-
console.log(
|
|
3878
|
-
`[ingest] skipped unsupported=${skipped.unsupported} too_large=${skipped.tooLarge} binary=${skipped.binary}`
|
|
3879
|
-
);
|
|
3880
|
-
console.log(`[ingest] wrote cache + db import files under .context/`);
|
|
3881
|
-
}
|
|
3882
|
-
|
|
3883
|
-
const isMainModule = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(__filename);
|
|
6
|
+
const isMainModule =
|
|
7
|
+
process.argv[1] &&
|
|
8
|
+
path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url));
|
|
3884
9
|
if (isMainModule) {
|
|
3885
10
|
main().catch((error) => {
|
|
3886
11
|
console.error(error instanceof Error ? error.message : String(error));
|
|
@@ -3888,23 +13,4 @@ if (isMainModule) {
|
|
|
3888
13
|
});
|
|
3889
14
|
}
|
|
3890
15
|
|
|
3891
|
-
export
|
|
3892
|
-
buildChunkAliasIndexes,
|
|
3893
|
-
buildSqlResourceReferenceMap,
|
|
3894
|
-
detectKind,
|
|
3895
|
-
extractSqlObjectReferencesFromContent,
|
|
3896
|
-
generateChunkDescription,
|
|
3897
|
-
generateConfigIncludeRelations,
|
|
3898
|
-
generateConfigTransformKeyRelations,
|
|
3899
|
-
generateMachineConfigRelations,
|
|
3900
|
-
generateConfigTransformRelations,
|
|
3901
|
-
generateModuleSummary,
|
|
3902
|
-
generateModules,
|
|
3903
|
-
generateNamedResourceRelations,
|
|
3904
|
-
generateProjects,
|
|
3905
|
-
generateSectionHandlerRelations,
|
|
3906
|
-
getChunkParserForExtension,
|
|
3907
|
-
parseFilesInWorkers,
|
|
3908
|
-
resolveIngestWorkerCount,
|
|
3909
|
-
resolveRelativeImportTargetId
|
|
3910
|
-
};
|
|
16
|
+
export * from "./lib/ingest/main.mjs";
|