archbyte 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/static/component-detector.js +71 -107
- package/dist/agents/static/connection-mapper.js +24 -25
- package/dist/agents/static/deep-drill.d.ts +72 -0
- package/dist/agents/static/deep-drill.js +388 -0
- package/dist/agents/static/doc-parser.js +73 -48
- package/dist/agents/static/env-detector.js +3 -6
- package/dist/agents/static/event-detector.js +20 -26
- package/dist/agents/static/infra-analyzer.js +15 -1
- package/dist/agents/static/structure-scanner.js +56 -57
- package/dist/agents/static/taxonomy.d.ts +19 -0
- package/dist/agents/static/taxonomy.js +147 -0
- package/dist/agents/tools/local-fs.js +5 -2
- package/dist/cli/run.js +1 -1
- package/dist/server/src/index.js +201 -68
- package/package.json +1 -1
- package/ui/dist/assets/index-BQouokNH.css +1 -0
- package/ui/dist/assets/index-QllGSFhe.js +72 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/index-BXsZyipz.js +0 -72
- package/ui/dist/assets/index-ow1c3Nxp.css +0 -1
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
// Static Analysis — Component Deep Drill
|
|
2
|
+
// Focused single-component analysis: runs 6 parallel scans on a component directory.
|
|
3
|
+
import { StaticToolkit } from "./utils.js";
|
|
4
|
+
import { categorizeDep } from "./taxonomy.js";
|
|
5
|
+
import { EXCLUDED_DIRS as SKIP_DIRS } from "./excluded-dirs.js";
|
|
6
|
+
// === Language mapping ===
|
|
7
|
+
const EXT_TO_LANGUAGE = {
|
|
8
|
+
".ts": "TypeScript", ".tsx": "TypeScript", ".js": "JavaScript", ".jsx": "JavaScript",
|
|
9
|
+
".mjs": "JavaScript", ".cjs": "JavaScript", ".py": "Python", ".rs": "Rust",
|
|
10
|
+
".go": "Go", ".java": "Java", ".kt": "Kotlin", ".rb": "Ruby",
|
|
11
|
+
".php": "PHP", ".cs": "C#", ".cpp": "C++", ".c": "C",
|
|
12
|
+
".swift": "Swift", ".dart": "Dart", ".vue": "Vue", ".svelte": "Svelte",
|
|
13
|
+
".css": "CSS", ".scss": "SCSS", ".html": "HTML", ".sql": "SQL",
|
|
14
|
+
".sh": "Shell", ".yaml": "YAML", ".yml": "YAML", ".json": "JSON",
|
|
15
|
+
".toml": "TOML", ".md": "Markdown",
|
|
16
|
+
};
|
|
17
|
+
const CODE_EXTENSIONS = new Set([
|
|
18
|
+
".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".rs", ".go",
|
|
19
|
+
".java", ".kt", ".rb", ".php", ".cs", ".cpp", ".c", ".swift", ".dart",
|
|
20
|
+
".vue", ".svelte",
|
|
21
|
+
]);
|
|
22
|
+
// === Entry point detection ===
|
|
23
|
+
const ENTRY_POINT_NAMES = new Set([
|
|
24
|
+
"index.ts", "index.js", "main.ts", "main.js", "app.ts", "app.js",
|
|
25
|
+
"server.ts", "server.js", "mod.ts", "mod.rs", "lib.rs", "main.rs",
|
|
26
|
+
"main.py", "app.py", "main.go", "__init__.py",
|
|
27
|
+
]);
|
|
28
|
+
// === Key file detection ===
|
|
29
|
+
function detectKeyFileRole(filePath) {
|
|
30
|
+
const name = filePath.split("/").pop()?.toLowerCase() || "";
|
|
31
|
+
if (ENTRY_POINT_NAMES.has(name))
|
|
32
|
+
return "entry-point";
|
|
33
|
+
if (name.includes("route") || name.includes("router"))
|
|
34
|
+
return "router";
|
|
35
|
+
if (name.includes("controller"))
|
|
36
|
+
return "controller";
|
|
37
|
+
if (name.includes("middleware"))
|
|
38
|
+
return "middleware";
|
|
39
|
+
if (name.includes("model") || name.includes("schema") || name.includes("entity"))
|
|
40
|
+
return "model";
|
|
41
|
+
if (name.includes("service"))
|
|
42
|
+
return "service";
|
|
43
|
+
if (name.includes("util") || name.includes("helper"))
|
|
44
|
+
return "utility";
|
|
45
|
+
if (name.includes("config") || name.includes("setting"))
|
|
46
|
+
return "config";
|
|
47
|
+
if (name.includes("test") || name.includes("spec"))
|
|
48
|
+
return "test";
|
|
49
|
+
if (name.includes("type") || name.includes("interface"))
|
|
50
|
+
return "types";
|
|
51
|
+
if (name === "package.json" || name === "cargo.toml" || name === "go.mod")
|
|
52
|
+
return "manifest";
|
|
53
|
+
if (name.includes("readme"))
|
|
54
|
+
return "docs";
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
const STRUCTURAL_PROBES = [
|
|
58
|
+
// Existing from component-detector
|
|
59
|
+
{ type: "db-query", label: "Database queries", pattern: "\\.query\\(|\\.execute\\(|\\.findOne\\(|SELECT.*FROM" },
|
|
60
|
+
{ type: "event-emitter", label: "Event emitters", pattern: "\\.emit\\(|\\.publish\\(|\\.dispatch\\(" },
|
|
61
|
+
{ type: "event-subscriber", label: "Event subscribers", pattern: "\\.subscribe\\(|\\.on\\(.*message|\\.consume\\(" },
|
|
62
|
+
{ type: "test-code", label: "Test code", pattern: "describe\\(|it\\(|test\\(|expect\\(" },
|
|
63
|
+
{ type: "auth-boundary", label: "Auth boundary", pattern: "middleware|authenticate|authorize|jwt\\.verify" },
|
|
64
|
+
// New probes
|
|
65
|
+
{ type: "caching", label: "Caching", pattern: "cache|Redis|setEx\\(" },
|
|
66
|
+
{ type: "http-client", label: "HTTP client", pattern: "fetch\\(|axios|got\\(|HttpClient" },
|
|
67
|
+
{ type: "logging", label: "Logging", pattern: "console\\.(log|warn|error)|logger\\.|winston|pino" },
|
|
68
|
+
{ type: "validation", label: "Validation", pattern: "validate|zod\\.|z\\.|Joi\\.|yup\\." },
|
|
69
|
+
{ type: "error-handling", label: "Error handling", pattern: "try\\s*\\{|catch\\s*\\(|throw new" },
|
|
70
|
+
{ type: "env-access", label: "Env access", pattern: "process\\.env|Deno\\.env|import\\.meta\\.env" },
|
|
71
|
+
{ type: "file-io", label: "File I/O", pattern: "readFile|writeFile|createReadStream|fs\\." },
|
|
72
|
+
];
|
|
73
|
+
const MAX_PROBE_MATCHES = 3;
|
|
74
|
+
// === Main ===
|
|
75
|
+
export async function runDeepDrill(projectRoot, componentId, componentName, componentPath) {
|
|
76
|
+
const start = Date.now();
|
|
77
|
+
const tk = new StaticToolkit(projectRoot);
|
|
78
|
+
// Run all 6 scans in parallel
|
|
79
|
+
const [tree, metrics, structure, deps, imports, patterns] = await Promise.all([
|
|
80
|
+
collectComponentTree(tk, componentPath),
|
|
81
|
+
collectFileMetrics(tk, componentPath),
|
|
82
|
+
detectInternalStructure(tk, componentPath),
|
|
83
|
+
detectDependencies(tk, componentPath),
|
|
84
|
+
buildImportGraph(tk, componentPath),
|
|
85
|
+
runStructuralProbes(tk, componentPath),
|
|
86
|
+
]);
|
|
87
|
+
return {
|
|
88
|
+
componentId,
|
|
89
|
+
componentName,
|
|
90
|
+
componentPath,
|
|
91
|
+
scannedAt: new Date().toISOString(),
|
|
92
|
+
durationMs: Date.now() - start,
|
|
93
|
+
fileTree: tree,
|
|
94
|
+
metrics,
|
|
95
|
+
structure,
|
|
96
|
+
dependencies: deps,
|
|
97
|
+
imports,
|
|
98
|
+
patterns,
|
|
99
|
+
// Connections are enriched by the server from architecture edges
|
|
100
|
+
connections: { outgoing: [], incoming: [] },
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
// === Scan 1: Component Tree ===
|
|
104
|
+
async function collectComponentTree(tk, basePath) {
|
|
105
|
+
const MAX_DEPTH = 4;
|
|
106
|
+
const MAX_ENTRIES = 500;
|
|
107
|
+
let entryCount = 0;
|
|
108
|
+
async function walk(dirPath, depth) {
|
|
109
|
+
if (depth > MAX_DEPTH || entryCount >= MAX_ENTRIES)
|
|
110
|
+
return [];
|
|
111
|
+
const entries = await tk.listDir(dirPath);
|
|
112
|
+
const result = [];
|
|
113
|
+
const sorted = entries.sort((a, b) => {
|
|
114
|
+
if (a.type !== b.type)
|
|
115
|
+
return a.type === "directory" ? -1 : 1;
|
|
116
|
+
return a.name.localeCompare(b.name);
|
|
117
|
+
});
|
|
118
|
+
for (const entry of sorted) {
|
|
119
|
+
if (entryCount >= MAX_ENTRIES)
|
|
120
|
+
break;
|
|
121
|
+
if (entry.type === "directory") {
|
|
122
|
+
if (entry.name.startsWith(".") && entry.name !== ".github")
|
|
123
|
+
continue;
|
|
124
|
+
if (SKIP_DIRS.has(entry.name))
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
const entryPath = dirPath === basePath ? `${basePath}/${entry.name}` : `${dirPath}/${entry.name}`;
|
|
128
|
+
entryCount++;
|
|
129
|
+
if (entry.type === "directory") {
|
|
130
|
+
const children = await walk(entryPath, depth + 1);
|
|
131
|
+
result.push({ path: entryPath, type: "directory", children });
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
result.push({ path: entryPath, type: "file" });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return result;
|
|
138
|
+
}
|
|
139
|
+
return walk(basePath, 0);
|
|
140
|
+
}
|
|
141
|
+
// === Scan 2: File Metrics ===
|
|
142
|
+
async function collectFileMetrics(tk, basePath) {
|
|
143
|
+
const files = await tk.globFiles("**/*", basePath);
|
|
144
|
+
const extCounts = new Map();
|
|
145
|
+
let totalLines = 0;
|
|
146
|
+
let codeFileCount = 0;
|
|
147
|
+
for (const file of files) {
|
|
148
|
+
const ext = getExtension(file);
|
|
149
|
+
if (ext) {
|
|
150
|
+
extCounts.set(ext, (extCounts.get(ext) || 0) + 1);
|
|
151
|
+
}
|
|
152
|
+
if (CODE_EXTENSIONS.has(ext)) {
|
|
153
|
+
codeFileCount++;
|
|
154
|
+
// Read file to count lines
|
|
155
|
+
const content = await tk.readFileSafe(file);
|
|
156
|
+
if (content) {
|
|
157
|
+
totalLines += content.split("\n").length;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const languages = [];
|
|
162
|
+
const langCounts = new Map();
|
|
163
|
+
for (const [ext, count] of extCounts) {
|
|
164
|
+
const lang = EXT_TO_LANGUAGE[ext];
|
|
165
|
+
if (lang) {
|
|
166
|
+
langCounts.set(lang, (langCounts.get(lang) || 0) + count);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
for (const [language, files] of langCounts) {
|
|
170
|
+
languages.push({ language, files });
|
|
171
|
+
}
|
|
172
|
+
languages.sort((a, b) => b.files - a.files);
|
|
173
|
+
return {
|
|
174
|
+
fileCount: files.length,
|
|
175
|
+
codeFileCount,
|
|
176
|
+
totalLines,
|
|
177
|
+
languages,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
// === Scan 3: Internal Structure ===
|
|
181
|
+
async function detectInternalStructure(tk, basePath) {
|
|
182
|
+
const entryPoints = [];
|
|
183
|
+
const keyFiles = [];
|
|
184
|
+
const publicExports = [];
|
|
185
|
+
const directories = [];
|
|
186
|
+
// Find entry points
|
|
187
|
+
const topEntries = await tk.listDir(basePath);
|
|
188
|
+
for (const entry of topEntries) {
|
|
189
|
+
if (entry.type === "directory" && !entry.name.startsWith(".") && !SKIP_DIRS.has(entry.name)) {
|
|
190
|
+
directories.push(entry.name);
|
|
191
|
+
}
|
|
192
|
+
if (entry.type === "file" && ENTRY_POINT_NAMES.has(entry.name)) {
|
|
193
|
+
entryPoints.push(`${basePath}/${entry.name}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Check src/ for entry points too
|
|
197
|
+
const srcEntries = await tk.listDir(`${basePath}/src`);
|
|
198
|
+
for (const entry of srcEntries) {
|
|
199
|
+
if (entry.type === "file" && ENTRY_POINT_NAMES.has(entry.name)) {
|
|
200
|
+
entryPoints.push(`${basePath}/src/${entry.name}`);
|
|
201
|
+
}
|
|
202
|
+
if (entry.type === "directory" && !entry.name.startsWith(".") && !SKIP_DIRS.has(entry.name)) {
|
|
203
|
+
directories.push(`src/${entry.name}`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
// Detect key files from all code files
|
|
207
|
+
const allFiles = await tk.globFiles("**/*.{ts,js,tsx,jsx,py,rs,go}", basePath);
|
|
208
|
+
for (const file of allFiles.slice(0, 100)) {
|
|
209
|
+
const role = detectKeyFileRole(file);
|
|
210
|
+
if (role) {
|
|
211
|
+
keyFiles.push({ path: file, role });
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// Extract public exports from entry points
|
|
215
|
+
for (const ep of entryPoints.slice(0, 3)) {
|
|
216
|
+
const content = await tk.readFileSafe(ep);
|
|
217
|
+
if (!content)
|
|
218
|
+
continue;
|
|
219
|
+
const lines = content.split("\n");
|
|
220
|
+
for (const line of lines) {
|
|
221
|
+
const exportMatch = line.match(/export\s+(?:async\s+)?(?:function|class|const|let|interface|type|enum)\s+(\w+)/);
|
|
222
|
+
if (exportMatch) {
|
|
223
|
+
publicExports.push(exportMatch[1]);
|
|
224
|
+
}
|
|
225
|
+
const defaultExport = line.match(/export\s+default\s+(?:async\s+)?(?:function|class)\s+(\w+)/);
|
|
226
|
+
if (defaultExport) {
|
|
227
|
+
publicExports.push(defaultExport[1]);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return { entryPoints, keyFiles: keyFiles.slice(0, 20), publicExports: publicExports.slice(0, 30), directories };
|
|
232
|
+
}
|
|
233
|
+
// === Scan 4: Dependencies ===
|
|
234
|
+
async function detectDependencies(tk, basePath) {
|
|
235
|
+
const production = [];
|
|
236
|
+
const development = [];
|
|
237
|
+
let manifest = null;
|
|
238
|
+
// Try package.json
|
|
239
|
+
const pkgJson = await tk.readJSON(`${basePath}/package.json`);
|
|
240
|
+
if (pkgJson) {
|
|
241
|
+
manifest = "package.json";
|
|
242
|
+
const deps = pkgJson.dependencies;
|
|
243
|
+
if (deps) {
|
|
244
|
+
for (const name of Object.keys(deps)) {
|
|
245
|
+
const cat = categorizeDep(name);
|
|
246
|
+
production.push({
|
|
247
|
+
name,
|
|
248
|
+
category: cat?.category ?? null,
|
|
249
|
+
displayName: cat?.displayName ?? name,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
const devDeps = pkgJson.devDependencies;
|
|
254
|
+
if (devDeps) {
|
|
255
|
+
for (const name of Object.keys(devDeps)) {
|
|
256
|
+
const cat = categorizeDep(name);
|
|
257
|
+
development.push({
|
|
258
|
+
name,
|
|
259
|
+
category: cat?.category ?? null,
|
|
260
|
+
displayName: cat?.displayName ?? name,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
// Try Cargo.toml
|
|
266
|
+
if (!manifest) {
|
|
267
|
+
const cargoContent = await tk.readFileSafe(`${basePath}/Cargo.toml`);
|
|
268
|
+
if (cargoContent) {
|
|
269
|
+
manifest = "Cargo.toml";
|
|
270
|
+
const depSection = cargoContent.match(/\[dependencies\]([\s\S]*?)(?:\[|$)/);
|
|
271
|
+
if (depSection) {
|
|
272
|
+
const lines = depSection[1].split("\n");
|
|
273
|
+
for (const line of lines) {
|
|
274
|
+
const m = line.match(/^(\w[\w-]*)\s*=/);
|
|
275
|
+
if (m) {
|
|
276
|
+
production.push({ name: m[1], category: null, displayName: m[1] });
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
// Try go.mod
|
|
283
|
+
if (!manifest) {
|
|
284
|
+
const goMod = await tk.readFileSafe(`${basePath}/go.mod`);
|
|
285
|
+
if (goMod) {
|
|
286
|
+
manifest = "go.mod";
|
|
287
|
+
const requireBlock = goMod.match(/require\s*\(([\s\S]*?)\)/);
|
|
288
|
+
if (requireBlock) {
|
|
289
|
+
const lines = requireBlock[1].split("\n");
|
|
290
|
+
for (const line of lines) {
|
|
291
|
+
const m = line.trim().match(/^(\S+)\s/);
|
|
292
|
+
if (m) {
|
|
293
|
+
production.push({ name: m[1], category: null, displayName: m[1].split("/").pop() || m[1] });
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
// Try pyproject.toml
|
|
300
|
+
if (!manifest) {
|
|
301
|
+
const pyproject = await tk.readFileSafe(`${basePath}/pyproject.toml`);
|
|
302
|
+
if (pyproject) {
|
|
303
|
+
manifest = "pyproject.toml";
|
|
304
|
+
const depSection = pyproject.match(/dependencies\s*=\s*\[([\s\S]*?)\]/);
|
|
305
|
+
if (depSection) {
|
|
306
|
+
const lines = depSection[1].split("\n");
|
|
307
|
+
for (const line of lines) {
|
|
308
|
+
const m = line.match(/"([^">=<\[]+)/);
|
|
309
|
+
if (m) {
|
|
310
|
+
production.push({ name: m[1].trim(), category: null, displayName: m[1].trim() });
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return { manifest, production, development };
|
|
317
|
+
}
|
|
318
|
+
// === Scan 5: Import Graph ===
|
|
319
|
+
async function buildImportGraph(tk, basePath) {
|
|
320
|
+
const internalMap = new Map();
|
|
321
|
+
const externalSet = new Set();
|
|
322
|
+
const grepResults = await tk.grepFiles("import\\s.*from\\s|require\\(", basePath);
|
|
323
|
+
for (const r of grepResults.slice(0, 300)) {
|
|
324
|
+
const imported = extractImportPath(r.content);
|
|
325
|
+
if (!imported)
|
|
326
|
+
continue;
|
|
327
|
+
if (imported.startsWith(".") || imported.startsWith("/")) {
|
|
328
|
+
// Internal import
|
|
329
|
+
if (!internalMap.has(imported))
|
|
330
|
+
internalMap.set(imported, new Set());
|
|
331
|
+
internalMap.get(imported).add(r.file);
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
// External package
|
|
335
|
+
const pkg = imported.startsWith("@") ? imported.split("/").slice(0, 2).join("/") : imported.split("/")[0];
|
|
336
|
+
externalSet.add(pkg);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
const internalImports = Array.from(internalMap.entries())
|
|
340
|
+
.map(([targetPath, sourceFiles]) => ({
|
|
341
|
+
targetPath,
|
|
342
|
+
sourceFiles: Array.from(sourceFiles).slice(0, 5),
|
|
343
|
+
}))
|
|
344
|
+
.slice(0, 30);
|
|
345
|
+
return {
|
|
346
|
+
internalImports,
|
|
347
|
+
externalPackages: Array.from(externalSet).sort(),
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
// === Scan 6: Structural Probes ===
|
|
351
|
+
async function runStructuralProbes(tk, basePath) {
|
|
352
|
+
const results = [];
|
|
353
|
+
const probePromises = STRUCTURAL_PROBES.map(async (probe) => {
|
|
354
|
+
const grepResults = await tk.grepFiles(probe.pattern, basePath);
|
|
355
|
+
if (grepResults.length === 0)
|
|
356
|
+
return null;
|
|
357
|
+
return {
|
|
358
|
+
type: probe.type,
|
|
359
|
+
label: probe.label,
|
|
360
|
+
matches: grepResults.slice(0, MAX_PROBE_MATCHES).map((r) => ({
|
|
361
|
+
file: r.file,
|
|
362
|
+
line: String(r.line ?? ""),
|
|
363
|
+
content: r.content.trim().slice(0, 120),
|
|
364
|
+
})),
|
|
365
|
+
count: grepResults.length,
|
|
366
|
+
};
|
|
367
|
+
});
|
|
368
|
+
const probeResults = await Promise.all(probePromises);
|
|
369
|
+
for (const r of probeResults) {
|
|
370
|
+
if (r)
|
|
371
|
+
results.push(r);
|
|
372
|
+
}
|
|
373
|
+
return results;
|
|
374
|
+
}
|
|
375
|
+
// === Helpers ===
|
|
376
|
+
function extractImportPath(line) {
|
|
377
|
+
const esMatch = line.match(/from\s+["']([^"']+)["']/);
|
|
378
|
+
if (esMatch)
|
|
379
|
+
return esMatch[1];
|
|
380
|
+
const cjsMatch = line.match(/require\(\s*["']([^"']+)["']\s*\)/);
|
|
381
|
+
if (cjsMatch)
|
|
382
|
+
return cjsMatch[1];
|
|
383
|
+
return null;
|
|
384
|
+
}
|
|
385
|
+
function getExtension(filePath) {
|
|
386
|
+
const match = filePath.match(/(\.[^./]+)$/);
|
|
387
|
+
return match ? match[1].toLowerCase() : "";
|
|
388
|
+
}
|
|
@@ -1,38 +1,7 @@
|
|
|
1
1
|
// Static Analysis — Doc Parser
|
|
2
2
|
// Deep-scans README, CLAUDE.md, architecture docs, API docs, mermaid diagrams
|
|
3
3
|
// to extract project description, components, connections, endpoints, and external deps
|
|
4
|
-
|
|
5
|
-
{ pattern: /aws\s*s3|amazon\s*s3|@aws-sdk\/client-s3/i, name: "AWS S3" },
|
|
6
|
-
{ pattern: /dynamodb|@aws-sdk\/client-dynamodb/i, name: "DynamoDB" },
|
|
7
|
-
{ pattern: /aws\s*lambda|@aws-sdk\/client-lambda/i, name: "AWS Lambda" },
|
|
8
|
-
{ pattern: /sqs|@aws-sdk\/client-sqs/i, name: "AWS SQS" },
|
|
9
|
-
{ pattern: /sns|@aws-sdk\/client-sns/i, name: "AWS SNS" },
|
|
10
|
-
{ pattern: /cloudflare\s*workers|wrangler/i, name: "Cloudflare Workers" },
|
|
11
|
-
{ pattern: /redis|ioredis|@upstash\/redis/i, name: "Redis" },
|
|
12
|
-
{ pattern: /postgres(?:ql)?|pg\b|@prisma|prisma|tokio-postgres|neon/i, name: "PostgreSQL" },
|
|
13
|
-
{ pattern: /mysql|mariadb/i, name: "MySQL" },
|
|
14
|
-
{ pattern: /mongodb|mongoose/i, name: "MongoDB" },
|
|
15
|
-
{ pattern: /firebase|firestore/i, name: "Firebase" },
|
|
16
|
-
{ pattern: /supabase/i, name: "Supabase" },
|
|
17
|
-
{ pattern: /stripe/i, name: "Stripe" },
|
|
18
|
-
{ pattern: /twilio/i, name: "Twilio" },
|
|
19
|
-
{ pattern: /sendgrid|@sendgrid/i, name: "SendGrid" },
|
|
20
|
-
{ pattern: /elasticsearch|opensearch/i, name: "Elasticsearch" },
|
|
21
|
-
{ pattern: /rabbitmq|amqplib/i, name: "RabbitMQ" },
|
|
22
|
-
{ pattern: /kafka|kafkajs/i, name: "Kafka" },
|
|
23
|
-
{ pattern: /nats\.io|nats/i, name: "NATS" },
|
|
24
|
-
{ pattern: /auth0/i, name: "Auth0" },
|
|
25
|
-
{ pattern: /clerk/i, name: "Clerk" },
|
|
26
|
-
{ pattern: /sentry/i, name: "Sentry" },
|
|
27
|
-
{ pattern: /datadog/i, name: "Datadog" },
|
|
28
|
-
{ pattern: /vercel/i, name: "Vercel" },
|
|
29
|
-
{ pattern: /netlify/i, name: "Netlify" },
|
|
30
|
-
{ pattern: /openai|gpt-4|gpt-3/i, name: "OpenAI" },
|
|
31
|
-
{ pattern: /anthropic|claude/i, name: "Anthropic" },
|
|
32
|
-
{ pattern: /mapbox/i, name: "Mapbox" },
|
|
33
|
-
{ pattern: /google\s*places|google\s*maps/i, name: "Google Maps/Places" },
|
|
34
|
-
{ pattern: /nginx/i, name: "NGINX" },
|
|
35
|
-
];
|
|
4
|
+
import { categorizeDep } from "./taxonomy.js";
|
|
36
5
|
export async function parseDocs(tk) {
|
|
37
6
|
const result = {
|
|
38
7
|
projectDescription: "",
|
|
@@ -46,7 +15,6 @@ export async function parseDocs(tk) {
|
|
|
46
15
|
const readme = await tk.readFileSafe("README.md");
|
|
47
16
|
if (readme) {
|
|
48
17
|
result.projectDescription = extractDescription(readme);
|
|
49
|
-
scanForExternalDeps(readme, result);
|
|
50
18
|
}
|
|
51
19
|
// Read all discovered doc files in parallel (batch of 15)
|
|
52
20
|
const contents = await Promise.all(docFiles.slice(0, 15).map(async (f) => ({
|
|
@@ -64,8 +32,6 @@ export async function parseDocs(tk) {
|
|
|
64
32
|
// Extract API endpoints from docs
|
|
65
33
|
const endpoints = extractAPIEndpoints(content);
|
|
66
34
|
result.apiEndpoints.push(...endpoints);
|
|
67
|
-
// Scan all docs for external service mentions
|
|
68
|
-
scanForExternalDeps(content, result);
|
|
69
35
|
}
|
|
70
36
|
// Parse OpenAPI spec if available
|
|
71
37
|
const [openApiYaml, openApiJson] = await Promise.all([
|
|
@@ -73,7 +39,7 @@ export async function parseDocs(tk) {
|
|
|
73
39
|
tk.readJSON("openapi.json"),
|
|
74
40
|
]);
|
|
75
41
|
parseOpenAPI(openApiYaml ?? openApiJson, result);
|
|
76
|
-
//
|
|
42
|
+
// Scan build manifests for external dependencies using taxonomy
|
|
77
43
|
await scanBuildDepsForServices(tk, result);
|
|
78
44
|
return result;
|
|
79
45
|
}
|
|
@@ -164,7 +130,6 @@ function extractDescription(readme) {
|
|
|
164
130
|
*/
|
|
165
131
|
function extractArchitectureNotes(path, content) {
|
|
166
132
|
const notes = [];
|
|
167
|
-
const lines = content.split("\n");
|
|
168
133
|
// Extract technology stack tables (markdown tables with | Component | Technology |)
|
|
169
134
|
const tableBlocks = extractMarkdownTables(content);
|
|
170
135
|
if (tableBlocks.length > 0) {
|
|
@@ -315,10 +280,20 @@ function extractConnectionDescriptions(content) {
|
|
|
315
280
|
}
|
|
316
281
|
return descs;
|
|
317
282
|
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
283
|
+
/**
|
|
284
|
+
* Scan external dependencies from manifests using taxonomy.
|
|
285
|
+
* Only reads build manifests (package.json, Cargo.toml, etc.) — never doc text.
|
|
286
|
+
*/
|
|
287
|
+
function scanDepsForExternals(depNames, result) {
|
|
288
|
+
for (const dep of depNames) {
|
|
289
|
+
const cat = categorizeDep(dep);
|
|
290
|
+
if (!cat)
|
|
291
|
+
continue;
|
|
292
|
+
if (cat.role === "external" || cat.role === "data" || cat.role === "messaging") {
|
|
293
|
+
const name = cat.displayName;
|
|
294
|
+
if (!result.externalDependencies.includes(name)) {
|
|
295
|
+
result.externalDependencies.push(name);
|
|
296
|
+
}
|
|
322
297
|
}
|
|
323
298
|
}
|
|
324
299
|
}
|
|
@@ -343,16 +318,66 @@ function parseOpenAPI(spec, result) {
|
|
|
343
318
|
}
|
|
344
319
|
}
|
|
345
320
|
async function scanBuildDepsForServices(tk, result) {
|
|
321
|
+
// Collect dep names from all package.json files
|
|
346
322
|
const pkg = await tk.readJSON("package.json");
|
|
347
323
|
if (pkg) {
|
|
348
|
-
const
|
|
324
|
+
const depNames = Object.keys({
|
|
349
325
|
...pkg.dependencies,
|
|
350
326
|
...pkg.devDependencies,
|
|
351
|
-
})
|
|
352
|
-
|
|
327
|
+
});
|
|
328
|
+
scanDepsForExternals(depNames, result);
|
|
329
|
+
}
|
|
330
|
+
// Check subdirectory package.json files too (e.g. cloud/package.json)
|
|
331
|
+
const rootEntries = await tk.listDir(".");
|
|
332
|
+
const subPkgs = await Promise.all(rootEntries
|
|
333
|
+
.filter((e) => e.type === "directory")
|
|
334
|
+
.slice(0, 15)
|
|
335
|
+
.map((e) => tk.readJSON(`${e.name}/package.json`)));
|
|
336
|
+
for (const subPkg of subPkgs) {
|
|
337
|
+
if (!subPkg)
|
|
338
|
+
continue;
|
|
339
|
+
const depNames = Object.keys({
|
|
340
|
+
...subPkg.dependencies,
|
|
341
|
+
...subPkg.devDependencies,
|
|
342
|
+
});
|
|
343
|
+
scanDepsForExternals(depNames, result);
|
|
344
|
+
}
|
|
345
|
+
// For non-JS manifests, extract dep names and check against taxonomy
|
|
346
|
+
const [cargo, goMod, requirements, pyProject] = await Promise.all([
|
|
347
|
+
tk.readFileSafe("Cargo.toml"),
|
|
348
|
+
tk.readFileSafe("go.mod"),
|
|
349
|
+
tk.readFileSafe("requirements.txt"),
|
|
350
|
+
tk.readFileSafe("pyproject.toml"),
|
|
351
|
+
]);
|
|
352
|
+
// Extract crate names from Cargo.toml [dependencies] section
|
|
353
|
+
if (cargo) {
|
|
354
|
+
const depSection = cargo.match(/\[dependencies\]([\s\S]*?)(?:\[|$)/);
|
|
355
|
+
if (depSection) {
|
|
356
|
+
const crateNames = [...depSection[1].matchAll(/^(\S+)\s*=/gm)].map(m => m[1]);
|
|
357
|
+
scanDepsForExternals(crateNames, result);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
// Extract module paths from go.mod require block
|
|
361
|
+
if (goMod) {
|
|
362
|
+
const requires = goMod.match(/require\s*\(([\s\S]*?)\)/);
|
|
363
|
+
if (requires) {
|
|
364
|
+
const modNames = [...requires[1].matchAll(/^\s*(\S+)/gm)].map(m => m[1]);
|
|
365
|
+
scanDepsForExternals(modNames, result);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
// Extract package names from requirements.txt
|
|
369
|
+
if (requirements) {
|
|
370
|
+
const pkgNames = requirements.split("\n")
|
|
371
|
+
.filter(l => l.trim() && !l.startsWith("#"))
|
|
372
|
+
.map(l => l.split(/[>=<!\[]/)[0].trim());
|
|
373
|
+
scanDepsForExternals(pkgNames, result);
|
|
374
|
+
}
|
|
375
|
+
// Extract package names from pyproject.toml dependencies
|
|
376
|
+
if (pyProject) {
|
|
377
|
+
const depSection = pyProject.match(/dependencies\s*=\s*\[([\s\S]*?)\]/);
|
|
378
|
+
if (depSection) {
|
|
379
|
+
const pkgNames = [...depSection[1].matchAll(/"([^">=<!\[]+)/g)].map(m => m[1].trim());
|
|
380
|
+
scanDepsForExternals(pkgNames, result);
|
|
381
|
+
}
|
|
353
382
|
}
|
|
354
|
-
// Also check Cargo.toml at root
|
|
355
|
-
const cargo = await tk.readFileSafe("Cargo.toml");
|
|
356
|
-
if (cargo)
|
|
357
|
-
scanForExternalDeps(cargo, result);
|
|
358
383
|
}
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
// Static Analysis — Environment Detector
|
|
2
2
|
// Detects environments, config patterns, and secrets management from .env files
|
|
3
|
-
|
|
4
|
-
"dotenv", "@aws-sdk/client-secrets-manager", "vault", "node-vault",
|
|
5
|
-
"@google-cloud/secret-manager", "@azure/keyvault-secrets", "infisical-sdk",
|
|
6
|
-
];
|
|
3
|
+
import { categorizeDep } from "./taxonomy.js";
|
|
7
4
|
export async function detectEnvironments(tk) {
|
|
8
5
|
const result = {
|
|
9
6
|
environments: [],
|
|
@@ -57,14 +54,14 @@ export async function detectEnvironments(tk) {
|
|
|
57
54
|
}
|
|
58
55
|
}
|
|
59
56
|
}
|
|
60
|
-
// Check for secrets management in deps
|
|
57
|
+
// Check for secrets management in deps using taxonomy
|
|
61
58
|
const pkg = await tk.readJSON("package.json");
|
|
62
59
|
if (pkg) {
|
|
63
60
|
const allDeps = Object.keys({
|
|
64
61
|
...pkg.dependencies,
|
|
65
62
|
...pkg.devDependencies,
|
|
66
63
|
});
|
|
67
|
-
result.hasSecrets = allDeps.some((d) =>
|
|
64
|
+
result.hasSecrets = allDeps.some((d) => categorizeDep(d)?.category === "secrets");
|
|
68
65
|
if (allDeps.includes("dotenv")) {
|
|
69
66
|
result.configPattern = result.configPattern ?? "dotenv";
|
|
70
67
|
}
|
|
@@ -1,54 +1,37 @@
|
|
|
1
1
|
// Static Analysis — Event Detector
|
|
2
2
|
// Detects event-driven architecture patterns from deps and code grep
|
|
3
3
|
import { EXCLUDED_DIRS_REGEX } from "./excluded-dirs.js";
|
|
4
|
-
|
|
5
|
-
{ dep: "kafkajs", technology: "Kafka" },
|
|
6
|
-
{ dep: "@confluentinc/kafka-javascript", technology: "Kafka" },
|
|
7
|
-
{ dep: "amqplib", technology: "RabbitMQ" },
|
|
8
|
-
{ dep: "bullmq", technology: "Redis Streams (BullMQ)" },
|
|
9
|
-
{ dep: "bull", technology: "Redis Streams (Bull)" },
|
|
10
|
-
{ dep: "bee-queue", technology: "Redis Streams (Bee-Queue)" },
|
|
11
|
-
{ dep: "socket.io", technology: "WebSocket (Socket.IO)" },
|
|
12
|
-
{ dep: "ws", technology: "WebSocket" },
|
|
13
|
-
{ dep: "@google-cloud/pubsub", technology: "Google Pub/Sub" },
|
|
14
|
-
{ dep: "@aws-sdk/client-sqs", technology: "AWS SQS" },
|
|
15
|
-
{ dep: "@aws-sdk/client-sns", technology: "AWS SNS" },
|
|
16
|
-
{ dep: "@aws-sdk/client-eventbridge", technology: "AWS EventBridge" },
|
|
17
|
-
{ dep: "nats", technology: "NATS" },
|
|
18
|
-
{ dep: "mqtt", technology: "MQTT" },
|
|
19
|
-
{ dep: "@azure/service-bus", technology: "Azure Service Bus" },
|
|
20
|
-
{ dep: "@azure/event-hubs", technology: "Azure Event Hubs" },
|
|
21
|
-
{ dep: "sse-channel", technology: "Server-Sent Events" },
|
|
22
|
-
{ dep: "better-sse", technology: "Server-Sent Events" },
|
|
23
|
-
];
|
|
4
|
+
import { categorizeDep } from "./taxonomy.js";
|
|
24
5
|
export async function detectEvents(tk) {
|
|
25
6
|
const result = {
|
|
26
7
|
hasEDA: false,
|
|
27
8
|
patterns: [],
|
|
28
9
|
events: [],
|
|
29
10
|
};
|
|
30
|
-
// Check package.json deps
|
|
11
|
+
// Check package.json deps against taxonomy (queue-client, realtime, sse)
|
|
31
12
|
const pkg = await tk.readJSON("package.json");
|
|
32
13
|
if (pkg) {
|
|
33
14
|
const allDeps = Object.keys({
|
|
34
15
|
...pkg.dependencies,
|
|
35
16
|
...pkg.devDependencies,
|
|
36
17
|
});
|
|
37
|
-
for (const
|
|
38
|
-
|
|
18
|
+
for (const dep of allDeps) {
|
|
19
|
+
const cat = categorizeDep(dep);
|
|
20
|
+
if (cat && (cat.category === "queue-client" || cat.category === "realtime" || cat.category === "sse")) {
|
|
39
21
|
result.hasEDA = true;
|
|
40
22
|
result.patterns.push({
|
|
41
|
-
technology:
|
|
42
|
-
dependency:
|
|
23
|
+
technology: cat.displayName,
|
|
24
|
+
dependency: dep,
|
|
43
25
|
});
|
|
44
26
|
}
|
|
45
27
|
}
|
|
46
28
|
}
|
|
47
29
|
// Grep for event patterns (filtered to exclude vendored code)
|
|
48
|
-
const [publishResults, subscribeResults, emitResults] = await Promise.all([
|
|
30
|
+
const [publishResults, subscribeResults, emitResults, sseResults] = await Promise.all([
|
|
49
31
|
tk.grepFiles("\\.publish\\(|\\.send\\(|\\.produce\\("),
|
|
50
32
|
tk.grepFiles("\\.subscribe\\(|\\.consume\\(|\\.on\\("),
|
|
51
33
|
tk.grepFiles("\\.emit\\(|\\.dispatch\\(|\\.trigger\\("),
|
|
34
|
+
tk.grepFiles("text/event-stream|new EventSource|sseClients|sse_clients"),
|
|
52
35
|
]);
|
|
53
36
|
// Exclude vendored/generated dirs from all results
|
|
54
37
|
const excludeDirs = EXCLUDED_DIRS_REGEX;
|
|
@@ -64,6 +47,17 @@ export async function detectEvents(tk) {
|
|
|
64
47
|
for (const r of filterVendored(emitResults).slice(0, 20)) {
|
|
65
48
|
result.events.push({ type: "emit", file: r.file, pattern: r.content.slice(0, 100) });
|
|
66
49
|
}
|
|
50
|
+
// SSE (Server-Sent Events) detection from code patterns (skip docs and agent prompts)
|
|
51
|
+
const filteredSSE = filterVendored(sseResults).filter((r) => !r.file.endsWith(".md") && !r.file.endsWith(".txt") &&
|
|
52
|
+
!r.file.startsWith("agents/"));
|
|
53
|
+
if (filteredSSE.length > 0) {
|
|
54
|
+
if (!result.patterns.some((p) => p.technology === "Server-Sent Events")) {
|
|
55
|
+
result.patterns.push({ technology: "Server-Sent Events", dependency: "(code pattern)" });
|
|
56
|
+
}
|
|
57
|
+
for (const r of filteredSSE.slice(0, 10)) {
|
|
58
|
+
result.events.push({ type: "broadcast", file: r.file, pattern: r.content.slice(0, 100) });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
67
61
|
if (result.events.length > 0) {
|
|
68
62
|
result.hasEDA = true;
|
|
69
63
|
}
|