crewforth 3.1.0 → 3.2.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +151 -0
- package/README.md +4 -3
- package/VERSION +1 -1
- package/adopt.sh +91 -11
- package/kit/AGENT_TEMPLATE.md +2 -1
- package/kit/CLAUDE.md +9 -8
- package/kit/agents/crew-commit-agent.md +7 -2
- package/kit/agents/crew-docs-agent.md +71 -0
- package/kit/agents/crew-planner.md +2 -0
- package/kit/agents/crew-refuter.md +77 -0
- package/kit/agents/crew-security-expert.md +3 -1
- package/kit/agents/crew-session-manager.md +11 -9
- package/kit/eval/doctor.sh +91 -3
- package/kit/eval/golden-routing.txt +9 -0
- package/kit/eval/lib/crew-map/crew-map.sh +26 -0
- package/kit/eval/lib/crew-map/detect.mjs +238 -0
- package/kit/eval/lib/crew-map/narrative.mjs +98 -0
- package/kit/eval/lib/crew-map/parse.mjs +305 -0
- package/kit/eval/lib/crew-map/project.mjs +163 -0
- package/kit/eval/lib/crew-map/rules/names.json +38 -0
- package/kit/eval/lib/crew-map/rules/packages.json +142 -0
- package/kit/eval/lib/crew-map/rules/python-stdlib.json +197 -0
- package/kit/eval/lib/crew-map/scan.mjs +214 -0
- package/kit/eval/lib/crew-map/stacks.mjs +441 -0
- package/kit/eval/lib/crew-map/surface.mjs +457 -0
- package/kit/eval/lib/findings/findings.mjs +410 -0
- package/kit/eval/lib/findings/findings.schema.json +122 -0
- package/kit/eval/lib/findings/findings.sh +29 -0
- package/kit/eval/lib/settings-json.awk +5 -0
- package/kit/eval/lib/star.sh +23 -7
- package/kit/eval/lib/statusline.settings.json +6 -0
- package/kit/eval/lib/statusline.sh +115 -0
- package/kit/eval/smoke-test.sh +465 -38
- package/kit/hooks/agent-outcome.sh +6 -3
- package/kit/hooks/guard-agent-model.sh +15 -5
- package/kit/hooks/guard-bash.sh +240 -182
- package/kit/hooks/guard-question.sh +73 -0
- package/kit/hooks/guard-write.sh +21 -0
- package/kit/hooks/prompt-approval.sh +320 -103
- package/kit/hooks/refuter-ledger.sh +80 -0
- package/kit/owned-blobs.tsv +4 -1
- package/kit/settings.json +33 -0
- package/kit/skills/crew-approve/SKILL.md +7 -0
- package/kit/skills/crew-doctor/SKILL.md +5 -1
- package/kit/skills/crew-handoff/SKILL.md +3 -2
- package/kit/skills/crew-map/SKILL.md +44 -0
- package/kit/skills/crew-review/SKILL.md +2 -0
- package/kit/skills/crew-update/SKILL.md +1 -1
- package/kit/skills/incident-runbook/SKILL.md +2 -2
- package/kit/skills/security-scan/references/verify.md +6 -3
- package/kit/studio/README.md +12 -0
- package/kit/studio/server/index.js +114 -11
- package/kit/studio/server/lib/crew-map.js +113 -0
- package/kit/studio/server/lib/model-outcomes.js +130 -15
- package/kit/studio/server/lib/pricing.js +3 -0
- package/kit/studio/server/lib/remote.js +99 -0
- package/kit/studio/server/lib/usage.js +74 -14
- package/kit/studio/web/app.js +214 -12
- package/kit/studio/web/canvas.js +44 -120
- package/kit/studio/web/graph-plan.js +10 -4
- package/kit/studio/web/index.html +10 -5
- package/kit/studio/web/map-plan.js +341 -0
- package/kit/studio/web/map.js +503 -0
- package/kit/studio/web/models-plan.js +61 -7
- package/kit/studio/web/models.js +19 -3
- package/kit/studio/web/style.css +125 -0
- package/kit/studio/web/usage.js +12 -0
- package/kit/studio/web/viewport.js +160 -0
- package/package.json +6 -5
- package/start.sh +10 -0
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
// stacks.mjs — modules, their declared dependencies, their imports and their references to each other, for the
|
|
2
|
+
// five stacks. Every fact carries the line it was read from; what cannot be read is returned as notMeasured.
|
|
3
|
+
//
|
|
4
|
+
// A module is { id, name, stack, dir, manifest, tech[], evidence[], deps[], refs[], imports[], entry, entryEv,
|
|
5
|
+
// aliases[], selfNames[], files[] }.
|
|
6
|
+
// deps { eco, name, version, dev, ev } declared in the manifest
|
|
7
|
+
// refs { to: <module name or manifest path>, by, ev, confidence } this module names another one
|
|
8
|
+
// imports { spec, root, ev, file } a source file imports a package
|
|
9
|
+
// `confidence` is 'evidence' when the line itself names the other module, 'inferred' when it was concluded
|
|
10
|
+
// (a namespace prefix, a Gradle line read by pattern).
|
|
11
|
+
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import { parseJsonc, parseToml, parseXml, xKids, xKid, xText, xLine, xFind, Unsupported } from './parse.mjs';
|
|
14
|
+
import { ev, evFirst, lineOf, isTestPath } from './project.mjs';
|
|
15
|
+
|
|
16
|
+
const P = path.posix;
|
|
17
|
+
const dirOf = (f) => { const d = P.dirname(f); return d === '.' ? '.' : d; };
|
|
18
|
+
const under = (file, dir) => dir === '.' || file === dir || file.startsWith(dir + '/');
|
|
19
|
+
const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
20
|
+
const isComment = (l) => /^\s*(\/\/|#|\*|\/\*|<!--)/.test(l);
|
|
21
|
+
|
|
22
|
+
export const SOURCE_EXT = {
|
|
23
|
+
node: /\.(m?[jt]sx?|cjs|cts|mts)$/, python: /\.py$/, go: /\.go$/, dotnet: /\.cs$/, java: /\.(java|kt)$/,
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
function mod(stack, name, manifest, dir) {
|
|
27
|
+
return { id: `mod:${stack}:${name}`, name, stack, dir, manifest, tech: new Set(), evidence: [], deps: [], refs: [],
|
|
28
|
+
imports: [], entry: false, entryEv: null, aliases: [], selfNames: [name], files: [] };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// A version the map may show. A dependency can be declared by URL (git+https://token@host/repo.git#v1), and a URL
|
|
32
|
+
// is never copied out: it reads as 'url', which no detector compares.
|
|
33
|
+
export function cleanVersion(v) {
|
|
34
|
+
const s = String(v == null ? '' : v).trim();
|
|
35
|
+
if (/:\/\/|@|^(git|github|gitlab|bitbucket|file|link|http)[+:]/i.test(s)) return 'url';
|
|
36
|
+
return s.slice(0, 64);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Packages that make a module something that is RUN (a server, a worker, a CLI), so it is an entry point even when
|
|
40
|
+
// nothing in the repository references it.
|
|
41
|
+
const RUNNABLE = { npm: ['express', 'fastify', '@nestjs/core', 'koa', '@hapi/hapi', 'next', 'nuxt', 'vite', 'react-scripts'],
|
|
42
|
+
pypi: ['fastapi', 'flask', 'django', 'uvicorn', 'gunicorn', 'celery', 'streamlit', 'aiohttp', 'starlette', 'typer', 'click'] };
|
|
43
|
+
function runnable(m) {
|
|
44
|
+
if (m.entry) return;
|
|
45
|
+
const d = m.deps.find((x) => (RUNNABLE[x.eco] || []).includes(x.eco === 'pypi' ? x.name.toLowerCase() : x.name));
|
|
46
|
+
if (d) { m.entry = true; m.entryEv = d.ev; }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Hand each source file to the deepest module of its stack whose directory holds it.
|
|
50
|
+
// A file under a sample project's own manifest (left out of discovery) belongs to that sample, so to nobody here.
|
|
51
|
+
const MANIFEST = /(^|\/)(package\.json|pyproject\.toml|setup\.cfg|requirements[^/]*\.txt|go\.mod|[^/]+\.(cs|fs|vb)proj|pom\.xml|build\.gradle(\.kts)?|settings\.gradle(\.kts)?)$/;
|
|
52
|
+
function own(modules, files, stack) {
|
|
53
|
+
const mine = modules.filter((m) => m.stack === stack).sort((a, b) => b.dir.length - a.dir.length);
|
|
54
|
+
const samples = files.filter((f) => MANIFEST.test(f) && isTestPath(f)).map(dirOf);
|
|
55
|
+
for (const f of files) {
|
|
56
|
+
if (!SOURCE_EXT[stack].test(f)) continue;
|
|
57
|
+
if (samples.some((d) => under(f, d))) continue;
|
|
58
|
+
const m = mine.find((x) => under(f, x.dir));
|
|
59
|
+
if (m) m.files.push(f);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ---- Node / TypeScript ---------------------------------------------------------------------------------------
|
|
64
|
+
function nodeStack(pj, out) {
|
|
65
|
+
const manifests = pj.manifests.filter((f) => P.basename(f) === 'package.json');
|
|
66
|
+
for (const f of manifests) {
|
|
67
|
+
const src = pj.read(f); if (!src) continue;
|
|
68
|
+
let doc; try { doc = parseJsonc(src.text); } catch (e) { out.notMeasured.push({ what: f, reason: `package.json could not be read: ${e.message}` }); continue; }
|
|
69
|
+
const v = doc.value || {}; const L = (k) => doc.lines.get(k.join('\u0000')) || 1;
|
|
70
|
+
const named = typeof v.name === 'string' && v.name.trim() !== '';
|
|
71
|
+
const name = named ? v.name : (dirOf(f) === '.' ? 'root' : dirOf(f));
|
|
72
|
+
const m = mod('node', name, f, dirOf(f));
|
|
73
|
+
const brace = lineOf(src.lines, /\{/) || 1;
|
|
74
|
+
m.evidence.push(named ? ev(f, L(['name']), v.name) : ev(f, brace, '{'));
|
|
75
|
+
for (const [sect, dev] of [['dependencies', false], ['devDependencies', true], ['peerDependencies', true], ['optionalDependencies', false]]) {
|
|
76
|
+
if (!v[sect] || typeof v[sect] !== 'object') continue;
|
|
77
|
+
for (const [dn, ver] of Object.entries(v[sect])) m.deps.push({ eco: 'npm', name: dn, version: cleanVersion(ver), dev, peer: sect === 'peerDependencies', ev: ev(f, L([sect, dn]), dn) });
|
|
78
|
+
}
|
|
79
|
+
for (const fw of ['next', 'react', 'vue', 'svelte', '@angular/core', 'express', 'fastify', '@nestjs/core', 'vite']) {
|
|
80
|
+
const d = m.deps.find((x) => x.name === fw && x.version !== 'url'); if (d) m.tech.add(`${fw}@${d.version.replace(/^[\^~>=<\s]+/, '')}`);
|
|
81
|
+
}
|
|
82
|
+
if (v.bin) { m.entry = true; m.entryEv = ev(f, L(['bin']), 'bin'); }
|
|
83
|
+
else if (v.scripts && (v.scripts.start || v.scripts.dev)) { const k = v.scripts.start ? 'start' : 'dev'; m.entry = true; m.entryEv = ev(f, L(['scripts', k]), k); }
|
|
84
|
+
else if (v.workspaces) { m.entry = true; m.entryEv = ev(f, L(['workspaces']), 'workspaces'); }
|
|
85
|
+
if (v.imports && typeof v.imports === 'object') m.aliases.push(...Object.keys(v.imports).map((k) => k.replace(/\*.*$/, '')));
|
|
86
|
+
runnable(m);
|
|
87
|
+
out.modules.push(m);
|
|
88
|
+
}
|
|
89
|
+
// tsconfig paths are aliases, not packages.
|
|
90
|
+
for (const f of pj.manifests.filter((x) => /(^|\/)tsconfig[^/]*\.json$|(^|\/)jsconfig\.json$/.test(x))) {
|
|
91
|
+
const src = pj.read(f); if (!src) continue;
|
|
92
|
+
let doc; try { doc = parseJsonc(src.text); } catch (e) { out.notMeasured.push({ what: f, reason: `could not be read: ${e.message}` }); continue; }
|
|
93
|
+
const paths = doc.value && doc.value.compilerOptions && doc.value.compilerOptions.paths;
|
|
94
|
+
if (!paths) continue;
|
|
95
|
+
const owner = out.modules.filter((m) => m.stack === 'node' && under(f, m.dir)).sort((a, b) => b.dir.length - a.dir.length)[0];
|
|
96
|
+
for (const k of Object.keys(paths)) (owner ? owner.aliases : out.globalAliases).push(k.replace(/\*.*$/, ''));
|
|
97
|
+
}
|
|
98
|
+
const mods = out.modules.filter((m) => m.stack === 'node');
|
|
99
|
+
own(out.modules, pj.files, 'node');
|
|
100
|
+
const byName = new Map(mods.map((m) => [m.name, m]));
|
|
101
|
+
// Is position `at` of a line inside a string literal? `'import x from "a"'` in a test is text, not an import.
|
|
102
|
+
const inString = (l, at) => {
|
|
103
|
+
let q = null;
|
|
104
|
+
for (let k = 0; k < at; k++) {
|
|
105
|
+
const ch = l[k];
|
|
106
|
+
if (q) { if (ch === '\\') k += 1; else if (ch === q) q = null; } else if (ch === '"' || ch === "'" || ch === '`') q = ch;
|
|
107
|
+
}
|
|
108
|
+
return q !== null;
|
|
109
|
+
};
|
|
110
|
+
const IMP = [/\bfrom\s+(['"])([^'"]+)\1/g, /^\s*import\s+(['"])([^'"]+)\1/g, /\brequire\(\s*(['"])([^'"]+)\1\s*\)/g, /\bimport\(\s*(['"])([^'"]+)\1\s*\)/g];
|
|
111
|
+
for (const m of mods) {
|
|
112
|
+
for (const d of m.deps) {
|
|
113
|
+
const t = byName.get(d.name);
|
|
114
|
+
if (t && t !== m) m.refs.push({ to: t.id, toMod: t, ev: d.ev, confidence: 'evidence' });
|
|
115
|
+
}
|
|
116
|
+
for (const f of m.files) {
|
|
117
|
+
const src = pj.read(f); if (!src) continue;
|
|
118
|
+
src.lines.forEach((l, i) => {
|
|
119
|
+
if (isComment(l)) return;
|
|
120
|
+
for (const re of IMP) {
|
|
121
|
+
re.lastIndex = 0;
|
|
122
|
+
for (const x of l.matchAll(re)) {
|
|
123
|
+
if (inString(l, x.index)) continue;
|
|
124
|
+
const spec = x[2];
|
|
125
|
+
if (spec.startsWith('.') || spec.startsWith('/')) {
|
|
126
|
+
const target = P.normalize(P.join(dirOf(f), spec));
|
|
127
|
+
const t = mods.filter((o) => under(target, o.dir)).sort((a, b) => b.dir.length - a.dir.length)[0];
|
|
128
|
+
if (t && t !== m) m.refs.push({ to: t.id, toMod: t, ev: ev(f, i + 1, spec), confidence: 'evidence' });
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
// Built at run time, or a bundler's virtual module (astro:content, virtual:x): not a package.
|
|
132
|
+
if (spec.includes('${') || /^[a-z][a-z0-9+.-]*:/.test(spec)) continue;
|
|
133
|
+
const root = spec.startsWith('@') ? spec.split('/').slice(0, 2).join('/') : spec.split('/')[0];
|
|
134
|
+
// A package is named the way npm names one; `from '[^']*…` inside a regex literal is not an import.
|
|
135
|
+
if (!/^(@[a-z0-9._~-]+\/)?[a-z0-9._~-]+$/i.test(root)) continue;
|
|
136
|
+
m.imports.push({ spec, root, ev: ev(f, i + 1, spec), file: f });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// ---- Python ---------------------------------------------------------------------------------------------------
|
|
145
|
+
const PEP508 = /^\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*(\[[^\]]*\])?\s*([^;#]*)/;
|
|
146
|
+
function pythonStack(pj, out) {
|
|
147
|
+
const dirs = new Map();
|
|
148
|
+
const get = (dir, manifest) => {
|
|
149
|
+
if (!dirs.has(dir)) {
|
|
150
|
+
const m = mod('python', dir === '.' ? 'root' : dir, manifest, dir);
|
|
151
|
+
dirs.set(dir, m);
|
|
152
|
+
}
|
|
153
|
+
return dirs.get(dir);
|
|
154
|
+
};
|
|
155
|
+
for (const f of pj.manifests) {
|
|
156
|
+
const b = P.basename(f);
|
|
157
|
+
const src = /^(pyproject\.toml|setup\.cfg|requirements[^/]*\.txt)$/.test(b) ? pj.read(f) : null;
|
|
158
|
+
if (!src) continue;
|
|
159
|
+
const m = get(dirOf(f), f);
|
|
160
|
+
if (b === 'pyproject.toml') {
|
|
161
|
+
let rows; try { rows = parseToml(src.text); } catch (e) { out.notMeasured.push({ what: f, reason: `pyproject.toml could not be read: ${e.message}` }); continue; }
|
|
162
|
+
m.manifest = f;
|
|
163
|
+
for (const r of rows) {
|
|
164
|
+
if (r.table === 'project' && r.key === 'name' && typeof r.value === 'string') { m.name = r.value; m.selfNames.push(r.value); m.evidence.push(ev(f, r.line, r.value)); }
|
|
165
|
+
if (r.table === 'tool.poetry' && r.key === 'name' && typeof r.value === 'string') { m.name = r.value; m.selfNames.push(r.value); m.evidence.push(ev(f, r.line, r.value)); }
|
|
166
|
+
if ((r.table === 'project' && r.key === 'dependencies') || (r.table === 'project.optional-dependencies')) {
|
|
167
|
+
const list = Array.isArray(r.value) ? r.value : [];
|
|
168
|
+
for (const s of list) {
|
|
169
|
+
const p = PEP508.exec(String(s)); if (!p) continue;
|
|
170
|
+
const line = lineOf(src.lines, new RegExp(`["']${esc(p[1])}`), r.line - 1) || r.line;
|
|
171
|
+
m.deps.push({ eco: 'pypi', name: p[1], version: cleanVersion(p[3]), dev: r.table !== 'project', ev: ev(f, line, p[1]) });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
if (/^tool\.poetry\.(dev-)?dependencies$|^tool\.poetry\.group\.[^.]+\.dependencies$/.test(r.table) && r.key !== 'python') {
|
|
175
|
+
const ver = typeof r.value === 'string' ? r.value : (r.value && r.value.version) || '';
|
|
176
|
+
m.deps.push({ eco: 'pypi', name: r.key, version: String(ver), dev: r.table !== 'tool.poetry.dependencies', ev: ev(f, r.line, r.key) });
|
|
177
|
+
}
|
|
178
|
+
if ((r.table === 'project.scripts' || r.table === 'tool.poetry.scripts') && !m.entry) { m.entry = true; m.entryEv = ev(f, r.line, r.key); }
|
|
179
|
+
}
|
|
180
|
+
} else if (b === 'setup.cfg') {
|
|
181
|
+
let inReq = false;
|
|
182
|
+
src.lines.forEach((l, i) => {
|
|
183
|
+
if (/^\s*install_requires\s*=/.test(l)) { inReq = true; return; }
|
|
184
|
+
if (inReq) {
|
|
185
|
+
if (/^\S/.test(l) || /^\s*\[/.test(l)) { inReq = false; return; }
|
|
186
|
+
const p = PEP508.exec(l); if (p && l.trim()) m.deps.push({ eco: 'pypi', name: p[1], version: cleanVersion(p[3]), dev: false, ev: ev(f, i + 1, p[1]) });
|
|
187
|
+
}
|
|
188
|
+
const n = /^\s*name\s*=\s*(\S+)/.exec(l); if (n && !inReq) { m.name = n[1]; m.selfNames.push(n[1]); }
|
|
189
|
+
});
|
|
190
|
+
} else {
|
|
191
|
+
const dev = /dev|test|lint/.test(b);
|
|
192
|
+
src.lines.forEach((l, i) => {
|
|
193
|
+
// An option line, or a requirement given by URL or path: no package name to read, and a URL is not copied.
|
|
194
|
+
if (/^\s*(#|-|$)/.test(l) || /^\s*([a-z][a-z0-9+.-]*:\/\/|git\+|\.{0,2}\/)/i.test(l)) return;
|
|
195
|
+
const p = PEP508.exec(l); if (p) m.deps.push({ eco: 'pypi', name: p[1], version: cleanVersion(p[3]), dev, ev: ev(f, i + 1, p[1]) });
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
if (!m.evidence.length) { const e = evFirst(f, src.lines); if (e) m.evidence.push(e); }
|
|
199
|
+
}
|
|
200
|
+
// A manifest with nothing in it (an empty requirements.txt) declares no module.
|
|
201
|
+
const mods = [...dirs.values()].filter((m) => m.evidence.length || m.deps.length);
|
|
202
|
+
for (const m of mods) { m.id = `mod:python:${m.name}`; if (!m.evidence.length) m.evidence.push(m.deps[0].ev); runnable(m); } // the declared name, once the manifests are read
|
|
203
|
+
out.modules.push(...mods);
|
|
204
|
+
own(out.modules, pj.files, 'python');
|
|
205
|
+
for (const m of mods) {
|
|
206
|
+
// Local top-level names: a package directory or a module file anywhere in this module is not a dependency.
|
|
207
|
+
const local = new Set();
|
|
208
|
+
for (const f of m.files) {
|
|
209
|
+
const rel = m.dir === '.' ? f : f.slice(m.dir.length + 1);
|
|
210
|
+
const segs = rel.split('/');
|
|
211
|
+
for (let k = 0; k < segs.length; k++) local.add(segs[k].replace(/\.py$/, ''));
|
|
212
|
+
}
|
|
213
|
+
m.localNames = local;
|
|
214
|
+
for (const f of m.files) {
|
|
215
|
+
const src = pj.read(f); if (!src) continue;
|
|
216
|
+
src.lines.forEach((l, i) => {
|
|
217
|
+
if (/^\s*__name__\s*==|if\s+__name__\s*==\s*['"]__main__['"]/.test(l) && !m.entry) { m.entry = true; m.entryEv = ev(f, i + 1, '__main__'); }
|
|
218
|
+
const a = /^\s*import\s+([\w.]+(?:\s+as\s+\w+)?(?:\s*,\s*[\w.]+(?:\s+as\s+\w+)?)*)\s*(?:#.*)?$/.exec(l);
|
|
219
|
+
if (a) for (const part of a[1].split(',')) { const spec = part.trim().split(/\s+/)[0]; m.imports.push({ spec, root: spec.split('.')[0], ev: ev(f, i + 1, spec), file: f }); }
|
|
220
|
+
const b2 = /^\s*from\s+([\w][\w.]*)\s+import\b/.exec(l);
|
|
221
|
+
if (b2) m.imports.push({ spec: b2[1], root: b2[1].split('.')[0], ev: ev(f, i + 1, b2[1]), file: f });
|
|
222
|
+
});
|
|
223
|
+
if (/(^|\/)(__main__|manage)\.py$/.test(f) && !m.entry) { m.entry = true; m.entryEv = evFirst(f, src.lines); m.entryFile = f; }
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// ---- Go -------------------------------------------------------------------------------------------------------
|
|
229
|
+
function goStack(pj, out) {
|
|
230
|
+
for (const f of pj.manifests.filter((x) => P.basename(x) === 'go.mod')) {
|
|
231
|
+
const src = pj.read(f); if (!src) continue;
|
|
232
|
+
let name = null; let inReq = false; const deps = [];
|
|
233
|
+
src.lines.forEach((l, i) => {
|
|
234
|
+
const mm = /^\s*module\s+(\S+)/.exec(l); if (mm) name = { v: mm[1], line: i + 1 };
|
|
235
|
+
if (/^\s*require\s*\(\s*$/.test(l)) { inReq = true; return; }
|
|
236
|
+
if (inReq && /^\s*\)\s*$/.test(l)) { inReq = false; return; }
|
|
237
|
+
const r = inReq ? /^\s*(\S+)\s+(\S+)(\s*\/\/\s*indirect)?/.exec(l) : /^\s*require\s+(\S+)\s+(\S+)(\s*\/\/\s*indirect)?/.exec(l);
|
|
238
|
+
if (r && !isComment(l)) deps.push({ eco: 'go', name: r[1], version: cleanVersion(r[2]), dev: !!r[3], indirect: !!r[3], ev: ev(f, i + 1, r[1]) });
|
|
239
|
+
});
|
|
240
|
+
if (!name) { out.notMeasured.push({ what: f, reason: 'go.mod has no module line' }); continue; }
|
|
241
|
+
const m = mod('go', name.v, f, dirOf(f));
|
|
242
|
+
m.evidence.push(ev(f, name.line, name.v)); m.deps = deps;
|
|
243
|
+
const gv = lineOf(src.lines, /^\s*go\s+\d/); if (gv) m.tech.add(`go@${src.lines[gv - 1].trim().split(/\s+/)[1]}`);
|
|
244
|
+
out.modules.push(m);
|
|
245
|
+
}
|
|
246
|
+
const mods = out.modules.filter((m) => m.stack === 'go');
|
|
247
|
+
own(out.modules, pj.files, 'go');
|
|
248
|
+
for (const m of mods) {
|
|
249
|
+
for (const f of m.files) {
|
|
250
|
+
const src = pj.read(f); if (!src) continue;
|
|
251
|
+
let block = false;
|
|
252
|
+
src.lines.forEach((l, i) => {
|
|
253
|
+
if (/^\s*package\s+main\b/.test(l) && !m.entry) { m.entry = true; m.entryEv = ev(f, i + 1, 'package main'); }
|
|
254
|
+
if (/^\s*import\s*\(\s*$/.test(l)) { block = true; return; }
|
|
255
|
+
if (block && /^\s*\)/.test(l)) { block = false; return; }
|
|
256
|
+
const x = block ? /^\s*(?:[\w.]+\s+)?"([^"]+)"/.exec(l) : /^\s*import\s+(?:[\w.]+\s+)?"([^"]+)"/.exec(l);
|
|
257
|
+
if (!x) return;
|
|
258
|
+
const spec = x[1];
|
|
259
|
+
const t = mods.filter((o) => spec === o.name || spec.startsWith(o.name + '/')).sort((a, b) => b.name.length - a.name.length)[0];
|
|
260
|
+
if (t) { if (t !== m) m.refs.push({ to: t.id, toMod: t, ev: ev(f, i + 1, spec), confidence: 'evidence' }); return; }
|
|
261
|
+
if (!spec.split('/')[0].includes('.')) return; // the standard library: no dot in the first element
|
|
262
|
+
m.imports.push({ spec, root: spec, ev: ev(f, i + 1, spec), file: f });
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// ---- .NET ---------------------------------------------------------------------------------------------------
|
|
269
|
+
function dotnetStack(pj, out) {
|
|
270
|
+
const central = new Map();
|
|
271
|
+
for (const f of pj.manifests.filter((x) => P.basename(x) === 'Directory.Packages.props')) {
|
|
272
|
+
const src = pj.read(f); if (!src) continue;
|
|
273
|
+
try { for (const p of xFind(parseXml(src.text), 'PackageVersion')) central.set(p.attrs.Include, p.attrs.Version || ''); }
|
|
274
|
+
catch (e) { out.notMeasured.push({ what: f, reason: `could not be read: ${e.message}` }); }
|
|
275
|
+
}
|
|
276
|
+
for (const f of pj.manifests.filter((x) => /\.(cs|fs|vb)proj$/.test(x))) {
|
|
277
|
+
const src = pj.read(f); if (!src) continue;
|
|
278
|
+
let x; try { x = parseXml(src.text); } catch (e) { out.notMeasured.push({ what: f, reason: `project file could not be read: ${e.message}` }); continue; }
|
|
279
|
+
const proj = xKid(x, 'Project'); if (!proj) { out.notMeasured.push({ what: f, reason: 'no <Project> element' }); continue; }
|
|
280
|
+
const name = P.basename(f).replace(/\.\w+proj$/, '');
|
|
281
|
+
const m = mod('dotnet', name, f, dirOf(f));
|
|
282
|
+
m.evidence.push(ev(f, proj.line, 'Project'));
|
|
283
|
+
const prop = (n) => { for (const pg of xKids(proj, 'PropertyGroup')) { const e = xKid(pg, n); if (e) return e; } return null; };
|
|
284
|
+
const tf = prop('TargetFramework') || prop('TargetFrameworks'); if (tf) m.tech.add(xText(tf));
|
|
285
|
+
const sdk = proj.attrs.Sdk || '';
|
|
286
|
+
if (sdk) m.tech.add(sdk);
|
|
287
|
+
for (const n of ['RootNamespace', 'AssemblyName']) { const e = prop(n); if (e && xText(e)) m.selfNames.push(xText(e)); }
|
|
288
|
+
const ot = prop('OutputType'); const tp = prop('IsTestProject');
|
|
289
|
+
if (/Sdk\.(Web|Worker|BlazorWebAssembly|Razor)/.test(sdk)) { m.entry = true; m.entryEv = ev(f, proj.line, sdk); }
|
|
290
|
+
else if (ot && /exe/i.test(xText(ot))) { m.entry = true; m.entryEv = ev(f, ot.line, 'OutputType'); }
|
|
291
|
+
else if (tp && /true/i.test(xText(tp))) { m.entry = true; m.entryEv = ev(f, tp.line, 'IsTestProject'); }
|
|
292
|
+
for (const ig of xKids(proj, 'ItemGroup')) {
|
|
293
|
+
for (const p of xKids(ig, 'PackageReference')) {
|
|
294
|
+
const n = p.attrs.Include || p.attrs.Update; if (!n) continue;
|
|
295
|
+
const v = p.attrs.Version || xText(xKid(p, 'Version')) || central.get(n) || '';
|
|
296
|
+
const at = p.attrLines.Include || p.attrLines.Update || p.line;
|
|
297
|
+
m.deps.push({ eco: 'nuget', name: n, version: cleanVersion(v), dev: false, central: !p.attrs.Version && !xKid(p, 'Version') && central.has(n), ev: ev(f, at, n) });
|
|
298
|
+
if (/^(Microsoft\.NET\.Test\.Sdk|xunit|NUnit|MSTest\.TestFramework)$/.test(n) && !m.entry) { m.entry = true; m.entryEv = ev(f, at, n); }
|
|
299
|
+
}
|
|
300
|
+
for (const r of xKids(ig, 'ProjectReference')) {
|
|
301
|
+
const inc = (r.attrs.Include || '').replace(/\\/g, '/');
|
|
302
|
+
if (!inc) continue;
|
|
303
|
+
m.refs.push({ to: P.normalize(P.join(dirOf(f), inc)), ev: ev(f, r.attrLines.Include || r.line, P.basename(inc)), confidence: 'evidence', byManifest: true });
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
out.modules.push(m);
|
|
307
|
+
}
|
|
308
|
+
const mods = out.modules.filter((m) => m.stack === 'dotnet');
|
|
309
|
+
// Windows and macOS resolve a reference whatever its case, so the build does too: an exact match first, then the
|
|
310
|
+
// one manifest that matches without case.
|
|
311
|
+
for (const m of mods) for (const r of m.refs) if (r.byManifest) {
|
|
312
|
+
const ci = mods.filter((o) => o.manifest.toLowerCase() === r.to.toLowerCase());
|
|
313
|
+
const t = mods.find((o) => o.manifest === r.to) || (ci.length === 1 ? ci[0] : null);
|
|
314
|
+
r.to = t ? t.id : `missing:${r.to}`; r.toMod = t;
|
|
315
|
+
}
|
|
316
|
+
own(out.modules, pj.files, 'dotnet');
|
|
317
|
+
// `using` a namespace another project owns: concluded from the name, so inferred, and only when no
|
|
318
|
+
// ProjectReference already says it.
|
|
319
|
+
const owners = mods.flatMap((o) => o.selfNames.map((n) => ({ n, o }))).sort((a, b) => b.n.length - a.n.length);
|
|
320
|
+
for (const m of mods) {
|
|
321
|
+
const seen = new Set(m.refs.map((r) => r.to));
|
|
322
|
+
for (const f of m.files) {
|
|
323
|
+
const src = pj.read(f); if (!src) continue;
|
|
324
|
+
src.lines.forEach((l, i) => {
|
|
325
|
+
const u = /^\s*(?:global\s+)?using\s+(?:static\s+)?([\w.]+)\s*;/.exec(l); if (!u) return;
|
|
326
|
+
const hit = owners.find(({ n }) => u[1] === n || u[1].startsWith(n + '.'));
|
|
327
|
+
if (hit && hit.o !== m && !seen.has(hit.o.id)) { seen.add(hit.o.id); m.refs.push({ to: hit.o.id, toMod: hit.o, ev: ev(f, i + 1, u[1]), confidence: 'inferred' }); }
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
if (mods.length) out.notMeasured.push({ what: '.NET package usage (dep-unused, dep-undeclared)', reason: 'a NuGet package name does not say which namespaces it provides, so imports cannot be matched to packages' });
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// ---- Java / Kotlin: Maven and Gradle ------------------------------------------------------------------------
|
|
335
|
+
function javaStack(pj, out) {
|
|
336
|
+
for (const f of pj.manifests.filter((x) => P.basename(x) === 'pom.xml')) {
|
|
337
|
+
const src = pj.read(f); if (!src) continue;
|
|
338
|
+
let x; try { x = parseXml(src.text); } catch (e) { out.notMeasured.push({ what: f, reason: `pom.xml could not be read: ${e.message}` }); continue; }
|
|
339
|
+
const p = xKid(x, 'project'); if (!p) continue;
|
|
340
|
+
const aid = xKid(p, 'artifactId'); if (!aid) { out.notMeasured.push({ what: f, reason: 'pom.xml has no artifactId' }); continue; }
|
|
341
|
+
const gid = xText(xKid(p, 'groupId')) || xText(xKid(xKid(p, 'parent'), 'groupId'));
|
|
342
|
+
const m = mod('java', xText(aid), f, dirOf(f));
|
|
343
|
+
m.selfNames.push(`${gid}:${xText(aid)}`);
|
|
344
|
+
m.evidence.push(ev(f, xLine(aid), xText(aid)));
|
|
345
|
+
const pk = xText(xKid(p, 'packaging')); if (pk) m.tech.add(`maven:${pk}`);
|
|
346
|
+
const deps = xKid(p, 'dependencies');
|
|
347
|
+
for (const d of xKids(deps, 'dependency')) {
|
|
348
|
+
const g = xText(xKid(d, 'groupId')); const a = xText(xKid(d, 'artifactId')); const v = xText(xKid(d, 'version'));
|
|
349
|
+
if (!a) continue;
|
|
350
|
+
const scope = xText(xKid(d, 'scope'));
|
|
351
|
+
m.deps.push({ eco: 'maven', name: `${g}:${a}`, version: cleanVersion(v), dev: scope === 'test', ev: ev(f, xLine(xKid(d, 'artifactId')), a) });
|
|
352
|
+
}
|
|
353
|
+
for (const mm of xKids(xKid(p, 'modules'), 'module')) m.refs.push({ to: `aggregates:${P.normalize(P.join(dirOf(f), xText(mm)))}`, ev: ev(f, xLine(mm), xText(mm)), confidence: 'evidence', aggregate: true });
|
|
354
|
+
for (const pl of xFind(p, 'artifactId')) if (xText(pl) === 'spring-boot-maven-plugin' && !m.entry) { m.entry = true; m.entryEv = ev(f, xLine(pl), 'spring-boot-maven-plugin'); }
|
|
355
|
+
out.modules.push(m);
|
|
356
|
+
}
|
|
357
|
+
const settings = pj.manifests.filter((x) => /(^|\/)settings\.gradle(\.kts)?$/.test(x));
|
|
358
|
+
for (const f of settings) {
|
|
359
|
+
const src = pj.read(f); if (!src) continue;
|
|
360
|
+
src.lines.forEach((l, i) => {
|
|
361
|
+
if (isComment(l) || !/^\s*include\b/.test(l)) return;
|
|
362
|
+
for (const q of l.matchAll(/["']:?([\w.:-]+)["']/g)) {
|
|
363
|
+
const name = q[1]; const dir = P.normalize(P.join(dirOf(f), name.replace(/:/g, '/')));
|
|
364
|
+
const build = [`${dir}/build.gradle.kts`, `${dir}/build.gradle`].find((b) => pj.files.includes(b)) || null;
|
|
365
|
+
const m = mod('java', name, build || f, dir); m.gradle = true;
|
|
366
|
+
m.evidence.push(ev(f, i + 1, name));
|
|
367
|
+
out.modules.push(m);
|
|
368
|
+
}
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
// A root build file with no settings: one module.
|
|
372
|
+
for (const f of pj.manifests.filter((x) => /(^|\/)build\.gradle(\.kts)?$/.test(x))) {
|
|
373
|
+
if (out.modules.some((m) => m.stack === 'java' && m.manifest === f)) continue;
|
|
374
|
+
if (out.modules.some((m) => m.stack === 'java' && m.gradle && m.dir === dirOf(f))) continue;
|
|
375
|
+
const first = evFirst(f, (pj.read(f) || { lines: [] }).lines);
|
|
376
|
+
if (!first) continue; // an empty build file declares nothing
|
|
377
|
+
const m = mod('java', dirOf(f) === '.' ? 'root' : dirOf(f), f, dirOf(f)); m.gradle = true;
|
|
378
|
+
m.evidence.push(first);
|
|
379
|
+
out.modules.push(m);
|
|
380
|
+
}
|
|
381
|
+
let catalog = 0;
|
|
382
|
+
const gmods = out.modules.filter((m) => m.stack === 'java' && m.gradle);
|
|
383
|
+
for (const m of gmods) {
|
|
384
|
+
if (!/build\.gradle/.test(m.manifest)) continue;
|
|
385
|
+
const src = pj.read(m.manifest); if (!src) continue;
|
|
386
|
+
m.tech.add(m.manifest.endsWith('.kts') ? 'gradle-kotlin-dsl' : 'gradle');
|
|
387
|
+
src.lines.forEach((l, i) => {
|
|
388
|
+
if (isComment(l)) return;
|
|
389
|
+
const pr = /\bproject\(\s*(?:path\s*[:=]\s*)?["']:?([\w.:-]+)["']\s*\)/.exec(l);
|
|
390
|
+
if (pr) { m.refs.push({ to: `gradle:${pr[1]}`, ev: ev(m.manifest, i + 1, pr[1]), confidence: 'inferred' }); return; }
|
|
391
|
+
const dep = /\b(implementation|api|compileOnly|runtimeOnly|testImplementation|kapt|ksp)\s*\(?\s*["']([\w.-]+):([\w.-]+)(?::([^"']+))?["']/.exec(l);
|
|
392
|
+
if (dep) { m.deps.push({ eco: 'maven', name: `${dep[2]}:${dep[3]}`, version: cleanVersion(dep[4] || ''), dev: /^test/.test(dep[1]), inferred: true, ev: ev(m.manifest, i + 1, dep[3]) }); return; }
|
|
393
|
+
if (/\b(implementation|api|compileOnly|runtimeOnly|testImplementation)\s*\(?\s*libs\./.test(l)) catalog++;
|
|
394
|
+
if (/org\.springframework\.boot/.test(l) && !m.entry) { m.entry = true; m.entryEv = ev(m.manifest, i + 1, 'org.springframework.boot'); }
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
if (catalog) out.notMeasured.push({ what: `Gradle version catalog references (${catalog})`, reason: 'libs.* entries live in a catalog this scanner does not resolve, so those dependencies are not listed' });
|
|
398
|
+
if (gmods.length) out.notMeasured.push({ what: 'Gradle build logic', reason: 'build.gradle(.kts) is code; dependencies and project references are read by pattern, so they are marked inferred' });
|
|
399
|
+
const mods = out.modules.filter((m) => m.stack === 'java');
|
|
400
|
+
for (const m of mods) {
|
|
401
|
+
for (const r of m.refs) {
|
|
402
|
+
if (r.to.startsWith('gradle:')) { const n = r.to.slice(7); const t = mods.find((o) => o.gradle && (o.name === n || o.name.endsWith(':' + n))); r.to = t ? t.id : `missing:${n}`; r.toMod = t; }
|
|
403
|
+
if (r.to.startsWith('aggregates:')) { const d = r.to.slice(11); const t = mods.find((o) => o.dir === d); r.to = t ? t.id : `missing:${d}`; r.toMod = t; }
|
|
404
|
+
}
|
|
405
|
+
for (const d of m.deps) {
|
|
406
|
+
// groupId:artifactId, both: an artifactId alone also names third-party artifacts (org.thirdparty:api).
|
|
407
|
+
const t = mods.find((o) => o !== m && o.selfNames.includes(d.name));
|
|
408
|
+
if (t) m.refs.push({ to: t.id, toMod: t, ev: d.ev, confidence: d.inferred ? 'inferred' : 'evidence' });
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
own(out.modules, pj.files, 'java');
|
|
412
|
+
for (const m of mods) {
|
|
413
|
+
if (m.entry) continue;
|
|
414
|
+
for (const f of m.files) {
|
|
415
|
+
const src = pj.read(f); if (!src) continue;
|
|
416
|
+
const i = src.lines.findIndex((l) => /static\s+void\s+main\s*\(|^\s*fun\s+main\s*\(|@SpringBootApplication/.test(l));
|
|
417
|
+
if (i >= 0) { m.entry = true; m.entryEv = ev(f, i + 1, src.lines[i].includes('@SpringBootApplication') ? '@SpringBootApplication' : 'main'); break; }
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
if (mods.length) out.notMeasured.push({ what: 'Java/Kotlin package usage (dep-unused, dep-undeclared)', reason: 'a Maven artifact does not say which packages it provides, so imports cannot be matched to artifacts' });
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
export function scanStacks(pj0) {
|
|
424
|
+
// A manifest under a test, fixture or example directory describes a sample project, not a module of this one.
|
|
425
|
+
// Its source files still belong to the module that holds them (a test imports that module's dependencies).
|
|
426
|
+
const pj = { ...pj0, manifests: pj0.files.filter((f) => !isTestPath(f)) };
|
|
427
|
+
const out = { modules: [], notMeasured: [], globalAliases: [] };
|
|
428
|
+
for (const fn of [nodeStack, pythonStack, goStack, dotnetStack, javaStack]) {
|
|
429
|
+
try { fn(pj, out); } catch (e) {
|
|
430
|
+
if (!(e instanceof Unsupported)) throw e;
|
|
431
|
+
out.notMeasured.push({ what: fn.name, reason: e.message });
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
// Two manifests with one name (a copy, a fork in a subdirectory) would share an id: the directory tells them apart.
|
|
435
|
+
const count = new Map();
|
|
436
|
+
for (const m of out.modules) count.set(m.id, (count.get(m.id) || 0) + 1);
|
|
437
|
+
for (const m of out.modules) if (count.get(m.id) > 1) m.id = `${m.id}@${m.dir}`;
|
|
438
|
+
// A reference holds its target module, so it follows the rename; the object is dropped once resolved.
|
|
439
|
+
for (const m of out.modules) for (const r of m.refs) { if (r.toMod) r.to = r.toMod.id; delete r.toMod; }
|
|
440
|
+
return out;
|
|
441
|
+
}
|