@bigsteele/the-prospect 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -17
- package/dist/check.js +3 -2
- package/dist/cli.js +116 -28
- package/dist/decisions.d.ts +93 -0
- package/dist/decisions.js +143 -0
- package/dist/detect/costs.js +50 -8
- package/dist/detect/database.d.ts +28 -0
- package/dist/detect/database.js +198 -0
- package/dist/detect/deadweight.js +60 -15
- package/dist/detect/deps.js +56 -0
- package/dist/detect/duplication.js +25 -2
- package/dist/detect/handrolled.js +84 -5
- package/dist/detect/stack.d.ts +8 -0
- package/dist/detect/stack.js +10 -2
- package/dist/detect/types.d.ts +76 -0
- package/dist/detect/vendors.js +108 -8
- package/dist/index.d.ts +15 -2
- package/dist/index.js +88 -2
- package/dist/northstar.js +15 -2
- package/dist/profile.d.ts +55 -0
- package/dist/profile.js +106 -0
- package/dist/report.js +183 -24
- package/dist/score.d.ts +3 -0
- package/dist/score.js +30 -9
- package/dist/verdicts.d.ts +80 -0
- package/dist/verdicts.js +144 -0
- package/dist/walk.d.ts +85 -1
- package/dist/walk.js +189 -5
- package/package.json +1 -1
- package/prompt/THE-PROSPECT.md +162 -118
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { scopeFor } from "../walk.js";
|
|
2
|
+
/**
|
|
3
|
+
* Strip comments so a shape quoted in prose is not read as a declaration.
|
|
4
|
+
*
|
|
5
|
+
* NOT used when looking for a caller check. The convention for stating who may
|
|
6
|
+
* call a definer function is a `-- caller-check:` MARKER COMMENT, so stripping
|
|
7
|
+
* comments first destroys the exact evidence being looked for - which it did,
|
|
8
|
+
* and the fixture's correctly-guarded function was flagged alongside the
|
|
9
|
+
* unguarded one. Declarations are read from stripped text; guards from raw.
|
|
10
|
+
*/
|
|
11
|
+
function stripComments(sql) {
|
|
12
|
+
return sql.replace(/\/\*[\s\S]*?\*\//g, " ").replace(/--[^\n]*/g, " ");
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Split into statements on semicolons that are not inside a dollar-quoted body.
|
|
16
|
+
* A function body is full of semicolons and is one statement; splitting naively
|
|
17
|
+
* turns every definer function into a dozen fragments and the caller check into
|
|
18
|
+
* a fragment of its own, which is how a guard goes missing.
|
|
19
|
+
*/
|
|
20
|
+
function statements(sql) {
|
|
21
|
+
const out = [];
|
|
22
|
+
let buf = "";
|
|
23
|
+
let i = 0;
|
|
24
|
+
let tag = null;
|
|
25
|
+
while (i < sql.length) {
|
|
26
|
+
if (!tag) {
|
|
27
|
+
const m = /^\$([A-Za-z_]*)\$/.exec(sql.slice(i));
|
|
28
|
+
if (m) {
|
|
29
|
+
tag = m[0];
|
|
30
|
+
buf += tag;
|
|
31
|
+
i += tag.length;
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (sql[i] === ";") {
|
|
35
|
+
out.push(buf);
|
|
36
|
+
buf = "";
|
|
37
|
+
i++;
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else if (sql.startsWith(tag, i)) {
|
|
42
|
+
buf += tag;
|
|
43
|
+
i += tag.length;
|
|
44
|
+
tag = null;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
buf += sql[i];
|
|
48
|
+
i++;
|
|
49
|
+
}
|
|
50
|
+
if (buf.trim())
|
|
51
|
+
out.push(buf);
|
|
52
|
+
return out;
|
|
53
|
+
}
|
|
54
|
+
const QUALIFIED = /(?:([a-z_][a-z0-9_]*)\.)?([a-z_][a-z0-9_]*)/i;
|
|
55
|
+
function qualify(raw) {
|
|
56
|
+
const m = QUALIFIED.exec(raw.replace(/"/g, ""));
|
|
57
|
+
if (!m)
|
|
58
|
+
return raw;
|
|
59
|
+
return `${m[1] ?? "public"}.${m[2]}`;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* A definer function states who may call it, one of two ways: a marker comment
|
|
63
|
+
* naming the predicate, or a runtime check in the body. Either is a statement
|
|
64
|
+
* somebody can be held to; neither is proof it is correct, and the finding says
|
|
65
|
+
* "states no caller check", never "is insecure".
|
|
66
|
+
*/
|
|
67
|
+
const CALLER_CHECK = /caller-check|is_account_member|is_platform_staff|auth\.uid\s*\(\s*\)|auth\.role\s*\(\s*\)|current_setting\s*\(|has_role|raise\s+exception|assert\b/i;
|
|
68
|
+
export async function detectDatabase(repo) {
|
|
69
|
+
// Migrations wherever they live; a `.sql.tmpl` shipped to a customer is the
|
|
70
|
+
// customer's database, so the template scope rules apply here as everywhere.
|
|
71
|
+
const files = scopeFor(repo.files, "database");
|
|
72
|
+
const rlsEnabled = new Set();
|
|
73
|
+
const rlsForced = new Set();
|
|
74
|
+
const tablesSeen = new Map();
|
|
75
|
+
const findings = [];
|
|
76
|
+
let policies = 0;
|
|
77
|
+
let definerFunctions = 0;
|
|
78
|
+
/**
|
|
79
|
+
* THE GRANTS ARE IN THE MIGRATIONS (0.2.1). The caller check was looked for
|
|
80
|
+
* inside the function body only, and 60 definer functions on the first real
|
|
81
|
+
* repository were reported as stating no check when each one was followed by
|
|
82
|
+
* `revoke all on function ... from public, anon, authenticated` - a caller
|
|
83
|
+
* check by another statement. Postgres grants EXECUTE to public by default,
|
|
84
|
+
* so a revoke from public/anon/authenticated is the statement that narrows
|
|
85
|
+
* who may call; a bare `grant execute ... to service_role` narrows nothing.
|
|
86
|
+
*/
|
|
87
|
+
const executeRevoked = new Set();
|
|
88
|
+
let guard;
|
|
89
|
+
// Two passes: every `alter table` and every `revoke` in the repository is
|
|
90
|
+
// collected first, because a table is very often created in one migration
|
|
91
|
+
// and secured in the next, and reading them in file order would flag every
|
|
92
|
+
// such table.
|
|
93
|
+
for (const f of files) {
|
|
94
|
+
const raw = await repo.read(f);
|
|
95
|
+
if (!raw)
|
|
96
|
+
continue;
|
|
97
|
+
const sql = stripComments(raw);
|
|
98
|
+
for (const m of sql.matchAll(/alter\s+table\s+(?:if\s+exists\s+)?([a-z0-9_."]+)\s+([a-z\s]*row\s+level\s+security)/gi)) {
|
|
99
|
+
const t = qualify(m[1]);
|
|
100
|
+
if (/force/i.test(m[2]))
|
|
101
|
+
rlsForced.add(t);
|
|
102
|
+
if (/enable/i.test(m[2]))
|
|
103
|
+
rlsEnabled.add(t);
|
|
104
|
+
}
|
|
105
|
+
for (const m of sql.matchAll(/revoke\s+(?:all|execute)(?:\s+privileges)?\s+on\s+function\s+([a-z0-9_."]+)\s*(?:\([^)]*\))?\s+from\s+([a-z_,\s"]+)/gi)) {
|
|
106
|
+
if (/\b(public|anon|authenticated)\b/i.test(m[2]))
|
|
107
|
+
executeRevoked.add(qualify(m[1]));
|
|
108
|
+
}
|
|
109
|
+
for (const g of sql.matchAll(/create\s+(?:or\s+replace\s+)?function\s+([a-z0-9_."]*(?:definer_guard|guard_scan|grant_scan)[a-z0-9_]*)\s*\(/gi)) {
|
|
110
|
+
const name = `${qualify(g[1])}()`;
|
|
111
|
+
// The one that names definers wins; otherwise the first guard seen.
|
|
112
|
+
if (!guard || (/definer/i.test(name) && !/definer/i.test(guard)))
|
|
113
|
+
guard = name;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
for (const f of files) {
|
|
117
|
+
const raw = await repo.read(f);
|
|
118
|
+
if (!raw)
|
|
119
|
+
continue;
|
|
120
|
+
const sql = stripComments(raw);
|
|
121
|
+
for (const m of sql.matchAll(/create\s+table\s+(?:if\s+not\s+exists\s+)?([a-z0-9_."]+)/gi)) {
|
|
122
|
+
const t = qualify(m[1]);
|
|
123
|
+
// `create policy p on public.x for select to public` contains the literal
|
|
124
|
+
// `table` nowhere, but a looser earlier pattern produced a table called
|
|
125
|
+
// `public.public` from exactly that line. A schema is not a table.
|
|
126
|
+
if (/^(public|auth|storage|extensions)\.(public|auth|storage)$/.test(t))
|
|
127
|
+
continue;
|
|
128
|
+
if (!tablesSeen.has(t))
|
|
129
|
+
tablesSeen.set(t, f);
|
|
130
|
+
}
|
|
131
|
+
// SPLIT THE RAW SQL, STRIP PER STATEMENT.
|
|
132
|
+
//
|
|
133
|
+
// The first version split the stripped text and the raw text separately and
|
|
134
|
+
// paired them by index, which silently drifts: removing comments removes
|
|
135
|
+
// semicolons inside them, so the two lists stop describing the same
|
|
136
|
+
// statements. On a real repository that reported 60 definer functions as
|
|
137
|
+
// stating no caller check when every one of them carried the marker - the
|
|
138
|
+
// failure mode this whole version exists to remove, arriving in the detector
|
|
139
|
+
// written to find it.
|
|
140
|
+
for (const withComments of statements(raw)) {
|
|
141
|
+
const stmt = stripComments(withComments);
|
|
142
|
+
if (/create\s+(or\s+replace\s+)?policy/i.test(stmt)) {
|
|
143
|
+
policies++;
|
|
144
|
+
const name = /create\s+(?:or\s+replace\s+)?policy\s+("?[a-z0-9_]+"?)/i.exec(stmt)?.[1]?.replace(/"/g, "");
|
|
145
|
+
// `to anon` hands the policy to unauthenticated callers. `to public`
|
|
146
|
+
// is the same reach by another name, since anon is a member of public.
|
|
147
|
+
if (name && /\bto\s+(anon|public)\b/i.test(stmt)) {
|
|
148
|
+
findings.push({
|
|
149
|
+
kind: "policy_reaches_anon",
|
|
150
|
+
subject: name,
|
|
151
|
+
file: f,
|
|
152
|
+
note: "a policy granted to anon or public is readable by an unauthenticated caller. Deliberate for genuinely public rows, and worth one look per policy.",
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (/create\s+(or\s+replace\s+)?function/i.test(stmt) && /security\s+definer/i.test(stmt)) {
|
|
157
|
+
definerFunctions++;
|
|
158
|
+
const name = /create\s+(?:or\s+replace\s+)?function\s+([a-z0-9_."]+)\s*\(/i.exec(stmt)?.[1];
|
|
159
|
+
if (name && !CALLER_CHECK.test(withComments) && !executeRevoked.has(qualify(name))) {
|
|
160
|
+
findings.push({
|
|
161
|
+
kind: "definer_without_check",
|
|
162
|
+
subject: qualify(name),
|
|
163
|
+
file: f,
|
|
164
|
+
note: "runs with the definer's rights, states no caller check in its body, and no migration revokes its EXECUTE from public, anon or authenticated. Either it is meant to be callable by anyone, or a grant applied outside the migrations is the only thing standing between it and one." +
|
|
165
|
+
(guard ? ` The repository carries \`${guard}\`, which checks the live grants; this scan reads only what the migrations state.` : ""),
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
for (const [t, f] of tablesSeen) {
|
|
172
|
+
if (!rlsEnabled.has(t)) {
|
|
173
|
+
findings.push({
|
|
174
|
+
kind: "table_without_rls",
|
|
175
|
+
subject: t,
|
|
176
|
+
file: f,
|
|
177
|
+
note: "no `enable row level security` anywhere in the migrations. Right for a lookup table nobody owns; a question for anything carrying one customer's rows.",
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
else if (!rlsForced.has(t)) {
|
|
181
|
+
findings.push({
|
|
182
|
+
kind: "rls_not_forced",
|
|
183
|
+
subject: t,
|
|
184
|
+
file: f,
|
|
185
|
+
note: "row level security is enabled but not forced, so the table owner bypasses it. Forcing it is one line and closes the gap.",
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return {
|
|
190
|
+
files: files.length,
|
|
191
|
+
tables: tablesSeen.size,
|
|
192
|
+
policies,
|
|
193
|
+
definer_functions: definerFunctions,
|
|
194
|
+
definer_execute_revoked: executeRevoked.size,
|
|
195
|
+
guard,
|
|
196
|
+
findings: findings.sort((a, b) => a.kind.localeCompare(b.kind) || a.subject.localeCompare(b.subject)).slice(0, 60),
|
|
197
|
+
};
|
|
198
|
+
}
|
|
@@ -13,9 +13,12 @@ function entrypoints(repo, manifests) {
|
|
|
13
13
|
for (const f of repo.files) {
|
|
14
14
|
if (/(^|\/)src\/(main|index|app|App)\.(t|j)sx?$/.test(f) ||
|
|
15
15
|
/(^|\/)(functions|api)\/[^/]+\/index\.(t|j)s$/.test(f) ||
|
|
16
|
+
/^api\/.*\.(m|c)?(t|j)sx?$/.test(f) || // a root api/ tree is a serverless router (Vercel): every file is a function
|
|
17
|
+
/(^|\/)netlify\/functions\/[^/]+\.(m|c)?(t|j)s$/.test(f) ||
|
|
16
18
|
/(^|\/)(pages|app)\/.*\.(t|j)sx?$/.test(f) || // file-based routers import nothing by name
|
|
17
19
|
/(^|\/)scripts?\/[^/]+\.(m|c)?(t|j)s$/.test(f) ||
|
|
18
20
|
/(^|\/)(index|server|worker|cli)\.(m|c)?(t|j)s$/.test(f) ||
|
|
21
|
+
NOT_BEHAVIOUR.test(f) || // a config file imports things (a playwright fixture, a vite plugin); it seeds the walk and is never itself listed
|
|
19
22
|
TEST_FILE.test(f) // tests reach code; code only tests reach is a different finding
|
|
20
23
|
) {
|
|
21
24
|
out.add(f);
|
|
@@ -101,32 +104,74 @@ export async function detectDeadweight(repo) {
|
|
|
101
104
|
const roots = entrypoints(repo, manifests);
|
|
102
105
|
const reached = new Set(roots);
|
|
103
106
|
const queue = [...roots];
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
reached.
|
|
113
|
-
|
|
107
|
+
const drain = async () => {
|
|
108
|
+
while (queue.length) {
|
|
109
|
+
const f = queue.pop();
|
|
110
|
+
const text = await repo.read(f);
|
|
111
|
+
if (!text)
|
|
112
|
+
continue;
|
|
113
|
+
for (const spec of specifiers(text)) {
|
|
114
|
+
const to = resolve(f, spec);
|
|
115
|
+
if (to && !reached.has(to)) {
|
|
116
|
+
reached.add(to);
|
|
117
|
+
queue.push(to);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
await drain();
|
|
123
|
+
const candidates = repo.files.filter((f) => CODE.test(f) && !TEST_FILE.test(f) && !NOT_RUNTIME.test(f) && !NOT_BEHAVIOUR.test(f));
|
|
124
|
+
// NAMED BY PATH (0.2.1). A build script that does
|
|
125
|
+
// `readFileSync(join(ROOT, 'kit', 'modes.src.js'))` reaches that file as
|
|
126
|
+
// surely as an import does, and the first real repository had its whole
|
|
127
|
+
// colour pipeline listed as unreached on exactly that. A file whose name a
|
|
128
|
+
// reached file quotes is reached, and its own imports are walked in turn.
|
|
129
|
+
const GENERIC_BASENAME = /^(index|main|app|utils?|types?|config|client|server|helpers?|constants?)\./i;
|
|
130
|
+
const namedBy = new Map();
|
|
131
|
+
for (let grew = true; grew;) {
|
|
132
|
+
grew = false;
|
|
133
|
+
for (const f of candidates) {
|
|
134
|
+
if (reached.has(f))
|
|
135
|
+
continue;
|
|
136
|
+
const base = f.split("/").pop();
|
|
137
|
+
if (base.length < 8 || GENERIC_BASENAME.test(base))
|
|
138
|
+
continue;
|
|
139
|
+
const re = new RegExp(`['"\`][^'"\`\\n]*(?<![a-z0-9_-])${base.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}['"\`]`);
|
|
140
|
+
for (const r of reached) {
|
|
141
|
+
if (r === f)
|
|
142
|
+
continue;
|
|
143
|
+
const text = await repo.read(r);
|
|
144
|
+
if (text && re.test(text)) {
|
|
145
|
+
reached.add(f);
|
|
146
|
+
namedBy.set(f, r);
|
|
147
|
+
queue.push(f);
|
|
148
|
+
grew = true;
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
114
151
|
}
|
|
115
152
|
}
|
|
153
|
+
await drain();
|
|
116
154
|
}
|
|
155
|
+
void namedBy;
|
|
156
|
+
// SERVED, NOT IMPORTED. Everything under public/ or static/ is handed to the
|
|
157
|
+
// browser as-is; nothing imports it because nothing has to.
|
|
158
|
+
const SERVED = /(^|\/)(public|static)\//i;
|
|
159
|
+
// A UI-kit scaffold beside a components.json: installed wholesale, imported
|
|
160
|
+
// one component at a time, never bundled when unimported.
|
|
161
|
+
const kit = repo.files.some((f) => /(^|\/)components\.json$/.test(f) && !/node_modules|fixtures?/.test(f));
|
|
117
162
|
const dead = [];
|
|
118
|
-
for (const f of
|
|
119
|
-
if (
|
|
120
|
-
continue;
|
|
121
|
-
if (reached.has(f))
|
|
163
|
+
for (const f of candidates) {
|
|
164
|
+
if (reached.has(f) || SERVED.test(f))
|
|
122
165
|
continue;
|
|
123
166
|
const text = await repo.read(f);
|
|
124
167
|
if (!text)
|
|
125
168
|
continue;
|
|
169
|
+
const scaffold = kit && /(^|\/)components\/ui\/[^/]+\.(t|j)sx?$/.test(f);
|
|
126
170
|
dead.push({
|
|
127
171
|
file: f,
|
|
128
172
|
loc: text.split("\n").length,
|
|
129
|
-
note: "no import path found from any entrypoint",
|
|
173
|
+
note: scaffold ? "a UI-kit scaffold component no file imports: never bundled, only read and searched" : "no import path found from any entrypoint",
|
|
174
|
+
...(scaffold ? { scaffold: true } : {}),
|
|
130
175
|
});
|
|
131
176
|
}
|
|
132
177
|
return {
|
package/dist/detect/deps.js
CHANGED
|
@@ -61,6 +61,62 @@ export async function detectDeps(repo) {
|
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
|
+
// MANIFESTS THAT ARE NOT package.json (0.2).
|
|
65
|
+
//
|
|
66
|
+
// The whole detector assumed npm, so a Python service with five pinned
|
|
67
|
+
// packages in requirements.txt read as a repository with no dependencies -
|
|
68
|
+
// and "no dependency with no reference found" then printed as a clean result
|
|
69
|
+
// rather than as a question never asked. Run across five repositories, that
|
|
70
|
+
// shape scored a six-file scraper at 96/100.
|
|
71
|
+
//
|
|
72
|
+
// Only formats simple enough to read without a parser, and each one's own
|
|
73
|
+
// reference test is the same as npm's: does any file name it.
|
|
74
|
+
for (const m of repo.files.filter((f) => /(^|\/)(requirements(-[a-z]+)?\.txt|pyproject\.toml|Gemfile|go\.mod|Cargo\.toml)$/.test(f) && !NOT_A_MANIFEST.test(f))) {
|
|
75
|
+
const text = await repo.read(m);
|
|
76
|
+
if (!text)
|
|
77
|
+
continue;
|
|
78
|
+
const base = m.split("/").pop();
|
|
79
|
+
if (/^requirements/.test(base)) {
|
|
80
|
+
for (const line of text.split("\n")) {
|
|
81
|
+
const t = line.trim();
|
|
82
|
+
if (!t || t.startsWith("#") || t.startsWith("-"))
|
|
83
|
+
continue;
|
|
84
|
+
const name = /^([A-Za-z0-9._-]+)/.exec(t)?.[1];
|
|
85
|
+
const version = /[=<>~!]=?\s*([0-9][^\s;#]*)/.exec(t)?.[1] ?? "";
|
|
86
|
+
if (name)
|
|
87
|
+
declared.push({ name, version, manifest: m, dev: /dev|test/i.test(base), imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
else if (base === "pyproject.toml") {
|
|
91
|
+
// `dependencies = ["fastapi>=0.1", ...]` under [project] or poetry's table.
|
|
92
|
+
for (const dm of text.matchAll(/^\s*["']?([A-Za-z0-9._-]+)["']?\s*=\s*["'][\^~>=<0-9][^"']*["']/gm)) {
|
|
93
|
+
declared.push({ name: dm[1], version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
94
|
+
}
|
|
95
|
+
for (const dm of text.matchAll(/["']([A-Za-z0-9._-]+)\s*[>=<~!][^"']*["']/g)) {
|
|
96
|
+
declared.push({ name: dm[1], version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
else if (base === "go.mod") {
|
|
100
|
+
for (const dm of text.matchAll(/^\s+([a-z0-9.\-]+\/[^\s]+)\s+v[^\s]+/gm)) {
|
|
101
|
+
declared.push({ name: dm[1], version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
else if (base === "Cargo.toml" || base === "Gemfile") {
|
|
105
|
+
const re = base === "Gemfile" ? /gem\s+["']([A-Za-z0-9._-]+)["']/g : /^\s*([A-Za-z0-9._-]+)\s*=/gm;
|
|
106
|
+
for (const dm of text.matchAll(re)) {
|
|
107
|
+
const name = dm[1];
|
|
108
|
+
if (base === "Cargo.toml" && /^(name|version|edition|authors|license|description|repository|edition2021)$/.test(name))
|
|
109
|
+
continue;
|
|
110
|
+
declared.push({ name, version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
// Deduplicate: pyproject's two patterns can both match one line.
|
|
115
|
+
const uniq = new Map();
|
|
116
|
+
for (const d of declared)
|
|
117
|
+
uniq.set(`${d.manifest}::${d.name}`, d);
|
|
118
|
+
declared.length = 0;
|
|
119
|
+
declared.push(...uniq.values());
|
|
64
120
|
if (declared.length === 0)
|
|
65
121
|
return [];
|
|
66
122
|
const byName = new Map();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { scopeFor } from "../walk.js";
|
|
2
2
|
const WINDOW = 30;
|
|
3
3
|
const MIN_FILES = 2;
|
|
4
4
|
const GENERATED = /(^|\/)(.*\.gen\.|.*\.generated\.|types\/supabase|database\.types)/i;
|
|
@@ -17,7 +17,11 @@ function meaningful(lines) {
|
|
|
17
17
|
return lines.filter((l) => l.length > 12).length >= WINDOW * 0.5;
|
|
18
18
|
}
|
|
19
19
|
export async function detectDuplication(repo) {
|
|
20
|
-
|
|
20
|
+
// The duplication scope, not the runtime one: a block copied into a fixture or
|
|
21
|
+
// a test is still a block copied, and the shipped templates are where copies
|
|
22
|
+
// matter most - a generator emits them into a customer's repository, where a
|
|
23
|
+
// shared import cannot follow.
|
|
24
|
+
const files = scopeFor(repo.files, "duplication").filter((f) => !GENERATED.test(f));
|
|
21
25
|
// hash of a window -> the places it occurs
|
|
22
26
|
const windows = new Map();
|
|
23
27
|
const fileLines = new Map();
|
|
@@ -69,10 +73,29 @@ export async function detectDuplication(repo) {
|
|
|
69
73
|
});
|
|
70
74
|
}
|
|
71
75
|
}
|
|
76
|
+
// PARALLEL ADAPTERS DUPLICATE BY DESIGN (0.2). Reading the shipped templates
|
|
77
|
+
// surfaced 25 clusters on the first real repository, and the largest - a
|
|
78
|
+
// 265-line pair - floored the level. It was HarnessMount.tsx.tmpl under
|
|
79
|
+
// next-app-clerk/ and under next-app-supabase/: the same file for two customer
|
|
80
|
+
// stacks, inlined because a generated file cannot import from the generator.
|
|
81
|
+
// Same relative path, sibling directories, a parent named for what it is.
|
|
82
|
+
const PARALLEL_PARENT = /(^|\/)(adapters?|templates?|shells?|shell-templates?|generators?|stacks?|targets?)\//i;
|
|
83
|
+
const parallel = (files) => {
|
|
84
|
+
if (files.length < 2 || !files.every((f) => PARALLEL_PARENT.test(f)))
|
|
85
|
+
return false;
|
|
86
|
+
// Strip the one path segment that names the sibling; the rest must agree.
|
|
87
|
+
const tails = files.map((f) => {
|
|
88
|
+
const m = PARALLEL_PARENT.exec(f);
|
|
89
|
+
const after = f.slice(m.index + m[0].length);
|
|
90
|
+
return after.split("/").slice(1).join("/");
|
|
91
|
+
});
|
|
92
|
+
return tails.every((t) => t.length > 0 && t === tails[0]);
|
|
93
|
+
};
|
|
72
94
|
return [...clusters.values()]
|
|
73
95
|
.map((c) => ({
|
|
74
96
|
...c,
|
|
75
97
|
deliberate: c.files.every((f) => deliberateFiles.has(f)) || c.files.filter((f) => deliberateFiles.has(f)).length >= c.files.length - 1,
|
|
98
|
+
parallel: parallel(c.files),
|
|
76
99
|
}))
|
|
77
100
|
.sort((x, y) => y.lines - x.lines)
|
|
78
101
|
.slice(0, 25);
|
|
@@ -1,9 +1,50 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { scopeFor } from "../walk.js";
|
|
2
|
+
/** How many of {named, implements, keeps state} a file showed. */
|
|
3
|
+
function legs(r, file, text) {
|
|
4
|
+
let n = 0;
|
|
5
|
+
if (r.name?.test(file))
|
|
6
|
+
n++;
|
|
7
|
+
if (r.content.test(text))
|
|
8
|
+
n++;
|
|
9
|
+
if (r.requires ? r.requires.test(text) : false)
|
|
10
|
+
n++;
|
|
11
|
+
return n;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The lines that actually implement the subsystem, not the file that holds it.
|
|
15
|
+
*
|
|
16
|
+
* `countLines(text)` counted whole files, so a 700-line dashboard that mentioned
|
|
17
|
+
* the rail once contributed 700 lines to its "size". The region is the span from
|
|
18
|
+
* the first corroborating match to the last, which is a floor on the real
|
|
19
|
+
* subsystem and never a ceiling on the file.
|
|
20
|
+
*/
|
|
21
|
+
function regionLines(r, text) {
|
|
22
|
+
const lines = text.split("\n");
|
|
23
|
+
const starts = [];
|
|
24
|
+
const ends = [];
|
|
25
|
+
// CONTENT ONLY (0.2.1). `requires` testifies that the state the job needs
|
|
26
|
+
// exists somewhere in the file; it does not bound the subsystem. Spanning to
|
|
27
|
+
// it turned one `.ilike(` on line 266 and the word "query" near the bottom
|
|
28
|
+
// into "1,002 lines of hand-rolled search" in an invite handler.
|
|
29
|
+
const g = new RegExp(r.content.source, r.content.flags.includes("g") ? r.content.flags : r.content.flags + "g");
|
|
30
|
+
for (const m of text.matchAll(g)) {
|
|
31
|
+
starts.push(text.slice(0, m.index).split("\n").length);
|
|
32
|
+
ends.push(text.slice(0, m.index + m[0].length).split("\n").length);
|
|
33
|
+
}
|
|
34
|
+
if (starts.length === 0)
|
|
35
|
+
return 0;
|
|
36
|
+
return Math.min(lines.length, Math.max(...ends) - Math.min(...starts) + 1);
|
|
37
|
+
}
|
|
2
38
|
const RULES = [
|
|
3
39
|
{
|
|
4
40
|
rail: "pdf",
|
|
5
41
|
name: /pdf|report.?pars|extract/i,
|
|
6
42
|
content: /%PDF-|getTextContent|pdftotext|pdf.{0,12}(parse|extract|text)|(parse|extract).{0,12}pdf/i,
|
|
43
|
+
// The third leg, which this rule always implied and never stated: pulling
|
|
44
|
+
// fields OUT. A file that merely names a PDF and mentions one is a download
|
|
45
|
+
// button; a file that names one, reads its text and runs capture groups over
|
|
46
|
+
// it is a parser. The owner's worked example shows all three.
|
|
47
|
+
requires: /\.(exec|match|matchAll)\(|\/[^/\n]{3,}\/[gimsuy]*\.exec|\((\?<[a-z]+>|\[)/i,
|
|
7
48
|
rail_sdk: /["'](pdf-parse|pdfjs-dist|@pdf-lib|unpdf|pdf2json)["']/,
|
|
8
49
|
},
|
|
9
50
|
{
|
|
@@ -15,7 +56,14 @@ const RULES = [
|
|
|
15
56
|
},
|
|
16
57
|
{
|
|
17
58
|
rail: "email-templating",
|
|
59
|
+
// NAME REQUIRED. `<table` within 4,000 characters of `.send(` described an
|
|
60
|
+
// ops dashboard on the first real repository, and the report gave it a size.
|
|
61
|
+
name: /email|mail(er)?|template|notif/i,
|
|
62
|
+
name_required: true,
|
|
18
63
|
content: /<(html|body|table)[\s>][\s\S]{0,4000}(sendEmail|sendMail|\.send\(|resend|smtp)/i,
|
|
64
|
+
// The third leg: a template is a template because something is interpolated
|
|
65
|
+
// into it. A static string is a string.
|
|
66
|
+
requires: /\$\{[^}]+\}|\{\{[^}]+\}\}|<%=?[\s\S]{0,80}%>|\.replace\(/,
|
|
19
67
|
rail_sdk: /["'](@react-email|react-email|mjml|maizzle|handlebars)["']/,
|
|
20
68
|
},
|
|
21
69
|
{
|
|
@@ -29,12 +77,26 @@ const RULES = [
|
|
|
29
77
|
rail: "queue-scheduler",
|
|
30
78
|
name: /queue|worker|scheduler|cron|jobs?/i,
|
|
31
79
|
content: /(setInterval|setTimeout)[\s\S]{0,300}(fetch|process|poll|dequeue|claim)|status\s*=\s*["']pending["'][\s\S]{0,400}(update|claim|lock)/i,
|
|
80
|
+
// The third leg: a queue keeps work somewhere and marks it done. A `sleep`
|
|
81
|
+
// helper in a device-flow poller is not a scheduler, and was reported as
|
|
82
|
+
// 1,265 lines of one.
|
|
83
|
+
requires: /(pending|queued|processing|claimed?|attempts?|retry|retries|dead.?letter)\b[\s\S]{0,400}(update|insert|set\(|save|delete|ack)/i,
|
|
32
84
|
rail_sdk: /["'](bullmq|bee-queue|agenda|bree|graphile-worker|@trigger\.dev|inngest|temporalio)["']/,
|
|
33
85
|
},
|
|
34
86
|
{
|
|
35
87
|
rail: "search",
|
|
36
|
-
|
|
37
|
-
|
|
88
|
+
// NAME REQUIRED, and `index` and `query` are not names (0.2.1): with them
|
|
89
|
+
// every `index.ts` announced itself as search, and one PostgREST
|
|
90
|
+
// `.ilike('email', email)` looking up an invitee was reported as 1,002 lines
|
|
91
|
+
// of a search engine. `tokenize(query)` inside a help-centre page is a
|
|
92
|
+
// filter box, not a search engine, and was reported as 260 lines of one.
|
|
93
|
+
name: /search|lookup|finder|indexer|full.?text/i,
|
|
94
|
+
name_required: true,
|
|
95
|
+
// AN ENGINE, NOT A FILTER. Something is tokenised and scored, an index is
|
|
96
|
+
// built and walked, or similarity is computed by hand. One LIKE is a WHERE
|
|
97
|
+
// clause; a database doing it is the rail, not the hand.
|
|
98
|
+
content: /\btokeni[sz]e\w*\b[\s\S]{0,600}\b(score|rank)\w*\b|\b(levenshtein|trigram|jaro|bm25|tf.?idf|inverted.?index|n.?grams?)\b|\b(build|make|create)(Search)?Index\s*\(|\bsearchIndex\b/i,
|
|
99
|
+
requires: /\b(query|term|needle|q)\b[\s\S]{0,600}\b(results?|hits|matches|ranked)\b/i,
|
|
38
100
|
rail_sdk: /["'](algoliasearch|meilisearch|typesense|@elastic|flexsearch|minisearch|fuse\.js)["']/,
|
|
39
101
|
},
|
|
40
102
|
{
|
|
@@ -46,7 +108,11 @@ const RULES = [
|
|
|
46
108
|
},
|
|
47
109
|
{
|
|
48
110
|
rail: "webhook-plumbing",
|
|
111
|
+
name: /webhook|hook|signature|signing/i,
|
|
49
112
|
content: /(createHmac|timingSafeEqual)[\s\S]{0,400}(signature|x-signature|svix|webhook)/i,
|
|
113
|
+
// The third leg: verifying a signature needs the secret and the header it
|
|
114
|
+
// arrived in. A file that only defines `timingSafeEqual` is a utility.
|
|
115
|
+
requires: /(headers?|req\.headers|request\.headers)[\s\S]{0,200}(signature|hmac)|process\.env\.[A-Z_]*(SECRET|SIGNING)/i,
|
|
50
116
|
rail_sdk: /["'](svix|@hono\/webhook)["']/,
|
|
51
117
|
},
|
|
52
118
|
{
|
|
@@ -63,7 +129,7 @@ const RULES = [
|
|
|
63
129
|
];
|
|
64
130
|
const countLines = (t) => t.split("\n").length;
|
|
65
131
|
export async function detectHandrolled(repo) {
|
|
66
|
-
const files =
|
|
132
|
+
const files = scopeFor(repo.files, "implementation");
|
|
67
133
|
// A rail SDK anywhere in the repository retires that rule everywhere: the
|
|
68
134
|
// decision to buy was already made, and re-litigating it is noise.
|
|
69
135
|
const railInUse = new Set();
|
|
@@ -90,8 +156,15 @@ export async function detectHandrolled(repo) {
|
|
|
90
156
|
const named = r.name ? r.name.test(f) : false;
|
|
91
157
|
if (r.name_required && !named)
|
|
92
158
|
continue;
|
|
159
|
+
// THREE LEGS IN ONE FILE, or this file is not part of the subsystem.
|
|
160
|
+
// A rule with no `name` or no `requires` can only ever show two, so it
|
|
161
|
+
// must carry both to contribute: a rail defined by content alone is the
|
|
162
|
+
// shape that produced "email templating" from a dashboard's `<table`.
|
|
163
|
+
if (legs(r, f, text) < 3)
|
|
164
|
+
continue;
|
|
93
165
|
const cur = byRail.get(r.rail) ?? { files: new Map(), signal: { file: f, line: "" }, named: false };
|
|
94
|
-
|
|
166
|
+
// The implementing region, not the whole file.
|
|
167
|
+
cur.files.set(f, regionLines(r, text));
|
|
95
168
|
cur.named = cur.named || named;
|
|
96
169
|
if (!cur.signal.line) {
|
|
97
170
|
const at = text.slice(0, m.index).split("\n").length;
|
|
@@ -108,5 +181,11 @@ export async function detectHandrolled(repo) {
|
|
|
108
181
|
confidence: (r.named ? "high" : "low"),
|
|
109
182
|
signal: r.signal,
|
|
110
183
|
}))
|
|
184
|
+
// CONFIDENCE NOW GATES (0.2). It was computed and never used: five
|
|
185
|
+
// low-confidence rails printed as fact, with sizes, under a heading that
|
|
186
|
+
// said "built by hand where the market sells a rail", and the score counted
|
|
187
|
+
// them. A finding nothing corroborates is a question; the report has a place
|
|
188
|
+
// for questions and it is not this list.
|
|
189
|
+
.filter((h) => h.confidence === "high" && h.loc > 0)
|
|
111
190
|
.sort((a, b) => b.loc - a.loc);
|
|
112
191
|
}
|
package/dist/detect/stack.d.ts
CHANGED
|
@@ -45,6 +45,14 @@ export interface WorkflowFact {
|
|
|
45
45
|
script_only: boolean;
|
|
46
46
|
}
|
|
47
47
|
export interface ConsolidationFact {
|
|
48
|
+
/** Stable id, so a verdict can name this finding: see verdicts.ts. */
|
|
49
|
+
id?: string;
|
|
50
|
+
/** A recorded decision that explains keeping both. On record is not a deduction. */
|
|
51
|
+
on_record?: {
|
|
52
|
+
file: string;
|
|
53
|
+
line: number;
|
|
54
|
+
excerpt: string;
|
|
55
|
+
};
|
|
48
56
|
/** The platform or service the repo already runs, that would remain. */
|
|
49
57
|
keep: string;
|
|
50
58
|
/** The candidate to cut. */
|
package/dist/detect/stack.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { scopeFor } from "../walk.js";
|
|
2
2
|
const HOSTING = [
|
|
3
3
|
{ platform: "Vercel", files: /(^|\/)vercel\.json$|(^|\/)\.vercel\/project\.json$/ },
|
|
4
4
|
{ platform: "Netlify", files: /(^|\/)netlify\.toml$/ },
|
|
@@ -154,7 +154,15 @@ export async function detectStack(repo) {
|
|
|
154
154
|
* directories, docs, fixtures and this package's own tree are excluded by
|
|
155
155
|
* the same rule the walker's detectors use. */
|
|
156
156
|
async function grepAny(repo, re, limit) {
|
|
157
|
-
|
|
157
|
+
// ASK FOR THE SCOPE BY NAME (0.2). This line used to re-derive its own, and
|
|
158
|
+
// omitted the one term `runtimeCode` had: test files. So `@auth0/nextjs-auth0`,
|
|
159
|
+
// quoted as a STRING inside a scanner's test describing a customer's codebase,
|
|
160
|
+
// read as this product importing Auth0 - and the report told an owner to drop a
|
|
161
|
+
// vendor they never had. That was the fourth self-contamination variant in this
|
|
162
|
+
// family, each one a call site that forgot a term the others remembered.
|
|
163
|
+
const files = scopeFor(repo.files, "vendor-presence")
|
|
164
|
+
.filter((f) => !/(^|\/)package(-lock)?\.json$/.test(f))
|
|
165
|
+
.slice(0, limit);
|
|
158
166
|
for (const f of files) {
|
|
159
167
|
const text = await repo.read(f);
|
|
160
168
|
if (text && re.test(text))
|