@bigsteele/the-prospect 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +23 -1
- package/dist/decisions.d.ts +93 -0
- package/dist/decisions.js +143 -0
- package/dist/detect/costs.js +2 -2
- package/dist/detect/database.d.ts +28 -0
- package/dist/detect/database.js +173 -0
- package/dist/detect/deps 2.d.ts +25 -0
- package/dist/detect/deps 2.js +198 -0
- package/dist/detect/deps.js +56 -0
- package/dist/detect/duplication.js +25 -2
- package/dist/detect/handrolled.js +73 -3
- package/dist/detect/stack.d.ts +6 -0
- package/dist/detect/stack.js +10 -2
- package/dist/detect/types 2.d.ts +106 -0
- package/dist/detect/types 2.js +11 -0
- package/dist/detect/types.d.ts +43 -0
- package/dist/detect/vendors.js +13 -6
- package/dist/index.d.ts +13 -2
- package/dist/index.js +65 -2
- package/dist/northstar.js +15 -2
- package/dist/profile.d.ts +55 -0
- package/dist/profile.js +106 -0
- package/dist/report.js +158 -18
- package/dist/score.d.ts +3 -0
- package/dist/score.js +24 -7
- package/dist/walk 2.d.ts +44 -0
- package/dist/walk 2.js +123 -0
- package/dist/walk.d.ts +85 -1
- package/dist/walk.js +189 -5
- package/package.json +1 -1
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
const CONFIG_FILE = /(^|\/)(tailwind|postcss|vite|next|nuxt|astro|svelte|webpack|rollup|babel|jest|vitest|playwright|eslint|prettier|tsup|drizzle|prisma)[^/]*\.(config\.)?(ts|js|mjs|cjs|json)$|(^|\/)\.(eslintrc|babelrc|prettierrc)(\.[a-z]+)?$|(^|\/)(package|turbo|nx|lerna)\.json$/i;
|
|
2
|
+
/** Nested manifests that describe fixtures or vendored copies, not this software. */
|
|
3
|
+
const NOT_A_MANIFEST = /(^|\/)(test|tests|fixtures?|__fixtures__|examples?|templates?)\//i;
|
|
4
|
+
/**
|
|
5
|
+
* Peers of the frameworks whose peers are used by the framework itself, for when
|
|
6
|
+
* node_modules is not installed and the real peerDependencies cannot be read.
|
|
7
|
+
*/
|
|
8
|
+
const FRAMEWORK_PEERS = {
|
|
9
|
+
next: ["react", "react-dom"],
|
|
10
|
+
gatsby: ["react", "react-dom"],
|
|
11
|
+
"@remix-run/react": ["react", "react-dom"],
|
|
12
|
+
expo: ["react", "react-native"],
|
|
13
|
+
"@monaco-editor/react": ["monaco-editor"],
|
|
14
|
+
"@tiptap/react": ["@tiptap/pm"],
|
|
15
|
+
"@tiptap/starter-kit": ["@tiptap/pm"],
|
|
16
|
+
};
|
|
17
|
+
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
18
|
+
function importSpecifiers(text) {
|
|
19
|
+
const out = [];
|
|
20
|
+
const patterns = [
|
|
21
|
+
/\bimport\s+(?:[^"'`]*?\s+from\s+)?["']([^"'\n]+)["']/g,
|
|
22
|
+
/\brequire\(\s*["']([^"'\n]+)["']\s*\)/g,
|
|
23
|
+
/\bimport\(\s*["']([^"'\n]+)["']\s*\)/g,
|
|
24
|
+
/\bexport\s+[^"'`\n]*?\s+from\s+["']([^"'\n]+)["']/g,
|
|
25
|
+
];
|
|
26
|
+
for (const re of patterns) {
|
|
27
|
+
for (const m of text.matchAll(re))
|
|
28
|
+
out.push(m[1]);
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
/** `@scope/pkg/deep/path` -> `@scope/pkg`; `pkg/deep` -> `pkg`. Relative and URL imports return null. */
|
|
33
|
+
export function packageOf(spec) {
|
|
34
|
+
if (spec.startsWith(".") || spec.startsWith("/") || /^(node:|https?:|npm:|jsr:)/.test(spec)) {
|
|
35
|
+
// Deno-style `npm:pkg@1` still names a package worth counting.
|
|
36
|
+
const npm = /^npm:(@?[^@/]+(?:\/[^@/]+)?)/.exec(spec);
|
|
37
|
+
return npm ? npm[1] : null;
|
|
38
|
+
}
|
|
39
|
+
const parts = spec.split("/");
|
|
40
|
+
return spec.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0];
|
|
41
|
+
}
|
|
42
|
+
export async function detectDeps(repo) {
|
|
43
|
+
const manifests = repo.files.filter((f) => /(^|\/)package\.json$/.test(f) && !/node_modules/.test(f) && !NOT_A_MANIFEST.test(f));
|
|
44
|
+
const declared = [];
|
|
45
|
+
const scriptsByManifest = new Map();
|
|
46
|
+
for (const m of manifests) {
|
|
47
|
+
const text = await repo.read(m);
|
|
48
|
+
if (!text)
|
|
49
|
+
continue;
|
|
50
|
+
let json;
|
|
51
|
+
try {
|
|
52
|
+
json = JSON.parse(text);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
scriptsByManifest.set(m, Object.values(json.scripts ?? {}).filter((v) => typeof v === "string"));
|
|
58
|
+
for (const [dev, block] of [[false, json.dependencies], [true, json.devDependencies]]) {
|
|
59
|
+
for (const [name, version] of Object.entries(block ?? {})) {
|
|
60
|
+
declared.push({ name, version, manifest: m, dev, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (declared.length === 0)
|
|
65
|
+
return [];
|
|
66
|
+
const byName = new Map();
|
|
67
|
+
for (const d of declared) {
|
|
68
|
+
const list = byName.get(d.name) ?? [];
|
|
69
|
+
list.push(d);
|
|
70
|
+
byName.set(d.name, list);
|
|
71
|
+
}
|
|
72
|
+
// Imports: every code file, tests included - a dependency only tests use is
|
|
73
|
+
// still referenced, and saying otherwise would be the lie this file bans.
|
|
74
|
+
const codeFiles = repo.files.filter((f) => /\.(ts|tsx|js|jsx|mjs|cjs|css|scss|sass|less|pcss)$/i.test(f) && !/node_modules/.test(f));
|
|
75
|
+
for (const f of codeFiles) {
|
|
76
|
+
const text = await repo.read(f);
|
|
77
|
+
if (!text)
|
|
78
|
+
continue;
|
|
79
|
+
const seen = new Set();
|
|
80
|
+
for (const spec of importSpecifiers(text)) {
|
|
81
|
+
const pkg = packageOf(spec);
|
|
82
|
+
if (pkg && !seen.has(pkg)) {
|
|
83
|
+
seen.add(pkg);
|
|
84
|
+
for (const d of byName.get(pkg) ?? [])
|
|
85
|
+
d.imported_by++;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// Stylesheets: `@import "tw-animate-css"`, Tailwind 4's `@plugin "pkg"`, and
|
|
89
|
+
// Sass's `@use` / `@forward`.
|
|
90
|
+
if (/\.(css|scss|sass|less|pcss)$/i.test(f)) {
|
|
91
|
+
for (const m of text.matchAll(/@(?:import|plugin|use|forward)\s+(?:url\(\s*)?["']([^"'\n]+)["']/g)) {
|
|
92
|
+
const pkg = packageOf(m[1].replace(/^~/, ""));
|
|
93
|
+
if (pkg && !seen.has(pkg)) {
|
|
94
|
+
seen.add(pkg);
|
|
95
|
+
for (const d of byName.get(pkg) ?? [])
|
|
96
|
+
d.imported_by++;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// Config mentions: plugins named as bare strings, and npm scripts that
|
|
102
|
+
// invoke a package's bin by name.
|
|
103
|
+
const configs = repo.files.filter((f) => CONFIG_FILE.test(f) && !/node_modules/.test(f));
|
|
104
|
+
for (const f of configs) {
|
|
105
|
+
const text = await repo.read(f);
|
|
106
|
+
if (!text)
|
|
107
|
+
continue;
|
|
108
|
+
for (const [name, list] of byName) {
|
|
109
|
+
const bare = name.replace(/^@[^/]+\//, "");
|
|
110
|
+
if (text.includes(`"${name}"`) || text.includes(`'${name}'`) || new RegExp(`\\b${escapeRe(bare)}\\b`).test(text)) {
|
|
111
|
+
for (const d of list)
|
|
112
|
+
if (d.manifest !== f)
|
|
113
|
+
d.config_mentions++;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// ── Used without an import ────────────────────────────────────────────────
|
|
118
|
+
const referenced = (d) => d.imported_by > 0 || d.config_mentions > 0 || !!d.required_by;
|
|
119
|
+
const installedCache = new Map();
|
|
120
|
+
const installed = async (d) => {
|
|
121
|
+
const key = `${d.manifest}|${d.name}`;
|
|
122
|
+
if (!installedCache.has(key))
|
|
123
|
+
installedCache.set(key, await repo.installed(d.manifest, d.name));
|
|
124
|
+
return installedCache.get(key);
|
|
125
|
+
};
|
|
126
|
+
// Run by bin name from the declaring manifest's own scripts. The loop above
|
|
127
|
+
// skips the declaring manifest, because its dependency list names every
|
|
128
|
+
// package, so its scripts are read here instead. The bin is not always the
|
|
129
|
+
// package name: react-email installs `email`.
|
|
130
|
+
for (const d of declared) {
|
|
131
|
+
if (referenced(d))
|
|
132
|
+
continue;
|
|
133
|
+
const scripts = scriptsByManifest.get(d.manifest) ?? [];
|
|
134
|
+
if (!scripts.length)
|
|
135
|
+
continue;
|
|
136
|
+
const bins = new Set([d.name.replace(/^@[^/]+\//, "")]);
|
|
137
|
+
const inst = await installed(d);
|
|
138
|
+
if (inst?.bin && typeof inst.bin === "object")
|
|
139
|
+
for (const b of Object.keys(inst.bin))
|
|
140
|
+
bins.add(b);
|
|
141
|
+
const hit = scripts.find((cmd) => [...bins].some((b) => new RegExp(`(^|[\\s;&|(])${escapeRe(b)}(\\s|$)`).test(cmd)));
|
|
142
|
+
if (hit)
|
|
143
|
+
d.required_by = `npm script: ${hit.length > 60 ? `${hit.slice(0, 57)}...` : hit}`;
|
|
144
|
+
}
|
|
145
|
+
// The JSX runtime. Every .tsx/.jsx file compiles to an import of `react` (or
|
|
146
|
+
// whatever jsxImportSource names) that no source file spells out.
|
|
147
|
+
if (repo.files.some((f) => /\.(tsx|jsx)$/.test(f) && !/node_modules/.test(f))) {
|
|
148
|
+
let source = "react";
|
|
149
|
+
for (const tc of repo.files.filter((f) => /(^|\/)(tsconfig|jsconfig)[^/]*\.json$/.test(f) && !NOT_A_MANIFEST.test(f))) {
|
|
150
|
+
const named = /"jsxImportSource"\s*:\s*"([^"]+)"/.exec((await repo.read(tc)) ?? "")?.[1];
|
|
151
|
+
if (named) {
|
|
152
|
+
source = packageOf(named) ?? source;
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
for (const d of byName.get(source) ?? [])
|
|
157
|
+
if (!referenced(d))
|
|
158
|
+
d.required_by = "JSX runtime";
|
|
159
|
+
}
|
|
160
|
+
// Capacitor's native build registers every installed plugin from package.json.
|
|
161
|
+
if (repo.files.some((f) => /(^|\/)capacitor\.config\.(ts|js|json)$/.test(f))) {
|
|
162
|
+
for (const d of declared) {
|
|
163
|
+
if (!referenced(d) && /^(@capacitor\/|@capacitor-community\/|cordova-plugin-)/.test(d.name)) {
|
|
164
|
+
d.required_by = "native build (Capacitor)";
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
// Peers of packages in use. Read from the installed manifest when there is one
|
|
169
|
+
// (optional peers excluded), from the short framework list when there is not.
|
|
170
|
+
// Repeated until nothing changes, because a peer can have peers.
|
|
171
|
+
for (let changed = true; changed;) {
|
|
172
|
+
changed = false;
|
|
173
|
+
for (const user of declared) {
|
|
174
|
+
if (!referenced(user))
|
|
175
|
+
continue;
|
|
176
|
+
const inst = await installed(user);
|
|
177
|
+
const peers = inst
|
|
178
|
+
? Object.keys(inst.peerDependencies ?? {}).filter((p) => !inst.peerDependenciesMeta?.[p]?.optional)
|
|
179
|
+
: (FRAMEWORK_PEERS[user.name] ?? []);
|
|
180
|
+
for (const peer of peers) {
|
|
181
|
+
for (const d of byName.get(peer) ?? []) {
|
|
182
|
+
// Resolvable from where the user is declared: the same manifest, or the root.
|
|
183
|
+
if (referenced(d) || (d.manifest !== user.manifest && d.manifest !== "package.json"))
|
|
184
|
+
continue;
|
|
185
|
+
d.required_by = `peer of ${user.name}`;
|
|
186
|
+
changed = true;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
for (const d of declared) {
|
|
192
|
+
// Types-only and tooling-adjacent packages are referenced by the compiler,
|
|
193
|
+
// not by an import statement; they never earn the flag.
|
|
194
|
+
const toolingShaped = /^@types\//.test(d.name) || /(^|-)(cli|eslint|prettier|typescript|vitest|vite|tsx?|husky|lint-staged|concurrently|nodemon)($|-)/.test(d.name);
|
|
195
|
+
d.no_reference_found = !referenced(d) && !toolingShaped;
|
|
196
|
+
}
|
|
197
|
+
return declared.sort((a, b) => Number(b.no_reference_found) - Number(a.no_reference_found) || a.name.localeCompare(b.name));
|
|
198
|
+
}
|
package/dist/detect/deps.js
CHANGED
|
@@ -61,6 +61,62 @@ export async function detectDeps(repo) {
|
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
|
+
// MANIFESTS THAT ARE NOT package.json (0.2).
|
|
65
|
+
//
|
|
66
|
+
// The whole detector assumed npm, so a Python service with five pinned
|
|
67
|
+
// packages in requirements.txt read as a repository with no dependencies -
|
|
68
|
+
// and "no dependency with no reference found" then printed as a clean result
|
|
69
|
+
// rather than as a question never asked. Run across five repositories, that
|
|
70
|
+
// shape scored a six-file scraper at 96/100.
|
|
71
|
+
//
|
|
72
|
+
// Only formats simple enough to read without a parser, and each one's own
|
|
73
|
+
// reference test is the same as npm's: does any file name it.
|
|
74
|
+
for (const m of repo.files.filter((f) => /(^|\/)(requirements(-[a-z]+)?\.txt|pyproject\.toml|Gemfile|go\.mod|Cargo\.toml)$/.test(f) && !NOT_A_MANIFEST.test(f))) {
|
|
75
|
+
const text = await repo.read(m);
|
|
76
|
+
if (!text)
|
|
77
|
+
continue;
|
|
78
|
+
const base = m.split("/").pop();
|
|
79
|
+
if (/^requirements/.test(base)) {
|
|
80
|
+
for (const line of text.split("\n")) {
|
|
81
|
+
const t = line.trim();
|
|
82
|
+
if (!t || t.startsWith("#") || t.startsWith("-"))
|
|
83
|
+
continue;
|
|
84
|
+
const name = /^([A-Za-z0-9._-]+)/.exec(t)?.[1];
|
|
85
|
+
const version = /[=<>~!]=?\s*([0-9][^\s;#]*)/.exec(t)?.[1] ?? "";
|
|
86
|
+
if (name)
|
|
87
|
+
declared.push({ name, version, manifest: m, dev: /dev|test/i.test(base), imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
else if (base === "pyproject.toml") {
|
|
91
|
+
// `dependencies = ["fastapi>=0.1", ...]` under [project] or poetry's table.
|
|
92
|
+
for (const dm of text.matchAll(/^\s*["']?([A-Za-z0-9._-]+)["']?\s*=\s*["'][\^~>=<0-9][^"']*["']/gm)) {
|
|
93
|
+
declared.push({ name: dm[1], version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
94
|
+
}
|
|
95
|
+
for (const dm of text.matchAll(/["']([A-Za-z0-9._-]+)\s*[>=<~!][^"']*["']/g)) {
|
|
96
|
+
declared.push({ name: dm[1], version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
else if (base === "go.mod") {
|
|
100
|
+
for (const dm of text.matchAll(/^\s+([a-z0-9.\-]+\/[^\s]+)\s+v[^\s]+/gm)) {
|
|
101
|
+
declared.push({ name: dm[1], version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
else if (base === "Cargo.toml" || base === "Gemfile") {
|
|
105
|
+
const re = base === "Gemfile" ? /gem\s+["']([A-Za-z0-9._-]+)["']/g : /^\s*([A-Za-z0-9._-]+)\s*=/gm;
|
|
106
|
+
for (const dm of text.matchAll(re)) {
|
|
107
|
+
const name = dm[1];
|
|
108
|
+
if (base === "Cargo.toml" && /^(name|version|edition|authors|license|description|repository|edition2021)$/.test(name))
|
|
109
|
+
continue;
|
|
110
|
+
declared.push({ name, version: "", manifest: m, dev: false, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
// Deduplicate: pyproject's two patterns can both match one line.
|
|
115
|
+
const uniq = new Map();
|
|
116
|
+
for (const d of declared)
|
|
117
|
+
uniq.set(`${d.manifest}::${d.name}`, d);
|
|
118
|
+
declared.length = 0;
|
|
119
|
+
declared.push(...uniq.values());
|
|
64
120
|
if (declared.length === 0)
|
|
65
121
|
return [];
|
|
66
122
|
const byName = new Map();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { scopeFor } from "../walk.js";
|
|
2
2
|
const WINDOW = 30;
|
|
3
3
|
const MIN_FILES = 2;
|
|
4
4
|
const GENERATED = /(^|\/)(.*\.gen\.|.*\.generated\.|types\/supabase|database\.types)/i;
|
|
@@ -17,7 +17,11 @@ function meaningful(lines) {
|
|
|
17
17
|
return lines.filter((l) => l.length > 12).length >= WINDOW * 0.5;
|
|
18
18
|
}
|
|
19
19
|
export async function detectDuplication(repo) {
|
|
20
|
-
|
|
20
|
+
// The duplication scope, not the runtime one: a block copied into a fixture or
|
|
21
|
+
// a test is still a block copied, and the shipped templates are where copies
|
|
22
|
+
// matter most - a generator emits them into a customer's repository, where a
|
|
23
|
+
// shared import cannot follow.
|
|
24
|
+
const files = scopeFor(repo.files, "duplication").filter((f) => !GENERATED.test(f));
|
|
21
25
|
// hash of a window -> the places it occurs
|
|
22
26
|
const windows = new Map();
|
|
23
27
|
const fileLines = new Map();
|
|
@@ -69,10 +73,29 @@ export async function detectDuplication(repo) {
|
|
|
69
73
|
});
|
|
70
74
|
}
|
|
71
75
|
}
|
|
76
|
+
// PARALLEL ADAPTERS DUPLICATE BY DESIGN (0.2). Reading the shipped templates
|
|
77
|
+
// surfaced 25 clusters on the first real repository, and the largest - a
|
|
78
|
+
// 265-line pair - floored the level. It was HarnessMount.tsx.tmpl under
|
|
79
|
+
// next-app-clerk/ and under next-app-supabase/: the same file for two customer
|
|
80
|
+
// stacks, inlined because a generated file cannot import from the generator.
|
|
81
|
+
// Same relative path, sibling directories, a parent named for what it is.
|
|
82
|
+
const PARALLEL_PARENT = /(^|\/)(adapters?|templates?|shells?|shell-templates?|generators?|stacks?|targets?)\//i;
|
|
83
|
+
const parallel = (files) => {
|
|
84
|
+
if (files.length < 2 || !files.every((f) => PARALLEL_PARENT.test(f)))
|
|
85
|
+
return false;
|
|
86
|
+
// Strip the one path segment that names the sibling; the rest must agree.
|
|
87
|
+
const tails = files.map((f) => {
|
|
88
|
+
const m = PARALLEL_PARENT.exec(f);
|
|
89
|
+
const after = f.slice(m.index + m[0].length);
|
|
90
|
+
return after.split("/").slice(1).join("/");
|
|
91
|
+
});
|
|
92
|
+
return tails.every((t) => t.length > 0 && t === tails[0]);
|
|
93
|
+
};
|
|
72
94
|
return [...clusters.values()]
|
|
73
95
|
.map((c) => ({
|
|
74
96
|
...c,
|
|
75
97
|
deliberate: c.files.every((f) => deliberateFiles.has(f)) || c.files.filter((f) => deliberateFiles.has(f)).length >= c.files.length - 1,
|
|
98
|
+
parallel: parallel(c.files),
|
|
76
99
|
}))
|
|
77
100
|
.sort((x, y) => y.lines - x.lines)
|
|
78
101
|
.slice(0, 25);
|
|
@@ -1,9 +1,47 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { scopeFor } from "../walk.js";
|
|
2
|
+
/** How many of {named, implements, keeps state} a file showed. */
|
|
3
|
+
function legs(r, file, text) {
|
|
4
|
+
let n = 0;
|
|
5
|
+
if (r.name?.test(file))
|
|
6
|
+
n++;
|
|
7
|
+
if (r.content.test(text))
|
|
8
|
+
n++;
|
|
9
|
+
if (r.requires ? r.requires.test(text) : false)
|
|
10
|
+
n++;
|
|
11
|
+
return n;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The lines that actually implement the subsystem, not the file that holds it.
|
|
15
|
+
*
|
|
16
|
+
* `countLines(text)` counted whole files, so a 700-line dashboard that mentioned
|
|
17
|
+
* the rail once contributed 700 lines to its "size". The region is the span from
|
|
18
|
+
* the first corroborating match to the last, which is a floor on the real
|
|
19
|
+
* subsystem and never a ceiling on the file.
|
|
20
|
+
*/
|
|
21
|
+
function regionLines(r, text) {
|
|
22
|
+
const lines = text.split("\n");
|
|
23
|
+
const hits = [];
|
|
24
|
+
for (const re of [r.content, r.requires].filter(Boolean)) {
|
|
25
|
+
const g = new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g");
|
|
26
|
+
for (const m of text.matchAll(g))
|
|
27
|
+
hits.push(text.slice(0, m.index).split("\n").length);
|
|
28
|
+
}
|
|
29
|
+
if (hits.length === 0)
|
|
30
|
+
return 0;
|
|
31
|
+
const first = Math.min(...hits);
|
|
32
|
+
const last = Math.max(...hits);
|
|
33
|
+
return Math.min(lines.length, last - first + 1);
|
|
34
|
+
}
|
|
2
35
|
const RULES = [
|
|
3
36
|
{
|
|
4
37
|
rail: "pdf",
|
|
5
38
|
name: /pdf|report.?pars|extract/i,
|
|
6
39
|
content: /%PDF-|getTextContent|pdftotext|pdf.{0,12}(parse|extract|text)|(parse|extract).{0,12}pdf/i,
|
|
40
|
+
// The third leg, which this rule always implied and never stated: pulling
|
|
41
|
+
// fields OUT. A file that merely names a PDF and mentions one is a download
|
|
42
|
+
// button; a file that names one, reads its text and runs capture groups over
|
|
43
|
+
// it is a parser. The owner's worked example shows all three.
|
|
44
|
+
requires: /\.(exec|match|matchAll)\(|\/[^/\n]{3,}\/[gimsuy]*\.exec|\((\?<[a-z]+>|\[)/i,
|
|
7
45
|
rail_sdk: /["'](pdf-parse|pdfjs-dist|@pdf-lib|unpdf|pdf2json)["']/,
|
|
8
46
|
},
|
|
9
47
|
{
|
|
@@ -15,7 +53,14 @@ const RULES = [
|
|
|
15
53
|
},
|
|
16
54
|
{
|
|
17
55
|
rail: "email-templating",
|
|
56
|
+
// NAME REQUIRED. `<table` within 4,000 characters of `.send(` described an
|
|
57
|
+
// ops dashboard on the first real repository, and the report gave it a size.
|
|
58
|
+
name: /email|mail(er)?|template|notif/i,
|
|
59
|
+
name_required: true,
|
|
18
60
|
content: /<(html|body|table)[\s>][\s\S]{0,4000}(sendEmail|sendMail|\.send\(|resend|smtp)/i,
|
|
61
|
+
// The third leg: a template is a template because something is interpolated
|
|
62
|
+
// into it. A static string is a string.
|
|
63
|
+
requires: /\$\{[^}]+\}|\{\{[^}]+\}\}|<%=?[\s\S]{0,80}%>|\.replace\(/,
|
|
19
64
|
rail_sdk: /["'](@react-email|react-email|mjml|maizzle|handlebars)["']/,
|
|
20
65
|
},
|
|
21
66
|
{
|
|
@@ -29,10 +74,18 @@ const RULES = [
|
|
|
29
74
|
rail: "queue-scheduler",
|
|
30
75
|
name: /queue|worker|scheduler|cron|jobs?/i,
|
|
31
76
|
content: /(setInterval|setTimeout)[\s\S]{0,300}(fetch|process|poll|dequeue|claim)|status\s*=\s*["']pending["'][\s\S]{0,400}(update|claim|lock)/i,
|
|
77
|
+
// The third leg: a queue keeps work somewhere and marks it done. A `sleep`
|
|
78
|
+
// helper in a device-flow poller is not a scheduler, and was reported as
|
|
79
|
+
// 1,265 lines of one.
|
|
80
|
+
requires: /(pending|queued|processing|claimed?|attempts?|retry|retries|dead.?letter)\b[\s\S]{0,400}(update|insert|set\(|save|delete|ack)/i,
|
|
32
81
|
rail_sdk: /["'](bullmq|bee-queue|agenda|bree|graphile-worker|@trigger\.dev|inngest|temporalio)["']/,
|
|
33
82
|
},
|
|
34
83
|
{
|
|
35
84
|
rail: "search",
|
|
85
|
+
// NAME REQUIRED. `tokenize(query)` inside a help-centre page is a filter box,
|
|
86
|
+
// not a search engine, and the old rule reported it as 260 lines of one.
|
|
87
|
+
name: /search|query|index|lookup|find/i,
|
|
88
|
+
name_required: true,
|
|
36
89
|
content: /(ilike|ILIKE|LIKE)\s*\(?\s*["'`]%|\.ilike\(|to_tsvector|\btokeni[sz]e\b[\s\S]{0,200}\bscore\b/i,
|
|
37
90
|
requires: /(search|query|match|find)/i,
|
|
38
91
|
rail_sdk: /["'](algoliasearch|meilisearch|typesense|@elastic|flexsearch|minisearch|fuse\.js)["']/,
|
|
@@ -46,7 +99,11 @@ const RULES = [
|
|
|
46
99
|
},
|
|
47
100
|
{
|
|
48
101
|
rail: "webhook-plumbing",
|
|
102
|
+
name: /webhook|hook|signature|signing/i,
|
|
49
103
|
content: /(createHmac|timingSafeEqual)[\s\S]{0,400}(signature|x-signature|svix|webhook)/i,
|
|
104
|
+
// The third leg: verifying a signature needs the secret and the header it
|
|
105
|
+
// arrived in. A file that only defines `timingSafeEqual` is a utility.
|
|
106
|
+
requires: /(headers?|req\.headers|request\.headers)[\s\S]{0,200}(signature|hmac)|process\.env\.[A-Z_]*(SECRET|SIGNING)/i,
|
|
50
107
|
rail_sdk: /["'](svix|@hono\/webhook)["']/,
|
|
51
108
|
},
|
|
52
109
|
{
|
|
@@ -63,7 +120,7 @@ const RULES = [
|
|
|
63
120
|
];
|
|
64
121
|
const countLines = (t) => t.split("\n").length;
|
|
65
122
|
export async function detectHandrolled(repo) {
|
|
66
|
-
const files =
|
|
123
|
+
const files = scopeFor(repo.files, "implementation");
|
|
67
124
|
// A rail SDK anywhere in the repository retires that rule everywhere: the
|
|
68
125
|
// decision to buy was already made, and re-litigating it is noise.
|
|
69
126
|
const railInUse = new Set();
|
|
@@ -90,8 +147,15 @@ export async function detectHandrolled(repo) {
|
|
|
90
147
|
const named = r.name ? r.name.test(f) : false;
|
|
91
148
|
if (r.name_required && !named)
|
|
92
149
|
continue;
|
|
150
|
+
// THREE LEGS IN ONE FILE, or this file is not part of the subsystem.
|
|
151
|
+
// A rule with no `name` or no `requires` can only ever show two, so it
|
|
152
|
+
// must carry both to contribute: a rail defined by content alone is the
|
|
153
|
+
// shape that produced "email templating" from a dashboard's `<table`.
|
|
154
|
+
if (legs(r, f, text) < 3)
|
|
155
|
+
continue;
|
|
93
156
|
const cur = byRail.get(r.rail) ?? { files: new Map(), signal: { file: f, line: "" }, named: false };
|
|
94
|
-
|
|
157
|
+
// The implementing region, not the whole file.
|
|
158
|
+
cur.files.set(f, regionLines(r, text));
|
|
95
159
|
cur.named = cur.named || named;
|
|
96
160
|
if (!cur.signal.line) {
|
|
97
161
|
const at = text.slice(0, m.index).split("\n").length;
|
|
@@ -108,5 +172,11 @@ export async function detectHandrolled(repo) {
|
|
|
108
172
|
confidence: (r.named ? "high" : "low"),
|
|
109
173
|
signal: r.signal,
|
|
110
174
|
}))
|
|
175
|
+
// CONFIDENCE NOW GATES (0.2). It was computed and never used: five
|
|
176
|
+
// low-confidence rails printed as fact, with sizes, under a heading that
|
|
177
|
+
// said "built by hand where the market sells a rail", and the score counted
|
|
178
|
+
// them. A finding nothing corroborates is a question; the report has a place
|
|
179
|
+
// for questions and it is not this list.
|
|
180
|
+
.filter((h) => h.confidence === "high" && h.loc > 0)
|
|
111
181
|
.sort((a, b) => b.loc - a.loc);
|
|
112
182
|
}
|
package/dist/detect/stack.d.ts
CHANGED
|
@@ -45,6 +45,12 @@ export interface WorkflowFact {
|
|
|
45
45
|
script_only: boolean;
|
|
46
46
|
}
|
|
47
47
|
export interface ConsolidationFact {
|
|
48
|
+
/** A recorded decision that explains keeping both. On record is not a deduction. */
|
|
49
|
+
on_record?: {
|
|
50
|
+
file: string;
|
|
51
|
+
line: number;
|
|
52
|
+
excerpt: string;
|
|
53
|
+
};
|
|
48
54
|
/** The platform or service the repo already runs, that would remain. */
|
|
49
55
|
keep: string;
|
|
50
56
|
/** The candidate to cut. */
|
package/dist/detect/stack.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { scopeFor } from "../walk.js";
|
|
2
2
|
const HOSTING = [
|
|
3
3
|
{ platform: "Vercel", files: /(^|\/)vercel\.json$|(^|\/)\.vercel\/project\.json$/ },
|
|
4
4
|
{ platform: "Netlify", files: /(^|\/)netlify\.toml$/ },
|
|
@@ -154,7 +154,15 @@ export async function detectStack(repo) {
|
|
|
154
154
|
* directories, docs, fixtures and this package's own tree are excluded by
|
|
155
155
|
* the same rule the walker's detectors use. */
|
|
156
156
|
async function grepAny(repo, re, limit) {
|
|
157
|
-
|
|
157
|
+
// ASK FOR THE SCOPE BY NAME (0.2). This line used to re-derive its own, and
|
|
158
|
+
// omitted the one term `runtimeCode` had: test files. So `@auth0/nextjs-auth0`,
|
|
159
|
+
// quoted as a STRING inside a scanner's test describing a customer's codebase,
|
|
160
|
+
// read as this product importing Auth0 - and the report told an owner to drop a
|
|
161
|
+
// vendor they never had. That was the fourth self-contamination variant in this
|
|
162
|
+
// family, each one a call site that forgot a term the others remembered.
|
|
163
|
+
const files = scopeFor(repo.files, "vendor-presence")
|
|
164
|
+
.filter((f) => !/(^|\/)package(-lock)?\.json$/.test(f))
|
|
165
|
+
.slice(0, limit);
|
|
158
166
|
for (const f of files) {
|
|
159
167
|
const text = await repo.read(f);
|
|
160
168
|
if (text && re.test(text))
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fact shapes. Family rule: a field is either read from the repository or
|
|
3
|
+
* absent. Nothing in these types is an opinion; opinions belong to the agent
|
|
4
|
+
* protocol, and every one it forms must cite a field from here.
|
|
5
|
+
*
|
|
6
|
+
* The one register rule encoded structurally: findings carry the honest verb.
|
|
7
|
+
* A dependency is not "unused", it has "no reference found" - the difference
|
|
8
|
+
* is a CLI-only tool that ships to production anyway versus a lie in a report
|
|
9
|
+
* a founder pays attention to.
|
|
10
|
+
*/
|
|
11
|
+
/** One declared dependency and every place it was actually seen. */
|
|
12
|
+
export interface DepFact {
|
|
13
|
+
name: string;
|
|
14
|
+
version: string;
|
|
15
|
+
/** Which manifest declared it, repository-relative. */
|
|
16
|
+
manifest: string;
|
|
17
|
+
dev: boolean;
|
|
18
|
+
/** Files whose import/require statements name this package. */
|
|
19
|
+
imported_by: number;
|
|
20
|
+
/** Config files that name it as a string (tailwind plugins, postcss, eslint). */
|
|
21
|
+
config_mentions: number;
|
|
22
|
+
/**
|
|
23
|
+
* Why it counts as used without an import or config mention, when that is the
|
|
24
|
+
* case: "peer of next", "npm script: email dev", "JSX runtime", "native build
|
|
25
|
+
* (Capacitor)".
|
|
26
|
+
*/
|
|
27
|
+
required_by?: string;
|
|
28
|
+
/** True when no import, config mention, script, or package requiring it was found. */
|
|
29
|
+
no_reference_found: boolean;
|
|
30
|
+
}
|
|
31
|
+
export type VendorCategory = "ai" | "email" | "sms" | "payments" | "database" | "auth" | "storage" | "crm" | "analytics" | "monitoring" | "render" | "search" | "queue" | "maps" | "calendar" | "other";
|
|
32
|
+
/** One external service the code talks to, with the evidence. */
|
|
33
|
+
export interface VendorFact {
|
|
34
|
+
service: string;
|
|
35
|
+
category: VendorCategory;
|
|
36
|
+
/** How it was seen: an SDK import, an outbound URL, an env var NAME. Values never. */
|
|
37
|
+
evidence: Array<{
|
|
38
|
+
kind: "sdk" | "url" | "env";
|
|
39
|
+
what: string;
|
|
40
|
+
file: string;
|
|
41
|
+
}>;
|
|
42
|
+
call_sites: number;
|
|
43
|
+
}
|
|
44
|
+
/** Two or more services doing the same category of work. */
|
|
45
|
+
export interface OverlapFact {
|
|
46
|
+
category: VendorCategory;
|
|
47
|
+
services: string[];
|
|
48
|
+
call_sites: number;
|
|
49
|
+
}
|
|
50
|
+
export type RailCategory = "pdf" | "rate-limiting" | "email-templating" | "auth-session" | "queue-scheduler" | "search" | "payments-logic" | "webhook-plumbing" | "parsing-ocr";
|
|
51
|
+
/** A subsystem built by hand where the market sells a rail. */
|
|
52
|
+
export interface HandrolledFact {
|
|
53
|
+
rail: RailCategory;
|
|
54
|
+
files: string[];
|
|
55
|
+
loc: number;
|
|
56
|
+
/** high: the file's name and its contents agree. low: contents only. */
|
|
57
|
+
confidence: "high" | "low";
|
|
58
|
+
/** The one line of evidence that convinced the detector, with its file. */
|
|
59
|
+
signal: {
|
|
60
|
+
file: string;
|
|
61
|
+
line: string;
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/** A cluster of near-identical code living in more than one file. */
|
|
65
|
+
export interface DuplicateFact {
|
|
66
|
+
files: string[];
|
|
67
|
+
/** Lines in the repeated block, after normalization. */
|
|
68
|
+
lines: number;
|
|
69
|
+
/** A header comment says the copy is deliberate ("copied from", "never imported"). */
|
|
70
|
+
deliberate: boolean;
|
|
71
|
+
/** First normalized line of the block, so a reader can find it. */
|
|
72
|
+
opens_with: string;
|
|
73
|
+
}
|
|
74
|
+
/** A runtime file no entrypoint reaches. */
|
|
75
|
+
export interface DeadFact {
|
|
76
|
+
file: string;
|
|
77
|
+
loc: number;
|
|
78
|
+
/** Why the detector believes nothing reaches it. */
|
|
79
|
+
note: string;
|
|
80
|
+
}
|
|
81
|
+
/** A call whose cost multiplies: per request, per row, or on a clock. */
|
|
82
|
+
export interface CostFact {
|
|
83
|
+
file: string;
|
|
84
|
+
shape: "per-request" | "per-row" | "per-schedule";
|
|
85
|
+
/** What is being called - a vendor host or an SDK call name. */
|
|
86
|
+
target: string;
|
|
87
|
+
/** The line that shows the shape (the loop, the handler, the interval). */
|
|
88
|
+
line: string;
|
|
89
|
+
}
|
|
90
|
+
/** Vocabulary the repository uses about its own domain, ranked by weight.
|
|
91
|
+
* The CLI never names the industry; it hands the agent the evidence. */
|
|
92
|
+
export interface Fingerprint {
|
|
93
|
+
terms: Array<{
|
|
94
|
+
term: string;
|
|
95
|
+
count: number;
|
|
96
|
+
sources: string[];
|
|
97
|
+
}>;
|
|
98
|
+
/** Where the terms came from: tables, routes, copy, manifest. */
|
|
99
|
+
note: string;
|
|
100
|
+
}
|
|
101
|
+
export interface NorthStar {
|
|
102
|
+
sentence: string | null;
|
|
103
|
+
source: "NORTH-STAR.md" | "planning" | "PRODUCT.md" | "heuristic" | "none";
|
|
104
|
+
confidence: "high" | "low" | "unknown";
|
|
105
|
+
note: string;
|
|
106
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fact shapes. Family rule: a field is either read from the repository or
|
|
3
|
+
* absent. Nothing in these types is an opinion; opinions belong to the agent
|
|
4
|
+
* protocol, and every one it forms must cite a field from here.
|
|
5
|
+
*
|
|
6
|
+
* The one register rule encoded structurally: findings carry the honest verb.
|
|
7
|
+
* A dependency is not "unused", it has "no reference found" - the difference
|
|
8
|
+
* is a CLI-only tool that ships to production anyway versus a lie in a report
|
|
9
|
+
* a founder pays attention to.
|
|
10
|
+
*/
|
|
11
|
+
export {};
|