@bigsteele/the-prospect 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -0
- package/dist/check.d.ts +26 -0
- package/dist/check.js +77 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +132 -0
- package/dist/detect/costs.d.ts +14 -0
- package/dist/detect/costs.js +46 -0
- package/dist/detect/deadweight.d.ts +22 -0
- package/dist/detect/deadweight.js +137 -0
- package/dist/detect/deps.d.ts +25 -0
- package/dist/detect/deps.js +198 -0
- package/dist/detect/duplication.d.ts +17 -0
- package/dist/detect/duplication.js +79 -0
- package/dist/detect/fingerprint.d.ts +16 -0
- package/dist/detect/fingerprint.js +95 -0
- package/dist/detect/handrolled.d.ts +23 -0
- package/dist/detect/handrolled.js +112 -0
- package/dist/detect/stack.d.ts +66 -0
- package/dist/detect/stack.js +164 -0
- package/dist/detect/types.d.ts +106 -0
- package/dist/detect/types.js +11 -0
- package/dist/detect/vendors.d.ts +26 -0
- package/dist/detect/vendors.js +125 -0
- package/dist/index.d.ts +36 -0
- package/dist/index.js +93 -0
- package/dist/northstar.d.ts +11 -0
- package/dist/northstar.js +95 -0
- package/dist/report.d.ts +16 -0
- package/dist/report.js +166 -0
- package/dist/score.d.ts +47 -0
- package/dist/score.js +93 -0
- package/dist/walk.d.ts +44 -0
- package/dist/walk.js +123 -0
- package/package.json +49 -0
- package/prompt/THE-PROSPECT.md +125 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
const CONFIG_FILE = /(^|\/)(tailwind|postcss|vite|next|nuxt|astro|svelte|webpack|rollup|babel|jest|vitest|playwright|eslint|prettier|tsup|drizzle|prisma)[^/]*\.(config\.)?(ts|js|mjs|cjs|json)$|(^|\/)\.(eslintrc|babelrc|prettierrc)(\.[a-z]+)?$|(^|\/)(package|turbo|nx|lerna)\.json$/i;
|
|
2
|
+
/** Nested manifests that describe fixtures or vendored copies, not this software. */
|
|
3
|
+
const NOT_A_MANIFEST = /(^|\/)(test|tests|fixtures?|__fixtures__|examples?|templates?)\//i;
|
|
4
|
+
/**
|
|
5
|
+
* Peers of the frameworks whose peers are used by the framework itself, for when
|
|
6
|
+
* node_modules is not installed and the real peerDependencies cannot be read.
|
|
7
|
+
*/
|
|
8
|
+
const FRAMEWORK_PEERS = {
|
|
9
|
+
next: ["react", "react-dom"],
|
|
10
|
+
gatsby: ["react", "react-dom"],
|
|
11
|
+
"@remix-run/react": ["react", "react-dom"],
|
|
12
|
+
expo: ["react", "react-native"],
|
|
13
|
+
"@monaco-editor/react": ["monaco-editor"],
|
|
14
|
+
"@tiptap/react": ["@tiptap/pm"],
|
|
15
|
+
"@tiptap/starter-kit": ["@tiptap/pm"],
|
|
16
|
+
};
|
|
17
|
+
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
18
|
+
function importSpecifiers(text) {
|
|
19
|
+
const out = [];
|
|
20
|
+
const patterns = [
|
|
21
|
+
/\bimport\s+(?:[^"'`]*?\s+from\s+)?["']([^"'\n]+)["']/g,
|
|
22
|
+
/\brequire\(\s*["']([^"'\n]+)["']\s*\)/g,
|
|
23
|
+
/\bimport\(\s*["']([^"'\n]+)["']\s*\)/g,
|
|
24
|
+
/\bexport\s+[^"'`\n]*?\s+from\s+["']([^"'\n]+)["']/g,
|
|
25
|
+
];
|
|
26
|
+
for (const re of patterns) {
|
|
27
|
+
for (const m of text.matchAll(re))
|
|
28
|
+
out.push(m[1]);
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
/** `@scope/pkg/deep/path` -> `@scope/pkg`; `pkg/deep` -> `pkg`. Relative and URL imports return null. */
|
|
33
|
+
export function packageOf(spec) {
|
|
34
|
+
if (spec.startsWith(".") || spec.startsWith("/") || /^(node:|https?:|npm:|jsr:)/.test(spec)) {
|
|
35
|
+
// Deno-style `npm:pkg@1` still names a package worth counting.
|
|
36
|
+
const npm = /^npm:(@?[^@/]+(?:\/[^@/]+)?)/.exec(spec);
|
|
37
|
+
return npm ? npm[1] : null;
|
|
38
|
+
}
|
|
39
|
+
const parts = spec.split("/");
|
|
40
|
+
return spec.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0];
|
|
41
|
+
}
|
|
42
|
+
export async function detectDeps(repo) {
|
|
43
|
+
const manifests = repo.files.filter((f) => /(^|\/)package\.json$/.test(f) && !/node_modules/.test(f) && !NOT_A_MANIFEST.test(f));
|
|
44
|
+
const declared = [];
|
|
45
|
+
const scriptsByManifest = new Map();
|
|
46
|
+
for (const m of manifests) {
|
|
47
|
+
const text = await repo.read(m);
|
|
48
|
+
if (!text)
|
|
49
|
+
continue;
|
|
50
|
+
let json;
|
|
51
|
+
try {
|
|
52
|
+
json = JSON.parse(text);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
scriptsByManifest.set(m, Object.values(json.scripts ?? {}).filter((v) => typeof v === "string"));
|
|
58
|
+
for (const [dev, block] of [[false, json.dependencies], [true, json.devDependencies]]) {
|
|
59
|
+
for (const [name, version] of Object.entries(block ?? {})) {
|
|
60
|
+
declared.push({ name, version, manifest: m, dev, imported_by: 0, config_mentions: 0, no_reference_found: false });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (declared.length === 0)
|
|
65
|
+
return [];
|
|
66
|
+
const byName = new Map();
|
|
67
|
+
for (const d of declared) {
|
|
68
|
+
const list = byName.get(d.name) ?? [];
|
|
69
|
+
list.push(d);
|
|
70
|
+
byName.set(d.name, list);
|
|
71
|
+
}
|
|
72
|
+
// Imports: every code file, tests included - a dependency only tests use is
|
|
73
|
+
// still referenced, and saying otherwise would be the lie this file bans.
|
|
74
|
+
const codeFiles = repo.files.filter((f) => /\.(ts|tsx|js|jsx|mjs|cjs|css|scss|sass|less|pcss)$/i.test(f) && !/node_modules/.test(f));
|
|
75
|
+
for (const f of codeFiles) {
|
|
76
|
+
const text = await repo.read(f);
|
|
77
|
+
if (!text)
|
|
78
|
+
continue;
|
|
79
|
+
const seen = new Set();
|
|
80
|
+
for (const spec of importSpecifiers(text)) {
|
|
81
|
+
const pkg = packageOf(spec);
|
|
82
|
+
if (pkg && !seen.has(pkg)) {
|
|
83
|
+
seen.add(pkg);
|
|
84
|
+
for (const d of byName.get(pkg) ?? [])
|
|
85
|
+
d.imported_by++;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// Stylesheets: `@import "tw-animate-css"`, Tailwind 4's `@plugin "pkg"`, and
|
|
89
|
+
// Sass's `@use` / `@forward`.
|
|
90
|
+
if (/\.(css|scss|sass|less|pcss)$/i.test(f)) {
|
|
91
|
+
for (const m of text.matchAll(/@(?:import|plugin|use|forward)\s+(?:url\(\s*)?["']([^"'\n]+)["']/g)) {
|
|
92
|
+
const pkg = packageOf(m[1].replace(/^~/, ""));
|
|
93
|
+
if (pkg && !seen.has(pkg)) {
|
|
94
|
+
seen.add(pkg);
|
|
95
|
+
for (const d of byName.get(pkg) ?? [])
|
|
96
|
+
d.imported_by++;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// Config mentions: plugins named as bare strings, and npm scripts that
|
|
102
|
+
// invoke a package's bin by name.
|
|
103
|
+
const configs = repo.files.filter((f) => CONFIG_FILE.test(f) && !/node_modules/.test(f));
|
|
104
|
+
for (const f of configs) {
|
|
105
|
+
const text = await repo.read(f);
|
|
106
|
+
if (!text)
|
|
107
|
+
continue;
|
|
108
|
+
for (const [name, list] of byName) {
|
|
109
|
+
const bare = name.replace(/^@[^/]+\//, "");
|
|
110
|
+
if (text.includes(`"${name}"`) || text.includes(`'${name}'`) || new RegExp(`\\b${escapeRe(bare)}\\b`).test(text)) {
|
|
111
|
+
for (const d of list)
|
|
112
|
+
if (d.manifest !== f)
|
|
113
|
+
d.config_mentions++;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// ── Used without an import ────────────────────────────────────────────────
|
|
118
|
+
const referenced = (d) => d.imported_by > 0 || d.config_mentions > 0 || !!d.required_by;
|
|
119
|
+
const installedCache = new Map();
|
|
120
|
+
const installed = async (d) => {
|
|
121
|
+
const key = `${d.manifest}|${d.name}`;
|
|
122
|
+
if (!installedCache.has(key))
|
|
123
|
+
installedCache.set(key, await repo.installed(d.manifest, d.name));
|
|
124
|
+
return installedCache.get(key);
|
|
125
|
+
};
|
|
126
|
+
// Run by bin name from the declaring manifest's own scripts. The loop above
|
|
127
|
+
// skips the declaring manifest, because its dependency list names every
|
|
128
|
+
// package, so its scripts are read here instead. The bin is not always the
|
|
129
|
+
// package name: react-email installs `email`.
|
|
130
|
+
for (const d of declared) {
|
|
131
|
+
if (referenced(d))
|
|
132
|
+
continue;
|
|
133
|
+
const scripts = scriptsByManifest.get(d.manifest) ?? [];
|
|
134
|
+
if (!scripts.length)
|
|
135
|
+
continue;
|
|
136
|
+
const bins = new Set([d.name.replace(/^@[^/]+\//, "")]);
|
|
137
|
+
const inst = await installed(d);
|
|
138
|
+
if (inst?.bin && typeof inst.bin === "object")
|
|
139
|
+
for (const b of Object.keys(inst.bin))
|
|
140
|
+
bins.add(b);
|
|
141
|
+
const hit = scripts.find((cmd) => [...bins].some((b) => new RegExp(`(^|[\\s;&|(])${escapeRe(b)}(\\s|$)`).test(cmd)));
|
|
142
|
+
if (hit)
|
|
143
|
+
d.required_by = `npm script: ${hit.length > 60 ? `${hit.slice(0, 57)}...` : hit}`;
|
|
144
|
+
}
|
|
145
|
+
// The JSX runtime. Every .tsx/.jsx file compiles to an import of `react` (or
|
|
146
|
+
// whatever jsxImportSource names) that no source file spells out.
|
|
147
|
+
if (repo.files.some((f) => /\.(tsx|jsx)$/.test(f) && !/node_modules/.test(f))) {
|
|
148
|
+
let source = "react";
|
|
149
|
+
for (const tc of repo.files.filter((f) => /(^|\/)(tsconfig|jsconfig)[^/]*\.json$/.test(f) && !NOT_A_MANIFEST.test(f))) {
|
|
150
|
+
const named = /"jsxImportSource"\s*:\s*"([^"]+)"/.exec((await repo.read(tc)) ?? "")?.[1];
|
|
151
|
+
if (named) {
|
|
152
|
+
source = packageOf(named) ?? source;
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
for (const d of byName.get(source) ?? [])
|
|
157
|
+
if (!referenced(d))
|
|
158
|
+
d.required_by = "JSX runtime";
|
|
159
|
+
}
|
|
160
|
+
// Capacitor's native build registers every installed plugin from package.json.
|
|
161
|
+
if (repo.files.some((f) => /(^|\/)capacitor\.config\.(ts|js|json)$/.test(f))) {
|
|
162
|
+
for (const d of declared) {
|
|
163
|
+
if (!referenced(d) && /^(@capacitor\/|@capacitor-community\/|cordova-plugin-)/.test(d.name)) {
|
|
164
|
+
d.required_by = "native build (Capacitor)";
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
// Peers of packages in use. Read from the installed manifest when there is one
|
|
169
|
+
// (optional peers excluded), from the short framework list when there is not.
|
|
170
|
+
// Repeated until nothing changes, because a peer can have peers.
|
|
171
|
+
for (let changed = true; changed;) {
|
|
172
|
+
changed = false;
|
|
173
|
+
for (const user of declared) {
|
|
174
|
+
if (!referenced(user))
|
|
175
|
+
continue;
|
|
176
|
+
const inst = await installed(user);
|
|
177
|
+
const peers = inst
|
|
178
|
+
? Object.keys(inst.peerDependencies ?? {}).filter((p) => !inst.peerDependenciesMeta?.[p]?.optional)
|
|
179
|
+
: (FRAMEWORK_PEERS[user.name] ?? []);
|
|
180
|
+
for (const peer of peers) {
|
|
181
|
+
for (const d of byName.get(peer) ?? []) {
|
|
182
|
+
// Resolvable from where the user is declared: the same manifest, or the root.
|
|
183
|
+
if (referenced(d) || (d.manifest !== user.manifest && d.manifest !== "package.json"))
|
|
184
|
+
continue;
|
|
185
|
+
d.required_by = `peer of ${user.name}`;
|
|
186
|
+
changed = true;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
for (const d of declared) {
|
|
192
|
+
// Types-only and tooling-adjacent packages are referenced by the compiler,
|
|
193
|
+
// not by an import statement; they never earn the flag.
|
|
194
|
+
const toolingShaped = /^@types\//.test(d.name) || /(^|-)(cli|eslint|prettier|typescript|vitest|vite|tsx?|husky|lint-staged|concurrently|nodemon)($|-)/.test(d.name);
|
|
195
|
+
d.no_reference_found = !referenced(d) && !toolingShaped;
|
|
196
|
+
}
|
|
197
|
+
return declared.sort((a, b) => Number(b.no_reference_found) - Number(a.no_reference_found) || a.name.localeCompare(b.name));
|
|
198
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Near-identical code living in more than one file.
|
|
3
|
+
*
|
|
4
|
+
* Rolling window over normalized lines, matched across files. A cluster is
|
|
5
|
+
* only worth a founder's attention when it is long enough to be a subsystem
|
|
6
|
+
* rather than an idiom, so the window is thirty lines and idiomatic repeats
|
|
7
|
+
* (import blocks, generated files) are filtered before hashing.
|
|
8
|
+
*
|
|
9
|
+
* Deliberate copies are downgraded, not hidden: this scan's own family
|
|
10
|
+
* copies its repository walker between packages on purpose, and the header
|
|
11
|
+
* comment saying so is the difference between a decision and an accident.
|
|
12
|
+
* A copy that ANNOUNCES itself is architecture; a copy that does not is a
|
|
13
|
+
* bug that has not happened yet.
|
|
14
|
+
*/
|
|
15
|
+
import type { Repo } from "../walk.js";
|
|
16
|
+
import type { DuplicateFact } from "./types.js";
|
|
17
|
+
export declare function detectDuplication(repo: Repo): Promise<DuplicateFact[]>;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { runtimeCode } from "../walk.js";
|
|
2
|
+
const WINDOW = 30;
|
|
3
|
+
const MIN_FILES = 2;
|
|
4
|
+
const GENERATED = /(^|\/)(.*\.gen\.|.*\.generated\.|types\/supabase|database\.types)/i;
|
|
5
|
+
const DELIBERATE = /copied from|never imported|do not import|kept in sync by hand/i;
|
|
6
|
+
/** Strip comments and blank lines; collapse whitespace. Lines too short to
|
|
7
|
+
* mean anything (braces, `else {`) stay in the window but are marked so a
|
|
8
|
+
* window of pure punctuation cannot match. */
|
|
9
|
+
function normalize(text) {
|
|
10
|
+
return text
|
|
11
|
+
.replace(/\/\*[\s\S]*?\*\//g, "")
|
|
12
|
+
.split("\n")
|
|
13
|
+
.map((l) => l.replace(/\/\/.*$/, "").trim().replace(/\s+/g, " "))
|
|
14
|
+
.filter((l) => l.length > 0);
|
|
15
|
+
}
|
|
16
|
+
function meaningful(lines) {
|
|
17
|
+
return lines.filter((l) => l.length > 12).length >= WINDOW * 0.5;
|
|
18
|
+
}
|
|
19
|
+
export async function detectDuplication(repo) {
|
|
20
|
+
const files = runtimeCode(repo.files).filter((f) => !GENERATED.test(f));
|
|
21
|
+
// hash of a window -> the places it occurs
|
|
22
|
+
const windows = new Map();
|
|
23
|
+
const fileLines = new Map();
|
|
24
|
+
const deliberateFiles = new Set();
|
|
25
|
+
for (const f of files) {
|
|
26
|
+
const text = await repo.read(f);
|
|
27
|
+
if (!text)
|
|
28
|
+
continue;
|
|
29
|
+
if (DELIBERATE.test(text.slice(0, 600)))
|
|
30
|
+
deliberateFiles.add(f);
|
|
31
|
+
const lines = normalize(text);
|
|
32
|
+
if (lines.length < WINDOW)
|
|
33
|
+
continue;
|
|
34
|
+
fileLines.set(f, lines);
|
|
35
|
+
for (let i = 0; i + WINDOW <= lines.length; i += 5) {
|
|
36
|
+
const slice = lines.slice(i, i + WINDOW);
|
|
37
|
+
if (!meaningful(slice))
|
|
38
|
+
continue;
|
|
39
|
+
const key = slice.join("\u0001");
|
|
40
|
+
const list = windows.get(key) ?? [];
|
|
41
|
+
list.push({ file: f, start: i });
|
|
42
|
+
windows.set(key, list);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// Group matching windows into per-file-set clusters, then extend each to
|
|
46
|
+
// its true length by walking forward while lines keep agreeing.
|
|
47
|
+
const clusters = new Map();
|
|
48
|
+
for (const [key, places] of windows) {
|
|
49
|
+
const distinct = [...new Set(places.map((p) => p.file))];
|
|
50
|
+
if (distinct.length < MIN_FILES)
|
|
51
|
+
continue;
|
|
52
|
+
const id = distinct.sort().join("|");
|
|
53
|
+
const first = places[0];
|
|
54
|
+
const a = fileLines.get(first.file);
|
|
55
|
+
const b0 = places.find((p) => p.file !== first.file);
|
|
56
|
+
const b = fileLines.get(b0.file);
|
|
57
|
+
let len = WINDOW;
|
|
58
|
+
while (first.start + len < a.length &&
|
|
59
|
+
b0.start + len < b.length &&
|
|
60
|
+
a[first.start + len] === b[b0.start + len]) {
|
|
61
|
+
len++;
|
|
62
|
+
}
|
|
63
|
+
const cur = clusters.get(id);
|
|
64
|
+
if (!cur || len > cur.lines) {
|
|
65
|
+
clusters.set(id, {
|
|
66
|
+
files: distinct,
|
|
67
|
+
lines: len,
|
|
68
|
+
opens_with: (a[first.start] ?? "").slice(0, 100),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return [...clusters.values()]
|
|
73
|
+
.map((c) => ({
|
|
74
|
+
...c,
|
|
75
|
+
deliberate: c.files.every((f) => deliberateFiles.has(f)) || c.files.filter((f) => deliberateFiles.has(f)).length >= c.files.length - 1,
|
|
76
|
+
}))
|
|
77
|
+
.sort((x, y) => y.lines - x.lines)
|
|
78
|
+
.slice(0, 25);
|
|
79
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The industry fingerprint: what this repository talks about.
|
|
3
|
+
*
|
|
4
|
+
* The CLI never names the industry - a deterministic scanner guessing
|
|
5
|
+
* "credit repair" from the word "dispute" would be the invented North Star
|
|
6
|
+
* problem all over again. Instead it collects the repository's own domain
|
|
7
|
+
* vocabulary - table names, route slugs, manifest prose, the nouns the
|
|
8
|
+
* schema is built from - ranks it, and hands the agent evidence it can
|
|
9
|
+
* confirm with the operator in one question.
|
|
10
|
+
*
|
|
11
|
+
* Programming vocabulary is stripped hard: "user", "id", "config" and their
|
|
12
|
+
* relatives fingerprint every repository on earth, which is to say none.
|
|
13
|
+
*/
|
|
14
|
+
import type { Repo } from "../walk.js";
|
|
15
|
+
import type { Fingerprint } from "./types.js";
|
|
16
|
+
export declare function detectFingerprint(repo: Repo): Promise<Fingerprint>;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/** Not this product's own vocabulary: fixtures, templates, examples, and
|
|
2
|
+
* sibling scan packages. The first host-repo run fingerprinted a credit-
|
|
3
|
+
* repair FIXTURE inside this very package as the host's industry. */
|
|
4
|
+
const NOT_DOMAIN = /(^|\/)(test|tests|__tests__|fixtures?|__fixtures__|templates?|examples?|samples?|golden|node_modules)\//i;
|
|
5
|
+
const STOP = new Set([
|
|
6
|
+
"the", "a", "an", "and", "or", "for", "with", "from", "this", "that", "your", "our", "are", "was", "has", "have", "will", "can", "not", "all", "any", "new", "get", "set", "use",
|
|
7
|
+
"user", "users", "id", "ids", "uuid", "data", "item", "items", "list", "lists", "page", "pages", "view", "views", "index", "main", "app", "apps", "config", "configs", "setting", "settings",
|
|
8
|
+
"table", "column", "row", "rows", "key", "keys", "value", "values", "type", "types", "status", "state", "states", "created", "updated", "deleted", "name", "email", "admin", "auth", "session", "sessions",
|
|
9
|
+
"token", "tokens", "test", "tests", "fixture", "fixtures", "util", "utils", "helper", "helpers", "component", "components", "hook", "hooks", "lib", "src", "public", "assets", "style", "styles",
|
|
10
|
+
"error", "errors", "message", "messages", "response", "request", "handler", "route", "routes", "api", "json", "html", "text", "file", "files", "folder", "null", "true", "false", "string", "number", "boolean",
|
|
11
|
+
"date", "time", "timestamp", "meta", "metadata", "count", "total", "default", "base", "core", "common", "shared", "client", "server", "function", "functions", "edge", "supabase", "react", "vite", "next",
|
|
12
|
+
"form", "forms", "input", "button", "modal", "card", "label", "field", "fields", "header", "footer", "nav", "layout", "theme", "dark", "light", "mobile", "desktop", "www", "com", "href", "src", "url", "urls",
|
|
13
|
+
]);
|
|
14
|
+
/** Fold plain plurals so "disputes" and "dispute" count as one term. */
|
|
15
|
+
const singular = (w) => w.length > 4 && w.endsWith("s") && !w.endsWith("ss") && !w.endsWith("us") && !w.endsWith("is")
|
|
16
|
+
? w.slice(0, -1)
|
|
17
|
+
: w;
|
|
18
|
+
const splitIdent = (s) => s
|
|
19
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
20
|
+
.toLowerCase()
|
|
21
|
+
.split(/[^a-z]+/)
|
|
22
|
+
.map(singular)
|
|
23
|
+
.filter((w) => w.length > 2 && !STOP.has(w));
|
|
24
|
+
export async function detectFingerprint(repo) {
|
|
25
|
+
const counts = new Map();
|
|
26
|
+
const bump = (term, weight, source) => {
|
|
27
|
+
const cur = counts.get(term) ?? { count: 0, sources: new Set() };
|
|
28
|
+
cur.count += weight;
|
|
29
|
+
cur.sources.add(source);
|
|
30
|
+
counts.set(term, cur);
|
|
31
|
+
};
|
|
32
|
+
// Schema: the strongest evidence. A table is a decision about the domain.
|
|
33
|
+
for (const f of repo.files.filter((x) => !NOT_DOMAIN.test(x) && (/(^|\/)(migrations?|schema|db)\/.*\.(sql|prisma)$|schema\.(sql|prisma)$/i.test(x)))) {
|
|
34
|
+
const text = await repo.read(f);
|
|
35
|
+
if (!text)
|
|
36
|
+
continue;
|
|
37
|
+
for (const m of text.matchAll(/\b(?:create\s+table(?:\s+if\s+not\s+exists)?|model)\s+["`]?(?:public\.)?([a-zA-Z_][a-zA-Z0-9_]*)/gi)) {
|
|
38
|
+
for (const w of splitIdent(m[1]))
|
|
39
|
+
bump(w, 5, "schema");
|
|
40
|
+
}
|
|
41
|
+
// Columns, whether the DDL is one line per column or one line per table.
|
|
42
|
+
for (const m of text.matchAll(/[,(\n]\s*["`]?([a-z_][a-z0-9_]*)["`]?\s+(?:text|varchar|int|bigint|boolean|timestamptz?|numeric|jsonb|uuid|date|serial)/gi)) {
|
|
43
|
+
for (const w of splitIdent(m[1]))
|
|
44
|
+
bump(w, 2, "schema");
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// Routes: what the product lets a person come and do.
|
|
48
|
+
for (const f of repo.files.filter((x) => !NOT_DOMAIN.test(x) && /\.(tsx?|jsx?)$/.test(x))) {
|
|
49
|
+
const text = await repo.read(f);
|
|
50
|
+
if (!text)
|
|
51
|
+
continue;
|
|
52
|
+
for (const m of text.matchAll(/path\s*[:=]\s*["'`]\/([a-z0-9/_:-]{3,60})["'`]|<Route[^>]*path=["'`]\/([a-z0-9/_:-]{3,60})["'`]/gi)) {
|
|
53
|
+
for (const seg of (m[1] ?? m[2] ?? "").split("/")) {
|
|
54
|
+
if (seg.startsWith(":"))
|
|
55
|
+
continue;
|
|
56
|
+
for (const w of splitIdent(seg))
|
|
57
|
+
bump(w, 3, "routes");
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
// The manifest's own words about itself.
|
|
62
|
+
for (const f of repo.files.filter((x) => /(^|\/)package\.json$/.test(x) && !NOT_DOMAIN.test(x))) {
|
|
63
|
+
const text = await repo.read(f);
|
|
64
|
+
if (!text)
|
|
65
|
+
continue;
|
|
66
|
+
try {
|
|
67
|
+
const j = JSON.parse(text);
|
|
68
|
+
for (const w of splitIdent(`${j.name ?? ""} ${j.description ?? ""}`))
|
|
69
|
+
bump(w, 2, "manifest");
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
// says nothing
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
// README prose, lightly weighted: it is advertising, but it is on-topic advertising.
|
|
76
|
+
const readme = repo.files.find((f) => /^README\.md$/i.test(f));
|
|
77
|
+
if (readme) {
|
|
78
|
+
const text = await repo.read(readme);
|
|
79
|
+
if (text) {
|
|
80
|
+
for (const w of splitIdent(text.slice(0, 3000)))
|
|
81
|
+
bump(w, 1, "readme");
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const terms = [...counts.entries()]
|
|
85
|
+
.map(([term, v]) => ({ term, count: v.count, sources: [...v.sources] }))
|
|
86
|
+
.filter((t) => t.count >= 4 || t.sources.length >= 2)
|
|
87
|
+
.sort((a, b) => b.count - a.count)
|
|
88
|
+
.slice(0, 40);
|
|
89
|
+
return {
|
|
90
|
+
terms,
|
|
91
|
+
note: terms.length >= 8
|
|
92
|
+
? "Ranked domain vocabulary from schema, routes, manifest and README. The agent names the industry from this and confirms it with the operator; the scanner never guesses."
|
|
93
|
+
: "The fingerprint is thin: little schema or route vocabulary to read. The agent must ask the operator what the business is before researching anything.",
|
|
94
|
+
};
|
|
95
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The buy-vs-build ledger: subsystems written by hand where the market sells
|
|
3
|
+
* a rail.
|
|
4
|
+
*
|
|
5
|
+
* The credit-repair founder parsing PDF credit reports by regex is not lazy.
|
|
6
|
+
* They built before report-access APIs were table stakes, and since then it
|
|
7
|
+
* has been nobody's job to notice. The exemplar repo hand-rolled a rate
|
|
8
|
+
* limiter, an email layer and a PDF pipeline for the same reason: each one
|
|
9
|
+
* was the fastest path on the day it was written.
|
|
10
|
+
*
|
|
11
|
+
* This detector does not say "replace it". Half of hand-rolled code is a
|
|
12
|
+
* correct decision - control, cost, no vendor risk. It says: here is the
|
|
13
|
+
* subsystem, here is its size, here is the rail category it competes with.
|
|
14
|
+
* The agent researches the actual rails; the report offers the trade, with
|
|
15
|
+
* two options, as a question.
|
|
16
|
+
*
|
|
17
|
+
* Precision first. A false "you hand-rolled auth" costs the whole report its
|
|
18
|
+
* credibility, so every rule needs the file's contents to testify, and most
|
|
19
|
+
* want the file's name to agree before claiming high confidence.
|
|
20
|
+
*/
|
|
21
|
+
import type { Repo } from "../walk.js";
|
|
22
|
+
import type { HandrolledFact } from "./types.js";
|
|
23
|
+
export declare function detectHandrolled(repo: Repo): Promise<HandrolledFact[]>;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { runtimeCode } from "../walk.js";
|
|
2
|
+
const RULES = [
|
|
3
|
+
{
|
|
4
|
+
rail: "pdf",
|
|
5
|
+
name: /pdf|report.?pars|extract/i,
|
|
6
|
+
content: /%PDF-|getTextContent|pdftotext|pdf.{0,12}(parse|extract|text)|(parse|extract).{0,12}pdf/i,
|
|
7
|
+
rail_sdk: /["'](pdf-parse|pdfjs-dist|@pdf-lib|unpdf|pdf2json)["']/,
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
rail: "rate-limiting",
|
|
11
|
+
name: /rate.?limit|throttle/i,
|
|
12
|
+
content: /rate.?limit/i,
|
|
13
|
+
requires: /(window|bucket|attempts?|count(er)?)\b[\s\S]{0,300}\b(now|Date\.now|timestamp|expires?|ttl)/i,
|
|
14
|
+
rail_sdk: /["'](@upstash\/ratelimit|rate-limiter-flexible|express-rate-limit|bottleneck|p-throttle)["']/,
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
rail: "email-templating",
|
|
18
|
+
content: /<(html|body|table)[\s>][\s\S]{0,4000}(sendEmail|sendMail|\.send\(|resend|smtp)/i,
|
|
19
|
+
rail_sdk: /["'](@react-email|react-email|mjml|maizzle|handlebars)["']/,
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
rail: "auth-session",
|
|
23
|
+
name: /auth|session|token/i,
|
|
24
|
+
content: /(randomBytes|randomUUID|getRandomValues)[\s\S]{0,400}(session|token)[\s\S]{0,400}(insert|INSERT|set\(|save|store)/i,
|
|
25
|
+
requires: /(expires?|max.?age|ttl|valid)/i,
|
|
26
|
+
rail_sdk: /["'](@clerk|@auth0|next-auth|@supabase\/auth|lucia|better-auth|passport)["']/,
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
rail: "queue-scheduler",
|
|
30
|
+
name: /queue|worker|scheduler|cron|jobs?/i,
|
|
31
|
+
content: /(setInterval|setTimeout)[\s\S]{0,300}(fetch|process|poll|dequeue|claim)|status\s*=\s*["']pending["'][\s\S]{0,400}(update|claim|lock)/i,
|
|
32
|
+
rail_sdk: /["'](bullmq|bee-queue|agenda|bree|graphile-worker|@trigger\.dev|inngest|temporalio)["']/,
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
rail: "search",
|
|
36
|
+
content: /(ilike|ILIKE|LIKE)\s*\(?\s*["'`]%|\.ilike\(|to_tsvector|\btokeni[sz]e\b[\s\S]{0,200}\bscore\b/i,
|
|
37
|
+
requires: /(search|query|match|find)/i,
|
|
38
|
+
rail_sdk: /["'](algoliasearch|meilisearch|typesense|@elastic|flexsearch|minisearch|fuse\.js)["']/,
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
rail: "payments-logic",
|
|
42
|
+
name: /invoice|billing|payment/i,
|
|
43
|
+
content: /(subtotal|line.?items?|invoice.?number)[\s\S]{0,400}(tax|total|amount)/i,
|
|
44
|
+
requires: /\*|\+|toFixed|Math\.round/,
|
|
45
|
+
rail_sdk: /["'](stripe|@stripe|square|@paypal|lemonsqueezy)["']/,
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
rail: "webhook-plumbing",
|
|
49
|
+
content: /(createHmac|timingSafeEqual)[\s\S]{0,400}(signature|x-signature|svix|webhook)/i,
|
|
50
|
+
rail_sdk: /["'](svix|@hono\/webhook)["']/,
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
rail: "parsing-ocr",
|
|
54
|
+
// Name evidence is REQUIRED here, not optional: two regex calls plus the
|
|
55
|
+
// word "report" appears in half the codebases on earth. The first host
|
|
56
|
+
// run cited a repository walker as parsing-ocr on exactly that.
|
|
57
|
+
name: /pars|extract|scrap|ocr/i,
|
|
58
|
+
name_required: true,
|
|
59
|
+
content: /(?:\.(?:match|exec|matchAll)\([\s\S]{0,2000}?){3,}/,
|
|
60
|
+
requires: /(document|statement|report|invoice|receipt|letter|form|upload)/i,
|
|
61
|
+
rail_sdk: /["'](tesseract\.js|@azure\/ai-form-recognizer|@aws-sdk\/client-textract|@google-cloud\/documentai)["']/,
|
|
62
|
+
},
|
|
63
|
+
];
|
|
64
|
+
const countLines = (t) => t.split("\n").length;
|
|
65
|
+
export async function detectHandrolled(repo) {
|
|
66
|
+
const files = runtimeCode(repo.files);
|
|
67
|
+
// A rail SDK anywhere in the repository retires that rule everywhere: the
|
|
68
|
+
// decision to buy was already made, and re-litigating it is noise.
|
|
69
|
+
const railInUse = new Set();
|
|
70
|
+
const texts = new Map();
|
|
71
|
+
for (const f of files) {
|
|
72
|
+
const text = await repo.read(f);
|
|
73
|
+
if (!text)
|
|
74
|
+
continue;
|
|
75
|
+
texts.set(f, text);
|
|
76
|
+
for (const r of RULES)
|
|
77
|
+
if (r.rail_sdk.test(text))
|
|
78
|
+
railInUse.add(r.rail);
|
|
79
|
+
}
|
|
80
|
+
const byRail = new Map();
|
|
81
|
+
for (const [f, text] of texts) {
|
|
82
|
+
for (const r of RULES) {
|
|
83
|
+
if (railInUse.has(r.rail))
|
|
84
|
+
continue;
|
|
85
|
+
const m = r.content.exec(text);
|
|
86
|
+
if (!m)
|
|
87
|
+
continue;
|
|
88
|
+
if (r.requires && !r.requires.test(text))
|
|
89
|
+
continue;
|
|
90
|
+
const named = r.name ? r.name.test(f) : false;
|
|
91
|
+
if (r.name_required && !named)
|
|
92
|
+
continue;
|
|
93
|
+
const cur = byRail.get(r.rail) ?? { files: new Map(), signal: { file: f, line: "" }, named: false };
|
|
94
|
+
cur.files.set(f, countLines(text));
|
|
95
|
+
cur.named = cur.named || named;
|
|
96
|
+
if (!cur.signal.line) {
|
|
97
|
+
const at = text.slice(0, m.index).split("\n").length;
|
|
98
|
+
cur.signal = { file: f, line: `${at}: ${text.split("\n")[at - 1]?.trim().slice(0, 120) ?? ""}` };
|
|
99
|
+
}
|
|
100
|
+
byRail.set(r.rail, cur);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return [...byRail.entries()]
|
|
104
|
+
.map(([rail, r]) => ({
|
|
105
|
+
rail,
|
|
106
|
+
files: [...r.files.keys()].slice(0, 10),
|
|
107
|
+
loc: [...r.files.values()].reduce((t, n) => t + n, 0),
|
|
108
|
+
confidence: (r.named ? "high" : "low"),
|
|
109
|
+
signal: r.signal,
|
|
110
|
+
}))
|
|
111
|
+
.sort((a, b) => b.loc - a.loc);
|
|
112
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The stack cut: platforms whose job another platform you already run can
|
|
3
|
+
* carry. Owner's own examples, verbatim in the plan: "you don't need GitHub
|
|
4
|
+
* Actions, you can use npx for what you are doing" and "you don't need
|
|
5
|
+
* Vercel since you are already using Cloudflare for X". A SaaS-cut audit,
|
|
6
|
+
* to reduce the bill and the number of places things can break.
|
|
7
|
+
*
|
|
8
|
+
* Two kinds of evidence, both deterministic:
|
|
9
|
+
*
|
|
10
|
+
* 1. PLATFORMS, read from config files - a vercel.json, a wrangler.toml,
|
|
11
|
+
* a netlify.toml, a Procfile, a .github/workflows tree. Two hosting
|
|
12
|
+
* platforms is the same "paid twice" finding the vendor detector makes
|
|
13
|
+
* for two AI providers, and it feeds the same overlap machinery.
|
|
14
|
+
* 2. CONSOLIDATION PAIRS, from a versioned house map: you run X, and Y -
|
|
15
|
+
* which the configs show you already have - covers X's job. The map
|
|
16
|
+
* states only what can be read (both sides exist); whether the kept
|
|
17
|
+
* platform's CURRENT offering truly carries the load is the research
|
|
18
|
+
* half's job, with a source and a date. The report words every pair
|
|
19
|
+
* as a question, never a verdict.
|
|
20
|
+
*
|
|
21
|
+
* CI gets special care. A publish workflow with a registry identity is
|
|
22
|
+
* doing a job a laptop cannot (trusted publishing IS the workflow), and a
|
|
23
|
+
* workflow that runs on every contributor's push has value one machine
|
|
24
|
+
* does not. The only workflows flagged are the ones whose every step is an
|
|
25
|
+
* npm script - the ones a pre-push line runs for free, faster, on the
|
|
26
|
+
* machine that already built the code.
|
|
27
|
+
*/
|
|
28
|
+
import type { Repo } from "../walk.js";
|
|
29
|
+
import type { OverlapFact } from "./types.js";
|
|
30
|
+
export interface PlatformFact {
|
|
31
|
+
platform: string;
|
|
32
|
+
kind: "hosting" | "ci" | "container" | "iac";
|
|
33
|
+
evidence: string[];
|
|
34
|
+
}
|
|
35
|
+
export interface WorkflowFact {
|
|
36
|
+
file: string;
|
|
37
|
+
triggers: string[];
|
|
38
|
+
/** Every `run:` step, first line each. */
|
|
39
|
+
runs: string[];
|
|
40
|
+
/** Publishing identity (id-token, registry, npm publish) - a job a laptop cannot do. */
|
|
41
|
+
publishes: boolean;
|
|
42
|
+
/** Deploys somewhere (wrangler, vercel, aws, ...). */
|
|
43
|
+
deploys: boolean;
|
|
44
|
+
/** Every run step is an npm/npx/node script - the npx-instead class. */
|
|
45
|
+
script_only: boolean;
|
|
46
|
+
}
|
|
47
|
+
export interface ConsolidationFact {
|
|
48
|
+
/** The platform or service the repo already runs, that would remain. */
|
|
49
|
+
keep: string;
|
|
50
|
+
/** The candidate to cut. */
|
|
51
|
+
candidate: string;
|
|
52
|
+
/** The job the kept one covers. */
|
|
53
|
+
covers: string;
|
|
54
|
+
evidence_keep: string;
|
|
55
|
+
evidence_candidate: string;
|
|
56
|
+
/** The question the report asks - suggestion register, decided by code. */
|
|
57
|
+
note: string;
|
|
58
|
+
}
|
|
59
|
+
export interface StackReading {
|
|
60
|
+
platforms: PlatformFact[];
|
|
61
|
+
workflows: WorkflowFact[];
|
|
62
|
+
consolidations: ConsolidationFact[];
|
|
63
|
+
/** Hosting/CI platform overlaps, same shape the vendor overlaps use. */
|
|
64
|
+
overlaps: OverlapFact[];
|
|
65
|
+
}
|
|
66
|
+
export declare function detectStack(repo: Repo): Promise<StackReading>;
|