cleartoship 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +264 -0
- package/action.yml +154 -0
- package/dist/banner.d.ts +1 -0
- package/dist/banner.js +14 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +103 -0
- package/dist/data/popular.d.ts +17 -0
- package/dist/data/popular.js +103 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +4 -0
- package/dist/report.d.ts +15 -0
- package/dist/report.js +317 -0
- package/dist/scan.d.ts +24 -0
- package/dist/scan.js +81 -0
- package/dist/scanners/community.d.ts +2 -0
- package/dist/scanners/community.js +155 -0
- package/dist/scanners/dependencies.d.ts +21 -0
- package/dist/scanners/dependencies.js +549 -0
- package/dist/scanners/index.d.ts +8 -0
- package/dist/scanners/index.js +13 -0
- package/dist/scanners/rls.d.ts +7 -0
- package/dist/scanners/rls.js +531 -0
- package/dist/scanners/secrets.d.ts +2 -0
- package/dist/scanners/secrets.js +439 -0
- package/dist/scanners/server-actions.d.ts +2 -0
- package/dist/scanners/server-actions.js +466 -0
- package/dist/types.d.ts +65 -0
- package/dist/types.js +10 -0
- package/dist/utils/ast.d.ts +13 -0
- package/dist/utils/ast.js +107 -0
- package/dist/utils/detect.d.ts +2 -0
- package/dist/utils/detect.js +77 -0
- package/dist/utils/entropy.d.ts +7 -0
- package/dist/utils/entropy.js +19 -0
- package/dist/utils/files.d.ts +13 -0
- package/dist/utils/files.js +154 -0
- package/dist/utils/osv.d.ts +26 -0
- package/dist/utils/osv.js +123 -0
- package/dist/utils/registry.d.ts +44 -0
- package/dist/utils/registry.js +195 -0
- package/dist/utils/sql.d.ts +20 -0
- package/dist/utils/sql.js +157 -0
- package/dist/utils/suppress.d.ts +19 -0
- package/dist/utils/suppress.js +56 -0
- package/dist/utils/traverse.d.ts +2 -0
- package/dist/utils/traverse.js +14 -0
- package/dist/vendor/gitleaks/rules.d.ts +13 -0
- package/dist/vendor/gitleaks/rules.js +1571 -0
- package/dist/vendor/guardvibe/index.d.ts +11 -0
- package/dist/vendor/guardvibe/index.js +60 -0
- package/dist/vendor/guardvibe/rules/advanced-security.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/advanced-security.js +418 -0
- package/dist/vendor/guardvibe/rules/ai-host-security.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/ai-host-security.js +190 -0
- package/dist/vendor/guardvibe/rules/ai-security.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/ai-security.js +413 -0
- package/dist/vendor/guardvibe/rules/ai-tool-runtime.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/ai-tool-runtime.js +194 -0
- package/dist/vendor/guardvibe/rules/api-security.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/api-security.js +154 -0
- package/dist/vendor/guardvibe/rules/auth.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/auth.js +222 -0
- package/dist/vendor/guardvibe/rules/cicd.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/cicd.js +108 -0
- package/dist/vendor/guardvibe/rules/core.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/core.js +484 -0
- package/dist/vendor/guardvibe/rules/cve-versions.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/cve-versions.js +1130 -0
- package/dist/vendor/guardvibe/rules/database.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/database.js +173 -0
- package/dist/vendor/guardvibe/rules/deployment.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/deployment.js +269 -0
- package/dist/vendor/guardvibe/rules/dockerfile.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/dockerfile.js +99 -0
- package/dist/vendor/guardvibe/rules/firebase.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/firebase.js +96 -0
- package/dist/vendor/guardvibe/rules/go.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/go.js +85 -0
- package/dist/vendor/guardvibe/rules/modern-stack.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/modern-stack.js +631 -0
- package/dist/vendor/guardvibe/rules/nextjs.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/nextjs.js +233 -0
- package/dist/vendor/guardvibe/rules/other-services.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/other-services.js +72 -0
- package/dist/vendor/guardvibe/rules/payments.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/payments.js +123 -0
- package/dist/vendor/guardvibe/rules/react-native.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/react-native.js +132 -0
- package/dist/vendor/guardvibe/rules/services.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/services.js +149 -0
- package/dist/vendor/guardvibe/rules/shell.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/shell.js +72 -0
- package/dist/vendor/guardvibe/rules/sql.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/sql.js +60 -0
- package/dist/vendor/guardvibe/rules/supply-chain.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/supply-chain.js +243 -0
- package/dist/vendor/guardvibe/rules/terraform.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/terraform.js +84 -0
- package/dist/vendor/guardvibe/rules/types.d.ts +14 -0
- package/dist/vendor/guardvibe/rules/types.js +11 -0
- package/dist/vendor/guardvibe/rules/web-security.d.ts +2 -0
- package/dist/vendor/guardvibe/rules/web-security.js +258 -0
- package/examples/security.yml +23 -0
- package/package.json +70 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { exists, read } from './files.js';
|
|
3
|
+
function hasDep(pkg, name) {
|
|
4
|
+
if (!pkg)
|
|
5
|
+
return false;
|
|
6
|
+
return Boolean(pkg.dependencies?.[name] ||
|
|
7
|
+
pkg.devDependencies?.[name] ||
|
|
8
|
+
pkg.peerDependencies?.[name]);
|
|
9
|
+
}
|
|
10
|
+
export function detectFramework(root, files) {
|
|
11
|
+
let pkg = null;
|
|
12
|
+
const raw = read(join(root, 'package.json'));
|
|
13
|
+
if (raw) {
|
|
14
|
+
try {
|
|
15
|
+
pkg = JSON.parse(raw);
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
/* malformed package.json is reported by the dependency scanner */
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const hasApp = exists(join(root, 'app')) ||
|
|
22
|
+
exists(join(root, 'src/app')) ||
|
|
23
|
+
files.some((f) => /\/(src\/)?app\/.*\/(page|layout|route)\.(t|j)sx?$/.test(f));
|
|
24
|
+
const hasPages = exists(join(root, 'pages')) || exists(join(root, 'src/pages'));
|
|
25
|
+
const isNext = hasDep(pkg, 'next') || hasApp || hasPages;
|
|
26
|
+
let nextjs = null;
|
|
27
|
+
if (isNext) {
|
|
28
|
+
if (hasApp && hasPages)
|
|
29
|
+
nextjs = 'both';
|
|
30
|
+
else if (hasApp)
|
|
31
|
+
nextjs = 'app-router';
|
|
32
|
+
else if (hasPages)
|
|
33
|
+
nextjs = 'pages-router';
|
|
34
|
+
else
|
|
35
|
+
nextjs = 'app-router';
|
|
36
|
+
}
|
|
37
|
+
const supabase = hasDep(pkg, '@supabase/supabase-js') ||
|
|
38
|
+
hasDep(pkg, '@supabase/ssr') ||
|
|
39
|
+
hasDep(pkg, '@supabase/auth-helpers-nextjs') ||
|
|
40
|
+
exists(join(root, 'supabase'));
|
|
41
|
+
const info = {
|
|
42
|
+
nextjs,
|
|
43
|
+
supabase,
|
|
44
|
+
prisma: hasDep(pkg, '@prisma/client') || exists(join(root, 'prisma')),
|
|
45
|
+
drizzle: hasDep(pkg, 'drizzle-orm'),
|
|
46
|
+
clerk: hasDep(pkg, '@clerk/nextjs') || hasDep(pkg, '@clerk/clerk-sdk-node'),
|
|
47
|
+
nextAuth: hasDep(pkg, 'next-auth') || hasDep(pkg, '@auth/core'),
|
|
48
|
+
stripe: hasDep(pkg, 'stripe'),
|
|
49
|
+
python: exists(join(root, 'requirements.txt')) ||
|
|
50
|
+
exists(join(root, 'pyproject.toml')),
|
|
51
|
+
describe() {
|
|
52
|
+
const parts = [];
|
|
53
|
+
if (this.nextjs === 'app-router')
|
|
54
|
+
parts.push('Next.js (App Router)');
|
|
55
|
+
else if (this.nextjs === 'pages-router')
|
|
56
|
+
parts.push('Next.js (Pages Router)');
|
|
57
|
+
else if (this.nextjs === 'both')
|
|
58
|
+
parts.push('Next.js (App + Pages Router)');
|
|
59
|
+
if (this.supabase)
|
|
60
|
+
parts.push('Supabase');
|
|
61
|
+
if (this.prisma)
|
|
62
|
+
parts.push('Prisma');
|
|
63
|
+
if (this.drizzle)
|
|
64
|
+
parts.push('Drizzle');
|
|
65
|
+
if (this.clerk)
|
|
66
|
+
parts.push('Clerk');
|
|
67
|
+
if (this.nextAuth)
|
|
68
|
+
parts.push('NextAuth');
|
|
69
|
+
if (this.stripe)
|
|
70
|
+
parts.push('Stripe');
|
|
71
|
+
if (this.python)
|
|
72
|
+
parts.push('Python');
|
|
73
|
+
return parts.length ? parts.join(' + ') : 'generic JavaScript/TypeScript';
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
return info;
|
|
77
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shannon entropy in bits per character. Real credentials are near-random and
|
|
3
|
+
* score high; words, paths and placeholders score low. gitleaks ships a
|
|
4
|
+
* threshold with many of its rules, and applying it is most of what keeps a
|
|
5
|
+
* broad credential ruleset from drowning a report in noise.
|
|
6
|
+
*/
|
|
7
|
+
export declare function shannonEntropy(value: string): number;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shannon entropy in bits per character. Real credentials are near-random and
|
|
3
|
+
* score high; words, paths and placeholders score low. gitleaks ships a
|
|
4
|
+
* threshold with many of its rules, and applying it is most of what keeps a
|
|
5
|
+
* broad credential ruleset from drowning a report in noise.
|
|
6
|
+
*/
|
|
7
|
+
export function shannonEntropy(value) {
|
|
8
|
+
if (value.length === 0)
|
|
9
|
+
return 0;
|
|
10
|
+
const counts = new Map();
|
|
11
|
+
for (const ch of value)
|
|
12
|
+
counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
|
13
|
+
let entropy = 0;
|
|
14
|
+
for (const count of counts.values()) {
|
|
15
|
+
const p = count / value.length;
|
|
16
|
+
entropy -= p * Math.log2(p);
|
|
17
|
+
}
|
|
18
|
+
return entropy;
|
|
19
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare function walk(root: string): string[];
|
|
2
|
+
export declare function read(file: string): string | null;
|
|
3
|
+
export declare function rel(root: string, file: string): string;
|
|
4
|
+
export declare function exists(p: string): boolean;
|
|
5
|
+
/** 1-indexed line number for a character offset. */
|
|
6
|
+
export declare function lineAt(source: string, index: number): number;
|
|
7
|
+
export declare function snippetAt(source: string, line: number): string;
|
|
8
|
+
export declare function isScript(file: string): boolean;
|
|
9
|
+
export declare function isSql(file: string): boolean;
|
|
10
|
+
/** Prose and agent-instruction files, where install commands get copy-pasted from. */
|
|
11
|
+
export declare function isProse(file: string): boolean;
|
|
12
|
+
/** Language tokens the vendored community rules match on. */
|
|
13
|
+
export declare function languagesFor(file: string): string[];
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { readdirSync, statSync, readFileSync, existsSync } from 'node:fs';
|
|
2
|
+
import { join, relative, sep } from 'node:path';
|
|
3
|
+
const SKIP_DIRS = new Set([
|
|
4
|
+
'node_modules', '.git', '.next', '.turbo', '.vercel', '.wrangler',
|
|
5
|
+
'dist', 'build', 'out', 'coverage', '.venv', 'venv', '__pycache__',
|
|
6
|
+
'.cache', '.pnpm-store', 'vendor', 'target', '.svelte-kit', '.nuxt',
|
|
7
|
+
'.cts-cache', '_reference',
|
|
8
|
+
]);
|
|
9
|
+
const SCAN_EXTS = new Set([
|
|
10
|
+
'.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts', '.sql',
|
|
11
|
+
'.json', '.txt', '.toml', '.env', '.yaml', '.yml',
|
|
12
|
+
// Prose files matter: AI agents copy `npm install <hallucination>` out of
|
|
13
|
+
// READMEs and agent instruction files long before it reaches a manifest.
|
|
14
|
+
'.md', '.mdc', '.mdx',
|
|
15
|
+
// Reachable by the vendored community ruleset, which covers more ecosystems
|
|
16
|
+
// than the AST scanners do.
|
|
17
|
+
'.py', '.go', '.sh', '.bash', '.tf', '.tfvars', '.rb', '.php',
|
|
18
|
+
]);
|
|
19
|
+
/** Extensionless files worth reading. */
|
|
20
|
+
const NAMED_FILES = new Set(['Dockerfile', 'Makefile', 'Procfile']);
|
|
21
|
+
/** Instruction files read by coding agents. No extension, but high signal. */
|
|
22
|
+
const AGENT_FILES = new Set([
|
|
23
|
+
'.cursorrules', '.windsurfrules', '.clinerules', '.aiderrules', '.goosehints',
|
|
24
|
+
]);
|
|
25
|
+
/** Files bigger than this are almost certainly bundles or fixtures, not source. */
|
|
26
|
+
const MAX_FILE_BYTES = 2_000_000;
|
|
27
|
+
export function walk(root) {
|
|
28
|
+
const found = [];
|
|
29
|
+
const stack = [root];
|
|
30
|
+
while (stack.length) {
|
|
31
|
+
const dir = stack.pop();
|
|
32
|
+
let entries;
|
|
33
|
+
try {
|
|
34
|
+
entries = readdirSync(dir);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
for (const entry of entries) {
|
|
40
|
+
const full = join(dir, entry);
|
|
41
|
+
let st;
|
|
42
|
+
try {
|
|
43
|
+
st = statSync(full);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (st.isDirectory()) {
|
|
49
|
+
if (SKIP_DIRS.has(entry))
|
|
50
|
+
continue;
|
|
51
|
+
stack.push(full);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (!st.isFile() || st.size > MAX_FILE_BYTES)
|
|
55
|
+
continue;
|
|
56
|
+
const dot = entry.lastIndexOf('.');
|
|
57
|
+
const ext = dot === -1 ? '' : entry.slice(dot);
|
|
58
|
+
// .env, .env.local, requirements.txt and friends have no useful extension.
|
|
59
|
+
if (SCAN_EXTS.has(ext) ||
|
|
60
|
+
AGENT_FILES.has(entry) ||
|
|
61
|
+
NAMED_FILES.has(entry) ||
|
|
62
|
+
entry.startsWith('Dockerfile') ||
|
|
63
|
+
entry.startsWith('.env') ||
|
|
64
|
+
entry === 'requirements.txt') {
|
|
65
|
+
found.push(full);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return found.sort();
|
|
70
|
+
}
|
|
71
|
+
export function read(file) {
|
|
72
|
+
try {
|
|
73
|
+
return readFileSync(file, 'utf8');
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
export function rel(root, file) {
|
|
80
|
+
const r = relative(root, file);
|
|
81
|
+
return r === '' ? '.' : r.split(sep).join('/');
|
|
82
|
+
}
|
|
83
|
+
export function exists(p) {
|
|
84
|
+
return existsSync(p);
|
|
85
|
+
}
|
|
86
|
+
/** 1-indexed line number for a character offset. */
|
|
87
|
+
export function lineAt(source, index) {
|
|
88
|
+
let line = 1;
|
|
89
|
+
for (let i = 0; i < index && i < source.length; i++) {
|
|
90
|
+
if (source.charCodeAt(i) === 10)
|
|
91
|
+
line++;
|
|
92
|
+
}
|
|
93
|
+
return line;
|
|
94
|
+
}
|
|
95
|
+
export function snippetAt(source, line) {
|
|
96
|
+
const text = source.split('\n')[line - 1] ?? '';
|
|
97
|
+
const trimmed = text.trim();
|
|
98
|
+
return trimmed.length > 160 ? trimmed.slice(0, 157) + '...' : trimmed;
|
|
99
|
+
}
|
|
100
|
+
const SCRIPT_EXTS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts']);
|
|
101
|
+
export function isScript(file) {
|
|
102
|
+
const dot = file.lastIndexOf('.');
|
|
103
|
+
return dot !== -1 && SCRIPT_EXTS.has(file.slice(dot));
|
|
104
|
+
}
|
|
105
|
+
export function isSql(file) {
|
|
106
|
+
return file.endsWith('.sql');
|
|
107
|
+
}
|
|
108
|
+
/** Prose and agent-instruction files, where install commands get copy-pasted from. */
|
|
109
|
+
export function isProse(file) {
|
|
110
|
+
const base = file.slice(file.lastIndexOf('/') + 1);
|
|
111
|
+
return /\.(md|mdc|mdx|txt)$/.test(base) || AGENT_FILES.has(base);
|
|
112
|
+
}
|
|
113
|
+
/** Language tokens the vendored community rules match on. */
|
|
114
|
+
export function languagesFor(file) {
|
|
115
|
+
const base = file.slice(file.lastIndexOf('/') + 1);
|
|
116
|
+
const ext = base.includes('.') ? base.slice(base.lastIndexOf('.')) : '';
|
|
117
|
+
if (base.startsWith('Dockerfile'))
|
|
118
|
+
return ['dockerfile'];
|
|
119
|
+
switch (ext) {
|
|
120
|
+
case '.ts':
|
|
121
|
+
case '.tsx':
|
|
122
|
+
case '.mts':
|
|
123
|
+
case '.cts':
|
|
124
|
+
return ['typescript', 'javascript'];
|
|
125
|
+
case '.js':
|
|
126
|
+
case '.jsx':
|
|
127
|
+
case '.mjs':
|
|
128
|
+
case '.cjs':
|
|
129
|
+
return ['javascript'];
|
|
130
|
+
case '.json':
|
|
131
|
+
return base === 'vercel.json' ? ['json', 'vercel-config'] : ['json'];
|
|
132
|
+
case '.sql':
|
|
133
|
+
return ['sql'];
|
|
134
|
+
case '.yml':
|
|
135
|
+
case '.yaml':
|
|
136
|
+
return ['yaml'];
|
|
137
|
+
case '.py':
|
|
138
|
+
return ['python'];
|
|
139
|
+
case '.go':
|
|
140
|
+
return ['go'];
|
|
141
|
+
case '.sh':
|
|
142
|
+
case '.bash':
|
|
143
|
+
return ['shell'];
|
|
144
|
+
case '.tf':
|
|
145
|
+
case '.tfvars':
|
|
146
|
+
return ['terraform'];
|
|
147
|
+
case '.rb':
|
|
148
|
+
return ['ruby'];
|
|
149
|
+
case '.php':
|
|
150
|
+
return ['php'];
|
|
151
|
+
default:
|
|
152
|
+
return [];
|
|
153
|
+
}
|
|
154
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface OsvQuery {
|
|
2
|
+
name: string;
|
|
3
|
+
version: string;
|
|
4
|
+
ecosystem: 'npm' | 'PyPI';
|
|
5
|
+
}
|
|
6
|
+
export interface OsvVulnerability {
|
|
7
|
+
id: string;
|
|
8
|
+
summary: string;
|
|
9
|
+
details?: string;
|
|
10
|
+
aliases: string[];
|
|
11
|
+
/** Highest CVSS base score found on the record, when it publishes one. */
|
|
12
|
+
cvss: number | null;
|
|
13
|
+
/** Lowest version that is not affected, when the record states one. */
|
|
14
|
+
fixedIn: string | null;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Returns, per input query index, the vulnerabilities affecting that exact
|
|
18
|
+
* version. Failures resolve to an empty list: a database outage must never be
|
|
19
|
+
* reported as "no vulnerabilities", so the caller is told separately.
|
|
20
|
+
*/
|
|
21
|
+
export declare function queryOsv(queries: OsvQuery[]): Promise<{
|
|
22
|
+
results: OsvVulnerability[][];
|
|
23
|
+
failed: boolean;
|
|
24
|
+
}>;
|
|
25
|
+
/** Maps a CVSS base score onto our severity vocabulary. */
|
|
26
|
+
export declare function severityFromCvss(cvss: number | null): 'critical' | 'high' | 'medium' | 'low';
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vulnerability lookups against OSV.dev — Google's Open Source Vulnerabilities
|
|
3
|
+
* database, the same data source behind osv-scanner (Apache-2.0). This is a
|
|
4
|
+
* public HTTP API, so nothing is vendored and no licence attaches; ClearToShip
|
|
5
|
+
* simply queries it.
|
|
6
|
+
*
|
|
7
|
+
* Authoritative CVE data replaces hand-maintained version regexes: a rule that
|
|
8
|
+
* hard-codes "next before 14.1.1" is stale the week after it is written.
|
|
9
|
+
*/
|
|
10
|
+
const OSV_BATCH = 'https://api.osv.dev/v1/querybatch';
|
|
11
|
+
const OSV_QUERY = 'https://api.osv.dev/v1/query';
|
|
12
|
+
const TIMEOUT_MS = 15_000;
|
|
13
|
+
function highestCvss(vuln) {
|
|
14
|
+
let best = null;
|
|
15
|
+
for (const s of vuln?.severity ?? []) {
|
|
16
|
+
// CVSS vectors are published as strings; the numeric score lives in
|
|
17
|
+
// database_specific or has to be derived. Prefer an explicit score.
|
|
18
|
+
const score = Number(s?.score);
|
|
19
|
+
if (Number.isFinite(score))
|
|
20
|
+
best = best === null ? score : Math.max(best, score);
|
|
21
|
+
}
|
|
22
|
+
const explicit = Number(vuln?.database_specific?.cvss?.score);
|
|
23
|
+
if (Number.isFinite(explicit))
|
|
24
|
+
best = best === null ? explicit : Math.max(best, explicit);
|
|
25
|
+
return best;
|
|
26
|
+
}
|
|
27
|
+
function firstFixed(vuln) {
|
|
28
|
+
for (const affected of vuln?.affected ?? []) {
|
|
29
|
+
for (const range of affected?.ranges ?? []) {
|
|
30
|
+
for (const event of range?.events ?? []) {
|
|
31
|
+
if (event?.fixed)
|
|
32
|
+
return String(event.fixed);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
function normalise(vuln) {
|
|
39
|
+
return {
|
|
40
|
+
id: String(vuln?.id ?? 'unknown'),
|
|
41
|
+
summary: String(vuln?.summary ?? vuln?.details ?? 'No summary published.').split('\n')[0],
|
|
42
|
+
details: typeof vuln?.details === 'string' ? vuln.details : undefined,
|
|
43
|
+
aliases: Array.isArray(vuln?.aliases) ? vuln.aliases.map(String) : [],
|
|
44
|
+
cvss: highestCvss(vuln),
|
|
45
|
+
fixedIn: firstFixed(vuln),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Returns, per input query index, the vulnerabilities affecting that exact
|
|
50
|
+
* version. Failures resolve to an empty list: a database outage must never be
|
|
51
|
+
* reported as "no vulnerabilities", so the caller is told separately.
|
|
52
|
+
*/
|
|
53
|
+
export async function queryOsv(queries) {
|
|
54
|
+
if (queries.length === 0)
|
|
55
|
+
return { results: [], failed: false };
|
|
56
|
+
const body = {
|
|
57
|
+
queries: queries.map((q) => ({
|
|
58
|
+
version: q.version,
|
|
59
|
+
package: { name: q.name, ecosystem: q.ecosystem },
|
|
60
|
+
})),
|
|
61
|
+
};
|
|
62
|
+
let batch;
|
|
63
|
+
try {
|
|
64
|
+
// cleartoship-ignore VG120 — OSV_BATCH is a module constant naming a
|
|
65
|
+
// fixed https://api.osv.dev endpoint; nothing user-controlled reaches it.
|
|
66
|
+
const res = await fetch(OSV_BATCH, {
|
|
67
|
+
method: 'POST',
|
|
68
|
+
headers: { 'content-type': 'application/json' },
|
|
69
|
+
body: JSON.stringify(body),
|
|
70
|
+
signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
71
|
+
});
|
|
72
|
+
if (!res.ok)
|
|
73
|
+
return { results: queries.map(() => []), failed: true };
|
|
74
|
+
batch = await res.json();
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return { results: queries.map(() => []), failed: true };
|
|
78
|
+
}
|
|
79
|
+
// querybatch returns ids only. Re-query the few packages that actually have
|
|
80
|
+
// hits to get summaries and fixed versions.
|
|
81
|
+
const results = queries.map(() => []);
|
|
82
|
+
const affected = [];
|
|
83
|
+
(batch?.results ?? []).forEach((r, i) => {
|
|
84
|
+
if (Array.isArray(r?.vulns) && r.vulns.length > 0)
|
|
85
|
+
affected.push(i);
|
|
86
|
+
});
|
|
87
|
+
await Promise.all(affected.map(async (i) => {
|
|
88
|
+
const q = queries[i];
|
|
89
|
+
try {
|
|
90
|
+
// cleartoship-ignore VG120 — OSV_QUERY is a module constant naming a
|
|
91
|
+
// fixed https://api.osv.dev endpoint; nothing user-controlled reaches it.
|
|
92
|
+
const res = await fetch(OSV_QUERY, {
|
|
93
|
+
method: 'POST',
|
|
94
|
+
headers: { 'content-type': 'application/json' },
|
|
95
|
+
body: JSON.stringify({
|
|
96
|
+
version: q.version,
|
|
97
|
+
package: { name: q.name, ecosystem: q.ecosystem },
|
|
98
|
+
}),
|
|
99
|
+
signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
100
|
+
});
|
|
101
|
+
if (!res.ok)
|
|
102
|
+
return;
|
|
103
|
+
const detail = await res.json();
|
|
104
|
+
results[i] = (detail?.vulns ?? []).map(normalise);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
/* keep the empty list for this package */
|
|
108
|
+
}
|
|
109
|
+
}));
|
|
110
|
+
return { results, failed: false };
|
|
111
|
+
}
|
|
112
|
+
/** Maps a CVSS base score onto our severity vocabulary. */
|
|
113
|
+
export function severityFromCvss(cvss) {
|
|
114
|
+
if (cvss === null)
|
|
115
|
+
return 'high';
|
|
116
|
+
if (cvss >= 9)
|
|
117
|
+
return 'critical';
|
|
118
|
+
if (cvss >= 7)
|
|
119
|
+
return 'high';
|
|
120
|
+
if (cvss >= 4)
|
|
121
|
+
return 'medium';
|
|
122
|
+
return 'low';
|
|
123
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export interface PackageFacts {
|
|
2
|
+
name: string;
|
|
3
|
+
ecosystem: 'npm' | 'pypi';
|
|
4
|
+
exists: boolean;
|
|
5
|
+
/** ISO date the package was first published. */
|
|
6
|
+
created?: string;
|
|
7
|
+
/** Downloads in the last week (npm only). */
|
|
8
|
+
weeklyDownloads?: number;
|
|
9
|
+
versions?: string[];
|
|
10
|
+
repository?: string;
|
|
11
|
+
maintainers?: number;
|
|
12
|
+
deprecated?: boolean;
|
|
13
|
+
/**
|
|
14
|
+
* npm removed the package for malware and serves HTTP 451. This is a much
|
|
15
|
+
* stronger signal than "does not exist" — the name was actively weaponised.
|
|
16
|
+
*/
|
|
17
|
+
securityHold?: boolean;
|
|
18
|
+
/**
|
|
19
|
+
* The package is gone from the registry but still has download traffic,
|
|
20
|
+
* meaning it was published, then unpublished. The name is free for anyone to
|
|
21
|
+
* take over and inherit those installs.
|
|
22
|
+
*/
|
|
23
|
+
unpublished?: boolean;
|
|
24
|
+
/** Set when the lookup itself failed (offline, rate limited, ...). */
|
|
25
|
+
error?: string;
|
|
26
|
+
}
|
|
27
|
+
export declare class Registry {
|
|
28
|
+
private readonly dir;
|
|
29
|
+
private readonly memo;
|
|
30
|
+
offline: boolean;
|
|
31
|
+
networkErrors: number;
|
|
32
|
+
constructor(cacheDir: string, offline?: boolean);
|
|
33
|
+
private cachePath;
|
|
34
|
+
private readCache;
|
|
35
|
+
private writeCache;
|
|
36
|
+
private getJson;
|
|
37
|
+
/** Weekly downloads, or undefined when the endpoint has nothing to say. */
|
|
38
|
+
private downloadsFor;
|
|
39
|
+
lookup(name: string, ecosystem: 'npm' | 'pypi'): Promise<PackageFacts>;
|
|
40
|
+
private lookupNpm;
|
|
41
|
+
private lookupPypi;
|
|
42
|
+
}
|
|
43
|
+
/** Runs `worker` over `items` with bounded concurrency, preserving order. */
|
|
44
|
+
export declare function pool<T, R>(items: T[], limit: number, worker: (item: T, index: number) => Promise<R>): Promise<R[]>;
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
5
|
+
const TIMEOUT_MS = 8000;
|
|
6
|
+
export class Registry {
|
|
7
|
+
dir;
|
|
8
|
+
memo = new Map();
|
|
9
|
+
offline;
|
|
10
|
+
networkErrors = 0;
|
|
11
|
+
constructor(cacheDir, offline = false) {
|
|
12
|
+
this.dir = cacheDir;
|
|
13
|
+
this.offline = offline;
|
|
14
|
+
if (!offline) {
|
|
15
|
+
try {
|
|
16
|
+
mkdirSync(this.dir, { recursive: true });
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
/* cache is an optimisation; carry on without it */
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
cachePath(key) {
|
|
24
|
+
return join(this.dir, createHash('sha256').update(key).digest('hex').slice(0, 40) + '.json');
|
|
25
|
+
}
|
|
26
|
+
readCache(key) {
|
|
27
|
+
try {
|
|
28
|
+
const raw = readFileSync(this.cachePath(key), 'utf8');
|
|
29
|
+
const parsed = JSON.parse(raw);
|
|
30
|
+
if (Date.now() - parsed.at > CACHE_TTL_MS)
|
|
31
|
+
return null;
|
|
32
|
+
return parsed.facts;
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
writeCache(key, facts) {
|
|
39
|
+
if (facts.error)
|
|
40
|
+
return;
|
|
41
|
+
try {
|
|
42
|
+
writeFileSync(this.cachePath(key), JSON.stringify({ at: Date.now(), facts }));
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
/* ignore */
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async getJson(url, accept) {
|
|
49
|
+
// cleartoship-ignore VG120 — the URL is built from a package name against a
|
|
50
|
+
// fixed registry host a few lines below; fetching it is this class's purpose.
|
|
51
|
+
const res = await fetch(url, {
|
|
52
|
+
headers: {
|
|
53
|
+
'user-agent': 'cleartoship (+https://cleartoship.app)',
|
|
54
|
+
...(accept ? { accept } : {}),
|
|
55
|
+
},
|
|
56
|
+
signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
57
|
+
});
|
|
58
|
+
if (!res.ok)
|
|
59
|
+
return res.status;
|
|
60
|
+
return await res.json();
|
|
61
|
+
}
|
|
62
|
+
/** Weekly downloads, or undefined when the endpoint has nothing to say. */
|
|
63
|
+
async downloadsFor(encoded) {
|
|
64
|
+
try {
|
|
65
|
+
const dl = await this.getJson(`https://api.npmjs.org/downloads/point/last-week/${encoded}`);
|
|
66
|
+
if (typeof dl === 'object' && dl && typeof dl.downloads === 'number')
|
|
67
|
+
return dl.downloads;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
/* the downloads API is flaky and non-essential */
|
|
71
|
+
}
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
async lookup(name, ecosystem) {
|
|
75
|
+
const key = `${ecosystem}:${name}`;
|
|
76
|
+
const memo = this.memo.get(key);
|
|
77
|
+
if (memo)
|
|
78
|
+
return memo;
|
|
79
|
+
const cached = this.readCache(key);
|
|
80
|
+
if (cached) {
|
|
81
|
+
this.memo.set(key, cached);
|
|
82
|
+
return cached;
|
|
83
|
+
}
|
|
84
|
+
if (this.offline) {
|
|
85
|
+
const facts = { name, ecosystem, exists: true, error: 'offline' };
|
|
86
|
+
this.memo.set(key, facts);
|
|
87
|
+
return facts;
|
|
88
|
+
}
|
|
89
|
+
let facts;
|
|
90
|
+
try {
|
|
91
|
+
facts = ecosystem === 'npm' ? await this.lookupNpm(name) : await this.lookupPypi(name);
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
this.networkErrors++;
|
|
95
|
+
facts = {
|
|
96
|
+
name,
|
|
97
|
+
ecosystem,
|
|
98
|
+
exists: true, // fail open: never claim a package is fake because the network blipped
|
|
99
|
+
error: err instanceof Error ? err.message : String(err),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
this.memo.set(key, facts);
|
|
103
|
+
this.writeCache(key, facts);
|
|
104
|
+
return facts;
|
|
105
|
+
}
|
|
106
|
+
async lookupNpm(name) {
|
|
107
|
+
const encoded = name.replace('/', '%2f');
|
|
108
|
+
const doc = await this.getJson(`https://registry.npmjs.org/${encoded}`, 'application/vnd.npm.install-v1+json');
|
|
109
|
+
// 451 is npm's "removed for malware" response. Never conflate it with 404.
|
|
110
|
+
if (doc === 451) {
|
|
111
|
+
return { name, ecosystem: 'npm', exists: false, securityHold: true };
|
|
112
|
+
}
|
|
113
|
+
if (doc === 404) {
|
|
114
|
+
// A name with no packument but real install traffic was published and then
|
|
115
|
+
// pulled, which leaves it open to takeover rather than merely invented.
|
|
116
|
+
const downloads = await this.downloadsFor(encoded);
|
|
117
|
+
return {
|
|
118
|
+
name,
|
|
119
|
+
ecosystem: 'npm',
|
|
120
|
+
exists: false,
|
|
121
|
+
weeklyDownloads: downloads,
|
|
122
|
+
unpublished: downloads !== undefined && downloads > 0,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
if (typeof doc === 'number')
|
|
126
|
+
throw new Error(`npm registry returned ${doc}`);
|
|
127
|
+
const versions = Object.keys(doc.versions ?? {});
|
|
128
|
+
const facts = { name, ecosystem: 'npm', exists: true, versions };
|
|
129
|
+
const downloads = await this.downloadsFor(encoded);
|
|
130
|
+
facts.weeklyDownloads = downloads;
|
|
131
|
+
// The full document carries publish dates and maintainer counts, but it is
|
|
132
|
+
// large. Only fetch it for packages that are not obviously well established.
|
|
133
|
+
if (downloads === undefined || downloads < 10_000) {
|
|
134
|
+
try {
|
|
135
|
+
const full = await this.getJson(`https://registry.npmjs.org/${encoded}`);
|
|
136
|
+
if (typeof full === 'object' && full) {
|
|
137
|
+
facts.created = full.time?.created;
|
|
138
|
+
facts.maintainers = Array.isArray(full.maintainers) ? full.maintainers.length : undefined;
|
|
139
|
+
const repo = full.repository;
|
|
140
|
+
facts.repository = typeof repo === 'string' ? repo : repo?.url;
|
|
141
|
+
const latest = full['dist-tags']?.latest;
|
|
142
|
+
facts.deprecated = Boolean(latest && full.versions?.[latest]?.deprecated);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
/* keep the abbreviated facts */
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return facts;
|
|
150
|
+
}
|
|
151
|
+
async lookupPypi(name) {
|
|
152
|
+
const doc = await this.getJson(`https://pypi.org/pypi/${encodeURIComponent(name)}/json`);
|
|
153
|
+
if (doc === 404)
|
|
154
|
+
return { name, ecosystem: 'pypi', exists: false };
|
|
155
|
+
if (doc === 451)
|
|
156
|
+
return { name, ecosystem: 'pypi', exists: false, securityHold: true };
|
|
157
|
+
if (typeof doc === 'number')
|
|
158
|
+
throw new Error(`PyPI returned ${doc}`);
|
|
159
|
+
const releases = doc.releases ?? {};
|
|
160
|
+
let earliest;
|
|
161
|
+
for (const files of Object.values(releases)) {
|
|
162
|
+
for (const f of files ?? []) {
|
|
163
|
+
const t = f?.upload_time_iso_8601 ?? f?.upload_time;
|
|
164
|
+
if (t && (!earliest || t < earliest))
|
|
165
|
+
earliest = t;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return {
|
|
169
|
+
name,
|
|
170
|
+
ecosystem: 'pypi',
|
|
171
|
+
exists: true,
|
|
172
|
+
created: earliest,
|
|
173
|
+
versions: Object.keys(releases),
|
|
174
|
+
repository: doc.info?.project_urls?.Source ??
|
|
175
|
+
doc.info?.project_urls?.Homepage ??
|
|
176
|
+
doc.info?.home_page ??
|
|
177
|
+
undefined,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/** Runs `worker` over `items` with bounded concurrency, preserving order. */
|
|
182
|
+
export async function pool(items, limit, worker) {
|
|
183
|
+
const results = new Array(items.length);
|
|
184
|
+
let cursor = 0;
|
|
185
|
+
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
186
|
+
for (;;) {
|
|
187
|
+
const i = cursor++;
|
|
188
|
+
if (i >= items.length)
|
|
189
|
+
return;
|
|
190
|
+
results[i] = await worker(items[i], i);
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
await Promise.all(runners);
|
|
194
|
+
return results;
|
|
195
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal PostgreSQL statement splitter. Deliberately not a full parser: it
|
|
3
|
+
* only needs to know where one statement ends, which means respecting string
|
|
4
|
+
* literals, dollar-quoted function bodies and both comment styles.
|
|
5
|
+
*/
|
|
6
|
+
export interface SqlStatement {
|
|
7
|
+
text: string;
|
|
8
|
+
/** 1-indexed line where the statement starts. */
|
|
9
|
+
line: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function splitStatements(source: string): SqlStatement[];
|
|
12
|
+
/** Reads a parenthesised group starting at `open` (index of the `(`). */
|
|
13
|
+
export declare function readBalanced(text: string, open: number): string | null;
|
|
14
|
+
/** Grabs the expression after a keyword such as USING or WITH CHECK. */
|
|
15
|
+
export declare function clauseAfter(text: string, keyword: RegExp): string | null;
|
|
16
|
+
/** Normalises `"public"."posts"` / `public.posts` / `posts` to `public.posts`. */
|
|
17
|
+
export declare function normaliseTable(raw: string): string;
|
|
18
|
+
export declare const QUALIFIED_NAME = "(?:\"[^\"]+\"|[A-Za-z_][A-Za-z0-9_$]*)(?:\\s*\\.\\s*(?:\"[^\"]+\"|[A-Za-z_][A-Za-z0-9_$]*))*";
|
|
19
|
+
/** True when a policy predicate lets everything through. */
|
|
20
|
+
export declare function isAlwaysTrue(expr: string | null): boolean;
|