@ecosyste-ms/critical 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +26 -0
- package/README.md +194 -0
- package/bin.js +88 -0
- package/critical-packages.db +0 -0
- package/lib/index.js +382 -0
- package/package.json +43 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Creative Commons Attribution-ShareAlike 4.0 International
|
|
2
|
+
|
|
3
|
+
Copyright (c) ecosyste.ms
|
|
4
|
+
|
|
5
|
+
This work is licensed under the Creative Commons Attribution-ShareAlike 4.0
|
|
6
|
+
International License. To view a copy of this license, visit
|
|
7
|
+
http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to Creative
|
|
8
|
+
Commons, PO Box 1866, Mountain View, CA 94042, USA.
|
|
9
|
+
|
|
10
|
+
You are free to:
|
|
11
|
+
|
|
12
|
+
Share - copy and redistribute the material in any medium or format
|
|
13
|
+
Adapt - remix, transform, and build upon the material for any purpose,
|
|
14
|
+
even commercially
|
|
15
|
+
|
|
16
|
+
Under the following terms:
|
|
17
|
+
|
|
18
|
+
Attribution - You must give appropriate credit, provide a link to the
|
|
19
|
+
license, and indicate if changes were made.
|
|
20
|
+
|
|
21
|
+
ShareAlike - If you remix, transform, or build upon the material, you
|
|
22
|
+
must distribute your contributions under the same license.
|
|
23
|
+
|
|
24
|
+
No additional restrictions - You may not apply legal terms or technological
|
|
25
|
+
measures that legally restrict others from
|
|
26
|
+
doing anything the license permits.
|
package/README.md
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# @ecosyste-ms/critical
|
|
2
|
+
|
|
3
|
+
SQLite database of critical open source packages from [ecosyste.ms](https://packages.ecosyste.ms).
|
|
4
|
+
|
|
5
|
+
The database is rebuilt daily and published to npm and as a GitHub release. Versions use CalVer (`YYYY.M.D`).
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @ecosyste-ms/critical
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The package includes a pre-built `critical-packages.db` file:
|
|
14
|
+
|
|
15
|
+
```javascript
|
|
16
|
+
import { databasePath } from '@ecosyste-ms/critical'
|
|
17
|
+
import Database from 'better-sqlite3'
|
|
18
|
+
|
|
19
|
+
const db = new Database(databasePath)
|
|
20
|
+
const pkg = db.prepare('SELECT * FROM packages WHERE name = ?').get('lodash')
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Download
|
|
24
|
+
|
|
25
|
+
You can also grab the database directly from the [releases page](../../releases/latest):
|
|
26
|
+
|
|
27
|
+
- `critical-packages.db` - uncompressed SQLite
|
|
28
|
+
- `critical-packages.db.gz` - gzip compressed
|
|
29
|
+
|
|
30
|
+
## Building
|
|
31
|
+
|
|
32
|
+
To build the database yourself:
|
|
33
|
+
|
|
34
|
+
```javascript
|
|
35
|
+
import { build, createDatabase } from '@ecosyste-ms/critical'
|
|
36
|
+
|
|
37
|
+
await build({
|
|
38
|
+
dbPath: 'critical-packages.db',
|
|
39
|
+
fetchVersionsData: true,
|
|
40
|
+
onProgress: console.log
|
|
41
|
+
})
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
From the command line:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
npx @ecosyste-ms/critical # full build with versions
|
|
48
|
+
npx @ecosyste-ms/critical --skip-versions # faster, packages only
|
|
49
|
+
npx @ecosyste-ms/critical -o my-db.db # custom output path
|
|
50
|
+
npx @ecosyste-ms/critical --stats # show database statistics
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Schema
|
|
54
|
+
|
|
55
|
+
### packages
|
|
56
|
+
|
|
57
|
+
| Column | Type | Description |
|
|
58
|
+
|--------|------|-------------|
|
|
59
|
+
| id | INTEGER | Primary key from ecosyste.ms |
|
|
60
|
+
| ecosystem | TEXT | Package ecosystem (npm, pypi, rubygems, etc.) |
|
|
61
|
+
| name | TEXT | Package name |
|
|
62
|
+
| purl | TEXT | Package URL (pkg:npm/lodash) |
|
|
63
|
+
| namespace | TEXT | Package namespace if applicable |
|
|
64
|
+
| description | TEXT | Package description |
|
|
65
|
+
| homepage | TEXT | Project homepage URL |
|
|
66
|
+
| repository_url | TEXT | Source repository URL |
|
|
67
|
+
| licenses | TEXT | SPDX license identifier |
|
|
68
|
+
| normalized_licenses | TEXT | JSON array of normalized license identifiers |
|
|
69
|
+
| latest_version | TEXT | Latest release version number |
|
|
70
|
+
| versions_count | INTEGER | Total number of versions |
|
|
71
|
+
| downloads | INTEGER | Download count |
|
|
72
|
+
| downloads_period | TEXT | Period for download count (e.g., last-month) |
|
|
73
|
+
| dependent_packages_count | INTEGER | Number of packages depending on this |
|
|
74
|
+
| dependent_repos_count | INTEGER | Number of repositories depending on this |
|
|
75
|
+
| first_release_at | TEXT | ISO 8601 timestamp of first release |
|
|
76
|
+
| latest_release_at | TEXT | ISO 8601 timestamp of latest release |
|
|
77
|
+
| last_synced_at | TEXT | When ecosyste.ms last synced this package |
|
|
78
|
+
| keywords | TEXT | Space-separated keywords for FTS |
|
|
79
|
+
|
|
80
|
+
Indexes: `(ecosystem, name)` unique, `purl`, `licenses`, `ecosystem`
|
|
81
|
+
|
|
82
|
+
### versions
|
|
83
|
+
|
|
84
|
+
| Column | Type | Description |
|
|
85
|
+
|--------|------|-------------|
|
|
86
|
+
| package_id | INTEGER | Foreign key to packages |
|
|
87
|
+
| number | TEXT | Version number |
|
|
88
|
+
|
|
89
|
+
Primary key: `(package_id, number)`
|
|
90
|
+
|
|
91
|
+
### advisories
|
|
92
|
+
|
|
93
|
+
| Column | Type | Description |
|
|
94
|
+
|--------|------|-------------|
|
|
95
|
+
| id | INTEGER | Auto-increment primary key |
|
|
96
|
+
| package_id | INTEGER | Foreign key to packages |
|
|
97
|
+
| uuid | TEXT | Advisory UUID (GHSA-xxxx-xxxx-xxxx) |
|
|
98
|
+
| url | TEXT | Advisory URL |
|
|
99
|
+
| title | TEXT | Advisory title |
|
|
100
|
+
| description | TEXT | Advisory description |
|
|
101
|
+
| severity | TEXT | Severity level (LOW, MODERATE, HIGH, CRITICAL) |
|
|
102
|
+
| published_at | TEXT | ISO 8601 publish timestamp |
|
|
103
|
+
| cvss_score | REAL | CVSS score |
|
|
104
|
+
|
|
105
|
+
Indexes: `package_id`, `uuid`, `severity`, `(package_id, uuid)` unique
|
|
106
|
+
|
|
107
|
+
### repo_metadata
|
|
108
|
+
|
|
109
|
+
| Column | Type | Description |
|
|
110
|
+
|--------|------|-------------|
|
|
111
|
+
| package_id | INTEGER | Primary key, foreign key to packages |
|
|
112
|
+
| owner | TEXT | Repository owner/organization |
|
|
113
|
+
| repo_name | TEXT | Repository name |
|
|
114
|
+
| full_name | TEXT | Full name (owner/repo) |
|
|
115
|
+
| host | TEXT | Host (github.com, gitlab.com, etc.) |
|
|
116
|
+
| language | TEXT | Primary language |
|
|
117
|
+
| stargazers_count | INTEGER | GitHub stars |
|
|
118
|
+
| forks_count | INTEGER | Fork count |
|
|
119
|
+
| open_issues_count | INTEGER | Open issues |
|
|
120
|
+
| archived | INTEGER | 1 if archived, 0 otherwise |
|
|
121
|
+
| fork | INTEGER | 1 if a fork, 0 otherwise |
|
|
122
|
+
|
|
123
|
+
Indexes: `full_name`, `owner`
|
|
124
|
+
|
|
125
|
+
### packages_fts
|
|
126
|
+
|
|
127
|
+
FTS5 virtual table for full-text search on `ecosystem`, `name`, `description`, and `keywords`.
|
|
128
|
+
|
|
129
|
+
### build_info
|
|
130
|
+
|
|
131
|
+
| Column | Type | Description |
|
|
132
|
+
|--------|------|-------------|
|
|
133
|
+
| id | INTEGER | Always 1 |
|
|
134
|
+
| built_at | TEXT | ISO 8601 build timestamp |
|
|
135
|
+
| package_count | INTEGER | Total packages |
|
|
136
|
+
| version_count | INTEGER | Total versions |
|
|
137
|
+
| advisory_count | INTEGER | Total advisories |
|
|
138
|
+
|
|
139
|
+
## Example Queries
|
|
140
|
+
|
|
141
|
+
SBOM enrichment by purl:
|
|
142
|
+
|
|
143
|
+
```sql
|
|
144
|
+
SELECT * FROM packages WHERE purl = 'pkg:npm/lodash';
|
|
145
|
+
|
|
146
|
+
-- Check if a specific version exists
|
|
147
|
+
SELECT * FROM versions v
|
|
148
|
+
JOIN packages p ON v.package_id = p.id
|
|
149
|
+
WHERE p.purl = 'pkg:npm/lodash' AND v.number = '4.17.21';
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Find packages with known vulnerabilities:
|
|
153
|
+
|
|
154
|
+
```sql
|
|
155
|
+
SELECT p.ecosystem, p.name, a.uuid, a.severity, a.title
|
|
156
|
+
FROM packages p
|
|
157
|
+
JOIN advisories a ON p.id = a.package_id
|
|
158
|
+
WHERE a.severity IN ('HIGH', 'CRITICAL');
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
License audit:
|
|
162
|
+
|
|
163
|
+
```sql
|
|
164
|
+
SELECT ecosystem, name, licenses
|
|
165
|
+
FROM packages
|
|
166
|
+
WHERE licenses NOT IN ('MIT', 'Apache-2.0', 'BSD-3-Clause');
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Full-text search:
|
|
170
|
+
|
|
171
|
+
```sql
|
|
172
|
+
SELECT p.* FROM packages p
|
|
173
|
+
JOIN packages_fts ON p.id = packages_fts.rowid
|
|
174
|
+
WHERE packages_fts MATCH 'http client';
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Packages by ecosystem:
|
|
178
|
+
|
|
179
|
+
```sql
|
|
180
|
+
SELECT * FROM packages WHERE ecosystem = 'npm';
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Most depended-on packages:
|
|
184
|
+
|
|
185
|
+
```sql
|
|
186
|
+
SELECT ecosystem, name, dependent_packages_count
|
|
187
|
+
FROM packages
|
|
188
|
+
ORDER BY dependent_packages_count DESC
|
|
189
|
+
LIMIT 100;
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## License
|
|
193
|
+
|
|
194
|
+
CC-BY-SA-4.0
|
package/bin.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { build, databasePath } from './lib/index.js'
|
|
3
|
+
import Database from 'better-sqlite3'
|
|
4
|
+
import { existsSync } from 'fs'
|
|
5
|
+
|
|
6
|
+
const args = process.argv.slice(2)
|
|
7
|
+
|
|
8
|
+
if (args.includes('--help') || args.includes('-h')) {
|
|
9
|
+
console.log(`Usage: critical [options]
|
|
10
|
+
|
|
11
|
+
Options:
|
|
12
|
+
--output, -o <path> Output database path (default: critical-packages.db)
|
|
13
|
+
--skip-versions Skip fetching version data (faster)
|
|
14
|
+
--stats Show database statistics
|
|
15
|
+
--help, -h Show this help message
|
|
16
|
+
|
|
17
|
+
Examples:
|
|
18
|
+
npx @ecosyste-ms/critical
|
|
19
|
+
npx @ecosyste-ms/critical --skip-versions
|
|
20
|
+
npx @ecosyste-ms/critical -o my-database.db
|
|
21
|
+
npx @ecosyste-ms/critical --stats`)
|
|
22
|
+
process.exit(0)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function getArg(flags) {
|
|
26
|
+
for (const flag of flags) {
|
|
27
|
+
const idx = args.indexOf(flag)
|
|
28
|
+
if (idx !== -1 && args[idx + 1]) {
|
|
29
|
+
return args[idx + 1]
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return null
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const dbPath = getArg(['--output', '-o']) || 'critical-packages.db'
|
|
36
|
+
|
|
37
|
+
if (args.includes('--stats')) {
|
|
38
|
+
const path = existsSync(dbPath) ? dbPath : databasePath
|
|
39
|
+
if (!existsSync(path)) {
|
|
40
|
+
console.error(`Database not found: ${path}`)
|
|
41
|
+
process.exit(1)
|
|
42
|
+
}
|
|
43
|
+
const db = new Database(path, { readonly: true })
|
|
44
|
+
const info = db.prepare('SELECT * FROM build_info WHERE id = 1').get()
|
|
45
|
+
const packageCount = db.prepare('SELECT COUNT(*) as count FROM packages').get().count
|
|
46
|
+
const versionCount = db.prepare('SELECT COUNT(*) as count FROM versions').get().count
|
|
47
|
+
const advisoryCount = db.prepare('SELECT COUNT(*) as count FROM advisories').get().count
|
|
48
|
+
const ecosystems = db.prepare(`
|
|
49
|
+
SELECT ecosystem, COUNT(*) as count
|
|
50
|
+
FROM packages
|
|
51
|
+
GROUP BY ecosystem
|
|
52
|
+
ORDER BY count DESC
|
|
53
|
+
`).all()
|
|
54
|
+
const severities = db.prepare(`
|
|
55
|
+
SELECT severity, COUNT(*) as count
|
|
56
|
+
FROM advisories
|
|
57
|
+
GROUP BY severity
|
|
58
|
+
ORDER BY count DESC
|
|
59
|
+
`).all()
|
|
60
|
+
|
|
61
|
+
console.log(`Database: ${path}`)
|
|
62
|
+
console.log(`Built: ${info?.built_at || 'unknown'}`)
|
|
63
|
+
console.log(`\nPackages: ${packageCount}`)
|
|
64
|
+
console.log(`Versions: ${versionCount}`)
|
|
65
|
+
console.log(`Advisories: ${advisoryCount}`)
|
|
66
|
+
console.log(`\nBy ecosystem:`)
|
|
67
|
+
for (const e of ecosystems) {
|
|
68
|
+
console.log(` ${e.ecosystem}: ${e.count}`)
|
|
69
|
+
}
|
|
70
|
+
console.log(`\nBy severity:`)
|
|
71
|
+
for (const s of severities) {
|
|
72
|
+
console.log(` ${s.severity || 'unknown'}: ${s.count}`)
|
|
73
|
+
}
|
|
74
|
+
db.close()
|
|
75
|
+
process.exit(0)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const skipVersions = args.includes('--skip-versions')
|
|
79
|
+
|
|
80
|
+
build({
|
|
81
|
+
dbPath,
|
|
82
|
+
fetchVersionsData: !skipVersions
|
|
83
|
+
})
|
|
84
|
+
.then(() => process.exit(0))
|
|
85
|
+
.catch(err => {
|
|
86
|
+
console.error(err)
|
|
87
|
+
process.exit(1)
|
|
88
|
+
})
|
|
Binary file
|
package/lib/index.js
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
import Database from 'better-sqlite3'
|
|
2
|
+
import { mkdir, unlink, readFile } from 'fs/promises'
|
|
3
|
+
import { dirname, join } from 'path'
|
|
4
|
+
import { fileURLToPath } from 'url'
|
|
5
|
+
|
|
6
|
+
const __dirname = dirname(fileURLToPath(import.meta.url))
|
|
7
|
+
const pkg = JSON.parse(await readFile(join(__dirname, '..', 'package.json'), 'utf8'))
|
|
8
|
+
const USER_AGENT = `${pkg.name}/${pkg.version}`
|
|
9
|
+
const databasePath = join(__dirname, '..', 'critical-packages.db')
|
|
10
|
+
|
|
11
|
+
const API_BASE = 'https://packages.ecosyste.ms/api/v1'
|
|
12
|
+
const PER_PAGE = 1000
|
|
13
|
+
const RATE_LIMIT_MS = 50
|
|
14
|
+
const CONCURRENCY = 10
|
|
15
|
+
|
|
16
|
+
function sleep(ms) {
|
|
17
|
+
return new Promise(resolve => setTimeout(resolve, ms))
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function fetchJson(url) {
|
|
21
|
+
const response = await fetch(url, {
|
|
22
|
+
headers: { 'User-Agent': USER_AGENT }
|
|
23
|
+
})
|
|
24
|
+
if (!response.ok) {
|
|
25
|
+
throw new Error(`HTTP ${response.status}: ${url}`)
|
|
26
|
+
}
|
|
27
|
+
return response.json()
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async function fetchAllCriticalPackages(onProgress) {
|
|
31
|
+
const packages = []
|
|
32
|
+
let page = 1
|
|
33
|
+
|
|
34
|
+
while (true) {
|
|
35
|
+
const url = `${API_BASE}/packages/critical?per_page=${PER_PAGE}&page=${page}`
|
|
36
|
+
onProgress?.(`Fetching page ${page}...`)
|
|
37
|
+
|
|
38
|
+
const batch = await fetchJson(url)
|
|
39
|
+
if (batch.length === 0) break
|
|
40
|
+
|
|
41
|
+
packages.push(...batch)
|
|
42
|
+
page++
|
|
43
|
+
|
|
44
|
+
await sleep(RATE_LIMIT_MS)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return packages
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async function fetchVersionNumbers(ecosystem, name) {
|
|
51
|
+
const registry = ecosystemToRegistry(ecosystem)
|
|
52
|
+
if (!registry) return []
|
|
53
|
+
|
|
54
|
+
const encodedName = encodeURIComponent(name)
|
|
55
|
+
const url = `${API_BASE}/registries/${registry}/packages/${encodedName}/version_numbers`
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
return await fetchJson(url)
|
|
59
|
+
} catch (err) {
|
|
60
|
+
return []
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function ecosystemToRegistry(ecosystem) {
|
|
65
|
+
const map = {
|
|
66
|
+
'npm': 'npmjs.org',
|
|
67
|
+
'pypi': 'pypi.org',
|
|
68
|
+
'rubygems': 'rubygems.org',
|
|
69
|
+
'go': 'proxy.golang.org',
|
|
70
|
+
'cargo': 'crates.io',
|
|
71
|
+
'maven': 'repo1.maven.org',
|
|
72
|
+
'nuget': 'nuget.org',
|
|
73
|
+
'packagist': 'packagist.org',
|
|
74
|
+
'hex': 'hex.pm',
|
|
75
|
+
'pub': 'pub.dev',
|
|
76
|
+
'hackage': 'hackage.haskell.org',
|
|
77
|
+
'cocoapods': 'cocoapods.org',
|
|
78
|
+
'conda': 'anaconda.org',
|
|
79
|
+
'clojars': 'clojars.org',
|
|
80
|
+
'puppet': 'forge.puppet.com',
|
|
81
|
+
'homebrew': 'formulae.brew.sh',
|
|
82
|
+
}
|
|
83
|
+
return map[ecosystem.toLowerCase()] || null
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function createDatabase(dbPath) {
|
|
87
|
+
const db = new Database(dbPath)
|
|
88
|
+
db.pragma('journal_mode = WAL')
|
|
89
|
+
|
|
90
|
+
db.exec(`
|
|
91
|
+
CREATE TABLE packages (
|
|
92
|
+
id INTEGER PRIMARY KEY,
|
|
93
|
+
ecosystem TEXT NOT NULL,
|
|
94
|
+
name TEXT NOT NULL,
|
|
95
|
+
purl TEXT,
|
|
96
|
+
namespace TEXT,
|
|
97
|
+
description TEXT,
|
|
98
|
+
homepage TEXT,
|
|
99
|
+
repository_url TEXT,
|
|
100
|
+
licenses TEXT,
|
|
101
|
+
normalized_licenses TEXT,
|
|
102
|
+
latest_version TEXT,
|
|
103
|
+
versions_count INTEGER,
|
|
104
|
+
downloads INTEGER,
|
|
105
|
+
downloads_period TEXT,
|
|
106
|
+
dependent_packages_count INTEGER,
|
|
107
|
+
dependent_repos_count INTEGER,
|
|
108
|
+
first_release_at TEXT,
|
|
109
|
+
latest_release_at TEXT,
|
|
110
|
+
last_synced_at TEXT,
|
|
111
|
+
keywords TEXT,
|
|
112
|
+
created_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
113
|
+
);
|
|
114
|
+
|
|
115
|
+
CREATE UNIQUE INDEX idx_packages_ecosystem_name ON packages(ecosystem, name);
|
|
116
|
+
CREATE INDEX idx_packages_purl ON packages(purl);
|
|
117
|
+
CREATE INDEX idx_packages_licenses ON packages(licenses);
|
|
118
|
+
CREATE INDEX idx_packages_ecosystem ON packages(ecosystem);
|
|
119
|
+
|
|
120
|
+
CREATE TABLE versions (
|
|
121
|
+
package_id INTEGER NOT NULL,
|
|
122
|
+
number TEXT NOT NULL,
|
|
123
|
+
PRIMARY KEY (package_id, number),
|
|
124
|
+
FOREIGN KEY (package_id) REFERENCES packages(id)
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
CREATE TABLE advisories (
|
|
128
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
129
|
+
package_id INTEGER NOT NULL,
|
|
130
|
+
uuid TEXT NOT NULL,
|
|
131
|
+
url TEXT,
|
|
132
|
+
title TEXT,
|
|
133
|
+
description TEXT,
|
|
134
|
+
severity TEXT,
|
|
135
|
+
published_at TEXT,
|
|
136
|
+
cvss_score REAL,
|
|
137
|
+
FOREIGN KEY (package_id) REFERENCES packages(id)
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
CREATE INDEX idx_advisories_package_id ON advisories(package_id);
|
|
141
|
+
CREATE INDEX idx_advisories_uuid ON advisories(uuid);
|
|
142
|
+
CREATE INDEX idx_advisories_severity ON advisories(severity);
|
|
143
|
+
CREATE UNIQUE INDEX idx_advisories_package_uuid ON advisories(package_id, uuid);
|
|
144
|
+
|
|
145
|
+
CREATE TABLE repo_metadata (
|
|
146
|
+
package_id INTEGER PRIMARY KEY,
|
|
147
|
+
owner TEXT,
|
|
148
|
+
repo_name TEXT,
|
|
149
|
+
full_name TEXT,
|
|
150
|
+
host TEXT,
|
|
151
|
+
language TEXT,
|
|
152
|
+
stargazers_count INTEGER,
|
|
153
|
+
forks_count INTEGER,
|
|
154
|
+
open_issues_count INTEGER,
|
|
155
|
+
archived INTEGER,
|
|
156
|
+
fork INTEGER,
|
|
157
|
+
FOREIGN KEY (package_id) REFERENCES packages(id)
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
CREATE INDEX idx_repo_full_name ON repo_metadata(full_name);
|
|
161
|
+
CREATE INDEX idx_repo_owner ON repo_metadata(owner);
|
|
162
|
+
|
|
163
|
+
CREATE VIRTUAL TABLE packages_fts USING fts5(
|
|
164
|
+
ecosystem,
|
|
165
|
+
name,
|
|
166
|
+
description,
|
|
167
|
+
keywords,
|
|
168
|
+
content=packages,
|
|
169
|
+
content_rowid=id
|
|
170
|
+
);
|
|
171
|
+
|
|
172
|
+
CREATE TRIGGER packages_ai AFTER INSERT ON packages BEGIN
|
|
173
|
+
INSERT INTO packages_fts(rowid, ecosystem, name, description, keywords)
|
|
174
|
+
VALUES (new.id, new.ecosystem, new.name, new.description, new.keywords);
|
|
175
|
+
END;
|
|
176
|
+
|
|
177
|
+
CREATE TRIGGER packages_ad AFTER DELETE ON packages BEGIN
|
|
178
|
+
INSERT INTO packages_fts(packages_fts, rowid, ecosystem, name, description, keywords)
|
|
179
|
+
VALUES ('delete', old.id, old.ecosystem, old.name, old.description, old.keywords);
|
|
180
|
+
END;
|
|
181
|
+
|
|
182
|
+
CREATE TRIGGER packages_au AFTER UPDATE ON packages BEGIN
|
|
183
|
+
INSERT INTO packages_fts(packages_fts, rowid, ecosystem, name, description, keywords)
|
|
184
|
+
VALUES ('delete', old.id, old.ecosystem, old.name, old.description, old.keywords);
|
|
185
|
+
INSERT INTO packages_fts(rowid, ecosystem, name, description, keywords)
|
|
186
|
+
VALUES (new.id, new.ecosystem, new.name, new.description, new.keywords);
|
|
187
|
+
END;
|
|
188
|
+
|
|
189
|
+
CREATE TABLE build_info (
|
|
190
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
191
|
+
built_at TEXT NOT NULL,
|
|
192
|
+
package_count INTEGER,
|
|
193
|
+
version_count INTEGER,
|
|
194
|
+
advisory_count INTEGER
|
|
195
|
+
);
|
|
196
|
+
`)
|
|
197
|
+
|
|
198
|
+
return db
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function insertPackage(db, pkg) {
|
|
202
|
+
const stmt = db.prepare(`
|
|
203
|
+
INSERT OR REPLACE INTO packages (
|
|
204
|
+
id, ecosystem, name, purl, namespace, description, homepage,
|
|
205
|
+
repository_url, licenses, normalized_licenses, latest_version,
|
|
206
|
+
versions_count, downloads, downloads_period, dependent_packages_count,
|
|
207
|
+
dependent_repos_count, first_release_at, latest_release_at,
|
|
208
|
+
last_synced_at, keywords
|
|
209
|
+
) VALUES (
|
|
210
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
211
|
+
)
|
|
212
|
+
`)
|
|
213
|
+
|
|
214
|
+
const keywords = Array.isArray(pkg.keywords_array) ? pkg.keywords_array.join(' ') : null
|
|
215
|
+
|
|
216
|
+
stmt.run(
|
|
217
|
+
pkg.id,
|
|
218
|
+
pkg.ecosystem,
|
|
219
|
+
pkg.name,
|
|
220
|
+
pkg.purl,
|
|
221
|
+
pkg.namespace,
|
|
222
|
+
pkg.description,
|
|
223
|
+
pkg.homepage,
|
|
224
|
+
pkg.repository_url,
|
|
225
|
+
pkg.licenses,
|
|
226
|
+
JSON.stringify(pkg.normalized_licenses),
|
|
227
|
+
pkg.latest_release_number,
|
|
228
|
+
pkg.versions_count,
|
|
229
|
+
pkg.downloads,
|
|
230
|
+
pkg.downloads_period,
|
|
231
|
+
pkg.dependent_packages_count,
|
|
232
|
+
pkg.dependent_repos_count,
|
|
233
|
+
pkg.first_release_published_at,
|
|
234
|
+
pkg.latest_release_published_at,
|
|
235
|
+
pkg.last_synced_at,
|
|
236
|
+
keywords
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
return pkg.id
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function insertRepoMetadata(db, packageId, repoMetadata, host) {
|
|
243
|
+
if (!repoMetadata) return
|
|
244
|
+
|
|
245
|
+
const stmt = db.prepare(`
|
|
246
|
+
INSERT OR REPLACE INTO repo_metadata (
|
|
247
|
+
package_id, owner, repo_name, full_name, host, language,
|
|
248
|
+
stargazers_count, forks_count, open_issues_count, archived, fork
|
|
249
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
250
|
+
`)
|
|
251
|
+
|
|
252
|
+
stmt.run(
|
|
253
|
+
packageId,
|
|
254
|
+
repoMetadata.owner,
|
|
255
|
+
repoMetadata.name,
|
|
256
|
+
repoMetadata.full_name,
|
|
257
|
+
host?.name,
|
|
258
|
+
repoMetadata.language,
|
|
259
|
+
repoMetadata.stargazers_count,
|
|
260
|
+
repoMetadata.forks_count,
|
|
261
|
+
repoMetadata.open_issues_count,
|
|
262
|
+
repoMetadata.archived ? 1 : 0,
|
|
263
|
+
repoMetadata.fork ? 1 : 0
|
|
264
|
+
)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function insertAdvisories(db, packageId, advisories) {
|
|
268
|
+
if (!advisories || advisories.length === 0) return
|
|
269
|
+
|
|
270
|
+
const stmt = db.prepare(`
|
|
271
|
+
INSERT OR REPLACE INTO advisories (
|
|
272
|
+
package_id, uuid, url, title, description, severity, published_at, cvss_score
|
|
273
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
274
|
+
`)
|
|
275
|
+
|
|
276
|
+
for (const advisory of advisories) {
|
|
277
|
+
if (!advisory || !advisory.uuid) continue
|
|
278
|
+
stmt.run(
|
|
279
|
+
packageId,
|
|
280
|
+
advisory.uuid,
|
|
281
|
+
advisory.url,
|
|
282
|
+
advisory.title,
|
|
283
|
+
advisory.description,
|
|
284
|
+
advisory.severity,
|
|
285
|
+
advisory.published_at,
|
|
286
|
+
advisory.cvss_score
|
|
287
|
+
)
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function insertVersions(db, packageId, versionNumbers) {
|
|
292
|
+
if (!versionNumbers || versionNumbers.length === 0) return
|
|
293
|
+
|
|
294
|
+
const stmt = db.prepare(`
|
|
295
|
+
INSERT OR REPLACE INTO versions (package_id, number) VALUES (?, ?)
|
|
296
|
+
`)
|
|
297
|
+
|
|
298
|
+
for (const number of versionNumbers) {
|
|
299
|
+
stmt.run(packageId, number)
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function updateBuildInfo(db) {
|
|
304
|
+
const packageCount = db.prepare('SELECT COUNT(*) as count FROM packages').get().count
|
|
305
|
+
const versionCount = db.prepare('SELECT COUNT(*) as count FROM versions').get().count
|
|
306
|
+
const advisoryCount = db.prepare('SELECT COUNT(*) as count FROM advisories').get().count
|
|
307
|
+
|
|
308
|
+
db.prepare(`
|
|
309
|
+
INSERT OR REPLACE INTO build_info (id, built_at, package_count, version_count, advisory_count)
|
|
310
|
+
VALUES (1, ?, ?, ?, ?)
|
|
311
|
+
`).run(new Date().toISOString(), packageCount, versionCount, advisoryCount)
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async function build(options = {}) {
|
|
315
|
+
const {
|
|
316
|
+
dbPath = 'critical-packages.db',
|
|
317
|
+
fetchVersionsData = true,
|
|
318
|
+
onProgress = console.log
|
|
319
|
+
} = options
|
|
320
|
+
|
|
321
|
+
await mkdir(dirname(dbPath) || '.', { recursive: true }).catch(() => {})
|
|
322
|
+
await unlink(dbPath).catch(() => {})
|
|
323
|
+
await unlink(dbPath + '-wal').catch(() => {})
|
|
324
|
+
await unlink(dbPath + '-shm').catch(() => {})
|
|
325
|
+
|
|
326
|
+
onProgress('Creating database...')
|
|
327
|
+
const db = createDatabase(dbPath)
|
|
328
|
+
|
|
329
|
+
onProgress('Fetching critical packages...')
|
|
330
|
+
const packages = await fetchAllCriticalPackages(onProgress)
|
|
331
|
+
onProgress(`Found ${packages.length} critical packages`)
|
|
332
|
+
|
|
333
|
+
const insertAll = db.transaction((pkgs) => {
|
|
334
|
+
for (const pkg of pkgs) {
|
|
335
|
+
insertPackage(db, pkg)
|
|
336
|
+
insertRepoMetadata(db, pkg.id, pkg.repo_metadata, pkg.host)
|
|
337
|
+
insertAdvisories(db, pkg.id, pkg.advisories)
|
|
338
|
+
}
|
|
339
|
+
})
|
|
340
|
+
|
|
341
|
+
onProgress('Inserting packages...')
|
|
342
|
+
insertAll(packages)
|
|
343
|
+
|
|
344
|
+
if (fetchVersionsData) {
|
|
345
|
+
onProgress('Fetching versions...')
|
|
346
|
+
let completed = 0
|
|
347
|
+
const total = packages.length
|
|
348
|
+
|
|
349
|
+
const processPackage = async (pkg) => {
|
|
350
|
+
const versions = await fetchVersionNumbers(pkg.ecosystem, pkg.name)
|
|
351
|
+
await sleep(RATE_LIMIT_MS)
|
|
352
|
+
return { pkg, versions }
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// Process in batches with concurrency limit
|
|
356
|
+
for (let i = 0; i < packages.length; i += CONCURRENCY) {
|
|
357
|
+
const batch = packages.slice(i, i + CONCURRENCY)
|
|
358
|
+
const results = await Promise.all(batch.map(processPackage))
|
|
359
|
+
|
|
360
|
+
db.transaction(() => {
|
|
361
|
+
for (const { pkg, versions } of results) {
|
|
362
|
+
if (versions.length > 0) {
|
|
363
|
+
insertVersions(db, pkg.id, versions)
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
})()
|
|
367
|
+
|
|
368
|
+
completed += batch.length
|
|
369
|
+
onProgress(`Fetched versions for ${completed}/${total} packages`)
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
updateBuildInfo(db)
|
|
374
|
+
|
|
375
|
+
const info = db.prepare('SELECT * FROM build_info WHERE id = 1').get()
|
|
376
|
+
onProgress(`Build complete: ${info.package_count} packages, ${info.version_count} versions, ${info.advisory_count} advisories`)
|
|
377
|
+
|
|
378
|
+
db.close()
|
|
379
|
+
return info
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
export { build, createDatabase, fetchAllCriticalPackages, fetchVersionNumbers, databasePath }
|
package/package.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@ecosyste-ms/critical",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "SQLite database of critical packages from ecosyste.ms",
|
|
5
|
+
"main": "./lib/index.js",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"bin": {
|
|
8
|
+
"critical": "bin.js"
|
|
9
|
+
},
|
|
10
|
+
"scripts": {
|
|
11
|
+
"build": "node bin.js",
|
|
12
|
+
"test": "node test/index.js"
|
|
13
|
+
},
|
|
14
|
+
"keywords": [
|
|
15
|
+
"ecosystems",
|
|
16
|
+
"packages",
|
|
17
|
+
"sqlite",
|
|
18
|
+
"critical",
|
|
19
|
+
"sbom",
|
|
20
|
+
"vulnerabilities"
|
|
21
|
+
],
|
|
22
|
+
"license": "CC-BY-SA-4.0",
|
|
23
|
+
"repository": {
|
|
24
|
+
"type": "git",
|
|
25
|
+
"url": "https://github.com/ecosyste-ms/critical.git"
|
|
26
|
+
},
|
|
27
|
+
"files": [
|
|
28
|
+
"lib",
|
|
29
|
+
"bin.js",
|
|
30
|
+
"README.md",
|
|
31
|
+
"LICENSE",
|
|
32
|
+
"critical-packages.db"
|
|
33
|
+
],
|
|
34
|
+
"publishConfig": {
|
|
35
|
+
"access": "public"
|
|
36
|
+
},
|
|
37
|
+
"engines": {
|
|
38
|
+
"node": ">=18.0.0"
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"better-sqlite3": "^11.7.0"
|
|
42
|
+
}
|
|
43
|
+
}
|