@ecosyste-ms/critical 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,26 @@
1
+ Creative Commons Attribution-ShareAlike 4.0 International
2
+
3
+ Copyright (c) ecosyste.ms
4
+
5
+ This work is licensed under the Creative Commons Attribution-ShareAlike 4.0
6
+ International License. To view a copy of this license, visit
7
+ http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to Creative
8
+ Commons, PO Box 1866, Mountain View, CA 94042, USA.
9
+
10
+ You are free to:
11
+
12
+ Share - copy and redistribute the material in any medium or format
13
+ Adapt - remix, transform, and build upon the material for any purpose,
14
+ even commercially
15
+
16
+ Under the following terms:
17
+
18
+ Attribution - You must give appropriate credit, provide a link to the
19
+ license, and indicate if changes were made.
20
+
21
+ ShareAlike - If you remix, transform, or build upon the material, you
22
+ must distribute your contributions under the same license.
23
+
24
+ No additional restrictions - You may not apply legal terms or technological
25
+ measures that legally restrict others from
26
+ doing anything the license permits.
package/README.md ADDED
@@ -0,0 +1,194 @@
1
+ # @ecosyste-ms/critical
2
+
3
+ SQLite database of critical open source packages from [ecosyste.ms](https://packages.ecosyste.ms).
4
+
5
+ The database is rebuilt daily and published to npm and as a GitHub release. Versions use CalVer (`YYYY.M.D`).
6
+
7
+ ## Install
8
+
9
+ ```bash
10
+ npm install @ecosyste-ms/critical
11
+ ```
12
+
13
+ The package includes a pre-built `critical-packages.db` file:
14
+
15
+ ```javascript
16
+ import { databasePath } from '@ecosyste-ms/critical'
17
+ import Database from 'better-sqlite3'
18
+
19
+ const db = new Database(databasePath)
20
+ const pkg = db.prepare('SELECT * FROM packages WHERE name = ?').get('lodash')
21
+ ```
22
+
23
+ ## Download
24
+
25
+ You can also grab the database directly from the [releases page](../../releases/latest):
26
+
27
+ - `critical-packages.db` - uncompressed SQLite
28
+ - `critical-packages.db.gz` - gzip compressed
29
+
30
+ ## Building
31
+
32
+ To build the database yourself:
33
+
34
+ ```javascript
35
+ import { build, createDatabase } from '@ecosyste-ms/critical'
36
+
37
+ await build({
38
+ dbPath: 'critical-packages.db',
39
+ fetchVersionsData: true,
40
+ onProgress: console.log
41
+ })
42
+ ```
43
+
44
+ From the command line:
45
+
46
+ ```bash
47
+ npx @ecosyste-ms/critical # full build with versions
48
+ npx @ecosyste-ms/critical --skip-versions # faster, packages only
49
+ npx @ecosyste-ms/critical -o my-db.db # custom output path
50
+ npx @ecosyste-ms/critical --stats # show database statistics
51
+ ```
52
+
53
+ ## Schema
54
+
55
+ ### packages
56
+
57
+ | Column | Type | Description |
58
+ |--------|------|-------------|
59
+ | id | INTEGER | Primary key from ecosyste.ms |
60
+ | ecosystem | TEXT | Package ecosystem (npm, pypi, rubygems, etc.) |
61
+ | name | TEXT | Package name |
62
+ | purl | TEXT | Package URL (pkg:npm/lodash) |
63
+ | namespace | TEXT | Package namespace if applicable |
64
+ | description | TEXT | Package description |
65
+ | homepage | TEXT | Project homepage URL |
66
+ | repository_url | TEXT | Source repository URL |
67
+ | licenses | TEXT | SPDX license identifier |
68
+ | normalized_licenses | TEXT | JSON array of normalized license identifiers |
69
+ | latest_version | TEXT | Latest release version number |
70
+ | versions_count | INTEGER | Total number of versions |
71
+ | downloads | INTEGER | Download count |
72
+ | downloads_period | TEXT | Period for download count (e.g., last-month) |
73
+ | dependent_packages_count | INTEGER | Number of packages depending on this |
74
+ | dependent_repos_count | INTEGER | Number of repositories depending on this |
75
+ | first_release_at | TEXT | ISO 8601 timestamp of first release |
76
+ | latest_release_at | TEXT | ISO 8601 timestamp of latest release |
77
+ | last_synced_at | TEXT | When ecosyste.ms last synced this package |
78
+ | keywords | TEXT | Space-separated keywords for FTS |
79
+
80
+ Indexes: `(ecosystem, name)` unique, `purl`, `licenses`, `ecosystem`
81
+
82
+ ### versions
83
+
84
+ | Column | Type | Description |
85
+ |--------|------|-------------|
86
+ | package_id | INTEGER | Foreign key to packages |
87
+ | number | TEXT | Version number |
88
+
89
+ Primary key: `(package_id, number)`
90
+
91
+ ### advisories
92
+
93
+ | Column | Type | Description |
94
+ |--------|------|-------------|
95
+ | id | INTEGER | Auto-increment primary key |
96
+ | package_id | INTEGER | Foreign key to packages |
97
+ | uuid | TEXT | Advisory UUID (GHSA-xxxx-xxxx-xxxx) |
98
+ | url | TEXT | Advisory URL |
99
+ | title | TEXT | Advisory title |
100
+ | description | TEXT | Advisory description |
101
+ | severity | TEXT | Severity level (LOW, MODERATE, HIGH, CRITICAL) |
102
+ | published_at | TEXT | ISO 8601 publish timestamp |
103
+ | cvss_score | REAL | CVSS score |
104
+
105
+ Indexes: `package_id`, `uuid`, `severity`, `(package_id, uuid)` unique
106
+
107
+ ### repo_metadata
108
+
109
+ | Column | Type | Description |
110
+ |--------|------|-------------|
111
+ | package_id | INTEGER | Primary key, foreign key to packages |
112
+ | owner | TEXT | Repository owner/organization |
113
+ | repo_name | TEXT | Repository name |
114
+ | full_name | TEXT | Full name (owner/repo) |
115
+ | host | TEXT | Host (github.com, gitlab.com, etc.) |
116
+ | language | TEXT | Primary language |
117
+ | stargazers_count | INTEGER | GitHub stars |
118
+ | forks_count | INTEGER | Fork count |
119
+ | open_issues_count | INTEGER | Open issues |
120
+ | archived | INTEGER | 1 if archived, 0 otherwise |
121
+ | fork | INTEGER | 1 if a fork, 0 otherwise |
122
+
123
+ Indexes: `full_name`, `owner`
124
+
125
+ ### packages_fts
126
+
127
+ FTS5 virtual table for full-text search on `ecosystem`, `name`, `description`, and `keywords`.
128
+
129
+ ### build_info
130
+
131
+ | Column | Type | Description |
132
+ |--------|------|-------------|
133
+ | id | INTEGER | Always 1 |
134
+ | built_at | TEXT | ISO 8601 build timestamp |
135
+ | package_count | INTEGER | Total packages |
136
+ | version_count | INTEGER | Total versions |
137
+ | advisory_count | INTEGER | Total advisories |
138
+
139
+ ## Example Queries
140
+
141
+ SBOM enrichment by purl:
142
+
143
+ ```sql
144
+ SELECT * FROM packages WHERE purl = 'pkg:npm/lodash';
145
+
146
+ -- Check if a specific version exists
147
+ SELECT * FROM versions v
148
+ JOIN packages p ON v.package_id = p.id
149
+ WHERE p.purl = 'pkg:npm/lodash' AND v.number = '4.17.21';
150
+ ```
151
+
152
+ Find packages with known vulnerabilities:
153
+
154
+ ```sql
155
+ SELECT p.ecosystem, p.name, a.uuid, a.severity, a.title
156
+ FROM packages p
157
+ JOIN advisories a ON p.id = a.package_id
158
+ WHERE a.severity IN ('HIGH', 'CRITICAL');
159
+ ```
160
+
161
+ License audit:
162
+
163
+ ```sql
164
+ SELECT ecosystem, name, licenses
165
+ FROM packages
166
+ WHERE licenses NOT IN ('MIT', 'Apache-2.0', 'BSD-3-Clause');
167
+ ```
168
+
169
+ Full-text search:
170
+
171
+ ```sql
172
+ SELECT p.* FROM packages p
173
+ JOIN packages_fts ON p.id = packages_fts.rowid
174
+ WHERE packages_fts MATCH 'http client';
175
+ ```
176
+
177
+ Packages by ecosystem:
178
+
179
+ ```sql
180
+ SELECT * FROM packages WHERE ecosystem = 'npm';
181
+ ```
182
+
183
+ Most depended-on packages:
184
+
185
+ ```sql
186
+ SELECT ecosystem, name, dependent_packages_count
187
+ FROM packages
188
+ ORDER BY dependent_packages_count DESC
189
+ LIMIT 100;
190
+ ```
191
+
192
+ ## License
193
+
194
+ CC-BY-SA-4.0
package/bin.js ADDED
@@ -0,0 +1,88 @@
1
+ #!/usr/bin/env node
2
+ import { build, databasePath } from './lib/index.js'
3
+ import Database from 'better-sqlite3'
4
+ import { existsSync } from 'fs'
5
+
6
+ const args = process.argv.slice(2)
7
+
8
+ if (args.includes('--help') || args.includes('-h')) {
9
+ console.log(`Usage: critical [options]
10
+
11
+ Options:
12
+ --output, -o <path> Output database path (default: critical-packages.db)
13
+ --skip-versions Skip fetching version data (faster)
14
+ --stats Show database statistics
15
+ --help, -h Show this help message
16
+
17
+ Examples:
18
+ npx @ecosyste-ms/critical
19
+ npx @ecosyste-ms/critical --skip-versions
20
+ npx @ecosyste-ms/critical -o my-database.db
21
+ npx @ecosyste-ms/critical --stats`)
22
+ process.exit(0)
23
+ }
24
+
25
+ function getArg(flags) {
26
+ for (const flag of flags) {
27
+ const idx = args.indexOf(flag)
28
+ if (idx !== -1 && args[idx + 1]) {
29
+ return args[idx + 1]
30
+ }
31
+ }
32
+ return null
33
+ }
34
+
35
+ const dbPath = getArg(['--output', '-o']) || 'critical-packages.db'
36
+
37
+ if (args.includes('--stats')) {
38
+ const path = existsSync(dbPath) ? dbPath : databasePath
39
+ if (!existsSync(path)) {
40
+ console.error(`Database not found: ${path}`)
41
+ process.exit(1)
42
+ }
43
+ const db = new Database(path, { readonly: true })
44
+ const info = db.prepare('SELECT * FROM build_info WHERE id = 1').get()
45
+ const packageCount = db.prepare('SELECT COUNT(*) as count FROM packages').get().count
46
+ const versionCount = db.prepare('SELECT COUNT(*) as count FROM versions').get().count
47
+ const advisoryCount = db.prepare('SELECT COUNT(*) as count FROM advisories').get().count
48
+ const ecosystems = db.prepare(`
49
+ SELECT ecosystem, COUNT(*) as count
50
+ FROM packages
51
+ GROUP BY ecosystem
52
+ ORDER BY count DESC
53
+ `).all()
54
+ const severities = db.prepare(`
55
+ SELECT severity, COUNT(*) as count
56
+ FROM advisories
57
+ GROUP BY severity
58
+ ORDER BY count DESC
59
+ `).all()
60
+
61
+ console.log(`Database: ${path}`)
62
+ console.log(`Built: ${info?.built_at || 'unknown'}`)
63
+ console.log(`\nPackages: ${packageCount}`)
64
+ console.log(`Versions: ${versionCount}`)
65
+ console.log(`Advisories: ${advisoryCount}`)
66
+ console.log(`\nBy ecosystem:`)
67
+ for (const e of ecosystems) {
68
+ console.log(` ${e.ecosystem}: ${e.count}`)
69
+ }
70
+ console.log(`\nBy severity:`)
71
+ for (const s of severities) {
72
+ console.log(` ${s.severity || 'unknown'}: ${s.count}`)
73
+ }
74
+ db.close()
75
+ process.exit(0)
76
+ }
77
+
78
+ const skipVersions = args.includes('--skip-versions')
79
+
80
+ build({
81
+ dbPath,
82
+ fetchVersionsData: !skipVersions
83
+ })
84
+ .then(() => process.exit(0))
85
+ .catch(err => {
86
+ console.error(err)
87
+ process.exit(1)
88
+ })
Binary file
package/lib/index.js ADDED
@@ -0,0 +1,382 @@
1
+ import Database from 'better-sqlite3'
2
+ import { mkdir, unlink, readFile } from 'fs/promises'
3
+ import { dirname, join } from 'path'
4
+ import { fileURLToPath } from 'url'
5
+
6
+ const __dirname = dirname(fileURLToPath(import.meta.url))
7
+ const pkg = JSON.parse(await readFile(join(__dirname, '..', 'package.json'), 'utf8'))
8
+ const USER_AGENT = `${pkg.name}/${pkg.version}`
9
+ const databasePath = join(__dirname, '..', 'critical-packages.db')
10
+
11
+ const API_BASE = 'https://packages.ecosyste.ms/api/v1'
12
+ const PER_PAGE = 1000
13
+ const RATE_LIMIT_MS = 50
14
+ const CONCURRENCY = 10
15
+
16
+ function sleep(ms) {
17
+ return new Promise(resolve => setTimeout(resolve, ms))
18
+ }
19
+
20
+ async function fetchJson(url) {
21
+ const response = await fetch(url, {
22
+ headers: { 'User-Agent': USER_AGENT }
23
+ })
24
+ if (!response.ok) {
25
+ throw new Error(`HTTP ${response.status}: ${url}`)
26
+ }
27
+ return response.json()
28
+ }
29
+
30
+ async function fetchAllCriticalPackages(onProgress) {
31
+ const packages = []
32
+ let page = 1
33
+
34
+ while (true) {
35
+ const url = `${API_BASE}/packages/critical?per_page=${PER_PAGE}&page=${page}`
36
+ onProgress?.(`Fetching page ${page}...`)
37
+
38
+ const batch = await fetchJson(url)
39
+ if (batch.length === 0) break
40
+
41
+ packages.push(...batch)
42
+ page++
43
+
44
+ await sleep(RATE_LIMIT_MS)
45
+ }
46
+
47
+ return packages
48
+ }
49
+
50
+ async function fetchVersionNumbers(ecosystem, name) {
51
+ const registry = ecosystemToRegistry(ecosystem)
52
+ if (!registry) return []
53
+
54
+ const encodedName = encodeURIComponent(name)
55
+ const url = `${API_BASE}/registries/${registry}/packages/${encodedName}/version_numbers`
56
+
57
+ try {
58
+ return await fetchJson(url)
59
+ } catch (err) {
60
+ return []
61
+ }
62
+ }
63
+
64
+ function ecosystemToRegistry(ecosystem) {
65
+ const map = {
66
+ 'npm': 'npmjs.org',
67
+ 'pypi': 'pypi.org',
68
+ 'rubygems': 'rubygems.org',
69
+ 'go': 'proxy.golang.org',
70
+ 'cargo': 'crates.io',
71
+ 'maven': 'repo1.maven.org',
72
+ 'nuget': 'nuget.org',
73
+ 'packagist': 'packagist.org',
74
+ 'hex': 'hex.pm',
75
+ 'pub': 'pub.dev',
76
+ 'hackage': 'hackage.haskell.org',
77
+ 'cocoapods': 'cocoapods.org',
78
+ 'conda': 'anaconda.org',
79
+ 'clojars': 'clojars.org',
80
+ 'puppet': 'forge.puppet.com',
81
+ 'homebrew': 'formulae.brew.sh',
82
+ }
83
+ return map[ecosystem.toLowerCase()] || null
84
+ }
85
+
86
+ function createDatabase(dbPath) {
87
+ const db = new Database(dbPath)
88
+ db.pragma('journal_mode = WAL')
89
+
90
+ db.exec(`
91
+ CREATE TABLE packages (
92
+ id INTEGER PRIMARY KEY,
93
+ ecosystem TEXT NOT NULL,
94
+ name TEXT NOT NULL,
95
+ purl TEXT,
96
+ namespace TEXT,
97
+ description TEXT,
98
+ homepage TEXT,
99
+ repository_url TEXT,
100
+ licenses TEXT,
101
+ normalized_licenses TEXT,
102
+ latest_version TEXT,
103
+ versions_count INTEGER,
104
+ downloads INTEGER,
105
+ downloads_period TEXT,
106
+ dependent_packages_count INTEGER,
107
+ dependent_repos_count INTEGER,
108
+ first_release_at TEXT,
109
+ latest_release_at TEXT,
110
+ last_synced_at TEXT,
111
+ keywords TEXT,
112
+ created_at TEXT DEFAULT CURRENT_TIMESTAMP
113
+ );
114
+
115
+ CREATE UNIQUE INDEX idx_packages_ecosystem_name ON packages(ecosystem, name);
116
+ CREATE INDEX idx_packages_purl ON packages(purl);
117
+ CREATE INDEX idx_packages_licenses ON packages(licenses);
118
+ CREATE INDEX idx_packages_ecosystem ON packages(ecosystem);
119
+
120
+ CREATE TABLE versions (
121
+ package_id INTEGER NOT NULL,
122
+ number TEXT NOT NULL,
123
+ PRIMARY KEY (package_id, number),
124
+ FOREIGN KEY (package_id) REFERENCES packages(id)
125
+ );
126
+
127
+ CREATE TABLE advisories (
128
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
129
+ package_id INTEGER NOT NULL,
130
+ uuid TEXT NOT NULL,
131
+ url TEXT,
132
+ title TEXT,
133
+ description TEXT,
134
+ severity TEXT,
135
+ published_at TEXT,
136
+ cvss_score REAL,
137
+ FOREIGN KEY (package_id) REFERENCES packages(id)
138
+ );
139
+
140
+ CREATE INDEX idx_advisories_package_id ON advisories(package_id);
141
+ CREATE INDEX idx_advisories_uuid ON advisories(uuid);
142
+ CREATE INDEX idx_advisories_severity ON advisories(severity);
143
+ CREATE UNIQUE INDEX idx_advisories_package_uuid ON advisories(package_id, uuid);
144
+
145
+ CREATE TABLE repo_metadata (
146
+ package_id INTEGER PRIMARY KEY,
147
+ owner TEXT,
148
+ repo_name TEXT,
149
+ full_name TEXT,
150
+ host TEXT,
151
+ language TEXT,
152
+ stargazers_count INTEGER,
153
+ forks_count INTEGER,
154
+ open_issues_count INTEGER,
155
+ archived INTEGER,
156
+ fork INTEGER,
157
+ FOREIGN KEY (package_id) REFERENCES packages(id)
158
+ );
159
+
160
+ CREATE INDEX idx_repo_full_name ON repo_metadata(full_name);
161
+ CREATE INDEX idx_repo_owner ON repo_metadata(owner);
162
+
163
+ CREATE VIRTUAL TABLE packages_fts USING fts5(
164
+ ecosystem,
165
+ name,
166
+ description,
167
+ keywords,
168
+ content=packages,
169
+ content_rowid=id
170
+ );
171
+
172
+ CREATE TRIGGER packages_ai AFTER INSERT ON packages BEGIN
173
+ INSERT INTO packages_fts(rowid, ecosystem, name, description, keywords)
174
+ VALUES (new.id, new.ecosystem, new.name, new.description, new.keywords);
175
+ END;
176
+
177
+ CREATE TRIGGER packages_ad AFTER DELETE ON packages BEGIN
178
+ INSERT INTO packages_fts(packages_fts, rowid, ecosystem, name, description, keywords)
179
+ VALUES ('delete', old.id, old.ecosystem, old.name, old.description, old.keywords);
180
+ END;
181
+
182
+ CREATE TRIGGER packages_au AFTER UPDATE ON packages BEGIN
183
+ INSERT INTO packages_fts(packages_fts, rowid, ecosystem, name, description, keywords)
184
+ VALUES ('delete', old.id, old.ecosystem, old.name, old.description, old.keywords);
185
+ INSERT INTO packages_fts(rowid, ecosystem, name, description, keywords)
186
+ VALUES (new.id, new.ecosystem, new.name, new.description, new.keywords);
187
+ END;
188
+
189
+ CREATE TABLE build_info (
190
+ id INTEGER PRIMARY KEY CHECK (id = 1),
191
+ built_at TEXT NOT NULL,
192
+ package_count INTEGER,
193
+ version_count INTEGER,
194
+ advisory_count INTEGER
195
+ );
196
+ `)
197
+
198
+ return db
199
+ }
200
+
201
+ function insertPackage(db, pkg) {
202
+ const stmt = db.prepare(`
203
+ INSERT OR REPLACE INTO packages (
204
+ id, ecosystem, name, purl, namespace, description, homepage,
205
+ repository_url, licenses, normalized_licenses, latest_version,
206
+ versions_count, downloads, downloads_period, dependent_packages_count,
207
+ dependent_repos_count, first_release_at, latest_release_at,
208
+ last_synced_at, keywords
209
+ ) VALUES (
210
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
211
+ )
212
+ `)
213
+
214
+ const keywords = Array.isArray(pkg.keywords_array) ? pkg.keywords_array.join(' ') : null
215
+
216
+ stmt.run(
217
+ pkg.id,
218
+ pkg.ecosystem,
219
+ pkg.name,
220
+ pkg.purl,
221
+ pkg.namespace,
222
+ pkg.description,
223
+ pkg.homepage,
224
+ pkg.repository_url,
225
+ pkg.licenses,
226
+ JSON.stringify(pkg.normalized_licenses),
227
+ pkg.latest_release_number,
228
+ pkg.versions_count,
229
+ pkg.downloads,
230
+ pkg.downloads_period,
231
+ pkg.dependent_packages_count,
232
+ pkg.dependent_repos_count,
233
+ pkg.first_release_published_at,
234
+ pkg.latest_release_published_at,
235
+ pkg.last_synced_at,
236
+ keywords
237
+ )
238
+
239
+ return pkg.id
240
+ }
241
+
242
+ function insertRepoMetadata(db, packageId, repoMetadata, host) {
243
+ if (!repoMetadata) return
244
+
245
+ const stmt = db.prepare(`
246
+ INSERT OR REPLACE INTO repo_metadata (
247
+ package_id, owner, repo_name, full_name, host, language,
248
+ stargazers_count, forks_count, open_issues_count, archived, fork
249
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
250
+ `)
251
+
252
+ stmt.run(
253
+ packageId,
254
+ repoMetadata.owner,
255
+ repoMetadata.name,
256
+ repoMetadata.full_name,
257
+ host?.name,
258
+ repoMetadata.language,
259
+ repoMetadata.stargazers_count,
260
+ repoMetadata.forks_count,
261
+ repoMetadata.open_issues_count,
262
+ repoMetadata.archived ? 1 : 0,
263
+ repoMetadata.fork ? 1 : 0
264
+ )
265
+ }
266
+
267
+ function insertAdvisories(db, packageId, advisories) {
268
+ if (!advisories || advisories.length === 0) return
269
+
270
+ const stmt = db.prepare(`
271
+ INSERT OR REPLACE INTO advisories (
272
+ package_id, uuid, url, title, description, severity, published_at, cvss_score
273
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
274
+ `)
275
+
276
+ for (const advisory of advisories) {
277
+ if (!advisory || !advisory.uuid) continue
278
+ stmt.run(
279
+ packageId,
280
+ advisory.uuid,
281
+ advisory.url,
282
+ advisory.title,
283
+ advisory.description,
284
+ advisory.severity,
285
+ advisory.published_at,
286
+ advisory.cvss_score
287
+ )
288
+ }
289
+ }
290
+
291
+ function insertVersions(db, packageId, versionNumbers) {
292
+ if (!versionNumbers || versionNumbers.length === 0) return
293
+
294
+ const stmt = db.prepare(`
295
+ INSERT OR REPLACE INTO versions (package_id, number) VALUES (?, ?)
296
+ `)
297
+
298
+ for (const number of versionNumbers) {
299
+ stmt.run(packageId, number)
300
+ }
301
+ }
302
+
303
+ function updateBuildInfo(db) {
304
+ const packageCount = db.prepare('SELECT COUNT(*) as count FROM packages').get().count
305
+ const versionCount = db.prepare('SELECT COUNT(*) as count FROM versions').get().count
306
+ const advisoryCount = db.prepare('SELECT COUNT(*) as count FROM advisories').get().count
307
+
308
+ db.prepare(`
309
+ INSERT OR REPLACE INTO build_info (id, built_at, package_count, version_count, advisory_count)
310
+ VALUES (1, ?, ?, ?, ?)
311
+ `).run(new Date().toISOString(), packageCount, versionCount, advisoryCount)
312
+ }
313
+
314
+ async function build(options = {}) {
315
+ const {
316
+ dbPath = 'critical-packages.db',
317
+ fetchVersionsData = true,
318
+ onProgress = console.log
319
+ } = options
320
+
321
+ await mkdir(dirname(dbPath) || '.', { recursive: true }).catch(() => {})
322
+ await unlink(dbPath).catch(() => {})
323
+ await unlink(dbPath + '-wal').catch(() => {})
324
+ await unlink(dbPath + '-shm').catch(() => {})
325
+
326
+ onProgress('Creating database...')
327
+ const db = createDatabase(dbPath)
328
+
329
+ onProgress('Fetching critical packages...')
330
+ const packages = await fetchAllCriticalPackages(onProgress)
331
+ onProgress(`Found ${packages.length} critical packages`)
332
+
333
+ const insertAll = db.transaction((pkgs) => {
334
+ for (const pkg of pkgs) {
335
+ insertPackage(db, pkg)
336
+ insertRepoMetadata(db, pkg.id, pkg.repo_metadata, pkg.host)
337
+ insertAdvisories(db, pkg.id, pkg.advisories)
338
+ }
339
+ })
340
+
341
+ onProgress('Inserting packages...')
342
+ insertAll(packages)
343
+
344
+ if (fetchVersionsData) {
345
+ onProgress('Fetching versions...')
346
+ let completed = 0
347
+ const total = packages.length
348
+
349
+ const processPackage = async (pkg) => {
350
+ const versions = await fetchVersionNumbers(pkg.ecosystem, pkg.name)
351
+ await sleep(RATE_LIMIT_MS)
352
+ return { pkg, versions }
353
+ }
354
+
355
+ // Process in batches with concurrency limit
356
+ for (let i = 0; i < packages.length; i += CONCURRENCY) {
357
+ const batch = packages.slice(i, i + CONCURRENCY)
358
+ const results = await Promise.all(batch.map(processPackage))
359
+
360
+ db.transaction(() => {
361
+ for (const { pkg, versions } of results) {
362
+ if (versions.length > 0) {
363
+ insertVersions(db, pkg.id, versions)
364
+ }
365
+ }
366
+ })()
367
+
368
+ completed += batch.length
369
+ onProgress(`Fetched versions for ${completed}/${total} packages`)
370
+ }
371
+ }
372
+
373
+ updateBuildInfo(db)
374
+
375
+ const info = db.prepare('SELECT * FROM build_info WHERE id = 1').get()
376
+ onProgress(`Build complete: ${info.package_count} packages, ${info.version_count} versions, ${info.advisory_count} advisories`)
377
+
378
+ db.close()
379
+ return info
380
+ }
381
+
382
+ export { build, createDatabase, fetchAllCriticalPackages, fetchVersionNumbers, databasePath }
package/package.json ADDED
@@ -0,0 +1,43 @@
1
+ {
2
+ "name": "@ecosyste-ms/critical",
3
+ "version": "1.0.0",
4
+ "description": "SQLite database of critical packages from ecosyste.ms",
5
+ "main": "./lib/index.js",
6
+ "type": "module",
7
+ "bin": {
8
+ "critical": "bin.js"
9
+ },
10
+ "scripts": {
11
+ "build": "node bin.js",
12
+ "test": "node test/index.js"
13
+ },
14
+ "keywords": [
15
+ "ecosystems",
16
+ "packages",
17
+ "sqlite",
18
+ "critical",
19
+ "sbom",
20
+ "vulnerabilities"
21
+ ],
22
+ "license": "CC-BY-SA-4.0",
23
+ "repository": {
24
+ "type": "git",
25
+ "url": "https://github.com/ecosyste-ms/critical.git"
26
+ },
27
+ "files": [
28
+ "lib",
29
+ "bin.js",
30
+ "README.md",
31
+ "LICENSE",
32
+ "critical-packages.db"
33
+ ],
34
+ "publishConfig": {
35
+ "access": "public"
36
+ },
37
+ "engines": {
38
+ "node": ">=18.0.0"
39
+ },
40
+ "dependencies": {
41
+ "better-sqlite3": "^11.7.0"
42
+ }
43
+ }