@ecosyste-ms/critical 1.0.20260328 → 1.1.20260514

package/bin.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { build, databasePath } from './lib/index.js'
-import Database from 'better-sqlite3'
+import { DatabaseSync } from 'node:sqlite'
 import { existsSync } from 'fs'
 
 const args = process.argv.slice(2)
@@ -40,7 +40,7 @@ if (args.includes('--stats')) {
     console.error(`Database not found: ${path}`)
     process.exit(1)
   }
-  const db = new Database(path, { readonly: true })
+  const db = new DatabaseSync(path, { readOnly: true })
   const info = db.prepare('SELECT * FROM build_info WHERE id = 1').get()
   const packageCount = db.prepare('SELECT COUNT(*) as count FROM packages').get().count
   const versionCount = db.prepare('SELECT COUNT(*) as count FROM versions').get().count

package/lib/index.js

@@ -1,4 +1,4 @@
-import Database from 'better-sqlite3'
+import { DatabaseSync } from 'node:sqlite'
 import { createReadStream, createWriteStream, existsSync } from 'fs'
 import { mkdir, unlink, readFile } from 'fs/promises'
 import { dirname, join } from 'path'
@@ -21,22 +21,39 @@ if (!existsSync(databasePath) && existsSync(gzPath)) {
 }
 
 const API_BASE = 'https://packages.ecosyste.ms/api/v1'
-const PER_PAGE = 1000
+const PER_PAGE = 100
 const RATE_LIMIT_MS = 50
 const CONCURRENCY = 10
+const MAX_RETRIES = 5
+const RETRY_BASE_MS = 1000
 
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms))
 }
 
 async function fetchJson(url) {
-  const response = await fetch(url, {
-    headers: { 'User-Agent': USER_AGENT }
-  })
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${url}`)
+  let lastErr
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const response = await fetch(url, {
+        headers: { 'User-Agent': USER_AGENT }
+      })
+      if (response.ok) return response.json()
+      // 4xx is a client error — don't retry. 5xx is server-side; retry.
+      if (response.status < 500) {
+        throw new Error(`HTTP ${response.status}: ${url}`)
+      }
+      lastErr = new Error(`HTTP ${response.status}: ${url}`)
+    } catch (err) {
+      // Network errors (fetch throws TypeError) are retryable.
+      if (err.message?.startsWith('HTTP ') && !err.message.match(/HTTP 5\d\d/)) throw err
+      lastErr = err
+    }
+    if (attempt < MAX_RETRIES) {
+      await sleep(RETRY_BASE_MS * Math.pow(2, attempt))
+    }
   }
-  return response.json()
+  throw lastErr
 }
 
 async function fetchAllCriticalPackages(onProgress) {
@@ -96,8 +113,8 @@ function ecosystemToRegistry(ecosystem) {
 }
 
 function createDatabase(dbPath) {
-  const db = new Database(dbPath)
-  db.pragma('journal_mode = WAL')
+  const db = new DatabaseSync(dbPath)
+  db.exec('PRAGMA journal_mode = WAL')
 
   db.exec(`
     CREATE TABLE packages (
@@ -263,14 +280,14 @@ function insertRepoMetadata(db, packageId, repoMetadata, host) {
 
   stmt.run(
     packageId,
-    repoMetadata.owner,
-    repoMetadata.name,
-    repoMetadata.full_name,
-    host?.name,
-    repoMetadata.language,
-    repoMetadata.stargazers_count,
-    repoMetadata.forks_count,
-    repoMetadata.open_issues_count,
+    repoMetadata.owner ?? null,
+    repoMetadata.name ?? null,
+    repoMetadata.full_name ?? null,
+    host?.name ?? null,
+    repoMetadata.language ?? null,
+    repoMetadata.stargazers_count ?? null,
+    repoMetadata.forks_count ?? null,
+    repoMetadata.open_issues_count ?? null,
     repoMetadata.archived ? 1 : 0,
     repoMetadata.fork ? 1 : 0
   )
@@ -342,16 +359,19 @@ async function build(options = {}) {
   const packages = await fetchAllCriticalPackages(onProgress)
   onProgress(`Found ${packages.length} critical packages`)
 
-  const insertAll = db.transaction((pkgs) => {
-    for (const pkg of pkgs) {
+  onProgress('Inserting packages...')
+  db.exec('BEGIN')
+  try {
+    for (const pkg of packages) {
       insertPackage(db, pkg)
       insertRepoMetadata(db, pkg.id, pkg.repo_metadata, pkg.host)
       insertAdvisories(db, pkg.id, pkg.advisories)
     }
-  })
-
-  onProgress('Inserting packages...')
-  insertAll(packages)
+    db.exec('COMMIT')
+  } catch (err) {
+    db.exec('ROLLBACK')
+    throw err
+  }
 
   if (fetchVersionsData) {
     onProgress('Fetching versions...')
@@ -369,13 +389,18 @@ async function build(options = {}) {
       const batch = packages.slice(i, i + CONCURRENCY)
       const results = await Promise.all(batch.map(processPackage))
 
-      db.transaction(() => {
+      db.exec('BEGIN')
+      try {
         for (const { pkg, versions } of results) {
           if (versions.length > 0) {
             insertVersions(db, pkg.id, versions)
           }
         }
-      })()
+        db.exec('COMMIT')
+      } catch (err) {
+        db.exec('ROLLBACK')
+        throw err
+      }
 
       completed += batch.length
       onProgress(`Fetched versions for ${completed}/${total} packages`)
@@ -391,4 +416,4 @@ async function build(options = {}) {
   return info
 }
 
-export { build, createDatabase, fetchAllCriticalPackages, fetchVersionNumbers, databasePath }
+export { build, createDatabase, fetchAllCriticalPackages, fetchVersionNumbers, databasePath, insertRepoMetadata }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ecosyste-ms/critical",
-  "version": "1.0.20260328",
+  "version": "1.1.20260514",
   "description": "SQLite database of critical packages from ecosyste.ms",
   "main": "./lib/index.js",
   "type": "module",
@@ -35,9 +35,6 @@
     "access": "public"
   },
   "engines": {
-    "node": ">=18.0.0"
-  },
-  "dependencies": {
-    "better-sqlite3": "^12.5.0"
+    "node": ">=22.5.0"
   }
 }