@vltpkg/security-archive 1.0.0-rc.24 → 1.0.0-rc.25

package/dist/index.js

@@ -1,4 +1,6 @@
+var _a;
 import { DatabaseSync } from 'node:sqlite';
+import { spawn } from 'node:child_process';
 import { mkdirSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { LRUCache } from 'lru-cache';
@@ -8,6 +10,7 @@ import { asDepID, splitDepID, baseDepID } from '@vltpkg/dep-id';
 import { error } from '@vltpkg/error-cause';
 import { XDG } from '@vltpkg/xdg';
 import { asPackageReportData } from "./types.js";
+import { __CODE_SPLIT_SCRIPT_NAME } from "./update-expired.js";
 export * from "./types.js";
 const SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true';
 const SOCKET_PUBLIC_API_TOKEN = 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api';
@@ -39,7 +42,6 @@ export const { version } = loadPackageJson(import.meta.filename, process.env.__V
  */
 export class SecurityArchive extends LRUCache {
     #expired = new Set();
-    #pUpdateExpired;
     #path;
     #retries;
     #nodesByName = new Map();
@@ -53,7 +55,7 @@ export class SecurityArchive extends LRUCache {
      * Creates a new security archive instance and starts the refresh process.
      */
     static async start(options) {
-        const archive = new SecurityArchive(options);
+        const archive = new _a(options);
         await archive.refresh(options);
         return archive;
     }
@@ -71,8 +73,8 @@ export class SecurityArchive extends LRUCache {
     }
     constructor(options = {}) {
         super({
-            max: SecurityArchive.defaultMax,
-            ttl: SecurityArchive.defaultTtl,
+            max: _a.defaultMax,
+            ttl: _a.defaultTtl,
             ...options,
             allowStale: true,
         });
@@ -141,35 +143,66 @@ export class SecurityArchive extends LRUCache {
                 db.prepare('DELETE FROM cache WHERE depID = ?').run(depID);
             }
         }
-        // TODO: we need to move this to a detached process in order for the
-        // cli commands that make usage of the security-archive, e.g: vlt ls,
-        // vlt query to not hang while waiting for stale-while-revalidate
-        // request to finish and close the db connection
-        this.#pUpdateExpired = this.#updateExpired(db);
+        // Spawn a detached process to revalidate expired entries in the
+        // background so that cli commands (vlt ls, vlt query) don't hang
+        // waiting for stale-while-revalidate requests to complete.
+        this.#spawnUpdateExpired();
     }
     /**
-     * Updates the database with renewed entries for expired packages.
+     * Spawns a detached child process to revalidate expired entries.
+     * The process runs independently — the main process does not wait for it.
      */
-    async #updateExpired(db) {
-        const expiredQueue = new Set();
+    #spawnUpdateExpired() {
+        const expired = [];
         for (const depID of this.#expired) {
             const node = this.#nodesByID.get(baseDepID(depID));
             /* c8 ignore start */
-            if (!node?.version) {
-                // skip any missing node or nodes without a version, this
+            if (!node?.version || !node.name) {
+                // skip any missing node or nodes without a version/name, this
                 // should not happen, but some optional dependencies might
                 // end up in this state when reading from the lockfile
                 continue;
             }
             /* c8 ignore stop */
-            const purl = `pkg:npm/${node.name}@${node.version}`;
-            expiredQueue.add({ purl });
+            expired.push({
+                depID: baseDepID(depID),
+                name: node.name,
+                version: node.version,
+            });
         }
-        if (expiredQueue.size > 0) {
-            const res = await pRetry(() => this.#retrieveRemoteData(expiredQueue), { retries: this.#retries });
-            const ids = this.#loadFromNDJSON(res);
-            this.#storeNewItemsToDatabase(db, ids);
+        if (!expired.length) {
+            return;
         }
+        const isDeno = globalThis.Deno !=
+            undefined;
+        const payload = {
+            dbPath: this.#path,
+            retries: this.#retries,
+            ttl: _a.defaultTtl,
+            expired,
+        };
+        const args = [];
+        // Deno on Windows does not support detached processes
+        /* c8 ignore next */
+        const detached = !(isDeno && process.platform === 'win32');
+        /* c8 ignore start */
+        if (isDeno) {
+            args.push('--unstable-node-globals', '--unstable-bare-node-builtins');
+        }
+        /* c8 ignore stop */
+        args.push(__CODE_SPLIT_SCRIPT_NAME);
+        const proc = spawn(process.execPath, args, {
+            detached,
+            stdio: ['pipe', 'ignore', 'ignore'],
+            env: { ...process.env },
+        });
+        proc.stdin.write(JSON.stringify(payload));
+        proc.stdin.end();
+        /* c8 ignore start */
+        if (detached) {
+            proc.unref();
+        }
+        /* c8 ignore stop */
     }
     /**
      * Queues up all required packages that are missing from the archive.
@@ -335,26 +368,18 @@ export class SecurityArchive extends LRUCache {
             this.#validateReportData();
         }
         finally {
-            // TODO: once we move the stale-while-revalidate to a detached process
-            // the this.#close method no longer needs to be async
-            void this.#close(db);
+            this.#close(db);
         }
     }
     /**
      * Closes the database connection and cleans up the internal state.
      */
-    async #close(db) {
-        // the revalidation of stale entries might not yet be completed
-        // so in case we have a pending promise for that, we need to wait
-        await this.#pUpdateExpired;
+    #close(db) {
         // close db connection
         db.exec('PRAGMA optimize');
         db.close();
         // clean up internal state
         this.#expired.clear();
-        this.#pUpdateExpired = undefined;
-        // TODO: at this point we could spawn a deref process to
-        // remove entries with expired ttl values and vaccum the db
     }
     /**
      * Outputs the current in-memory cache as a JSON object.
@@ -367,3 +392,4 @@ export class SecurityArchive extends LRUCache {
         return obj;
     }
 }
+_a = SecurityArchive;

package/dist/update-expired.d.ts

@@ -0,0 +1,32 @@
+import type { DepID } from '@vltpkg/dep-id';
+import type EventEmitter from 'node:events';
+export declare const __CODE_SPLIT_SCRIPT_NAME: string;
+export declare const version: string;
+/**
+ * Serialized payload sent to the detached process via stdin.
+ */
+export type UpdateExpiredPayload = {
+    /** Path to the sqlite database */
+    dbPath: string;
+    /** Number of retries for fetching remote data */
+    retries: number;
+    /** TTL in ms to use for new entries */
+    ttl: number;
+    /** Expired entries to revalidate */
+    expired: ExpiredEntry[];
+};
+/**
+ * Minimal serialized information for each expired entry.
+ * Only carries what's needed for the API call + DB write.
+ */
+export type ExpiredEntry = {
+    depID: DepID;
+    name: string;
+    version: string;
+};
+/**
+ * Main entrypoint for the detached process.
+ * Reads a JSON payload from stdin, fetches updated security data,
+ * and writes it back to the SQLite database.
+ */
+export declare const main: (input?: EventEmitter) => Promise<boolean>;

package/dist/update-expired.js

@@ -0,0 +1,153 @@
+import { DatabaseSync } from 'node:sqlite';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import pRetry, { AbortError } from 'p-retry';
+import { asDepID, baseDepID } from '@vltpkg/dep-id';
+import { error } from '@vltpkg/error-cause';
+import { loadPackageJson } from 'package-json-from-dist';
+import { asPackageReportData } from "./types.js";
+export const __CODE_SPLIT_SCRIPT_NAME = import.meta.filename;
+const SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true';
+const SOCKET_PUBLIC_API_TOKEN = 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api';
+export const { version } = loadPackageJson(import.meta.filename, process.env.__VLT_INTERNAL_CLI_PACKAGE_JSON);
+/* c8 ignore start */
+const isMain = (path) => path === __CODE_SPLIT_SCRIPT_NAME ||
+    path === pathToFileURL(__CODE_SPLIT_SCRIPT_NAME).toString();
+/* c8 ignore stop */
+/**
+ * Fetches security information from the socket.dev API.
+ */
+const retrieveRemoteData = async (queue, retries) => {
+    return pRetry(async () => {
+        const req = await fetch(SOCKET_API_V0_URL, {
+            method: 'POST',
+            headers: {
+                Authorization: `Basic ${Buffer.from(`${SOCKET_PUBLIC_API_TOKEN}:`).toString('base64url')}`,
+                'User-Agent': `@vltpkg/security-archive/${version}`,
+            },
+            body: JSON.stringify({
+                components: Array.from(queue),
+            }),
+        });
+        if (req.status === 404) {
+            throw new AbortError('Missing API');
+        }
+        if (!req.ok || !(req.status >= 200 && req.status <= 299)) {
+            throw error('Failed to fetch security data', {
+                response: req,
+            });
+        }
+        const str = await req.text();
+        return str.trim() + '\n';
+    }, { retries });
+};
+/**
+ * Main entrypoint for the detached process.
+ * Reads a JSON payload from stdin, fetches updated security data,
+ * and writes it back to the SQLite database.
+ */
+export const main = async (input = process.stdin) => {
+    const payload = await new Promise((resolve, reject) => {
+        const chunks = [];
+        let chunkLen = 0;
+        input.on('data', (chunk) => {
+            chunks.push(chunk);
+            chunkLen += chunk.length;
+        });
+        input.on('end', () => {
+            try {
+                const raw = Buffer.concat(chunks, chunkLen).toString();
+                resolve(JSON.parse(raw));
+                /* c8 ignore start */
+            }
+            catch (err) {
+                reject(err);
+            }
+            /* c8 ignore stop */
+        });
+        /* c8 ignore next */
+        input.on('error', reject);
+    });
+    const { dbPath, retries, ttl, expired } = payload;
+    if (!expired.length) {
+        return false;
+    }
+    // Build purl queue from expired entries
+    const expiredQueue = new Set();
+    // Build a lookup map: name@version -> depID
+    const entryByNameVersion = new Map();
+    for (const entry of expired) {
+        const purl = `pkg:npm/${entry.name}@${entry.version}`;
+        expiredQueue.add({ purl });
+        entryByNameVersion.set(`${entry.name}@${entry.version}`, entry);
+    }
+    // Fetch updated data
+    const res = await retrieveRemoteData(expiredQueue, retries);
+    // Parse NDJSON response and collect results
+    const now = Date.now();
+    const results = [];
+    const json = res.split('}\n');
+    for (const line of json) {
+        if (!line.trim())
+            continue;
+        const data = JSON.parse(line + '}');
+        const scope = data.namespace ? `${data.namespace}/` : '';
+        const name = `${scope}${data.name}`;
+        const key = `${name}@${data.version}`;
+        const entry = entryByNameVersion.get(key);
+        if (!entry) {
+            // eslint-disable-next-line no-console
+            console.warn(`security-archive: failed to find entry for ${key} found in the response.`);
+            continue;
+        }
+        // Calculate average score from all score components
+        const scoreComponents = [
+            data.score.license,
+            data.score.maintenance,
+            data.score.quality,
+            data.score.supplyChain,
+            data.score.vulnerability,
+        ];
+        const newAverageScore = Number((scoreComponents.reduce((sum, score) => sum + score, 0) /
+            scoreComponents.length).toFixed(2));
+        const reportData = asPackageReportData({
+            ...data,
+            score: { ...data.score, overall: newAverageScore },
+        });
+        results.push({
+            depID: baseDepID(asDepID(entry.depID)),
+            report: JSON.stringify(reportData),
+            start: now,
+            ttl,
+        });
+    }
+    if (!results.length) {
+        return false;
+    }
+    // Write to database
+    mkdirSync(dirname(dbPath), { recursive: true });
+    const db = new DatabaseSync(dbPath);
+    db.exec('CREATE TABLE IF NOT EXISTS cache ' +
+        '(depID TEXT PRIMARY KEY, report TEXT, ttl INTEGER, start INTEGER) ' +
+        'WITHOUT ROWID');
+    db.exec('PRAGMA journal_mode = WAL');
+    db.exec('PRAGMA synchronous = NORMAL');
+    const dbWrite = db.prepare('INSERT OR REPLACE INTO cache (depID, report, start, ttl) ' +
+        'VALUES (?, ?, ?, ?)');
+    for (const { depID, report, start, ttl: entryTtl } of results) {
+        dbWrite.run(depID, report, start, entryTtl);
+    }
+    db.exec('PRAGMA optimize');
+    db.close();
+    return true;
+};
+/* c8 ignore start */
+if (isMain(process.argv[1])) {
+    process.title = 'vlt-security-archive-update';
+    const res = await main(process.stdin);
+    if (!res) {
+        process.exit(1);
+    }
+}
+/* c8 ignore stop */

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vltpkg/security-archive",
   "description": "A database of security information of packages",
-  "version": "1.0.0-rc.24",
+  "version": "1.0.0-rc.25",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/vltpkg/vltpkg.git",
@@ -12,11 +12,11 @@
     "email": "support@vlt.sh"
   },
   "dependencies": {
-    "@vltpkg/dep-id": "1.0.0-rc.24",
-    "@vltpkg/error-cause": "1.0.0-rc.24",
-    "@vltpkg/spec": "1.0.0-rc.24",
-    "@vltpkg/types": "1.0.0-rc.24",
-    "@vltpkg/xdg": "1.0.0-rc.24",
+    "@vltpkg/dep-id": "1.0.0-rc.25",
+    "@vltpkg/error-cause": "1.0.0-rc.25",
+    "@vltpkg/spec": "1.0.0-rc.25",
+    "@vltpkg/types": "1.0.0-rc.25",
+    "@vltpkg/xdg": "1.0.0-rc.25",
     "lru-cache": "^11.2.4",
     "p-retry": "^7.1.1",
     "package-json-from-dist": "^1.0.1"
@@ -63,6 +63,11 @@
       "import": {
         "default": "./dist/browser.js"
       }
+    },
+    "./update-expired": {
+      "import": {
+        "default": "./dist/update-expired.js"
+      }
     }
   },
   "files": [