@vltpkg/security-archive 1.3.8 → 1.5.0

package/README.md

@@ -23,7 +23,9 @@ const graph = actual.load({
   projectRoot: process.cwd(),
 })
 
-const archive = await SecurityArchive.start({ graph, specOptions })
+const archive = await SecurityArchive.start({
+  nodes: [...graph.nodes.values()],
+})
 
 if (archive.ok) {
   for (const node of graph.nodes.values()) {
@@ -36,3 +38,14 @@ if (archive.ok) {
   console.warn('Failed to start the SecurityArchive')
 }
 ```
+
+## Options
+
+- `path`: sqlite db location. Defaults to the vlt XDG cache dir.
+- `retries`: retry attempts on failed API requests. Defaults to `3`.
+- `timeout`: max ms to wait for the API, retries included. Defaults to
+  `VLT_SECURITY_ARCHIVE_TIMEOUT` env or `30000`. Must be a positive
+  integer, other values are ignored. On timeout a warning is printed,
+  missing packages stay unscanned, `ok` is `false`, `timedOut` is
+  `true` and nothing is cached.
+- `ttl`: ms to cache newly fetched entries for. Defaults to 3 hours.

package/dist/index.d.ts

@@ -3,19 +3,6 @@ import type { DepID } from '@vltpkg/dep-id';
 import type { PackageReportData, SecurityArchiveLike, SecurityArchiveRefreshOptions } from './types.ts';
 export * from './types.ts';
 export { npmRegistryURL, usesNpmRegistry } from './browser.ts';
-export type JSONItemResponse = {
-    namespace?: `@{string}`;
-    name: string;
-    version: string;
-    score: {
-        overall: number;
-        license: number;
-        maintenance: number;
-        quality: number;
-        supplyChain: number;
-        vulnerability: number;
-    };
-};
 export type DBReadEntry = {
     depID: string;
     now: number;
@@ -37,6 +24,14 @@ export type SecurityArchiveOptions = LRUCache.OptionsBase<DepID, PackageReportDa
      * Number of retries attempts to reach the remote security API.
      */
     retries?: number;
+    /**
+     * Max ms to wait for the remote security API, incl. retries.
+     * Defaults to `VLT_SECURITY_ARCHIVE_TIMEOUT` env or
+     * {@link SecurityArchive.defaultTimeout}. Positive integers only,
+     * other values are ignored. On timeout, missing packages are left
+     * unscanned and a warning is printed.
+     */
+    timeout?: number;
 };
 /**
  * A database of security information for given packages from a list of nodes.
@@ -44,7 +39,8 @@ export type SecurityArchiveOptions = LRUCache.OptionsBase<DepID, PackageReportDa
  * Using the SecurityArchive.refresh() method will update the local cache
  * with information from the socket.dev APIs or load from the local storage
  * if available. Information about package security is then available
- * using the SecurityArchive.get() method.
+ * using the SecurityArchive.get() method. Remote requests are bounded
+ * by the `timeout` option.
  */
 export declare class SecurityArchive extends LRUCache<DepID, PackageReportData> implements SecurityArchiveLike {
     #private;
@@ -53,6 +49,10 @@ export declare class SecurityArchive extends LRUCache<DepID, PackageReportData>
      * for all public registry packages from the initial list of nodes.
      */
     ok: boolean;
+    /**
+     * True if the last refresh() timed out waiting for the remote API.
+     */
+    timedOut: boolean;
     /**
      * Creates a new security archive instance and starts the refresh process.
      */
@@ -65,6 +65,10 @@ export declare class SecurityArchive extends LRUCache<DepID, PackageReportData>
      * By default, entries are cached for 3 hours.
      */
     static get defaultTtl(): number;
+    /**
+     * By default, remote requests time out after 30 seconds.
+     */
+    static get defaultTimeout(): number;
     constructor(options?: SecurityArchiveOptions);
     /**
      * Get the report data for a node, keyed by its base {@link DepID}.

package/dist/index.js

@@ -9,25 +9,34 @@ import { asDepID, baseDepID } from '@vltpkg/dep-id';
 import { error } from '@vltpkg/error-cause';
 import { userAgent } from '@vltpkg/user-agent';
 import { XDG } from '@vltpkg/xdg';
-import { asPackageReportData } from "./types.js";
+import { asPackageReportData, getItemScore } from "./types.js";
 import { __CODE_SPLIT_SCRIPT_NAME } from "./update-expired.js";
 import { usesNpmRegistry } from "./browser.js";
 export * from "./types.js";
 export { npmRegistryURL, usesNpmRegistry } from "./browser.js";
 const SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true';
 const SOCKET_PUBLIC_API_TOKEN = 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api';
+// AbortSignal.timeout throws on <0/NaN and overflows to 1ms above 2^31-1
+const asTimeout = (raw) => {
+    const n = Number(raw);
+    return Number.isInteger(n) && n > 0 && n <= 2_147_483_647 ?
+        n
+        : undefined;
+};
 /**
  * A database of security information for given packages from a list of nodes.
  *
  * Using the SecurityArchive.refresh() method will update the local cache
  * with information from the socket.dev APIs or load from the local storage
  * if available. Information about package security is then available
- * using the SecurityArchive.get() method.
+ * using the SecurityArchive.get() method. Remote requests are bounded
+ * by the `timeout` option.
  */
 export class SecurityArchive extends LRUCache {
     #expired = new Set();
     #path;
     #retries;
+    #timeout;
     #nodesByName = new Map();
     #nodesByID = new Map();
     /**
@@ -35,6 +44,10 @@ export class SecurityArchive extends LRUCache {
      * for all public registry packages from the initial list of nodes.
      */
     ok = false;
+    /**
+     * True if the last refresh() timed out waiting for the remote API.
+     */
+    timedOut = false;
     /**
      * Creates a new security archive instance and starts the refresh process.
      */
@@ -55,6 +68,12 @@ export class SecurityArchive extends LRUCache {
     static get defaultTtl() {
         return 1000 * 60 * 60 * 3;
     }
+    /**
+     * By default, remote requests time out after 30 seconds.
+     */
+    static get defaultTimeout() {
+        return 30_000;
+    }
     constructor(options = {}) {
         super({
             max: _a.defaultMax,
@@ -65,6 +84,10 @@ export class SecurityArchive extends LRUCache {
         this.#path =
             options.path ?? new XDG('vlt').cache('security-archive.db');
         this.#retries = options.retries ?? 3;
+        this.#timeout =
+            asTimeout(options.timeout) ??
+                asTimeout(process.env.VLT_SECURITY_ARCHIVE_TIMEOUT) ??
+                _a.defaultTimeout;
     }
     /**
      * Get the report data for a node, keyed by its base {@link DepID}.
@@ -187,6 +210,7 @@ export class SecurityArchive extends LRUCache {
         const payload = {
             dbPath: this.#path,
             retries: this.#retries,
+            timeout: this.#timeout,
             ttl: _a.defaultTtl,
             expired,
         };
@@ -235,7 +259,7 @@ export class SecurityArchive extends LRUCache {
      * Fetches security information from the socket.dev API.
      * Returns a string of NDJSON data.
      */
-    async #retrieveRemoteData(queue) {
+    async #retrieveRemoteData(queue, signal) {
         // fetch information from the socket.dev API
         const req = await fetch(SOCKET_API_V0_URL, {
             method: 'POST',
@@ -246,6 +270,7 @@ export class SecurityArchive extends LRUCache {
             body: JSON.stringify({
                 components: Array.from(queue),
             }),
+            signal,
         });
         // on missing valid auth or API, it should abort the retry logic
         if (req.status === 404) {
@@ -258,6 +283,23 @@ export class SecurityArchive extends LRUCache {
         const str = await req.text();
         return str.trim() + '\n';
     }
+    /**
+     * Fetches remote data within the configured timeout, retries included.
+     * Returns `undefined` on timeout so that queued packages stay unscanned.
+     */
+    async #fetchRemoteData(queue) {
+        const signal = AbortSignal.timeout(this.#timeout);
+        try {
+            return await pRetry(() => this.#retrieveRemoteData(queue, signal), { retries: this.#retries, signal });
+        }
+        catch (err) {
+            if (!signal.aborted)
+                throw err;
+            this.timedOut = true;
+            // eslint-disable-next-line no-console
+            console.warn(`security-archive: timed out after ${this.#timeout}ms fetching security data; ${queue.size} package(s) left unscanned (use :scanned to exclude them). Set VLT_SECURITY_ARCHIVE_TIMEOUT=<ms> to wait longer.`);
+        }
+    }
     /**
      * Parses and load NDJSON data into the in-memory cache.
      * Returns a set of dep ids that were successfully loaded.
@@ -270,30 +312,25 @@ export class SecurityArchive extends LRUCache {
             if (!line.trim())
                 continue;
             const data = JSON.parse(line + '}');
+            // skip the purlError and summary wrapper items the API may
+            // include in the stream, they do not describe a package
+            if (data._type)
+                continue;
             const scope = data.namespace ? `${data.namespace}/` : '';
             const name = `${scope}${data.name}`;
-            const node = this.#retrieveNodeByNameVersion(name, data.version);
+            const node = this.#retrieveNodeByNameVersion(name, String(data.version));
             if (node) {
+                const score = getItemScore(data);
+                // a package that socket.dev has not finished analyzing yet
+                // comes back without a score, leave it out of the archive so
+                // that it gets requested again on the next run
+                if (!score)
+                    continue;
                 const normalizedId = baseDepID(node.id);
-                fetchedDepIDs.add(baseDepID(node.id));
-                // Calculate average score from all score components
-                const scoreComponents = [
-                    data.score.license,
-                    data.score.maintenance,
-                    data.score.quality,
-                    data.score.supplyChain,
-                    data.score.vulnerability,
-                ];
-                const newAverageScore = Number((scoreComponents.reduce((sum, score) => sum + score, 0) /
-                    scoreComponents.length).toFixed(2));
-                // Add average score to the score object
-                const scoreWithNewAverage = {
-                    ...data.score,
-                    overall: newAverageScore,
-                };
+                fetchedDepIDs.add(normalizedId);
                 this.set(normalizedId, asPackageReportData({
                     ...data,
-                    score: scoreWithNewAverage,
+                    score,
                 }));
             }
             else {
@@ -350,6 +387,7 @@ export class SecurityArchive extends LRUCache {
     async refresh({ nodes }) {
         // should start by clearing the current in-memory cache
         this.clear();
+        this.timedOut = false;
         const db = this.#openDatabase();
         // indexes nodes by name and id for quick lookup
         this.#nodesByName.clear();
@@ -367,10 +405,12 @@ export class SecurityArchive extends LRUCache {
             const queue = this.#queueUpRequiredPackages();
             // only reach for the remote API if there are packages queued up
             if (queue.size > 0) {
-                // Parse the response data
-                const res = await pRetry(() => this.#retrieveRemoteData(queue), { retries: this.#retries });
-                const ids = this.#loadFromNDJSON(res);
-                this.#storeNewItemsToDatabase(db, ids);
+                const res = await this.#fetchRemoteData(queue);
+                // on timeout nothing is loaded nor cached
+                if (res !== undefined) {
+                    const ids = this.#loadFromNDJSON(res);
+                    this.#storeNewItemsToDatabase(db, ids);
+                }
             }
             // validates the refresh process was successful
             this.#validateReportData();

package/dist/types.d.ts

@@ -89,3 +89,30 @@ export type PackageReportData = {
 };
 export declare const isPackageReportData: (o: unknown) => o is PackageReportData;
 export declare const asPackageReportData: (o: unknown) => PackageReportData;
+/**
+ * A single line of the NDJSON stream returned by the socket.dev
+ * batch purl API. Besides package artifacts, the stream can carry
+ * `purlError` and `summary` wrapper items, and a package that was
+ * published recently enough to not have been analyzed yet comes back
+ * as an artifact with no `score` object at all.
+ */
+export type JSONItemResponse = {
+    _type?: 'purlError' | 'summary';
+    namespace?: `@${string}`;
+    name?: string;
+    version?: string;
+    score?: {
+        overall?: number;
+        license: number;
+        maintenance: number;
+        quality: number;
+        supplyChain: number;
+        vulnerability: number;
+    };
+};
+/**
+ * Returns the score of a response item with its `overall` value
+ * recalculated as the average of the individual components, or
+ * `undefined` when the item is not a scored package artifact.
+ */
+export declare const getItemScore: (data: JSONItemResponse) => PackageScore | undefined;

package/dist/types.js

@@ -27,3 +27,23 @@ export const asPackageReportData = (o) => {
     }
     return o;
 };
+/**
+ * Returns the score of a response item with its `overall` value
+ * recalculated as the average of the individual components, or
+ * `undefined` when the item is not a scored package artifact.
+ */
+export const getItemScore = (data) => {
+    if (data._type || !data.score)
+        return undefined;
+    const { license, maintenance, quality, supplyChain, vulnerability, } = data.score;
+    const scoreComponents = [
+        license,
+        maintenance,
+        quality,
+        supplyChain,
+        vulnerability,
+    ];
+    const overall = Number((scoreComponents.reduce((sum, score) => sum + score, 0) /
+        scoreComponents.length).toFixed(2));
+    return { ...data.score, overall };
+};

package/dist/update-expired.d.ts

@@ -9,6 +9,8 @@ export type UpdateExpiredPayload = {
     dbPath: string;
     /** Number of retries for fetching remote data */
     retries: number;
+    /** Max ms for the remote request, incl. retries */
+    timeout: number;
     /** TTL in ms to use for new entries */
     ttl: number;
     /** Expired entries to revalidate */

package/dist/update-expired.js

@@ -6,7 +6,7 @@ import pRetry, { AbortError } from 'p-retry';
 import { asDepID, baseDepID } from '@vltpkg/dep-id';
 import { error } from '@vltpkg/error-cause';
 import { userAgent } from '@vltpkg/user-agent';
-import { asPackageReportData } from "./types.js";
+import { asPackageReportData, getItemScore } from "./types.js";
 export const __CODE_SPLIT_SCRIPT_NAME = import.meta.filename;
 const SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true';
 const SOCKET_PUBLIC_API_TOKEN = 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api';
@@ -17,7 +17,8 @@ const isMain = (path) => path === __CODE_SPLIT_SCRIPT_NAME ||
 /**
  * Fetches security information from the socket.dev API.
  */
-const retrieveRemoteData = async (queue, retries) => {
+const retrieveRemoteData = async (queue, retries, timeout) => {
+    const signal = AbortSignal.timeout(timeout);
     return pRetry(async () => {
         const req = await fetch(SOCKET_API_V0_URL, {
             method: 'POST',
@@ -28,6 +29,7 @@ const retrieveRemoteData = async (queue, retries) => {
             body: JSON.stringify({
                 components: Array.from(queue),
             }),
+            signal,
         });
         if (req.status === 404) {
             throw new AbortError('Missing API');
@@ -39,7 +41,7 @@ const retrieveRemoteData = async (queue, retries) => {
         }
         const str = await req.text();
         return str.trim() + '\n';
-    }, { retries });
+    }, { retries, signal });
 };
 /**
  * Main entrypoint for the detached process.
@@ -68,7 +70,7 @@ export const main = async (input = process.stdin) => {
         /* c8 ignore next */
         input.on('error', reject);
     });
-    const { dbPath, retries, ttl, expired } = payload;
+    const { dbPath, retries, timeout, ttl, expired } = payload;
     if (!expired.length) {
         return false;
     }
@@ -82,7 +84,7 @@ export const main = async (input = process.stdin) => {
         entryByNameVersion.set(`${entry.name}@${entry.version}`, entry);
     }
     // Fetch updated data
-    const res = await retrieveRemoteData(expiredQueue, retries);
+    const res = await retrieveRemoteData(expiredQueue, retries, timeout);
     // Parse NDJSON response and collect results
     const now = Date.now();
     const results = [];
@@ -91,6 +93,10 @@ export const main = async (input = process.stdin) => {
         if (!line.trim())
             continue;
         const data = JSON.parse(line + '}');
+        // skip the purlError and summary wrapper items the API may
+        // include in the stream, they do not describe a package
+        if (data._type)
+            continue;
         const scope = data.namespace ? `${data.namespace}/` : '';
         const name = `${scope}${data.name}`;
         const key = `${name}@${data.version}`;
@@ -100,19 +106,15 @@ export const main = async (input = process.stdin) => {
             console.warn(`security-archive: failed to find entry for ${key} found in the response.`);
             continue;
         }
-        // Calculate average score from all score components
-        const scoreComponents = [
-            data.score.license,
-            data.score.maintenance,
-            data.score.quality,
-            data.score.supplyChain,
-            data.score.vulnerability,
-        ];
-        const newAverageScore = Number((scoreComponents.reduce((sum, score) => sum + score, 0) /
-            scoreComponents.length).toFixed(2));
+        const score = getItemScore(data);
+        // a package that socket.dev has not finished analyzing yet comes
+        // back without a score, keep the stale entry rather than replacing
+        // it with an invalid one
+        if (!score)
+            continue;
         const reportData = asPackageReportData({
             ...data,
-            score: { ...data.score, overall: newAverageScore },
+            score,
         });
         results.push({
             depID: baseDepID(asDepID(entry.depID)),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vltpkg/security-archive",
   "description": "A database of security information of packages",
-  "version": "1.3.8",
+  "version": "1.5.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/vltpkg/vltpkg.git",
@@ -13,12 +13,12 @@
     "url": "http://vlt.sh"
   },
   "dependencies": {
-    "@vltpkg/dep-id": "1.3.8",
-    "@vltpkg/error-cause": "1.3.8",
-    "@vltpkg/spec": "1.3.8",
-    "@vltpkg/types": "1.3.8",
-    "@vltpkg/user-agent": "1.3.8",
-    "@vltpkg/xdg": "1.3.8",
+    "@vltpkg/dep-id": "1.5.0",
+    "@vltpkg/error-cause": "1.5.0",
+    "@vltpkg/spec": "1.5.0",
+    "@vltpkg/types": "1.5.0",
+    "@vltpkg/user-agent": "1.5.0",
+    "@vltpkg/xdg": "1.5.0",
     "lru-cache": "^11.2.4",
     "p-retry": "^7.1.1"
   },