@vltpkg/security-archive 1.4.0 → 1.6.0

package/README.md

@@ -23,7 +23,9 @@ const graph = actual.load({
   projectRoot: process.cwd(),
 })
 
-const archive = await SecurityArchive.start({ graph, specOptions })
+const archive = await SecurityArchive.start({
+  nodes: [...graph.nodes.values()],
+})
 
 if (archive.ok) {
   for (const node of graph.nodes.values()) {
@@ -36,3 +38,14 @@ if (archive.ok) {
   console.warn('Failed to start the SecurityArchive')
 }
 ```
+
+## Options
+
+- `path`: sqlite db location. Defaults to the vlt XDG cache dir.
+- `retries`: retry attempts on failed API requests. Defaults to `3`.
+- `timeout`: max ms to wait for the API, retries included. Defaults to
+  `VLT_SECURITY_ARCHIVE_TIMEOUT` env or `30000`. Must be a positive
+  integer, other values are ignored. On timeout a warning is printed,
+  missing packages stay unscanned, `ok` is `false`, `timedOut` is
+  `true` and nothing is cached.
+- `ttl`: ms to cache newly fetched entries for. Defaults to 3 hours.

package/dist/index.d.ts

@@ -24,6 +24,14 @@ export type SecurityArchiveOptions = LRUCache.OptionsBase<DepID, PackageReportDa
      * Number of retries attempts to reach the remote security API.
      */
     retries?: number;
+    /**
+     * Max ms to wait for the remote security API, incl. retries.
+     * Defaults to `VLT_SECURITY_ARCHIVE_TIMEOUT` env or
+     * {@link SecurityArchive.defaultTimeout}. Positive integers only,
+     * other values are ignored. On timeout, missing packages are left
+     * unscanned and a warning is printed.
+     */
+    timeout?: number;
 };
 /**
  * A database of security information for given packages from a list of nodes.
@@ -31,7 +39,8 @@ export type SecurityArchiveOptions = LRUCache.OptionsBase<DepID, PackageReportDa
  * Using the SecurityArchive.refresh() method will update the local cache
  * with information from the socket.dev APIs or load from the local storage
  * if available. Information about package security is then available
- * using the SecurityArchive.get() method.
+ * using the SecurityArchive.get() method. Remote requests are bounded
+ * by the `timeout` option.
  */
 export declare class SecurityArchive extends LRUCache<DepID, PackageReportData> implements SecurityArchiveLike {
     #private;
@@ -40,6 +49,10 @@ export declare class SecurityArchive extends LRUCache<DepID, PackageReportData>
      * for all public registry packages from the initial list of nodes.
      */
     ok: boolean;
+    /**
+     * True if the last refresh() timed out waiting for the remote API.
+     */
+    timedOut: boolean;
     /**
      * Creates a new security archive instance and starts the refresh process.
      */
@@ -52,6 +65,10 @@ export declare class SecurityArchive extends LRUCache<DepID, PackageReportData>
      * By default, entries are cached for 3 hours.
      */
     static get defaultTtl(): number;
+    /**
+     * By default, remote requests time out after 30 seconds.
+     */
+    static get defaultTimeout(): number;
     constructor(options?: SecurityArchiveOptions);
     /**
      * Get the report data for a node, keyed by its base {@link DepID}.

package/dist/index.js

@@ -16,18 +16,27 @@ export * from "./types.js";
 export { npmRegistryURL, usesNpmRegistry } from "./browser.js";
 const SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true';
 const SOCKET_PUBLIC_API_TOKEN = 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api';
+// AbortSignal.timeout throws on <0/NaN and overflows to 1ms above 2^31-1
+const asTimeout = (raw) => {
+    const n = Number(raw);
+    return Number.isInteger(n) && n > 0 && n <= 2_147_483_647 ?
+        n
+        : undefined;
+};
 /**
  * A database of security information for given packages from a list of nodes.
  *
  * Using the SecurityArchive.refresh() method will update the local cache
  * with information from the socket.dev APIs or load from the local storage
  * if available. Information about package security is then available
- * using the SecurityArchive.get() method.
+ * using the SecurityArchive.get() method. Remote requests are bounded
+ * by the `timeout` option.
  */
 export class SecurityArchive extends LRUCache {
     #expired = new Set();
     #path;
     #retries;
+    #timeout;
     #nodesByName = new Map();
     #nodesByID = new Map();
     /**
@@ -35,6 +44,10 @@ export class SecurityArchive extends LRUCache {
      * for all public registry packages from the initial list of nodes.
      */
     ok = false;
+    /**
+     * True if the last refresh() timed out waiting for the remote API.
+     */
+    timedOut = false;
     /**
      * Creates a new security archive instance and starts the refresh process.
      */
@@ -55,6 +68,12 @@ export class SecurityArchive extends LRUCache {
     static get defaultTtl() {
         return 1000 * 60 * 60 * 3;
     }
+    /**
+     * By default, remote requests time out after 30 seconds.
+     */
+    static get defaultTimeout() {
+        return 30_000;
+    }
     constructor(options = {}) {
         super({
             max: _a.defaultMax,
@@ -65,6 +84,10 @@ export class SecurityArchive extends LRUCache {
         this.#path =
             options.path ?? new XDG('vlt').cache('security-archive.db');
         this.#retries = options.retries ?? 3;
+        this.#timeout =
+            asTimeout(options.timeout) ??
+                asTimeout(process.env.VLT_SECURITY_ARCHIVE_TIMEOUT) ??
+                _a.defaultTimeout;
     }
     /**
      * Get the report data for a node, keyed by its base {@link DepID}.
@@ -187,6 +210,7 @@ export class SecurityArchive extends LRUCache {
         const payload = {
             dbPath: this.#path,
             retries: this.#retries,
+            timeout: this.#timeout,
             ttl: _a.defaultTtl,
             expired,
         };
@@ -235,7 +259,7 @@ export class SecurityArchive extends LRUCache {
      * Fetches security information from the socket.dev API.
      * Returns a string of NDJSON data.
      */
-    async #retrieveRemoteData(queue) {
+    async #retrieveRemoteData(queue, signal) {
         // fetch information from the socket.dev API
         const req = await fetch(SOCKET_API_V0_URL, {
             method: 'POST',
@@ -246,6 +270,7 @@ export class SecurityArchive extends LRUCache {
             body: JSON.stringify({
                 components: Array.from(queue),
             }),
+            signal,
         });
         // on missing valid auth or API, it should abort the retry logic
         if (req.status === 404) {
@@ -258,6 +283,23 @@ export class SecurityArchive extends LRUCache {
         const str = await req.text();
         return str.trim() + '\n';
     }
+    /**
+     * Fetches remote data within the configured timeout, retries included.
+     * Returns `undefined` on timeout so that queued packages stay unscanned.
+     */
+    async #fetchRemoteData(queue) {
+        const signal = AbortSignal.timeout(this.#timeout);
+        try {
+            return await pRetry(() => this.#retrieveRemoteData(queue, signal), { retries: this.#retries, signal });
+        }
+        catch (err) {
+            if (!signal.aborted)
+                throw err;
+            this.timedOut = true;
+            // eslint-disable-next-line no-console
+            console.warn(`security-archive: timed out after ${this.#timeout}ms fetching security data; ${queue.size} package(s) left unscanned (use :scanned to exclude them). Set VLT_SECURITY_ARCHIVE_TIMEOUT=<ms> to wait longer.`);
+        }
+    }
     /**
      * Parses and load NDJSON data into the in-memory cache.
      * Returns a set of dep ids that were successfully loaded.
@@ -345,6 +387,7 @@ export class SecurityArchive extends LRUCache {
     async refresh({ nodes }) {
         // should start by clearing the current in-memory cache
         this.clear();
+        this.timedOut = false;
         const db = this.#openDatabase();
         // indexes nodes by name and id for quick lookup
         this.#nodesByName.clear();
@@ -362,10 +405,12 @@ export class SecurityArchive extends LRUCache {
             const queue = this.#queueUpRequiredPackages();
             // only reach for the remote API if there are packages queued up
             if (queue.size > 0) {
-                // Parse the response data
-                const res = await pRetry(() => this.#retrieveRemoteData(queue), { retries: this.#retries });
-                const ids = this.#loadFromNDJSON(res);
-                this.#storeNewItemsToDatabase(db, ids);
+                const res = await this.#fetchRemoteData(queue);
+                // on timeout nothing is loaded nor cached
+                if (res !== undefined) {
+                    const ids = this.#loadFromNDJSON(res);
+                    this.#storeNewItemsToDatabase(db, ids);
+                }
             }
             // validates the refresh process was successful
             this.#validateReportData();

package/dist/update-expired.d.ts

@@ -9,6 +9,8 @@ export type UpdateExpiredPayload = {
     dbPath: string;
     /** Number of retries for fetching remote data */
     retries: number;
+    /** Max ms for the remote request, incl. retries */
+    timeout: number;
     /** TTL in ms to use for new entries */
     ttl: number;
     /** Expired entries to revalidate */

package/dist/update-expired.js

@@ -17,7 +17,8 @@ const isMain = (path) => path === __CODE_SPLIT_SCRIPT_NAME ||
 /**
  * Fetches security information from the socket.dev API.
  */
-const retrieveRemoteData = async (queue, retries) => {
+const retrieveRemoteData = async (queue, retries, timeout) => {
+    const signal = AbortSignal.timeout(timeout);
     return pRetry(async () => {
         const req = await fetch(SOCKET_API_V0_URL, {
             method: 'POST',
@@ -28,6 +29,7 @@ const retrieveRemoteData = async (queue, retries) => {
             body: JSON.stringify({
                 components: Array.from(queue),
             }),
+            signal,
         });
         if (req.status === 404) {
             throw new AbortError('Missing API');
@@ -39,7 +41,7 @@ const retrieveRemoteData = async (queue, retries) => {
         }
         const str = await req.text();
         return str.trim() + '\n';
-    }, { retries });
+    }, { retries, signal });
 };
 /**
  * Main entrypoint for the detached process.
@@ -68,7 +70,7 @@ export const main = async (input = process.stdin) => {
         /* c8 ignore next */
         input.on('error', reject);
     });
-    const { dbPath, retries, ttl, expired } = payload;
+    const { dbPath, retries, timeout, ttl, expired } = payload;
     if (!expired.length) {
         return false;
     }
@@ -82,7 +84,7 @@ export const main = async (input = process.stdin) => {
         entryByNameVersion.set(`${entry.name}@${entry.version}`, entry);
     }
     // Fetch updated data
-    const res = await retrieveRemoteData(expiredQueue, retries);
+    const res = await retrieveRemoteData(expiredQueue, retries, timeout);
     // Parse NDJSON response and collect results
     const now = Date.now();
     const results = [];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vltpkg/security-archive",
   "description": "A database of security information of packages",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/vltpkg/vltpkg.git",
@@ -13,12 +13,12 @@
     "url": "http://vlt.sh"
   },
   "dependencies": {
-    "@vltpkg/dep-id": "1.4.0",
-    "@vltpkg/error-cause": "1.4.0",
-    "@vltpkg/spec": "1.4.0",
-    "@vltpkg/types": "1.4.0",
-    "@vltpkg/user-agent": "1.4.0",
-    "@vltpkg/xdg": "1.4.0",
+    "@vltpkg/dep-id": "1.6.0",
+    "@vltpkg/error-cause": "1.6.0",
+    "@vltpkg/spec": "1.6.0",
+    "@vltpkg/types": "1.6.0",
+    "@vltpkg/user-agent": "1.6.0",
+    "@vltpkg/xdg": "1.6.0",
     "lru-cache": "^11.2.4",
     "p-retry": "^7.1.1"
   },