@vltpkg/security-archive 0.0.0-4

package/LICENSE

@@ -0,0 +1,15 @@
+Copyright (c) vlt technology, Inc.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+   Subject to the terms and conditions of this license, each copyright holder and contributor hereby grants to those receiving rights under this license a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except for failure to satisfy the conditions of this license) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer this software, where such license applies only to those patent claims, already acquired or hereafter acquired, licensable by such copyright holder or contributor that are necessarily infringed by:
+
+(a) their Contribution(s) (the licensed copyrights of copyright holders and non-copyrightable additions of contributors, in source or binary form) alone; or
+(b) combination of their Contribution(s) with the work of authorship to which such Contribution(s) was added by such copyright holder or contributor, if, at the time the Contribution is added, such addition causes such combination to be necessarily infringed. The patent license shall not apply to any other combinations which include the Contribution.
+Except as expressly stated above, no rights or licenses from any copyright holder or contributor is granted under this license, whether expressly, by implication, estoppel or otherwise.
+
+DISCLAIMER
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/README.md

@@ -0,0 +1,38 @@
+# @vltpkg/security-archive
+
+A key/value storage that holds security data for unique package
+versions that are coming from a public registry.
+
+This package serves as the backend for `@vltpkg/query` when using
+pseudo-selectors that rely on security data.
+
+Security data is provided in partnership with
+[Socket](https://socket.dev/).
+
+## Usage
+
+```
+import { actual } from '@vltpkg/graph'
+import { SecurityArchive } from '@vltpkg/security-archive'
+
+const specOptions = {
+  registry: 'https://registry.npmjs.org/',
+}
+const graph = actual.load({
+  ...specOptions,
+  projectRoot: process.cwd(),
+})
+
+const archive = await SecurityArchive.start({ graph, specOptions })
+
+if (archive.ok) {
+  for (const node of graph.nodes.values()) {
+    const securityData = archive.get(node.id)
+    if (securityData) {
+      console.log('securityData', securityData)
+    }
+  }
+} else {
+  console.warn('Failed to start the SecurityArchive')
+}
+```

package/dist/esm/browser.d.ts

@@ -0,0 +1,13 @@
+import type { PackageReportData, SecurityArchiveLike } from './types.ts';
+import type { DepID } from '@vltpkg/dep-id';
+export type * from './types.ts';
+/**
+ * A database of security information for given packages in a graph.
+ */
+export declare class SecurityArchive extends Map<DepID, PackageReportData> implements SecurityArchiveLike {
+    /**
+     * Loads a security archive from a valid JSON dump.
+     */
+    static load(dump: unknown): SecurityArchive;
+}
+//# sourceMappingURL=browser.d.ts.map

package/dist/esm/browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAE3C,mBAAmB,YAAY,CAAA;AAsB/B;;GAEG;AACH,qBAAa,eACX,SAAQ,GAAG,CAAC,KAAK,EAAE,iBAAiB,CACpC,YAAW,mBAAmB;IAE9B;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO;CAQ1B"}

package/dist/esm/browser.js

@@ -0,0 +1,29 @@
+import { asDepID } from '@vltpkg/dep-id/browser';
+import { error } from '@vltpkg/error-cause';
+import { asPackageReportData } from "./types.js";
+const isObj = (o) => !!o && typeof o === 'object';
+const isSecurityArchiveJSON = (json) => isObj(json) &&
+    Object.entries(json).every(([key, value]) => typeof key === 'string' && isObj(value));
+const asSecurityArchiveJSON = (json) => {
+    if (!isSecurityArchiveJSON(json)) {
+        throw error('Invalid security archive JSON', { found: json });
+    }
+    return json;
+};
+/**
+ * A database of security information for given packages in a graph.
+ */
+export class SecurityArchive extends Map {
+    /**
+     * Loads a security archive from a valid JSON dump.
+     */
+    static load(dump) {
+        const archive = new SecurityArchive();
+        const json = asSecurityArchiveJSON(dump);
+        for (const [key, value] of Object.entries(json)) {
+            archive.set(asDepID(key), asPackageReportData(value));
+        }
+        return archive;
+    }
+}
+//# sourceMappingURL=browser.js.map

package/dist/esm/browser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAA;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAUhD,MAAM,KAAK,GAAG,CAAC,CAAU,EAAgC,EAAE,CACzD,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAA;AAE9B,MAAM,qBAAqB,GAAG,CAC5B,IAAa,EACsB,EAAE,CACrC,KAAK,CAAC,IAAI,CAAC;IACX,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CACxB,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,CAC1D,CAAA;AAEH,MAAM,qBAAqB,GAAG,CAC5B,IAAa,EACc,EAAE;IAC7B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAC/D,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,OAAO,eACX,SAAQ,GAA6B;IAGrC;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,IAAa;QACvB,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAA;QACrC,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAA;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;CACF","sourcesContent":["import { asDepID } from '@vltpkg/dep-id/browser'\nimport { error } from '@vltpkg/error-cause'\nimport { asPackageReportData } from './types.ts'\nimport type { JSONField } from '@vltpkg/types'\nimport type {\n  PackageReportData,\n  SecurityArchiveLike,\n} from './types.ts'\nimport type { DepID } from '@vltpkg/dep-id'\n\nexport type * from './types.ts'\n\nconst isObj = (o: unknown): o is Record<string, unknown> =>\n  !!o && typeof o === 'object'\n\nconst isSecurityArchiveJSON = (\n  json: unknown,\n): json is Record<string, JSONField> =>\n  isObj(json) &&\n  Object.entries(json).every(\n    ([key, value]) => typeof key === 'string' && isObj(value),\n  )\n\nconst asSecurityArchiveJSON = (\n  json: unknown,\n): Record<string, JSONField> => {\n  if (!isSecurityArchiveJSON(json)) {\n    throw error('Invalid security archive JSON', { found: json })\n  }\n  return json\n}\n\n/**\n * A database of security information for given packages in a graph.\n */\nexport class SecurityArchive\n  extends Map<DepID, PackageReportData>\n  implements SecurityArchiveLike\n{\n  /**\n   * Loads a security archive from a valid JSON dump.\n   */\n  static load(dump: unknown) {\n    const archive = new SecurityArchive()\n    const json = asSecurityArchiveJSON(dump)\n    for (const [key, value] of Object.entries(json)) {\n      archive.set(asDepID(key), asPackageReportData(value))\n    }\n    return archive\n  }\n}\n"]}

package/dist/esm/index.d.ts

@@ -0,0 +1,74 @@
+import { LRUCache } from 'lru-cache';
+import type { DepID } from '@vltpkg/dep-id';
+import type { PackageReportData, SecurityArchiveLike, SecurityArchiveRefreshOptions } from './types.ts';
+export type * from './types.ts';
+export declare const targetSecurityRegisty = "https://registry.npmjs.org/";
+export type JSONItemResponse = {
+    namespace?: `@{string}`;
+    name: string;
+    version: string;
+};
+export type DBReadEntry = {
+    depID: string;
+    report: string;
+    start: number;
+    ttl: number;
+};
+export type DBWriteEntry = [string, string, number, number];
+export type SecurityArchiveOptions = LRUCache.OptionsBase<DepID, PackageReportData, unknown> & {
+    /**
+     * Security archive does not supports a fetch-on-demand model.
+     */
+    fetchMethod?: undefined;
+    /**
+     * An optional value for the path in which to store the sqlite db.
+     */
+    path?: string;
+    /**
+     * Number of retries attempts to reach the remote security API.
+     */
+    retries?: number;
+};
+export declare const version: string;
+/**
+ * A database of security information for given packages in a graph.
+ *
+ * Using the SecurityArchive.refresh() method will update the local cache
+ * with information from the socket.dev APIs or load from the local storage
+ * if available. Information about package security is then available
+ * using the SecurityArchive.get() method.
+ */
+export declare class SecurityArchive extends LRUCache<DepID, PackageReportData> implements SecurityArchiveLike {
+    #private;
+    /**
+     * True if the refresh process was successful and report data
+     * is available for all public registry packages in the graph.
+     */
+    ok: boolean;
+    /**
+     * Creates a new security archive instance and starts the refresh process.
+     */
+    static start(options: SecurityArchiveOptions & SecurityArchiveRefreshOptions): Promise<SecurityArchive>;
+    /**
+     * By default, limits to 100K entries in the in-memory archive.
+     */
+    static get defaultMax(): number;
+    /**
+     * By default, entries are cached for 3 hours.
+     */
+    static get defaultTtl(): number;
+    constructor(options?: SecurityArchiveOptions);
+    /**
+     * Starts the security archive by providing a {@link GraphLike} instance,
+     * its registry-based nodes are going to be used as valid potential entries.
+     *
+     * Any entry that is missing from the persisted cached values are going
+     * to be requested in a batch-request to the remote socket.dev API.
+     */
+    refresh({ graph, specOptions, }: SecurityArchiveRefreshOptions): Promise<void>;
+    /**
+     * Outputs the current in-memory cache as a JSON object.
+     */
+    toJSON(): Record<DepID, PackageReportData> | undefined;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAOpC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAG3C,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,6BAA6B,EAC9B,MAAM,YAAY,CAAA;AAEnB,mBAAmB,YAAY,CAAA;AAM/B,eAAO,MAAM,qBAAqB,gCAAgC,CAAA;AAElE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,CAAC,EAAE,WAAW,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;AAE3D,MAAM,MAAM,sBAAsB,GAAG,QAAQ,CAAC,WAAW,CACvD,KAAK,EACL,iBAAiB,EACjB,OAAO,CACR,GAAG;IACF;;OAEG;IACH,WAAW,CAAC,EAAE,SAAS,CAAA;IACvB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB,CAAA;AAyBD,eAAO,MAAQ,OAAO,QAKrB,CAAA;AAED;;;;;;;GAOG;AACH,qBAAa,eACX,SAAQ,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CACzC,YAAW,mBAAmB;;IAM9B;;;OAGG;IACH,EAAE,UAAQ;IAEV;;OAEG;WACU,KAAK,CAChB,OAAO,EAAE,sBAAsB,GAAG,6BAA6B;IAOjE;;OAEG;IACH,MAAM,KAAK,UAAU,WAEpB;IAED;;OAEG;IACH,MAAM,KAAK,UAAU,WAEpB;gBAEW,OAAO,GAAE,sBAA2B;IAkMhD;;;;;;OAMG;IACG,OAAO,CAAC,EACZ,KAAK,EACL,WAAW,GACZ,EAAE,6BAA6B;IA+BhC;;OAEG;IACH,MAAM;CASP"}

package/dist/esm/index.js

@@ -0,0 +1,278 @@
+// eslint-disable-next-line import/no-unresolved
+import { DatabaseSync } from 'node:sqlite';
+import { LRUCache } from 'lru-cache';
+import pRetry, { AbortError } from 'p-retry';
+import { loadPackageJson } from 'package-json-from-dist';
+import { asDepID, splitDepID, joinDepIDTuple } from '@vltpkg/dep-id';
+import { error } from '@vltpkg/error-cause';
+import { XDG } from '@vltpkg/xdg';
+import { asPackageReportData } from "./types.js";
+const SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true';
+const SOCKET_PUBLIC_API_TOKEN = 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api';
+export const targetSecurityRegisty = 'https://registry.npmjs.org/';
+/**
+ * Only the public npm registry has information available in the
+ * socket.dev API. This function checks if a given depID is pointing
+ * to the public npm registry and returns `true` if it does.
+ */
+const usesTargetRegistry = (depID, specOptions) => {
+    const [nodeType, nodeRegistry] = splitDepID(depID);
+    if (nodeType !== 'registry') {
+        return false;
+    }
+    const reg = nodeRegistry === '' ?
+        specOptions.registry
+        : specOptions.registries?.[nodeRegistry];
+    return reg === targetSecurityRegisty;
+};
+// Loads the version number to be used in the User-Agent header
+export const { version } = loadPackageJson(import.meta.filename, process.env.__VLT_INTERNAL_CLI_PACKAGE_JSON);
+/**
+ * A database of security information for given packages in a graph.
+ *
+ * Using the SecurityArchive.refresh() method will update the local cache
+ * with information from the socket.dev APIs or load from the local storage
+ * if available. Information about package security is then available
+ * using the SecurityArchive.get() method.
+ */
+export class SecurityArchive extends LRUCache {
+    #fetchedDepIDs = new Set();
+    #path;
+    #retries;
+    /**
+     * True if the refresh process was successful and report data
+     * is available for all public registry packages in the graph.
+     */
+    ok = false;
+    /**
+     * Creates a new security archive instance and starts the refresh process.
+     */
+    static async start(options) {
+        const archive = new SecurityArchive(options);
+        await archive.refresh(options);
+        return archive;
+    }
+    /**
+     * By default, limits to 100K entries in the in-memory archive.
+     */
+    static get defaultMax() {
+        return 100_000;
+    }
+    /**
+     * By default, entries are cached for 3 hours.
+     */
+    static get defaultTtl() {
+        return 1000 * 60 * 60 * 3;
+    }
+    constructor(options = {}) {
+        super({
+            max: SecurityArchive.defaultMax,
+            ttl: SecurityArchive.defaultTtl,
+            ...options,
+        });
+        this.#path =
+            options.path ?? new XDG('vlt').cache('security-archive.db');
+        this.#retries = options.retries ?? 3;
+    }
+    /**
+     * Opens a connection to the database at the given path.
+     */
+    #openDatabase() {
+        const db = new DatabaseSync(this.#path);
+        db.exec('CREATE TABLE IF NOT EXISTS cache ' +
+            '(depID TEXT PRIMARY KEY, report TEXT, ttl INTEGER, start INTEGER) ' +
+            // WITHOUT ROWID models a key=value storage,
+            // enforcing depID must be unique and not null
+            // while using less space and more efficient lookups
+            // https://www.sqlite.org/withoutrowid.html
+            'WITHOUT ROWID');
+        db.exec('PRAGMA journal_mode = TRUNCATE');
+        db.exec('PRAGMA synchronous = NORMAL');
+        return db;
+    }
+    /**
+     * Loads all valid items found in the database into the in-memory cache.
+     */
+    #loadItemsFromDatabase(db, graph) {
+        const depIDs = [];
+        for (const node of graph.nodes.values()) {
+            depIDs.push(`'${node.id}'`);
+        }
+        // retrieves from the db packages using their dep-id reference
+        // and only include entries that have a valid TTL value
+        const dbRead = db.prepare('SELECT depID, report, start, ttl, ' +
+            `(SELECT UNIXEPOCH('subsecond')) as now ` +
+            'FROM cache ' +
+            `WHERE depID IN (${depIDs.join(',')}) ` +
+            'GROUP BY depID HAVING SUM(start + ttl) > now');
+        for (const entry of dbRead.all()) {
+            const { depID, report, start, ttl } = entry;
+            try {
+                this.set(asDepID(depID), asPackageReportData(JSON.parse(report)), {
+                    ttl,
+                    start,
+                });
+            }
+            catch {
+                // prune any invalid entries from the database
+                db.prepare('DELETE FROM cache WHERE depID = ?').run(depID);
+            }
+        }
+    }
+    /**
+     * Queues up all required packages that are missing from the archive.
+     */
+    #queueUpRequiredPackages(graph, specOptions) {
+        const queue = new Set();
+        const nodes = graph.nodes.values();
+        for (const node of nodes) {
+            // only queue up valid public registry
+            // references that are missing from the archive
+            if (usesTargetRegistry(node.id, specOptions) &&
+                !this.has(node.id)) {
+                const purl = `pkg:npm/${node.name}@${node.version}`;
+                queue.add({ purl });
+            }
+        }
+        return queue;
+    }
+    /**
+     * Fetches security information from the socket.dev API.
+     * Returns a string of NDJSON data.
+     */
+    async #retrieveRemoteData(queue) {
+        // fetch information from the socket.dev API
+        const req = await fetch(SOCKET_API_V0_URL, {
+            method: 'POST',
+            headers: {
+                Authorization: `Basic ${Buffer.from(`${SOCKET_PUBLIC_API_TOKEN}:`).toString('base64url')}`,
+                'User-Agent': `@vltpkg/security-archive/${version}`,
+            },
+            body: JSON.stringify({
+                components: Array.from(queue),
+            }),
+        });
+        // on missing valid auth or API, it should abort the retry logic
+        if (req.status === 404) {
+            throw new AbortError('Missing API');
+        }
+        // on any other error throws an error that will trigger retry
+        if (!req.ok || !(req.status >= 200 && req.status <= 299)) {
+            throw error('Failed to fetch security data', { response: req });
+        }
+        const str = await req.text();
+        return str.trim() + '\n';
+    }
+    /**
+     * Parses and load NDJSON data into the in-memory cache.
+     */
+    #loadFromNDJSON(str, graph) {
+        const json = str.split('}\n');
+        for (const line of json) {
+            if (!line.trim())
+                continue;
+            const data = JSON.parse(line + '}');
+            const scope = data.namespace ? `${data.namespace}/` : '';
+            const depID = joinDepIDTuple([
+                'registry',
+                '',
+                `${scope}${data.name}@${data.version}`,
+            ]);
+            const aliasedDepID = joinDepIDTuple([
+                'registry',
+                'npm',
+                `${scope}${data.name}@${data.version}`,
+            ]);
+            const node = graph.nodes.get(depID) ?? graph.nodes.get(aliasedDepID);
+            if (node) {
+                this.#fetchedDepIDs.add(node.id);
+                this.set(node.id, asPackageReportData(data));
+            }
+            else {
+                // eslint-disable-next-line no-console
+                console.warn(`security-archive: failed to find node for ${scope}${data.name}@${data.version} found in the response.`);
+            }
+        }
+    }
+    /**
+     * Store new in-memory items to the database.
+     */
+    #storeNewItemsToDatabase(db) {
+        const insertData = [];
+        for (const depID of this.#fetchedDepIDs) {
+            const entry = this.info(depID);
+            if (entry?.start && entry.ttl) {
+                insertData.push([
+                    depID,
+                    JSON.stringify(entry.value),
+                    entry.start,
+                    entry.ttl,
+                ]);
+            }
+        }
+        const dbWrite = db.prepare('INSERT OR REPLACE INTO cache (depID, report, start, ttl) ' +
+            'VALUES (?, ?, ?, ?)');
+        for (const data of insertData) {
+            dbWrite.run(...data);
+        }
+    }
+    /**
+     * Validates that all public-registry packages in the graph
+     * have a valid report data in the current in-memory cache.
+     */
+    #validateReportData(graph, specOptions) {
+        for (const node of graph.nodes.values()) {
+            if (usesTargetRegistry(node.id, specOptions)) {
+                if (!this.has(node.id)) {
+                    this.ok = false;
+                    return;
+                }
+            }
+        }
+        this.ok = true;
+    }
+    /**
+     * Starts the security archive by providing a {@link GraphLike} instance,
+     * its registry-based nodes are going to be used as valid potential entries.
+     *
+     * Any entry that is missing from the persisted cached values are going
+     * to be requested in a batch-request to the remote socket.dev API.
+     */
+    async refresh({ graph, specOptions, }) {
+        // should start by clearing the current in-memory cache
+        this.clear();
+        const db = this.#openDatabase();
+        try {
+            this.#loadItemsFromDatabase(db, graph);
+            const queue = this.#queueUpRequiredPackages(graph, specOptions);
+            // only reach for the remote API if there are packages queued up
+            if (queue.size > 0) {
+                // Parse the response data
+                const res = await pRetry(() => this.#retrieveRemoteData(queue), { retries: this.#retries });
+                this.#loadFromNDJSON(res, graph);
+                this.#storeNewItemsToDatabase(db);
+            }
+            // validates the refresh process was successful
+            this.#validateReportData(graph, specOptions);
+        }
+        finally {
+            db.exec('PRAGMA optimize');
+            db.close();
+            // TODO: at this point we could spawn a deref process to
+            // remove entries with expired ttl values and vaccum the db
+        }
+    }
+    /**
+     * Outputs the current in-memory cache as a JSON object.
+     */
+    toJSON() {
+        if (!this.ok)
+            return undefined;
+        const obj = {};
+        for (const [key, value] of this.dump()) {
+            obj[key] = value.value;
+        }
+        return obj;
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAYhD,MAAM,iBAAiB,GAAG,4CAA4C,CAAA;AACtE,MAAM,uBAAuB,GAC3B,yDAAyD,CAAA;AAE3D,MAAM,CAAC,MAAM,qBAAqB,GAAG,6BAA6B,CAAA;AAoClE;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,CACzB,KAAY,EACZ,WAAwB,EACf,EAAE;IACX,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAElD,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,GAAG,GACP,YAAY,KAAK,EAAE,CAAC,CAAC;QACnB,WAAW,CAAC,QAAQ;QACtB,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,YAAY,CAAC,CAAA;IAC1C,OAAO,GAAG,KAAK,qBAAqB,CAAA;AACtC,CAAC,CAAA;AAED,+DAA+D;AAC/D,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CACxC,MAAM,CAAC,IAAI,CAAC,QAAQ,EACpB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAG5C,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,eACX,SAAQ,QAAkC;IAG1C,cAAc,GAAG,IAAI,GAAG,EAAS,CAAA;IACjC,KAAK,CAAQ;IACb,QAAQ,CAAQ;IAEhB;;;OAGG;IACH,EAAE,GAAG,KAAK,CAAA;IAEV;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,KAAK,CAChB,OAA+D;QAE/D,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC9B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,KAAK,UAAU;QACnB,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,KAAK,UAAU;QACnB,OAAO,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAC3B,CAAC;IAED,YAAY,UAAkC,EAAE;QAC9C,KAAK,CAAC;YACJ,GAAG,EAAE,eAAe,CAAC,UAAU;YAC/B,GAAG,EAAE,eAAe,CAAC,UAAU;YAC/B,GAAG,OAAO;SACX,CAAC,CAAA;QAEF,IAAI,CAAC,KAAK;YACR,OAAO,CAAC,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;QAC7D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACvC,EAAE,CAAC,IAAI,CACL,mCAAmC;YACjC,oEAAoE;YACpE,4CAA4C;YAC5C,8CAA8C;YAC9C,oDAAoD;YACpD,2CAA2C;YAC3C,eAAe,CAClB,CAAA;QACD,EAAE,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAA;QACzC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,EAAgB,EAAE,KAAgB;QACvD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAA;QAC7B,CAAC;QACD,8DAA8D;QAC9D,uDAAuD;QACvD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,oCAAoC;YAClC,yCAAyC;YACzC,aAAa;YACb,mBAAmB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI;YACvC,8CAA8C,CACjD,CAAA;QACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,EAAmB,EAAE,CAAC;YAClD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,KAAK,CAAA;YAC3C,IAAI,CAAC;gBACH,IAAI,CAAC,GAAG,CACN,OAAO,CAAC,KAAK,CAAC,EACd,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EACvC;oBACE,GAAG;oBACH,KAAK;iBACN,CACF,CAAA;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;gBAC9C,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,wBAAwB,CACtB,KAAgB,EAChB,WAAwB;QAExB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAA;QAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAA;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,sCAAsC;YACtC,+CAA+C;YAC/C,IACE,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;gBACxC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAClB,CAAC;gBACD,MAAM,IAAI,GAAG,WAAW,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;gBACnD,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YACrB,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB,CACvB,KAAkC;QAElC,4CAA4C;QAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,uBAAuB,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;gBAC1F,YAAY,EAAE,4BAA4B,OAAO,EAAE;aACpD;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;aAC9B,CAAC;SACH,CAAC,CAAA;QACF,gEAAgE;QAChE,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,IAAI,UAAU,CAAC,aAAa,CAAC,CAAA;QACrC,CAAC;QACD,6DAA6D;QAC7D,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,KAAK,CAAC,+BAA+B,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAA;QACjE,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;QAC5B,OAAO,GAAG,CAAC,IAAI,EAAE,GAAG,IAAI,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,GAAW,EAAE,KAAgB;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAQ;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAqB,CAAA;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;YACxD,MAAM,KAAK,GAAG,cAAc,CAAC;gBAC3B,UAAU;gBACV,EAAE;gBACF,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE;aACvC,CAAC,CAAA;YACF,MAAM,YAAY,GAAG,cAAc,CAAC;gBAClC,UAAU;gBACV,KAAK;gBACL,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE;aACvC,CAAC,CAAA;YACF,MAAM,IAAI,GACR,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;YACzD,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAA;YAC9C,CAAC;iBAAM,CAAC;gBACN,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CACV,6CAA6C,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,yBAAyB,CACxG,CAAA;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,wBAAwB,CAAC,EAAgB;QACvC,MAAM,UAAU,GAAmB,EAAE,CAAA;QACrC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9B,IAAI,KAAK,EAAE,KAAK,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK;oBACL,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC;oBAC3B,KAAK,CAAC,KAAK;oBACX,KAAK,CAAC,GAAG;iBACV,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACxB,2DAA2D;YACzD,qBAAqB,CACxB,CAAA;QACD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,KAAgB,EAAE,WAAwB;QAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;oBACvB,IAAI,CAAC,EAAE,GAAG,KAAK,CAAA;oBACf,OAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IAChB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,EACZ,KAAK,EACL,WAAW,GACmB;QAC9B,uDAAuD;QACvD,IAAI,CAAC,KAAK,EAAE,CAAA;QAEZ,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QAE/B,IAAI,CAAC;YACH,IAAI,CAAC,sBAAsB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;YAEtC,MAAM,KAAK,GAAG,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAA;YAC/D,gEAAgE;YAChE,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACnB,0BAA0B;gBAC1B,MAAM,GAAG,GAAG,MAAM,MAAM,CACtB,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EACrC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAC3B,CAAA;gBACD,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;gBAChC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC,CAAA;YACnC,CAAC;YAED,+CAA+C;YAC/C,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAA;QAC9C,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;YAC1B,EAAE,CAAC,KAAK,EAAE,CAAA;YACV,wDAAwD;YACxD,2DAA2D;QAC7D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,SAAS,CAAA;QAE9B,MAAM,GAAG,GAAqC,EAAE,CAAA;QAChD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACxB,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;CACF","sourcesContent":["// eslint-disable-next-line import/no-unresolved\nimport { DatabaseSync } from 'node:sqlite'\nimport { LRUCache } from 'lru-cache'\nimport pRetry, { AbortError } from 'p-retry'\nimport { loadPackageJson } from 'package-json-from-dist'\nimport { asDepID, splitDepID, joinDepIDTuple } from '@vltpkg/dep-id'\nimport { error } from '@vltpkg/error-cause'\nimport { XDG } from '@vltpkg/xdg'\nimport { asPackageReportData } from './types.ts'\nimport type { DepID } from '@vltpkg/dep-id'\nimport type { GraphLike } from '@vltpkg/graph'\nimport type { SpecOptions } from '@vltpkg/spec'\nimport type {\n  PackageReportData,\n  SecurityArchiveLike,\n  SecurityArchiveRefreshOptions,\n} from './types.ts'\n\nexport type * from './types.ts'\n\nconst SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true'\nconst SOCKET_PUBLIC_API_TOKEN =\n  'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api'\n\nexport const targetSecurityRegisty = 'https://registry.npmjs.org/'\n\nexport type JSONItemResponse = {\n  namespace?: `@{string}`\n  name: string\n  version: string\n}\n\nexport type DBReadEntry = {\n  depID: string\n  report: string\n  start: number\n  ttl: number\n}\n\nexport type DBWriteEntry = [string, string, number, number]\n\nexport type SecurityArchiveOptions = LRUCache.OptionsBase<\n  DepID,\n  PackageReportData,\n  unknown\n> & {\n  /**\n   * Security archive does not supports a fetch-on-demand model.\n   */\n  fetchMethod?: undefined\n  /**\n   * An optional value for the path in which to store the sqlite db.\n   */\n  path?: string\n  /**\n   * Number of retries attempts to reach the remote security API.\n   */\n  retries?: number\n}\n\n/**\n * Only the public npm registry has information available in the\n * socket.dev API. This function checks if a given depID is pointing\n * to the public npm registry and returns `true` if it does.\n */\nconst usesTargetRegistry = (\n  depID: DepID,\n  specOptions: SpecOptions,\n): boolean => {\n  const [nodeType, nodeRegistry] = splitDepID(depID)\n\n  if (nodeType !== 'registry') {\n    return false\n  }\n\n  const reg =\n    nodeRegistry === '' ?\n      specOptions.registry\n    : specOptions.registries?.[nodeRegistry]\n  return reg === targetSecurityRegisty\n}\n\n// Loads the version number to be used in the User-Agent header\nexport const { version } = loadPackageJson(\n  import.meta.filename,\n  process.env.__VLT_INTERNAL_CLI_PACKAGE_JSON,\n) as {\n  version: string\n}\n\n/**\n * A database of security information for given packages in a graph.\n *\n * Using the SecurityArchive.refresh() method will update the local cache\n * with information from the socket.dev APIs or load from the local storage\n * if available. Information about package security is then available\n * using the SecurityArchive.get() method.\n */\nexport class SecurityArchive\n  extends LRUCache<DepID, PackageReportData>\n  implements SecurityArchiveLike\n{\n  #fetchedDepIDs = new Set<DepID>()\n  #path: string\n  #retries: number\n\n  /**\n   * True if the refresh process was successful and report data\n   * is available for all public registry packages in the graph.\n   */\n  ok = false\n\n  /**\n   * Creates a new security archive instance and starts the refresh process.\n   */\n  static async start(\n    options: SecurityArchiveOptions & SecurityArchiveRefreshOptions,\n  ) {\n    const archive = new SecurityArchive(options)\n    await archive.refresh(options)\n    return archive\n  }\n\n  /**\n   * By default, limits to 100K entries in the in-memory archive.\n   */\n  static get defaultMax() {\n    return 100_000\n  }\n\n  /**\n   * By default, entries are cached for 3 hours.\n   */\n  static get defaultTtl() {\n    return 1000 * 60 * 60 * 3\n  }\n\n  constructor(options: SecurityArchiveOptions = {}) {\n    super({\n      max: SecurityArchive.defaultMax,\n      ttl: SecurityArchive.defaultTtl,\n      ...options,\n    })\n\n    this.#path =\n      options.path ?? new XDG('vlt').cache('security-archive.db')\n    this.#retries = options.retries ?? 3\n  }\n\n  /**\n   * Opens a connection to the database at the given path.\n   */\n  #openDatabase(): DatabaseSync {\n    const db = new DatabaseSync(this.#path)\n    db.exec(\n      'CREATE TABLE IF NOT EXISTS cache ' +\n        '(depID TEXT PRIMARY KEY, report TEXT, ttl INTEGER, start INTEGER) ' +\n        // WITHOUT ROWID models a key=value storage,\n        // enforcing depID must be unique and not null\n        // while using less space and more efficient lookups\n        // https://www.sqlite.org/withoutrowid.html\n        'WITHOUT ROWID',\n    )\n    db.exec('PRAGMA journal_mode = TRUNCATE')\n    db.exec('PRAGMA synchronous = NORMAL')\n    return db\n  }\n\n  /**\n   * Loads all valid items found in the database into the in-memory cache.\n   */\n  #loadItemsFromDatabase(db: DatabaseSync, graph: GraphLike): void {\n    const depIDs: string[] = []\n    for (const node of graph.nodes.values()) {\n      depIDs.push(`'${node.id}'`)\n    }\n    // retrieves from the db packages using their dep-id reference\n    // and only include entries that have a valid TTL value\n    const dbRead = db.prepare(\n      'SELECT depID, report, start, ttl, ' +\n        `(SELECT UNIXEPOCH('subsecond')) as now ` +\n        'FROM cache ' +\n        `WHERE depID IN (${depIDs.join(',')}) ` +\n        'GROUP BY depID HAVING SUM(start + ttl) > now',\n    )\n    for (const entry of dbRead.all() as DBReadEntry[]) {\n      const { depID, report, start, ttl } = entry\n      try {\n        this.set(\n          asDepID(depID),\n          asPackageReportData(JSON.parse(report)),\n          {\n            ttl,\n            start,\n          },\n        )\n      } catch {\n        // prune any invalid entries from the database\n        db.prepare('DELETE FROM cache WHERE depID = ?').run(depID)\n      }\n    }\n  }\n\n  /**\n   * Queues up all required packages that are missing from the archive.\n   */\n  #queueUpRequiredPackages(\n    graph: GraphLike,\n    specOptions: SpecOptions,\n  ): Set<Record<'purl', string>> {\n    const queue = new Set<Record<'purl', string>>()\n    const nodes = graph.nodes.values()\n    for (const node of nodes) {\n      // only queue up valid public registry\n      // references that are missing from the archive\n      if (\n        usesTargetRegistry(node.id, specOptions) &&\n        !this.has(node.id)\n      ) {\n        const purl = `pkg:npm/${node.name}@${node.version}`\n        queue.add({ purl })\n      }\n    }\n    return queue\n  }\n\n  /**\n   * Fetches security information from the socket.dev API.\n   * Returns a string of NDJSON data.\n   */\n  async #retrieveRemoteData(\n    queue: Set<Record<'purl', string>>,\n  ): Promise<string> {\n    // fetch information from the socket.dev API\n    const req = await fetch(SOCKET_API_V0_URL, {\n      method: 'POST',\n      headers: {\n        Authorization: `Basic ${Buffer.from(`${SOCKET_PUBLIC_API_TOKEN}:`).toString('base64url')}`,\n        'User-Agent': `@vltpkg/security-archive/${version}`,\n      },\n      body: JSON.stringify({\n        components: Array.from(queue),\n      }),\n    })\n    // on missing valid auth or API, it should abort the retry logic\n    if (req.status === 404) {\n      throw new AbortError('Missing API')\n    }\n    // on any other error throws an error that will trigger retry\n    if (!req.ok || !(req.status >= 200 && req.status <= 299)) {\n      throw error('Failed to fetch security data', { response: req })\n    }\n\n    const str = await req.text()\n    return str.trim() + '\\n'\n  }\n\n  /**\n   * Parses and load NDJSON data into the in-memory cache.\n   */\n  #loadFromNDJSON(str: string, graph: GraphLike): void {\n    const json = str.split('}\\n')\n    for (const line of json) {\n      if (!line.trim()) continue\n      const data = JSON.parse(line + '}') as JSONItemResponse\n      const scope = data.namespace ? `${data.namespace}/` : ''\n      const depID = joinDepIDTuple([\n        'registry',\n        '',\n        `${scope}${data.name}@${data.version}`,\n      ])\n      const aliasedDepID = joinDepIDTuple([\n        'registry',\n        'npm',\n        `${scope}${data.name}@${data.version}`,\n      ])\n      const node =\n        graph.nodes.get(depID) ?? graph.nodes.get(aliasedDepID)\n      if (node) {\n        this.#fetchedDepIDs.add(node.id)\n        this.set(node.id, asPackageReportData(data))\n      } else {\n        // eslint-disable-next-line no-console\n        console.warn(\n          `security-archive: failed to find node for ${scope}${data.name}@${data.version} found in the response.`,\n        )\n      }\n    }\n  }\n\n  /**\n   * Store new in-memory items to the database.\n   */\n  #storeNewItemsToDatabase(db: DatabaseSync): void {\n    const insertData: DBWriteEntry[] = []\n    for (const depID of this.#fetchedDepIDs) {\n      const entry = this.info(depID)\n      if (entry?.start && entry.ttl) {\n        insertData.push([\n          depID,\n          JSON.stringify(entry.value),\n          entry.start,\n          entry.ttl,\n        ])\n      }\n    }\n    const dbWrite = db.prepare(\n      'INSERT OR REPLACE INTO cache (depID, report, start, ttl) ' +\n        'VALUES (?, ?, ?, ?)',\n    )\n    for (const data of insertData) {\n      dbWrite.run(...data)\n    }\n  }\n\n  /**\n   * Validates that all public-registry packages in the graph\n   * have a valid report data in the current in-memory cache.\n   */\n  #validateReportData(graph: GraphLike, specOptions: SpecOptions) {\n    for (const node of graph.nodes.values()) {\n      if (usesTargetRegistry(node.id, specOptions)) {\n        if (!this.has(node.id)) {\n          this.ok = false\n          return\n        }\n      }\n    }\n    this.ok = true\n  }\n\n  /**\n   * Starts the security archive by providing a {@link GraphLike} instance,\n   * its registry-based nodes are going to be used as valid potential entries.\n   *\n   * Any entry that is missing from the persisted cached values are going\n   * to be requested in a batch-request to the remote socket.dev API.\n   */\n  async refresh({\n    graph,\n    specOptions,\n  }: SecurityArchiveRefreshOptions) {\n    // should start by clearing the current in-memory cache\n    this.clear()\n\n    const db = this.#openDatabase()\n\n    try {\n      this.#loadItemsFromDatabase(db, graph)\n\n      const queue = this.#queueUpRequiredPackages(graph, specOptions)\n      // only reach for the remote API if there are packages queued up\n      if (queue.size > 0) {\n        // Parse the response data\n        const res = await pRetry(\n          () => this.#retrieveRemoteData(queue),\n          { retries: this.#retries },\n        )\n        this.#loadFromNDJSON(res, graph)\n        this.#storeNewItemsToDatabase(db)\n      }\n\n      // validates the refresh process was successful\n      this.#validateReportData(graph, specOptions)\n    } finally {\n      db.exec('PRAGMA optimize')\n      db.close()\n      // TODO: at this point we could spawn a deref process to\n      // remove entries with expired ttl values and vaccum the db\n    }\n  }\n\n  /**\n   * Outputs the current in-memory cache as a JSON object.\n   */\n  toJSON() {\n    if (!this.ok) return undefined\n\n    const obj: Record<DepID, PackageReportData> = {}\n    for (const [key, value] of this.dump()) {\n      obj[key] = value.value\n    }\n    return obj\n  }\n}\n"]}

package/dist/esm/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

package/dist/esm/types.d.ts

@@ -0,0 +1,60 @@
+import type { DepID } from '@vltpkg/dep-id';
+import type { GraphLike } from '@vltpkg/graph';
+import type { SpecOptions } from '@vltpkg/spec';
+/**
+ * Parameter options for initializing a security archive.
+ */
+export type SecurityArchiveRefreshOptions = {
+    /**
+     * A @link{GraphLike} instance to find what packages the
+     * security archive should have.
+     */
+    graph: GraphLike;
+    /**
+     * A @link{SpecOptions} instance to use for resolving dependencies.
+     */
+    specOptions: SpecOptions;
+};
+/**
+ * An interface for interacting with a security archive.
+ */
+export interface SecurityArchiveLike {
+    get: (depId: DepID) => PackageReportData | undefined;
+    set: (depId: DepID, data: PackageReportData) => void;
+    delete: (depId: DepID) => void;
+    has: (depId: DepID) => boolean;
+    clear: () => void;
+}
+/**
+ * Package alert extra information.
+ */
+export type PackageAlertProps = {
+    lastPublish: string;
+};
+/**
+ * A known alert for a given package.
+ */
+export type PackageAlert = {
+    key: string;
+    type: string;
+    severity: 'low' | 'middle' | 'high' | 'critical';
+    category: string;
+    props: PackageAlertProps;
+};
+/**
+ * The report data for a given package.
+ */
+export type PackageReportData = {
+    id: string;
+    author: string[];
+    size: number;
+    type: 'npm';
+    namespace?: `@${string}`;
+    name: string;
+    version: string;
+    license: string;
+    alerts: PackageAlert[];
+};
+export declare const isPackageReportData: (o: unknown) => o is PackageReportData;
+export declare const asPackageReportData: (o: unknown) => PackageReportData;
+//# sourceMappingURL=types.d.ts.map

package/dist/esm/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAE/C;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C;;;OAGG;IACH,KAAK,EAAE,SAAS,CAAA;IAChB;;OAEG;IACH,WAAW,EAAE,WAAW,CAAA;CACzB,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,iBAAiB,GAAG,SAAS,CAAA;IACpD,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAA;IACpD,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;IAC9B,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,OAAO,CAAA;IAC9B,KAAK,EAAE,MAAM,IAAI,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAA;IAChD,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,iBAAiB,CAAA;CACzB,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,KAAK,CAAA;IACX,SAAS,CAAC,EAAE,IAAI,MAAM,EAAE,CAAA;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,YAAY,EAAE,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,mBAAmB,MAC3B,OAAO,KACT,CAAC,IAAI,iBAQU,CAAA;AAElB,eAAO,MAAM,mBAAmB,MAC3B,OAAO,KACT,iBAKF,CAAA"}

package/dist/esm/types.js

@@ -0,0 +1,16 @@
+import { error } from '@vltpkg/error-cause';
+export const isPackageReportData = (o) => typeof o === 'object' &&
+    o != null &&
+    'id' in o &&
+    'type' in o &&
+    'name' in o &&
+    'version' in o &&
+    'alerts' in o &&
+    o.type === 'npm';
+export const asPackageReportData = (o) => {
+    if (!isPackageReportData(o)) {
+        throw error('Invalid package report data', { found: o });
+    }
+    return o;
+};
+//# sourceMappingURL=types.js.map

package/dist/esm/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAgE3C,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,CAAU,EACc,EAAE,CAC1B,OAAO,CAAC,KAAK,QAAQ;IACrB,CAAC,IAAI,IAAI;IACT,IAAI,IAAI,CAAC;IACT,MAAM,IAAI,CAAC;IACX,MAAM,IAAI,CAAC;IACX,SAAS,IAAI,CAAC;IACd,QAAQ,IAAI,CAAC;IACb,CAAC,CAAC,IAAI,KAAK,KAAK,CAAA;AAElB,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,CAAU,EACS,EAAE;IACrB,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;IAC1D,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC,CAAA","sourcesContent":["import { error } from '@vltpkg/error-cause'\nimport type { DepID } from '@vltpkg/dep-id'\nimport type { GraphLike } from '@vltpkg/graph'\nimport type { SpecOptions } from '@vltpkg/spec'\n\n/**\n * Parameter options for initializing a security archive.\n */\nexport type SecurityArchiveRefreshOptions = {\n  /**\n   * A @link{GraphLike} instance to find what packages the\n   * security archive should have.\n   */\n  graph: GraphLike\n  /**\n   * A @link{SpecOptions} instance to use for resolving dependencies.\n   */\n  specOptions: SpecOptions\n}\n\n/**\n * An interface for interacting with a security archive.\n */\nexport interface SecurityArchiveLike {\n  get: (depId: DepID) => PackageReportData | undefined\n  set: (depId: DepID, data: PackageReportData) => void\n  delete: (depId: DepID) => void\n  has: (depId: DepID) => boolean\n  clear: () => void\n}\n\n/**\n * Package alert extra information.\n */\nexport type PackageAlertProps = {\n  lastPublish: string\n}\n\n/**\n * A known alert for a given package.\n */\nexport type PackageAlert = {\n  key: string\n  type: string\n  severity: 'low' | 'middle' | 'high' | 'critical'\n  category: string\n  props: PackageAlertProps\n}\n\n/**\n * The report data for a given package.\n */\nexport type PackageReportData = {\n  id: string\n  author: string[]\n  size: number\n  type: 'npm'\n  namespace?: `@${string}`\n  name: string\n  version: string\n  license: string\n  alerts: PackageAlert[]\n}\n\nexport const isPackageReportData = (\n  o: unknown,\n): o is PackageReportData =>\n  typeof o === 'object' &&\n  o != null &&\n  'id' in o &&\n  'type' in o &&\n  'name' in o &&\n  'version' in o &&\n  'alerts' in o &&\n  o.type === 'npm'\n\nexport const asPackageReportData = (\n  o: unknown,\n): PackageReportData => {\n  if (!isPackageReportData(o)) {\n    throw error('Invalid package report data', { found: o })\n  }\n  return o\n}\n"]}

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "@vltpkg/security-archive",
+  "description": "A database of security information of packages",
+  "version": "0.0.0-4",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vltpkg/vltpkg.git",
+    "directory": "src/security-archive"
+  },
+  "tshy": {
+    "selfLink": false,
+    "liveDev": true,
+    "dialects": [
+      "esm"
+    ],
+    "exports": {
+      "./package.json": "./package.json",
+      ".": "./src/index.ts"
+    }
+  },
+  "dependencies": {
+    "lru-cache": "^11.0.2",
+    "p-retry": "^6.2.1",
+    "package-json-from-dist": "^1.0.0",
+    "@vltpkg/dep-id": "0.0.0-4",
+    "@vltpkg/error-cause": "0.0.0-4",
+    "@vltpkg/spec": "0.0.0-4",
+    "@vltpkg/graph": "0.0.0-4",
+    "@vltpkg/types": "0.0.0-4",
+    "@vltpkg/xdg": "0.0.0-4"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.20.0",
+    "@types/eslint__js": "^8.42.3",
+    "@types/node": "^22.13.1",
+    "eslint": "^9.20.0",
+    "prettier": "^3.4.2",
+    "tap": "^21.1.0",
+    "tshy": "^3.0.2",
+    "typedoc": "0.27.6",
+    "typescript": "5.7.3",
+    "typescript-eslint": "^8.23.0"
+  },
+  "license": "BSD-2-Clause-Patent",
+  "engines": {
+    "node": ">=22"
+  },
+  "tap": {
+    "extends": "../../tap-config.yaml"
+  },
+  "prettier": "../../.prettierrc.js",
+  "module": "./dist/esm/index.js",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": {
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      }
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "format": "prettier --write . --log-level warn --ignore-path ../../.prettierignore --cache",
+    "format:check": "prettier --check . --ignore-path ../../.prettierignore --cache",
+    "lint": "eslint . --fix",
+    "lint:check": "eslint .",
+    "snap": "tap",
+    "test": "tap",
+    "posttest": "tsc --noEmit",
+    "typecheck": "tsc --noEmit"
+  }
+}