@renovatebot/osv-offline-db 2.1.1 → 2.2.0

package/dist/lib/db.d.ts

@@ -1,11 +1,18 @@
 import { Ecosystem } from './ecosystem';
-import type { Osv } from '..';
+import type { Vulnerability } from './osv';
 export declare class OsvOfflineDb {
+    private disposed;
     static readonly rootDirectory: string;
-    private db;
+    private _data;
+    private _db;
+    private _pendingAcs;
+    private readonly _exitHandler;
     private constructor();
     private _load;
     private _init;
+    private _buildIndex;
     static create(): OsvOfflineDb;
-    query(ecosystem: Ecosystem, packageName: string): Promise<Osv.Vulnerability[]>;
+    query(ecosystem: Ecosystem, packageName: string): Promise<Vulnerability[]>;
+    [Symbol.dispose](): void;
+    private _unload;
 }

package/dist/lib/db.js

@@ -4,48 +4,176 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OsvOfflineDb = void 0;
-const os_1 = require("os");
-const path_1 = __importDefault(require("path"));
-const nedb_1 = __importDefault(require("@seald-io/nedb"));
-const ecosystem_1 = require("./ecosystem");
+const promises_1 = require("node:fs/promises");
+const node_fs_1 = require("node:fs");
+const node_util_1 = require("node:util");
+const node_path_1 = __importDefault(require("node:path"));
+const node_os_1 = require("node:os");
+const node_readline_1 = __importDefault(require("node:readline"));
 const purl_helper_1 = require("./purl-helper");
 const debug_1 = __importDefault(require("debug"));
 const logger = (0, debug_1.default)('osv-offline:db');
+const readAsync = (0, node_util_1.promisify)(node_fs_1.read);
 class OsvOfflineDb {
-    static rootDirectory = process.env.OSV_OFFLINE_ROOT_DIR ?? path_1.default.join((0, os_1.tmpdir)(), 'osv-offline');
-    db = {};
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    constructor() { }
-    _load(ecosystem) {
-        return (this.db[ecosystem] ??= this._init(ecosystem));
+    disposed = false;
+    static rootDirectory = process.env.OSV_OFFLINE_ROOT_DIR ?? node_path_1.default.join((0, node_os_1.tmpdir)(), 'osv-offline');
+    _data = {};
+    _db = {};
+    _pendingAcs = new Set();
+    _exitHandler = () => {
+        if (this.disposed) {
+            return;
+        }
+        logger('Databases are not disposed! Please explicitly dispose.');
+        this[Symbol.dispose]();
+    };
+    constructor() {
+        process.on('exit', this._exitHandler);
+    }
+    async _load(ecosystem) {
+        const cached = this._data[ecosystem];
+        if (cached) {
+            const fileStat = await (0, promises_1.stat)(cached.filePath).catch(() => null);
+            // Check if the data currently in memory is still the same object we checked against
+            // If not, another request has already handled the reload/unload
+            if (this._data[ecosystem] !== cached) {
+                return (this._db[ecosystem] ??= this._init(ecosystem));
+            }
+            if (fileStat?.mtimeMs !== cached.mtime) {
+                logger(fileStat
+                    ? `File changed for ecosystem '${ecosystem}', reloading ...`
+                    : `File removed for ecosystem '${ecosystem}', unloading ...`);
+                // Delete references before unloading to prevent concurrent callers from picking up stale data
+                delete this._data[ecosystem];
+                delete this._db[ecosystem];
+                this._unload(cached);
+            }
+        }
+        return (this._db[ecosystem] ??= this._init(ecosystem));
     }
     async _init(ecosystem) {
-        if (!ecosystem_1.ecosystems.includes(ecosystem)) {
-            throw new Error(`Ecosystem not supported`);
+        logger(`Initializing ecosystem '${ecosystem}' ...`);
+        const ac = new AbortController();
+        this._pendingAcs.add(ac);
+        try {
+            const filePath = node_path_1.default.join(OsvOfflineDb.rootDirectory, `${ecosystem.toLowerCase()}.nedb`);
+            const fileStat = await (0, promises_1.stat)(filePath).catch(() => null);
+            if (!fileStat) {
+                logger(`Missing data for ecosystem '${ecosystem}'`);
+                delete this._db[ecosystem];
+                return null;
+            }
+            const data = {
+                filePath,
+                fd: (0, node_fs_1.openSync)(filePath, 'r'),
+                index: new Map(),
+                mtime: fileStat.mtimeMs,
+            };
+            await this._buildIndex(ac, data);
+            if (ac.signal.aborted) {
+                logger(`Initializing ecosystem '${ecosystem}' aborted.`);
+                delete this._db[ecosystem];
+                this._unload(data);
+                return null;
+            }
+            this._data[ecosystem] = data;
+            logger(`Initializing ecosystem '${ecosystem}' done.`);
+            return data;
         }
-        logger(`Initializing database for ecosystem: ${ecosystem}`);
-        const db = new nedb_1.default({
-            filename: path_1.default.join(OsvOfflineDb.rootDirectory, `${ecosystem.toLowerCase()}.nedb`),
+        finally {
+            this._pendingAcs.delete(ac);
+        }
+    }
+    async _buildIndex(ac, data) {
+        const fileStream = (0, node_fs_1.createReadStream)(data.filePath);
+        const rl = node_readline_1.default.createInterface({
+            input: fileStream,
+            crlfDelay: Infinity,
         });
-        await db.loadDatabaseAsync();
-        return db;
+        let currentOffset = 0;
+        const affectedPackageNames = new Set();
+        for await (const line of rl) {
+            if (ac.signal.aborted) {
+                fileStream.destroy();
+                return;
+            }
+            const lineByteLength = Buffer.byteLength(line, 'utf8');
+            try {
+                const record = JSON.parse(line);
+                if (record.affected) {
+                    for (const affected of record.affected) {
+                        const packageName = affected.package?.name;
+                        if (packageName && !affectedPackageNames.has(packageName)) {
+                            affectedPackageNames.add(packageName);
+                            let pointers = data.index.get(packageName);
+                            if (!pointers) {
+                                pointers = [];
+                                data.index.set(packageName, pointers);
+                            }
+                            pointers.push({ offset: currentOffset, length: lineByteLength });
+                        }
+                    }
+                    affectedPackageNames.clear();
+                }
+            }
+            catch {
+                // Skip malformed lines
+            }
+            currentOffset += lineByteLength + 1;
+        }
     }
     static create() {
         const osvOfflineDb = new OsvOfflineDb();
         return osvOfflineDb;
     }
     async query(ecosystem, packageName) {
-        return await (await this._load(ecosystem)).findAsync({
-            affected: {
-                $elemMatch: {
-                    package: {
-                        name: packageName,
-                        ecosystem,
-                        purl: (0, purl_helper_1.packageToPurl)(ecosystem, packageName),
-                    },
-                },
-            },
-        });
+        if (this.disposed) {
+            throw new Error('Database disposed');
+        }
+        const data = await this._load(ecosystem);
+        if (!data) {
+            return [];
+        }
+        const pointers = data.index.get(packageName);
+        if (!pointers || pointers.length === 0) {
+            return [];
+        }
+        const advisories = await Promise.all(pointers.map(async ({ offset, length }) => {
+            const buffer = Buffer.allocUnsafe(length);
+            const { bytesRead } = await readAsync(data.fd, buffer, 0, length, offset);
+            return JSON.parse(buffer.toString('utf8', 0, bytesRead));
+        }));
+        if (this.disposed)
+            return [];
+        const targetPurl = (0, purl_helper_1.packageToPurl)(ecosystem, packageName);
+        return advisories.filter((vuln) => vuln.affected?.some((a) => a.package?.name === packageName &&
+            a.package.ecosystem === ecosystem &&
+            a.package.purl === targetPurl));
+    }
+    [Symbol.dispose]() {
+        if (this.disposed) {
+            return;
+        }
+        logger(`Disposing databases ...`);
+        this.disposed = true;
+        process.off('exit', this._exitHandler);
+        for (const ac of this._pendingAcs) {
+            ac.abort();
+        }
+        this._pendingAcs.clear();
+        for (const d of Object.values(this._data)) {
+            this._unload(d);
+        }
+        this._db = {};
+        this._data = {};
+    }
+    _unload(d) {
+        try {
+            (0, node_fs_1.closeSync)(d.fd);
+        }
+        catch {
+            // ignore
+        }
     }
 }
 exports.OsvOfflineDb = OsvOfflineDb;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@renovatebot/osv-offline-db",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
@@ -13,7 +13,6 @@
     "url": "https://github.com/renovatebot/osv-offline.git"
   },
   "dependencies": {
-    "@seald-io/nedb": "^4.1.2",
     "debug": "^4.4.3"
   },
   "devDependencies": {
@@ -21,7 +20,7 @@
     "@tsconfig/strictest": "2.0.8",
     "@types/debug": "4.1.12",
     "@types/fs-extra": "11.0.4",
-    "@types/node": "22.19.11",
+    "@types/node": "22.19.13",
     "fs-extra": "11.3.3"
   },
   "engines": {