@renovatebot/osv-offline 3.0.12 → 3.1.0

package/README.md

@@ -4,6 +4,15 @@
 [![Build status](https://img.shields.io/github/actions/workflow/status/renovatebot/osv-offline/build.yml?branch=main&style=for-the-badge)](https://github.com/renovatebot/osv-offline/actions/workflows/build.yml)
 [![MIT license](https://img.shields.io/badge/license-MIT-blue?style=for-the-badge)](./LICENSE)
 
+## Environment variables
+
+| Name                           | Description                                                                                                                             | Default                                                                               |
+| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |
+| `OSV_OFFLINE_ROOT_DIR`         | Directory of the downloaded databases                                                                                                   | `<tmpdir>/osv-offline`                                                                |
+| `OSV_OFFLINE_DATABASE_URL`     | URL of the database zip                                                                                                                 | `https://github.com/renovatebot/osv-offline/releases/latest/download/osv-offline.zip` |
+| `OSV_OFFLINE_DISABLE_DOWNLOAD` | Set to `true` to use the databases in `OSV_OFFLINE_ROOT_DIR` without downloading them                                                   | -                                                                                     |
+| `OSV_OFFLINE_MAX_AGE`          | How old the downloaded databases may be before they are downloaded again, as a [`ms`](https://github.com/vercel/ms) duration, e.g. `4h` | `1d`                                                                                  |
+
 ## License
 
 This package is licensed under [the MIT license](https://opensource.org/licenses/MIT).

package/dist/lib/download.js

@@ -4,17 +4,35 @@ import { pipeline } from 'node:stream/promises';
 import { OsvOfflineDb } from '@renovatebot/osv-offline-db';
 import path from 'path';
 import { DateTime } from 'luxon';
-import AdmZip from 'adm-zip';
+import ms from 'ms';
 import { failure, success } from "./types.js";
+import { extractZip } from "./unzip.js";
 import debug from 'debug';
 const logger = debug('osv-offline:download');
+const defaultMaxAge = '1d';
+/**
+ * Returns how many milliseconds old the local databases may be before they are downloaded again: `OSV_OFFLINE_MAX_AGE` (e.g. `4h`), or one day.
+ */
+function getMaxAge() {
+    const value = process.env.OSV_OFFLINE_MAX_AGE;
+    if (!value) {
+        return ms(defaultMaxAge);
+    }
+    // returns `undefined` for an invalid value, despite the type
+    const maxAge = ms(value);
+    if (maxAge === undefined) {
+        logger(`Invalid OSV_OFFLINE_MAX_AGE '${value}', using ${defaultMaxAge}.`);
+        return ms(defaultMaxAge);
+    }
+    return maxAge;
+}
 export async function tryDownloadDb() {
     await fs.ensureDir(OsvOfflineDb.rootDirectory);
     if (process.env.OSV_OFFLINE_DISABLE_DOWNLOAD?.toLowerCase() === 'true') {
         logger('Skipping database download.');
         return success();
     }
-    // if local database exists and is less than a day old, don't do any network requests
+    // if local database exists and is younger than the maximum age, don't do any network requests
     let stats;
     try {
         stats = await fs.stat(path.join(OsvOfflineDb.rootDirectory, `osv-offline.zip`));
@@ -23,27 +41,40 @@ export async function tryDownloadDb() {
         // ignored
     }
     if (stats !== undefined &&
-        DateTime.utc().diff(DateTime.fromJSDate(stats.mtime)).as('days') < 1) {
+        DateTime.utc().diff(DateTime.fromJSDate(stats.mtime)).toMillis() <
+            getMaxAge()) {
         logger('Skipping download, databases are up-to-date.');
         return success();
     }
     // only download databases if local databases are missing or remote is newer
+    // download and extract into a temporary directory first, so a failed attempt leaves the current databases untouched
+    const tmpDir = await fs.mkdtemp(path.join(OsvOfflineDb.rootDirectory, '.download-'));
     try {
         logger('Downloading databases ...');
         const databaseUrl = process.env.OSV_OFFLINE_DATABASE_URL ??
             'https://github.com/renovatebot/osv-offline/releases/latest/download/osv-offline.zip';
         const stream = got.stream(databaseUrl);
-        const zipPath = path.join(OsvOfflineDb.rootDirectory, 'osv-offline.zip');
-        const writeStream = fs.createWriteStream(zipPath);
+        const tmpZipPath = path.join(tmpDir, 'osv-offline.zip');
+        const writeStream = fs.createWriteStream(tmpZipPath);
         await pipeline(stream, writeStream);
         logger('Downloading databases done.');
         logger('Extracting databases ...');
-        const zip = new AdmZip(zipPath);
-        zip.extractAllTo(OsvOfflineDb.rootDirectory);
+        const extractDir = path.join(tmpDir, 'extracted');
+        await fs.ensureDir(extractDir);
+        const files = await extractZip(tmpZipPath, extractDir);
+        // renaming replaces existing databases, while readers which already opened them keep reading the old files
+        for (const file of files) {
+            await fs.rename(path.join(extractDir, file), path.join(OsvOfflineDb.rootDirectory, file));
+        }
+        // the zip's modification time marks the databases as up-to-date, so it's moved last
+        await fs.rename(tmpZipPath, path.join(OsvOfflineDb.rootDirectory, 'osv-offline.zip'));
         logger('Extracting databases done.');
     }
     catch (err) {
         return failure(err);
     }
+    finally {
+        await fs.remove(tmpDir);
+    }
     return success();
 }

package/dist/lib/unzip.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Extracts the files of a zip archive into `targetDir`, streaming each one to disk and verifying its CRC-32.
+ *
+ * Only supports files in the root of the archive.
+ *
+ * @returns the names of the extracted files
+ */
+export declare function extractZip(zipPath: string, targetDir: string): Promise<string[]>;

package/dist/lib/unzip.js

@@ -0,0 +1,63 @@
+import { createWriteStream } from 'node:fs';
+import path from 'node:path';
+import { Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { crc32 } from 'node:zlib';
+import { openPromise } from 'yauzl';
+/**
+ * Extracts the files of a zip archive into `targetDir`, streaming each one to disk and verifying its CRC-32.
+ *
+ * Only supports files in the root of the archive.
+ *
+ * @returns the names of the extracted files
+ */
+export async function extractZip(zipPath, targetDir) {
+    const zipFile = await openPromise(zipPath, {
+        lazyEntries: true,
+        autoClose: false,
+    });
+    try {
+        const files = [];
+        for await (const entry of zipFile.eachEntry()) {
+            if (entry.fileName.endsWith('/')) {
+                // directory entry
+                continue;
+            }
+            if (entry.fileName.includes('/')) {
+                throw new Error(`Unsupported zip entry name: ${entry.fileName}`);
+            }
+            await pipeline(await zipFile.openReadStreamPromise(entry), new Crc32Verifier(entry), createWriteStream(path.join(targetDir, entry.fileName)));
+            files.push(entry.fileName);
+        }
+        return files;
+    }
+    finally {
+        zipFile.close();
+    }
+}
+/**
+ * Passes the data through, and fails when its CRC-32 differs from the zip entry's.
+ *
+ * yauzl only verifies the size of an entry.
+ *
+ * @see https://github.com/thejoshwolfe/yauzl/issues/49
+ */
+class Crc32Verifier extends Transform {
+    entry;
+    crc = 0;
+    constructor(entry) {
+        super();
+        this.entry = entry;
+    }
+    _transform(chunk, _encoding, callback) {
+        this.crc = crc32(chunk, this.crc);
+        callback(null, chunk);
+    }
+    _flush(callback) {
+        if (this.crc !== this.entry.crc32) {
+            callback(new Error(`CRC-32 mismatch for ${this.entry.fileName}`));
+            return;
+        }
+        callback();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@renovatebot/osv-offline",
-  "version": "3.0.12",
+  "version": "3.1.0",
   "main": "dist/index.js",
   "license": "MIT",
   "type": "module",
@@ -12,19 +12,23 @@
     "url": "https://github.com/renovatebot/osv-offline.git"
   },
   "dependencies": {
-    "@renovatebot/osv-offline-db": "3.0.12",
-    "adm-zip": "~0.6.1",
+    "@renovatebot/osv-offline-db": "3.1.0",
     "debug": "^4.4.3",
     "fs-extra": "^11.4.1",
     "got": "^15.1.0",
-    "luxon": "^3.7.2"
+    "luxon": "^3.7.2",
+    "ms": "^2.1.3",
+    "yauzl": "^3.4.0"
   },
   "devDependencies": {
-    "@types/adm-zip": "0.5.8",
+    "@types/adm-zip": "0.6.0",
     "@types/debug": "4.1.13",
     "@types/fs-extra": "11.0.4",
-    "@types/luxon": "3.7.5",
-    "@types/node": "22.20.4"
+    "@types/luxon": "3.7.6",
+    "@types/ms": "2.1.0",
+    "@types/node": "22.20.5",
+    "@types/yauzl": "3.4.0",
+    "adm-zip": "0.6.1"
   },
   "engines": {
     "node": ">=22.12.0",