@pnpm/store.index 1000.0.0-0

package/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015-2016 Rico Sta. Cruz and other contributors
+Copyright (c) 2016-2026 Zoltan Kochan and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,29 @@
+# @pnpm/store.index
+
+> SQLite-backed index for the pnpm content-addressable store
+
+## Why SQLite instead of individual index files?
+
+Previously, pnpm stored package metadata as individual JSON files under
+`$STORE/index/`. Each resolved package had its own file, keyed by its integrity
+hash. This worked but had several downsides at scale:
+
+- **Filesystem overhead.** Every lookup required `open` / `read` / `close`
+  syscalls, and every write needed an atomic `write` + `rename` per entry.
+  On repositories with thousands of dependencies the accumulated I/O was
+  significant.
+- **Space inefficiency.** Small metadata entries still consumed a minimum
+  filesystem block each (typically 4 KiB), wasting space.
+Storing all entries in a single SQLite database (`$STORE/index.db`) addresses
+these issues:
+
+- **Fewer syscalls.** Reads and writes go through SQLite's page cache and
+  memory-mapped I/O instead of individual file operations.
+- **Space efficiency.** Small entries share database pages instead of each
+  occupying a full filesystem block.
+- **Batch writes.** Multiple entries can be inserted in a single transaction,
+  reducing disk flushes.
+
+## License
+
+MIT

package/lib/index.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Pack data for storage using msgpackr.
+ * Use this when data will be packed in one thread and stored by another,
+ * to ensure the same Packr instance is used for pack and unpack within each thread.
+ */
+export declare function packForStorage(data: unknown): Uint8Array;
+/**
+ * Create a store index key from an integrity hash and package id.
+ * The key is `${integrity}\t${pkgId}` — tab-separated.
+ * Integrity strings never contain tabs, so this is unambiguous.
+ */
+export declare function storeIndexKey(integrity: string, pkgId: string): string;
+export declare function gitHostedStoreIndexKey(pkgId: string, opts: {
+    built: boolean;
+}): string;
+/**
+ * Close all open StoreIndex instances.
+ * Useful in tests that need to remove the store directory.
+ */
+export declare function closeAllStoreIndexes(): void;
+export declare class StoreIndex {
+    private db;
+    private closed;
+    private pendingWrites;
+    private flushScheduled;
+    private stmtGet;
+    private stmtSet;
+    private stmtDel;
+    private stmtHas;
+    private stmtAll;
+    private readonly exitHandler;
+    constructor(storeDir: string);
+    get(key: string): unknown | undefined;
+    set(key: string, data: unknown): void;
+    delete(key: string): boolean;
+    has(key: string): boolean;
+    /**
+     * Iterate over all index entries.
+     * Yields [key, data] pairs where key is `integrity\tpkgId`.
+     */
+    entries(): IterableIterator<[string, unknown]>;
+    /**
+     * Queue pre-packed writes to be flushed on the next tick.
+     * Used by the fetch phase for throughput.
+     */
+    queueWrites(writes: Array<{
+        key: string;
+        buffer: Uint8Array;
+    }>): void;
+    /**
+     * Flush all pending queued writes immediately.
+     */
+    flush(): void;
+    /**
+     * Write multiple pre-packed entries in a single transaction.
+     * The buffers must already be msgpack-encoded.
+     */
+    setRawMany(entries: Array<{
+        key: string;
+        buffer: Uint8Array;
+    }>): void;
+    /**
+     * Delete multiple index entries in a single transaction,
+     * then VACUUM to reclaim disk space.
+     */
+    deleteMany(keys: string[]): void;
+    close(): void;
+}

package/lib/index.js

@@ -0,0 +1,254 @@
+import { createRequire } from 'module';
+import fs from 'fs';
+import { Packr } from 'msgpackr';
+// Use createRequire to load node:sqlite because it is a prefix-only builtin
+// that Jest's ESM module resolver cannot handle.
+const req = createRequire(import.meta.url);
+const { DatabaseSync } = req('node:sqlite');
+const packr = new Packr({
+    useRecords: true,
+    moreTypes: true,
+});
+const SQLITE_BUSY = 5;
+const RETRY_DELAY_MS = 50;
+const MAX_RETRIES = 100; // ~5 seconds total
+function sqliteRetry(fn) {
+    for (let attempt = 0;; attempt++) {
+        try {
+            return fn();
+        }
+        catch (err) {
+            if (isSqliteBusy(err) && attempt < MAX_RETRIES) {
+                sleepSync(RETRY_DELAY_MS);
+                continue;
+            }
+            throw err;
+        }
+    }
+}
+function isSqliteBusy(err) {
+    // errcode may be an extended error code (e.g. SQLITE_BUSY_RECOVERY = 261),
+    // so mask off the upper bits to get the primary error code.
+    return (err?.errcode & 0xFF) === SQLITE_BUSY;
+}
+const sleepBuffer = new Int32Array(new SharedArrayBuffer(4));
+function sleepSync(ms) {
+    Atomics.wait(sleepBuffer, 0, 0, ms);
+}
+/**
+ * Pack data for storage using msgpackr.
+ * Use this when data will be packed in one thread and stored by another,
+ * to ensure the same Packr instance is used for pack and unpack within each thread.
+ */
+export function packForStorage(data) {
+    return packr.pack(data);
+}
+/**
+ * Create a store index key from an integrity hash and package id.
+ * The key is `${integrity}\t${pkgId}` — tab-separated.
+ * Integrity strings never contain tabs, so this is unambiguous.
+ */
+export function storeIndexKey(integrity, pkgId) {
+    return `${integrity}\t${pkgId}`;
+}
+export function gitHostedStoreIndexKey(pkgId, opts) {
+    return storeIndexKey(pkgId, opts.built ? 'built' : 'not-built');
+}
+const openInstances = new Set();
+/**
+ * Close all open StoreIndex instances.
+ * Useful in tests that need to remove the store directory.
+ */
+export function closeAllStoreIndexes() {
+    for (const si of openInstances) {
+        si.close();
+    }
+}
+export class StoreIndex {
+    db;
+    closed = false;
+    pendingWrites = [];
+    flushScheduled = false;
+    stmtGet;
+    stmtSet;
+    stmtDel;
+    stmtHas;
+    stmtAll;
+    exitHandler;
+    constructor(storeDir) {
+        const dbPath = `${storeDir}/index.db`;
+        fs.mkdirSync(storeDir, { recursive: true });
+        this.db = new DatabaseSync(dbPath);
+        // Set busy_timeout FIRST so SQLite's internal busy handler is active
+        // during all subsequent operations. On Windows, file locking is mandatory
+        // and concurrent processes (e.g. parallel dlx calls) will contend.
+        this.db.exec('PRAGMA busy_timeout=5000');
+        sqliteRetry(() => {
+            this.db.exec('PRAGMA journal_mode=WAL');
+            this.db.exec('PRAGMA synchronous=NORMAL');
+            // Increase memory map size to 512MB
+            this.db.exec('PRAGMA mmap_size=536870912');
+            // Increase page cache size to ~32MB
+            this.db.exec('PRAGMA cache_size=-32000');
+            this.db.exec('PRAGMA temp_store=MEMORY');
+            // Increase wal autocheckpoint interval to reduce I/O during heavy writes
+            this.db.exec('PRAGMA wal_autocheckpoint=10000');
+            this.db.exec(`
+        CREATE TABLE IF NOT EXISTS package_index (
+          key TEXT PRIMARY KEY,
+          data BLOB NOT NULL
+        ) WITHOUT ROWID
+      `);
+        });
+        this.stmtGet = this.db.prepare('SELECT data FROM package_index WHERE key = ?');
+        this.stmtSet = this.db.prepare('INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)');
+        this.stmtDel = this.db.prepare('DELETE FROM package_index WHERE key = ?');
+        this.stmtHas = this.db.prepare('SELECT 1 FROM package_index WHERE key = ?');
+        this.stmtAll = this.db.prepare('SELECT key, data FROM package_index');
+        this.exitHandler = () => this.close();
+        process.on('exit', this.exitHandler);
+        openInstances.add(this);
+    }
+    get(key) {
+        const row = sqliteRetry(() => this.stmtGet.get(key));
+        if (row) {
+            return packr.unpack(row.data);
+        }
+        return undefined;
+    }
+    set(key, data) {
+        const buffer = packr.pack(data);
+        sqliteRetry(() => {
+            this.stmtSet.run(key, buffer);
+        });
+    }
+    delete(key) {
+        let result;
+        sqliteRetry(() => {
+            result = this.stmtDel.run(key);
+        });
+        return result.changes > 0;
+    }
+    has(key) {
+        return sqliteRetry(() => this.stmtHas.get(key)) != null;
+    }
+    /**
+     * Iterate over all index entries.
+     * Yields [key, data] pairs where key is `integrity\tpkgId`.
+     */
+    *entries() {
+        for (const row of this.stmtAll.iterate()) {
+            yield [row.key, packr.unpack(row.data)];
+        }
+    }
+    /**
+     * Queue pre-packed writes to be flushed on the next tick.
+     * Used by the fetch phase for throughput.
+     */
+    queueWrites(writes) {
+        for (const w of writes) {
+            this.pendingWrites.push(w);
+        }
+        if (!this.flushScheduled) {
+            this.flushScheduled = true;
+            process.nextTick(() => this.flush());
+        }
+    }
+    /**
+     * Flush all pending queued writes immediately.
+     */
+    flush() {
+        this.flushScheduled = false;
+        if (this.pendingWrites.length === 0)
+            return;
+        this.setRawMany(this.pendingWrites);
+        this.pendingWrites = [];
+    }
+    /**
+     * Write multiple pre-packed entries in a single transaction.
+     * The buffers must already be msgpack-encoded.
+     */
+    setRawMany(entries) {
+        if (this.closed || entries.length === 0)
+            return;
+        if (entries.length === 1) {
+            sqliteRetry(() => {
+                this.stmtSet.run(entries[0].key, entries[0].buffer);
+            });
+            return;
+        }
+        sqliteRetry(() => {
+            this.db.exec('BEGIN IMMEDIATE');
+            let committed = false;
+            try {
+                for (const { key, buffer } of entries) {
+                    this.stmtSet.run(key, buffer);
+                }
+                this.db.exec('COMMIT');
+                committed = true;
+            }
+            finally {
+                if (!committed) {
+                    try {
+                        this.db.exec('ROLLBACK');
+                    }
+                    catch { }
+                }
+            }
+        });
+    }
+    /**
+     * Delete multiple index entries in a single transaction,
+     * then VACUUM to reclaim disk space.
+     */
+    deleteMany(keys) {
+        if (keys.length === 0)
+            return;
+        if (keys.length === 1) {
+            this.delete(keys[0]);
+            this.db.exec('VACUUM');
+            return;
+        }
+        sqliteRetry(() => {
+            this.db.exec('BEGIN IMMEDIATE');
+            let committed = false;
+            try {
+                for (const key of keys) {
+                    this.stmtDel.run(key);
+                }
+                this.db.exec('COMMIT');
+                committed = true;
+            }
+            finally {
+                if (!committed) {
+                    try {
+                        this.db.exec('ROLLBACK');
+                    }
+                    catch { }
+                }
+            }
+        });
+        this.db.exec('VACUUM');
+    }
+    close() {
+        if (this.closed)
+            return;
+        this.flush();
+        this.closed = true;
+        openInstances.delete(this);
+        process.removeListener('exit', this.exitHandler);
+        try {
+            this.db.exec('PRAGMA optimize');
+        }
+        catch {
+            // PRAGMA optimize is a performance hint; safe to ignore if the DB is locked.
+        }
+        try {
+            this.db.close();
+        }
+        catch {
+            // The DB may be locked by another connection; the OS will reclaim it on process exit.
+        }
+    }
+}
+//# sourceMappingURL=index.js.map

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@pnpm/store.index",
+  "version": "1000.0.0-0",
+  "description": "SQLite-backed index for the pnpm content-addressable store",
+  "keywords": [
+    "pnpm",
+    "pnpm11",
+    "store"
+  ],
+  "license": "MIT",
+  "funding": "https://opencollective.com/pnpm",
+  "repository": "https://github.com/pnpm/pnpm/tree/main/store/index",
+  "homepage": "https://github.com/pnpm/pnpm/tree/main/store/index#readme",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "dependencies": {
+    "msgpackr": "^1.11.2"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.11",
+    "tempy": "3.0.0",
+    "@pnpm/store.index": "1000.0.0-0"
+  },
+  "engines": {
+    "node": ">=22.13"
+  },
+  "jest": {
+    "preset": "@pnpm/jest-config"
+  },
+  "scripts": {
+    "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
+    "_test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "compile": "tsgo --build && pnpm run lint --fix"
+  }
+}