@vltpkg/tar 1.0.0-rc.23 → 1.0.0-rc.25

package/dist/find-tar-dir.d.ts

@@ -0,0 +1 @@
+export declare const findTarDir: (path: string | undefined, tarDir?: string) => string | undefined;

package/dist/find-tar-dir.js

@@ -0,0 +1,22 @@
+// usually this will be 'package/', but could also be anything
+// eg, github tarballs are ${user}-${project}-${committish}
+// if it starts with `./` then all entries must as well.
+export const findTarDir = (path, tarDir) => {
+    if (tarDir !== undefined)
+        return tarDir;
+    if (!path)
+        return undefined;
+    const i = path.indexOf('/', path.startsWith('./') ? 2 : 0);
+    if (i === -1)
+        return undefined;
+    const chomp = path.substring(0, i);
+    if (chomp === '.' ||
+        chomp === '..' ||
+        chomp === '' ||
+        chomp === './.' ||
+        chomp === './..' ||
+        chomp === './') {
+        return undefined;
+    }
+    return chomp + '/';
+};

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from './unpack.ts';
+export * from './pool.ts';
+export * from './unpack-request.ts';

package/dist/index.js

@@ -0,0 +1,3 @@
+export * from "./unpack.js";
+export * from "./pool.js";
+export * from "./unpack-request.js";

package/dist/pool.d.ts

@@ -0,0 +1,40 @@
+import { UnpackRequest } from './unpack-request.ts';
+import { Worker } from './worker.ts';
+export * from './worker.ts';
+/**
+ * Automatically expanding/contracting set of workers to maximize parallelism
+ * of unpack operations up to 1 less than the number of CPUs (or 1).
+ *
+ * `pool.unpack(tarData, target)` will perform the unpack operation
+ * synchronously, in one of these workers, and returns a promise when the
+ * worker has confirmed completion of the task.
+ */
+export declare class Pool {
+    #private;
+    /**
+     * Number of workers to emplly. Defaults to 1 less than the number of
+     * CPUs, or 1.
+     */
+    jobs: number;
+    /**
+     * Set of currently active worker threads
+     */
+    workers: Set<Worker>;
+    /**
+     * Queue of requests awaiting an available worker
+     */
+    queue: UnpackRequest[];
+    /**
+     * Requests that have been assigned to a worker, but have not yet
+     * been confirmed completed.
+     */
+    pending: Map<number, UnpackRequest>;
+    /**
+     * Provide the tardata to be unpacked, and the location where it's to be
+     * placed. Will create a new worker up to the `jobs` value, and then start
+     * pushing in the queue for workers to pick up as they become available.
+     *
+     * Returned promise resolves when the provided tarball has been extracted.
+     */
+    unpack(tarData: Buffer, target: string): Promise<void>;
+}

package/dist/pool.js

@@ -0,0 +1,87 @@
+import { error } from '@vltpkg/error-cause';
+import { asError } from '@vltpkg/types';
+import os from 'node:os';
+import { UnpackRequest } from "./unpack-request.js";
+import { isResponseOK, Worker } from "./worker.js";
+export * from "./worker.js";
+/**
+ * Automatically expanding/contracting set of workers to maximize parallelism
+ * of unpack operations up to 1 less than the number of CPUs (or 1).
+ *
+ * `pool.unpack(tarData, target)` will perform the unpack operation
+ * synchronously, in one of these workers, and returns a promise when the
+ * worker has confirmed completion of the task.
+ */
+export class Pool {
+    /**
+     * Number of workers to emplly. Defaults to 1 less than the number of
+     * CPUs, or 1.
+     */
+    /* c8 ignore next */
+    jobs = 8 * (Math.max(os.availableParallelism(), 2) - 1);
+    /**
+     * Set of currently active worker threads
+     */
+    workers = new Set();
+    /**
+     * Queue of requests awaiting an available worker
+     */
+    queue = [];
+    /**
+     * Requests that have been assigned to a worker, but have not yet
+     * been confirmed completed.
+     */
+    pending = new Map();
+    // handle a message from the worker
+    #onMessage(w, m) {
+        const { id } = m;
+        // a request has been met or failed, report and either
+        // pick up the next item in the queue, or terminate worker
+        const ur = this.pending.get(id);
+        /* c8 ignore next */
+        if (!ur)
+            return;
+        if (isResponseOK(m)) {
+            ur.resolve();
+            /* c8 ignore start - nearly impossible in normal circumstances */
+        }
+        else {
+            ur.reject(error(asError(m.error, 'failed without error message').message, {
+                found: m,
+                cause: m.error,
+            }));
+        }
+        /* c8 ignore stop */
+        const next = this.queue.shift();
+        if (!next) {
+            this.workers.delete(w);
+        }
+        else {
+            void w.process(next);
+        }
+    }
+    // create a new worker
+    #createWorker(req) {
+        const w = new Worker((m) => this.#onMessage(w, m));
+        this.workers.add(w);
+        void w.process(req);
+    }
+    /**
+     * Provide the tardata to be unpacked, and the location where it's to be
+     * placed. Will create a new worker up to the `jobs` value, and then start
+     * pushing in the queue for workers to pick up as they become available.
+     *
+     * Returned promise resolves when the provided tarball has been extracted.
+     */
+    async unpack(tarData, target) {
+        const ur = new UnpackRequest(tarData, target);
+        this.pending.set(ur.id, ur);
+        if (this.workers.size < this.jobs) {
+            this.#createWorker(ur);
+        }
+        else {
+            this.queue.push(ur);
+        }
+        return ur.promise;
+    }
+}

package/dist/unpack-request.d.ts

@@ -0,0 +1,9 @@
+export declare class UnpackRequest {
+    id: number;
+    tarData: Buffer;
+    target: string;
+    resolve: () => void;
+    reject: (reason?: any) => void;
+    promise: Promise<void>;
+    constructor(tarData: Buffer, target: string);
+}

package/dist/unpack-request.js

@@ -0,0 +1,16 @@
+let ID = 1;
+export class UnpackRequest {
+    id = ID++;
+    tarData;
+    target;
+    resolve;
+    reject;
+    promise = new Promise((res, rej) => {
+        this.resolve = res;
+        this.reject = rej;
+    });
+    constructor(tarData, target) {
+        this.tarData = tarData;
+        this.target = target;
+    }
+}

package/dist/unpack.d.ts

@@ -0,0 +1 @@
+export declare const unpack: (tarData: Buffer, target: string) => Promise<void>;

package/dist/unpack.js

@@ -0,0 +1,175 @@
+import { error } from '@vltpkg/error-cause';
+import { randomBytes } from 'node:crypto';
+import { lstat, mkdir, rename, writeFile } from 'node:fs/promises';
+import { basename, dirname, resolve, sep } from 'node:path';
+import { rimraf } from 'rimraf';
+import { Header } from 'tar/header';
+import { Pax } from 'tar/pax';
+import { unzip as unzipCB } from 'node:zlib';
+import { findTarDir } from "./find-tar-dir.js";
+const unzip = async (input) => new Promise((res, rej) => 
+/* c8 ignore start */
+unzipCB(input, (er, result) => (er ? rej(er) : res(result))));
+const exists = async (path) => {
+    try {
+        await lstat(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+};
+let id = 1;
+const tmp = randomBytes(6).toString('hex') + '.';
+const tmpSuffix = () => tmp + String(id++);
+const checkFs = (h, tarDir, target) => {
+    /* c8 ignore start - impossible */
+    if (!h.path)
+        return false;
+    if (!tarDir)
+        return false;
+    /* c8 ignore stop */
+    h.path = h.path.replace(/[\\/]+/g, '/');
+    // packages should always be in a 'package' tarDir in the archive
+    if (!h.path.startsWith(tarDir))
+        return false;
+    // Package root
+    const absoluteBasePath = target;
+    const itemAbsolutePath = resolve(target, h.path.slice(tarDir.length));
+    if (!itemAbsolutePath.startsWith(absoluteBasePath)) {
+        return false;
+    }
+    return true;
+};
+const write = async (path, body, executable = false) => {
+    await mkdirp(dirname(path));
+    // if the mode is world-executable, then make it executable
+    // this is needed for some packages that have a file that is
+    // not a declared bin, but still used as a cli executable.
+    await writeFile(path, body, {
+        mode: executable ? 0o777 : 0o666,
+    });
+};
+const made = new Set();
+const making = new Map();
+const mkdirp = async (d) => {
+    if (!made.has(d)) {
+        const m = making.get(d) ??
+            mkdir(d, { recursive: true, mode: 0o777 }).then(() => making.delete(d));
+        making.set(d, m);
+        await m;
+        made.add(d);
+    }
+};
+export const unpack = async (tarData, target) => {
+    const isGzip = tarData[0] === 0x1f && tarData[1] === 0x8b;
+    await unpackUnzipped(isGzip ? await unzip(tarData) : tarData, target);
+};
+const unpackUnzipped = async (buffer, target) => {
+    /* c8 ignore start */
+    const isGzip = buffer[0] === 0x1f && buffer[1] === 0x8b;
+    if (isGzip) {
+        throw error('still gzipped after unzipping', {
+            found: isGzip,
+            wanted: false,
+        });
+    }
+    /* c8 ignore stop */
+    // another real quick gutcheck before we get started
+    if (buffer.length % 512 !== 0) {
+        throw error('Invalid tarball: length not divisible by 512', {
+            found: buffer.length,
+        });
+    }
+    if (buffer.length < 1024) {
+        throw error('Invalid tarball: not terminated by 1024 null bytes', { found: buffer.length });
+    }
+    // make sure the last kb is all zeros
+    for (let i = buffer.length - 1024; i < buffer.length; i++) {
+        if (buffer[i] !== 0) {
+            throw error('Invalid tarball: not terminated by 1024 null bytes', { found: buffer.subarray(i, i + 10) });
+        }
+    }
+    const tmp = dirname(target) + sep + '.' + basename(target) + '.' + tmpSuffix();
+    const og = tmp + '.ORIGINAL';
+    await Promise.all([rimraf(tmp), rimraf(og)]);
+    let succeeded = false;
+    try {
+        let tarDir = undefined;
+        let offset = 0;
+        let h;
+        let ex = undefined;
+        let gex = undefined;
+        while (offset < buffer.length &&
+            !(h = new Header(buffer, offset, ex, gex)).nullBlock) {
+            offset += 512;
+            ex = undefined;
+            gex = undefined;
+            const size = h.size ?? 0;
+            const body = buffer.subarray(offset, offset + size);
+            // skip invalid headers
+            if (!h.cksumValid)
+                continue;
+            offset += 512 * Math.ceil(size / 512);
+            // TODO: tarDir might not be named "package/"
+            // find the first tarDir in the first entry, and use that.
+            switch (h.type) {
+                case 'File':
+                    if (!tarDir)
+                        tarDir = findTarDir(h.path, tarDir);
+                    /* c8 ignore next */
+                    if (!tarDir)
+                        continue;
+                    if (!checkFs(h, tarDir, tmp))
+                        continue;
+                    await write(resolve(tmp, h.path.substring(tarDir.length)), body, 
+                    // if it's world-executable, it's an executable
+                    // otherwise, make it read-only.
+                    1 === ((h.mode ?? 0x666) & 1));
+                    break;
+                case 'Directory':
+                    /* c8 ignore next 2 */
+                    if (!tarDir)
+                        tarDir = findTarDir(h.path, tarDir);
+                    if (!tarDir)
+                        continue;
+                    if (!checkFs(h, tarDir, tmp))
+                        continue;
+                    await mkdirp(resolve(tmp, h.path.substring(tarDir.length)));
+                    break;
+                case 'GlobalExtendedHeader':
+                    gex = Pax.parse(body.toString(), gex, true);
+                    break;
+                case 'ExtendedHeader':
+                case 'OldExtendedHeader':
+                    ex = Pax.parse(body.toString(), ex, false);
+                    break;
+                case 'NextFileHasLongPath':
+                case 'OldGnuLongPath':
+                    ex ??= Object.create(null);
+                    ex.path = body.toString().replace(/\0.*/, '');
+                    break;
+            }
+        }
+        const targetExists = await exists(target);
+        if (targetExists)
+            await rename(target, og);
+        await rename(tmp, target);
+        if (targetExists)
+            await rimraf(og);
+        succeeded = true;
+    }
+    finally {
+        // do not handle error or obscure throw site, just do the cleanup
+        // if it didn't complete successfully.
+        if (!succeeded) {
+            /* c8 ignore start */
+            if (await exists(og)) {
+                await rimraf(target);
+                await rename(og, target);
+            }
+            /* c8 ignore stop */
+            await rimraf(tmp);
+        }
+    }
+};

package/dist/worker.d.ts

@@ -0,0 +1,19 @@
+import type { UnpackRequest } from './unpack-request.ts';
+export type ResponseError = {
+    id: number;
+    error: unknown;
+};
+export type ResponseOK = {
+    id: number;
+    ok: true;
+};
+export declare const isResponseOK: (o: unknown) => o is ResponseOK;
+/**
+ * Basically just a queue of unpack requests,
+ * to keep them throttled to a reasonable amount of parallelism
+ */
+export declare class Worker {
+    onMessage: (m: ResponseError | ResponseOK) => void;
+    constructor(onMessage: (m: ResponseError | ResponseOK) => void);
+    process(req: UnpackRequest): Promise<void>;
+}

package/dist/worker.js

@@ -0,0 +1,25 @@
+import { unpack } from "./unpack.js";
+const isObj = (o) => !!o && typeof o === 'object';
+export const isResponseOK = (o) => isObj(o) && typeof o.id === 'number' && o.ok === true;
+/**
+ * Basically just a queue of unpack requests,
+ * to keep them throttled to a reasonable amount of parallelism
+ */
+export class Worker {
+    onMessage;
+    constructor(onMessage) {
+        this.onMessage = onMessage;
+    }
+    async process(req) {
+        const { target, tarData, id } = req;
+        try {
+            await unpack(tarData, target);
+            const m = { id, ok: true };
+            this.onMessage(m);
+        }
+        catch (error) {
+            const m = { id, error };
+            this.onMessage(m);
+        }
+    }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vltpkg/tar",
   "description": "An extremely limited and very fast tar extractor",
-  "version": "1.0.0-rc.23",
+  "version": "1.0.0-rc.25",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/vltpkg/vltpkg.git",
@@ -12,8 +12,8 @@
     "email": "support@vlt.sh"
   },
   "dependencies": {
-    "@vltpkg/error-cause": "1.0.0-rc.23",
-    "@vltpkg/types": "1.0.0-rc.23",
+    "@vltpkg/error-cause": "1.0.0-rc.25",
+    "@vltpkg/types": "1.0.0-rc.25",
     "rimraf": "^6.1.2",
     "tar": "^7.5.2"
   },