@vltpkg/registry-client 0.0.0-3 → 0.0.0-31

package/dist/esm/cache-entry.js

@@ -20,10 +20,11 @@
 // of the file.
 import { error } from '@vltpkg/error-cause';
 import ccp from 'cache-control-parser';
-import { createHash } from 'crypto';
-import { inspect } from 'util';
-import { gunzipSync } from 'zlib';
+import { createHash } from 'node:crypto';
+import { inspect } from 'node:util';
+import { gunzipSync } from 'node:zlib';
 import { getRawHeader, setRawHeader } from "./raw-header.js";
+import { getDecodedValue, getEncondedValue, } from "./string-encoding.js";
 const readSize = (buf, offset) => {
     const a = buf[offset];
     const b = buf[offset + 1];
@@ -45,66 +46,181 @@ const readSize = (buf, offset) => {
 const kCustomInspect = Symbol.for('nodejs.util.inspect.custom');
 export class CacheEntry {
     #statusCode;
+    /** The raw headers as an array of buffers */
     #headers;
-    #body = [];
+    /** The body buffer, used if the content length is known. */
+    #body;
+    /**
+     * If the content length is unknown we save the body in multiple parts
+     * in order to only concatenate once at the end and save extra memory copies.
+     */
+    #bodyParts = [];
+    /** Used to track the length of the body while reading chunks */
     #bodyLength = 0;
+    /** The total length of the body, if known */
+    #contentLength;
     #integrity;
     #integrityActual;
     #json;
-    constructor(statusCode, headers, integrity) {
-        this.#integrity = integrity;
-        this.#statusCode = statusCode;
+    #trustIntegrity;
+    #staleWhileRevalidateFactor;
+    constructor(statusCode, headers, { body, integrity, trustIntegrity = false, 'stale-while-revalidate-factor': staleWhileRevalidateFactor = 60, contentLength, } = {}) {
         this.#headers = headers;
+        this.#statusCode = statusCode;
+        this.#trustIntegrity = trustIntegrity;
+        this.#staleWhileRevalidateFactor = staleWhileRevalidateFactor;
+        if (integrity)
+            this.integrity = integrity;
+        // if content-legnth is known then we'll only allocate that much memory
+        // and we'll avoid copying memory around when adding new chunks.
+        if (contentLength != null && typeof contentLength === 'number') {
+            this.#contentLength = contentLength;
+        }
+        // if a body is provided then use that, in this case the `addBody`
+        // method should no longer be used.
+        if (body) {
+            const buffer = new ArrayBuffer(body.byteLength);
+            this.#body = new Uint8Array(buffer, 0, body.byteLength);
+            this.#body.set(body, 0);
+            this.#bodyLength = body.byteLength;
+            /* c8 ignore start */
+        }
+        else if (this.#contentLength) {
+            const buffer = new ArrayBuffer(this.#contentLength);
+            this.#body = new Uint8Array(buffer, 0, this.#contentLength);
+            this.#bodyLength = 0;
+        }
+        /* c8 ignore stop */
     }
     get #headersAsObject() {
         const ret = [];
         for (let i = 0; i < this.#headers.length - 1; i += 2) {
-            const key = String(this.#headers[i]);
-            const val = String(this.#headers[i + 1]);
+            const key = getDecodedValue(this.#headers[i]);
+            const val = getDecodedValue(this.#headers[i + 1]);
             ret.push([key, val]);
         }
         return ret;
     }
-    [kCustomInspect](depth, options) {
-        const str = inspect({
-            statusCode: this.statusCode,
+    toJSON() {
+        const { statusCode, valid, staleWhileRevalidate, cacheControl, date, contentType, integrity, maxAge, isGzip, isJSON, } = this;
+        /* c8 ignore start */
+        const age = date ?
+            Math.floor((Date.now() - date.getTime()) / 1000)
+            : undefined;
+        const expires = date ? new Date(date.getTime() + this.maxAge * 1000) : undefined;
+        /* c8 ignore end */
+        return Object.fromEntries(Object.entries({
+            statusCode,
             headers: this.#headersAsObject,
-            text: this.text(),
-        }, {
+            contentType,
+            integrity,
+            date,
+            expires,
+            cacheControl,
+            valid,
+            staleWhileRevalidate,
+            age,
+            maxAge,
+            isGzip,
+            isJSON,
+        }).filter(([_, v]) => v !== undefined));
+    }
+    [kCustomInspect](depth, options) {
+        const str = inspect(this.toJSON(), {
             depth,
             ...options,
         });
         return `@vltpkg/registry-client.CacheEntry ${str}`;
     }
+    #date;
+    get date() {
+        if (this.#date)
+            return this.#date;
+        const dh = this.getHeaderString('date');
+        if (dh)
+            this.#date = new Date(dh);
+        return this.#date;
+    }
+    #maxAge;
+    get maxAge() {
+        if (this.#maxAge !== undefined)
+            return this.#maxAge;
+        // see if the max-age has not yet been crossed
+        // default to 5m if maxage is not set, as some registries
+        // do not set a cache control header at all.
+        const cc = this.cacheControl;
+        this.#maxAge = cc['max-age'] || cc['s-maxage'] || 300;
+        return this.#maxAge;
+    }
+    #cacheControl;
+    get cacheControl() {
+        if (this.#cacheControl)
+            return this.#cacheControl;
+        const cc = this.getHeaderString('cache-control');
+        this.#cacheControl = cc ? ccp.parse(cc) : {};
+        return this.#cacheControl;
+    }
+    #staleWhileRevalidate;
+    get staleWhileRevalidate() {
+        if (this.#staleWhileRevalidate !== undefined)
+            return this.#staleWhileRevalidate;
+        if (this.valid || !this.date)
+            return true;
+        const swv = this.cacheControl['stale-while-revalidate'] ??
+            this.maxAge * this.#staleWhileRevalidateFactor;
+        this.#staleWhileRevalidate =
+            this.date.getTime() + swv * 1000 > Date.now();
+        return this.#staleWhileRevalidate;
+    }
+    #contentType;
+    get contentType() {
+        if (this.#contentType !== undefined)
+            return this.#contentType;
+        this.#contentType = this.getHeaderString('content-type') ?? '';
+        return this.#contentType;
+    }
     /**
      * `true` if the entry represents a cached response that is still
      * valid to use.
      */
+    #valid;
     get valid() {
-        const cc_ = this.getHeader('cache-control')?.toString();
-        const cc = cc_ ? ccp.parse(cc_) : {};
-        const ct = this.getHeader('content-type')?.toString() ?? '';
-        const dh = this.getHeader('date')?.toString();
+        if (this.#valid !== undefined)
+            return this.#valid;
         // immutable = never changes
-        if (cc.immutable)
-            return true;
+        if (this.cacheControl.immutable)
+            return (this.#valid = true);
         // some registries do text/json, some do application/json,
         // some do application/vnd.npm.install-v1+json
         // If it's NOT json, it's an immutable tarball
-        if (ct !== '' && !/\bjson\b/.test(ct))
-            return true;
+        const ct = this.contentType;
+        if (ct && !/\bjson\b/.test(ct))
+            return (this.#valid = true);
         // see if the max-age has not yet been crossed
         // default to 5m if maxage is not set, as some registries
         // do not set a cache control header at all.
-        const ma = cc['max-age'] || cc['s-maxage'] || 300;
-        if (ma && dh) {
-            return Date.parse(dh) + ma * 1000 > Date.now();
-        }
-        return false;
+        if (!this.date)
+            return (this.#valid = false);
+        this.#valid =
+            this.date.getTime() + this.maxAge * 1000 > Date.now();
+        return this.#valid;
     }
+    /**
+     * Add contents to the entry body.
+     */
     addBody(b) {
-        this.#body.push(b);
-        this.#bodyLength += b.byteLength;
+        // when the content length is uknown we store each chunk in an array that
+        // later on is concatenate into a single buffer, otherwise we just append
+        // the new chunk of bytes to the already allocated buffer keeping track
+        // of the current offset in the `this.#bodyLength` property.
+        if (!this.#body) {
+            this.#bodyParts.push(b);
+            this.#bodyLength += b.byteLength;
+        }
+        else {
+            this.#body.set(b, this.#bodyLength);
+            this.#bodyLength += b.byteLength;
+        }
     }
     get statusCode() {
         return this.#statusCode;
@@ -113,27 +229,68 @@ export class CacheEntry {
         return this.#headers;
     }
     /**
-     * check that the sri integrity string that was provided to the ctor
+     * Returns the body as a single Uint8Array, concatenating parts if needed.
+     */
+    get _body() {
+        // if the body is known we'll just use that
+        if (this.#body)
+            return this.#body;
+        // otherwise we concatenate the body parts into a single buffer
+        const buffer = new ArrayBuffer(this.#bodyLength);
+        const b = new Uint8Array(buffer, 0, this.#bodyLength);
+        let off = 0;
+        for (const part of this.#bodyParts) {
+            b.set(part, off);
+            off += part.byteLength;
+        }
+        return b;
+    }
+    /**
+     * Check that the sri integrity string that was provided to the ctor
      * matches the body that we actually received. This should only be called
      * AFTER the entire body has been completely downloaded.
      *
+     * This method **will throw** if the integrity values do not match.
+     *
      * Note that this will *usually* not be true if the value is coming out of
      * the cache, because the cache entries are un-gzipped in place. It should
      * _only_ be called for artifacts that come from an actual http response.
+     *
+     * Returns true if anything was actually verified.
      */
-    checkIntegrity() {
+    checkIntegrity(context = {}) {
         if (!this.#integrity)
             return false;
-        return this.integrityActual === this.#integrity;
+        if (this.integrityActual !== this.#integrity) {
+            throw error('Integrity check failure', {
+                code: 'EINTEGRITY',
+                response: this,
+                wanted: this.#integrity,
+                found: this.integrityActual,
+                ...context,
+            });
+        }
+        return true;
     }
     get integrityActual() {
         if (this.#integrityActual)
             return this.#integrityActual;
         const hash = createHash('sha512');
-        for (const buf of this.#body)
-            hash.update(buf);
-        this.#integrityActual = `sha512-${hash.digest('base64')}`;
-        return this.#integrityActual;
+        hash.update(this._body);
+        const i = `sha512-${hash.digest('base64')}`;
+        this.integrityActual = i;
+        return i;
+    }
+    set integrityActual(i) {
+        this.#integrityActual = i;
+        this.setHeader('integrity', i);
+    }
+    set integrity(i) {
+        if (!this.#integrity && i) {
+            this.#integrity = i;
+            if (this.#trustIntegrity)
+                this.integrityActual = i;
+        }
     }
     get integrity() {
         return this.#integrity;
@@ -144,6 +301,15 @@ export class CacheEntry {
     getHeader(h) {
         return getRawHeader(this.#headers, h);
     }
+    /**
+     * Give it a key, and it'll return the decoded string of that header value
+     */
+    getHeaderString(h) {
+        const value = getRawHeader(this.#headers, h);
+        if (value) {
+            return getDecodedValue(value);
+        }
+    }
     /**
      * Set a header to a specific value
      */
@@ -154,14 +320,7 @@ export class CacheEntry {
      * Return the body of the entry as a Buffer
      */
     buffer() {
-        const b = this.#body[0];
-        if (!b)
-            return Buffer.allocUnsafe(0);
-        if (this.#body.length === 1)
-            return b;
-        const cat = Buffer.concat(this.#body, this.#bodyLength);
-        this.#body = [cat];
-        return cat;
+        return Buffer.from(this._body.buffer, this._body.byteOffset, this._body.byteLength);
     }
     // return the buffer if it's a tarball, or the parsed
     // JSON if it's not.
@@ -172,7 +331,7 @@ export class CacheEntry {
     get isJSON() {
         if (this.#isJSON !== undefined)
             return this.#isJSON;
-        const ct = this.getHeader('content-type')?.toString();
+        const ct = this.getHeaderString('content-type');
         // if it says it's json, assume json
         if (ct)
             return (this.#isJSON = /\bjson\b/.test(ct));
@@ -190,10 +349,10 @@ export class CacheEntry {
     get isGzip() {
         if (this.#isGzip !== undefined)
             return this.#isGzip;
-        const ce = this.getHeader('content-encoding')?.toString();
+        const ce = this.getHeaderString('content-encoding');
         if (ce && !/\bgzip\b/.test(ce))
             return (this.#isGzip = false);
-        const buf = this.buffer();
+        const buf = this._body;
         if (buf.length < 2)
             return false;
         this.#isGzip = buf[0] === 0x1f && buf[1] === 0x8b;
@@ -216,14 +375,16 @@ export class CacheEntry {
             // we know that if we know it's gzip, that the body has been
             // flattened to a single buffer, so save the extra call.
             /* c8 ignore start */
-            if (this.#body[0] == null)
+            if (this._body.length === 0)
                 throw error('Invalid buffer, cant unzip');
             /* c8 ignore stop */
-            const b = gunzipSync(this.#body[0]);
+            const b = gunzipSync(this._body);
             this.setHeader('content-encoding', 'identity');
-            this.#body = [b];
-            this.#bodyLength = b.byteLength;
-            this.setHeader('content-length', String(this.#bodyLength));
+            const u8 = new Uint8Array(b.buffer, b.byteOffset, b.byteLength);
+            this.#body = u8;
+            this.#bodyLength = u8.byteLength;
+            this.#contentLength = u8.byteLength;
+            this.setHeader('content-length', String(this.#contentLength));
             this.#isGzip = false;
             return true;
         }
@@ -235,7 +396,7 @@ export class CacheEntry {
      */
     text() {
         this.unzip();
-        return this.buffer().toString();
+        return getDecodedValue(this._body);
     }
     /**
      * Parse the entry body as JSON and return the result
@@ -243,7 +404,8 @@ export class CacheEntry {
     json() {
         if (this.#json !== undefined)
             return this.#json;
-        const obj = JSON.parse(this.text());
+        const text = this.text();
+        const obj = JSON.parse(text || '{}');
         this.#json = obj;
         return obj;
     }
@@ -254,71 +416,103 @@ export class CacheEntry {
      */
     static decode(buffer) {
         if (buffer.length < 4) {
-            return new CacheEntry(0, []);
+            return emptyCacheEntry;
         }
         const headSize = readSize(buffer, 0);
         if (buffer.length < headSize) {
-            return new CacheEntry(0, []);
+            return emptyCacheEntry;
         }
-        const statusCode = Number(buffer.subarray(4, 7).toString());
+        const statusCode = Number(getDecodedValue(buffer.subarray(4, 7)));
         const headersBuffer = buffer.subarray(7, headSize);
-        // walk through the headers array, building up the rawHeaders Buffer[]
+        // walk through the headers array, building up the rawHeaders
         const headers = [];
         let i = 0;
+        let integrity = undefined;
         while (i < headersBuffer.length - 4) {
             const size = readSize(headersBuffer, i);
-            headers.push(headersBuffer.subarray(i + 4, i + size));
+            const val = headersBuffer.subarray(i + 4, i + size);
+            // if the last one was the key integrity, then this one is the value
+            if (headers.length % 2 === 1) {
+                const k = getDecodedValue(headers[headers.length - 1]).toLowerCase();
+                if (k === 'integrity')
+                    integrity = getDecodedValue(val);
+            }
+            headers.push(val);
             i += size;
         }
-        const c = new CacheEntry(statusCode, headers);
         const body = buffer.subarray(headSize);
-        c.#body = [body];
-        c.#bodyLength = body.byteLength;
-        c.setHeader('content-length', String(c.#bodyLength));
+        const c = new CacheEntry(statusCode, setRawHeader(headers, 'content-length', String(body.byteLength)), {
+            body,
+            integrity,
+            trustIntegrity: true,
+            contentLength: body.byteLength,
+        });
         if (c.isJSON) {
             try {
                 c.json();
             }
             catch {
-                return new CacheEntry(0, []);
+                return emptyCacheEntry;
             }
         }
         return c;
     }
+    static isGzipEntry(buffer) {
+        if (buffer.length < 4)
+            return false;
+        const headSize = readSize(buffer, 0);
+        const gzipBytes = buffer.subarray(headSize, headSize + 2);
+        return gzipBytes[0] === 0x1f && gzipBytes[1] === 0x8b;
+    }
     /**
      * Encode the entry as a single Buffer for writing to the cache
      */
-    // TODO: should this maybe not concat, and just return Buffer[]?
-    // Then we can writev it to the cache file and save the memory copy
     encode() {
-        // store json results as a serialized object.
         if (this.isJSON)
             this.json();
-        const sb = Buffer.from(String(this.#statusCode));
-        const chunks = [sb];
-        let headLength = sb.byteLength + 4;
+        const statusStr = String(this.#statusCode);
+        const statusBytes = getEncondedValue(statusStr);
+        // compute headLength = 4 (length field itself) + statusBytes + Σ(4 + headerLen) for each header item
+        let headLength = 4 + statusBytes.byteLength;
+        for (const h of this.#headers)
+            headLength += 4 + h.byteLength;
+        // allocate and fill head length prefix (big-endian)
+        const headLenBytes = new Uint8Array(4);
+        headLenBytes[0] = (headLength >> 24) & 0xff;
+        headLenBytes[1] = (headLength >> 16) & 0xff;
+        headLenBytes[2] = (headLength >> 8) & 0xff;
+        headLenBytes[3] = headLength & 0xff;
+        // header chunks: [len, bytes] for each header item
+        const headerChunks = [];
         for (const h of this.#headers) {
-            const hlBuf = Buffer.allocUnsafe(4);
-            const hl = h.byteLength + 4;
-            headLength += hl;
-            hlBuf.set([
-                (hl >> 24) & 0xff,
-                (hl >> 16) & 0xff,
-                (hl >> 8) & 0xff,
-                hl & 0xff,
-            ], 0);
-            chunks.push(hlBuf, h);
+            const l = headLenBytes.byteLength + h.byteLength;
+            const lb = new Uint8Array(4);
+            lb[0] = (l >> 24) & 0xff;
+            lb[1] = (l >> 16) & 0xff;
+            lb[2] = (l >> 8) & 0xff;
+            lb[3] = l & 0xff;
+            headerChunks.push(lb, h);
+        }
+        // total size
+        const total = headLenBytes.byteLength +
+            statusBytes.byteLength +
+            headerChunks.reduce((n, b) => n + b.byteLength, 0) +
+            this._body.byteLength;
+        // returns the concatenate buffer with all the pieces
+        const outBuffer = new ArrayBuffer(total);
+        const out = Buffer.from(outBuffer, 0, total);
+        let off = 0;
+        out.set(headLenBytes, off);
+        off += headLenBytes.byteLength;
+        out.set(statusBytes, off);
+        off += statusBytes.byteLength;
+        for (const chunk of headerChunks) {
+            out.set(chunk, off);
+            off += chunk.byteLength;
         }
-        const hlBuf = Buffer.allocUnsafe(4);
-        hlBuf.set([
-            (headLength >> 24) & 0xff,
-            (headLength >> 16) & 0xff,
-            (headLength >> 8) & 0xff,
-            headLength & 0xff,
-        ], 0);
-        chunks.unshift(hlBuf);
-        chunks.push(...this.#body);
-        return Buffer.concat(chunks, headLength + this.#bodyLength);
+        out.set(this._body, off);
+        return out;
     }
 }
+const emptyCacheEntry = new CacheEntry(0, [], { contentLength: 0 });
 //# sourceMappingURL=cache-entry.js.map