@atproto/common 0.4.2 → 0.4.3

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @atproto/common
 
+## 0.4.3
+
+### Patch Changes
+
+- [#2770](https://github.com/bluesky-social/atproto/pull/2770) [`a07b21151`](https://github.com/bluesky-social/atproto/commit/a07b21151f1850340c4b7797ebb11521b1a6cdf3) Thanks [@matthieusieben](https://github.com/matthieusieben)! - add streamToNodeBuffer utility to convert Uint8Array (async) iterables to Buffer
+
+- Updated dependencies [[`a07b21151`](https://github.com/bluesky-social/atproto/commit/a07b21151f1850340c4b7797ebb11521b1a6cdf3), [`a07b21151`](https://github.com/bluesky-social/atproto/commit/a07b21151f1850340c4b7797ebb11521b1a6cdf3), [`eb20ff64a`](https://github.com/bluesky-social/atproto/commit/eb20ff64a2d8e3061c652e1e247bf9b0fe3c41a6)]:
+  - @atproto/common-web@0.3.1
+
 ## 0.4.2
 
 ### Patch Changes

package/dist/streams.d.ts

@@ -1,10 +1,11 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { Stream, Readable, Transform, TransformCallback } from 'stream';
+import { Duplex, Readable, Stream, Transform, TransformCallback } from 'node:stream';
 export declare const forwardStreamErrors: (...streams: Stream[]) => void;
 export declare const cloneStream: (stream: Readable) => Readable;
 export declare const streamSize: (stream: Readable) => Promise<number>;
-export declare const streamToBytes: (stream: Readable) => Promise<Uint8Array>;
+export declare const streamToBytes: (stream: AsyncIterable<Uint8Array>) => Promise<Uint8Array>;
+export declare const streamToNodeBuffer: (stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) => Promise<Buffer>;
 export declare const byteIterableToStream: (iter: AsyncIterable<Uint8Array>) => Readable;
 export declare const bytesToStream: (bytes: Uint8Array) => Readable;
 export declare class MaxSizeChecker extends Transform {
@@ -14,4 +15,11 @@ export declare class MaxSizeChecker extends Transform {
     constructor(maxSize: number, createError: () => Error);
     _transform(chunk: Uint8Array, _enc: BufferEncoding, cb: TransformCallback): void;
 }
+export declare function decodeStream(stream: Readable, contentEncoding?: string): Readable;
+export declare function decodeStream(stream: AsyncIterable<Uint8Array>, contentEncoding?: string): AsyncIterable<Uint8Array> | Readable;
+/**
+ * Create a series of decoding streams based on the content-encoding header. The
+ * resulting streams should be piped together to decode the content.
+ */
+export declare function createDecoders(contentEncoding?: string): Duplex[];
 //# sourceMappingURL=streams.d.ts.map

package/dist/streams.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":";;AAAA,OAAO,EACL,MAAM,EACN,QAAQ,EAER,SAAS,EACT,iBAAiB,EAClB,MAAM,QAAQ,CAAA;AAEf,eAAO,MAAM,mBAAmB,eAAgB,MAAM,EAAE,SAOvD,CAAA;AAED,eAAO,MAAM,WAAW,WAAY,QAAQ,KAAG,QAI9C,CAAA;AAED,eAAO,MAAM,UAAU,WAAkB,QAAQ,KAAG,QAAQ,MAAM,CAMjE,CAAA;AAED,eAAO,MAAM,aAAa,WAAkB,QAAQ,KAAG,QAAQ,UAAU,CAMxE,CAAA;AAED,eAAO,MAAM,oBAAoB,SACzB,cAAc,UAAU,CAAC,KAC9B,QAEF,CAAA;AAED,eAAO,MAAM,aAAa,UAAW,UAAU,KAAG,QAKjD,CAAA;AAED,qBAAa,cAAe,SAAQ,SAAS;IAGlC,OAAO,EAAE,MAAM;IACf,WAAW,EAAE,MAAM,KAAK;IAHjC,SAAS,SAAI;gBAEJ,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,KAAK;IAIjC,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,iBAAiB;CAQ1E"}
+{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":";;AAAA,OAAO,EACL,MAAM,EAGN,QAAQ,EACR,MAAM,EACN,SAAS,EACT,iBAAiB,EAClB,MAAM,aAAa,CAAA;AAGpB,eAAO,MAAM,mBAAmB,eAAgB,MAAM,EAAE,SAOvD,CAAA;AAED,eAAO,MAAM,WAAW,WAAY,QAAQ,KAAG,QAI9C,CAAA;AAED,eAAO,MAAM,UAAU,WAAkB,QAAQ,KAAG,QAAQ,MAAM,CAMjE,CAAA;AAED,eAAO,MAAM,aAAa,WAAkB,cAAc,UAAU,CAAC,wBAInB,CAAA;AAGlD,eAAO,MAAM,kBAAkB,WACrB,SAAS,UAAU,CAAC,GAAG,cAAc,UAAU,CAAC,KACvD,QAAQ,MAAM,CAYhB,CAAA;AAED,eAAO,MAAM,oBAAoB,SACzB,cAAc,UAAU,CAAC,KAC9B,QAEF,CAAA;AAED,eAAO,MAAM,aAAa,UAAW,UAAU,KAAG,QAKjD,CAAA;AAED,qBAAa,cAAe,SAAQ,SAAS;IAGlC,OAAO,EAAE,MAAM;IACf,WAAW,EAAE,MAAM,KAAK;IAHjC,SAAS,SAAI;gBAEJ,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,KAAK;IAIjC,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,iBAAiB;CAQ1E;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,QAAQ,EAChB,eAAe,CAAC,EAAE,MAAM,GACvB,QAAQ,CAAA;AACX,wBAAgB,YAAY,CAC1B,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,EACjC,eAAe,CAAC,EAAE,MAAM,GACvB,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAA;AAUvC;;;GAGG;AACH,wBAAgB,cAAc,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAcjE"}

package/dist/streams.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MaxSizeChecker = exports.bytesToStream = exports.byteIterableToStream = exports.streamToBytes = exports.streamSize = exports.cloneStream = exports.forwardStreamErrors = void 0;
-const stream_1 = require("stream");
+exports.createDecoders = exports.decodeStream = exports.MaxSizeChecker = exports.bytesToStream = exports.byteIterableToStream = exports.streamToNodeBuffer = exports.streamToBytes = exports.streamSize = exports.cloneStream = exports.forwardStreamErrors = void 0;
+const node_stream_1 = require("node:stream");
+const node_zlib_1 = require("node:zlib");
 const forwardStreamErrors = (...streams) => {
     for (let i = 1; i < streams.length; ++i) {
         const prev = streams[i - 1];
@@ -11,7 +12,7 @@ const forwardStreamErrors = (...streams) => {
 };
 exports.forwardStreamErrors = forwardStreamErrors;
 const cloneStream = (stream) => {
-    const passthrough = new stream_1.PassThrough();
+    const passthrough = new node_stream_1.PassThrough();
     (0, exports.forwardStreamErrors)(stream, passthrough);
     return stream.pipe(passthrough);
 };
@@ -19,31 +20,45 @@ exports.cloneStream = cloneStream;
 const streamSize = async (stream) => {
     let size = 0;
     for await (const chunk of stream) {
-        size += chunk.length;
+        size += Buffer.byteLength(chunk);
     }
     return size;
 };
 exports.streamSize = streamSize;
-const streamToBytes = async (stream) => {
-    const bufs = [];
-    for await (const bytes of stream) {
-        bufs.push(bytes);
+const streamToBytes = async (stream) => 
+// @NOTE Though Buffer is a sub-class of Uint8Array, we have observed
+// inconsistencies when using a Buffer in place of Uint8Array. For this
+// reason, we convert the Buffer to a Uint8Array.
+new Uint8Array(await (0, exports.streamToNodeBuffer)(stream));
+exports.streamToBytes = streamToBytes;
+// streamToBuffer identifier name already taken by @atproto/common-web
+const streamToNodeBuffer = async (stream) => {
+    const chunks = [];
+    let totalLength = 0; // keep track of total length for Buffer.concat
+    for await (const chunk of stream) {
+        if (chunk instanceof Uint8Array) {
+            chunks.push(chunk);
+            totalLength += Buffer.byteLength(chunk);
+        }
+        else {
+            throw new TypeError('expected Uint8Array');
+        }
     }
-    return new Uint8Array(Buffer.concat(bufs));
+    return Buffer.concat(chunks, totalLength);
 };
-exports.streamToBytes = streamToBytes;
+exports.streamToNodeBuffer = streamToNodeBuffer;
 const byteIterableToStream = (iter) => {
-    return stream_1.Readable.from(iter, { objectMode: false });
+    return node_stream_1.Readable.from(iter, { objectMode: false });
 };
 exports.byteIterableToStream = byteIterableToStream;
 const bytesToStream = (bytes) => {
-    const stream = new stream_1.Readable();
+    const stream = new node_stream_1.Readable();
     stream.push(bytes);
     stream.push(null);
     return stream;
 };
 exports.bytesToStream = bytesToStream;
-class MaxSizeChecker extends stream_1.Transform {
+class MaxSizeChecker extends node_stream_1.Transform {
     constructor(maxSize, createError) {
         super();
         Object.defineProperty(this, "maxSize", {
@@ -76,4 +91,50 @@ class MaxSizeChecker extends stream_1.Transform {
     }
 }
 exports.MaxSizeChecker = MaxSizeChecker;
+function decodeStream(stream, contentEncoding) {
+    const decoders = createDecoders(contentEncoding);
+    if (decoders.length === 0)
+        return stream;
+    return (0, node_stream_1.pipeline)([stream, ...decoders], () => { });
+}
+exports.decodeStream = decodeStream;
+/**
+ * Create a series of decoding streams based on the content-encoding header. The
+ * resulting streams should be piped together to decode the content.
+ */
+function createDecoders(contentEncoding) {
+    const decoders = [];
+    if (contentEncoding) {
+        const encodings = contentEncoding.split(',');
+        for (const encoding of encodings) {
+            const normalizedEncoding = normalizeEncoding(encoding);
+            if (normalizedEncoding === 'identity')
+                continue;
+            decoders.push(createDecoder(normalizedEncoding));
+        }
+    }
+    return decoders.reverse();
+}
+exports.createDecoders = createDecoders;
+function normalizeEncoding(encoding) {
+    // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1
+    // > All content-coding values are case-insensitive...
+    return encoding.trim().toLowerCase();
+}
+function createDecoder(normalizedEncoding) {
+    switch (normalizedEncoding) {
+        // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.2
+        case 'gzip':
+        case 'x-gzip':
+            return (0, node_zlib_1.createGunzip)();
+        case 'deflate':
+            return (0, node_zlib_1.createInflate)();
+        case 'br':
+            return (0, node_zlib_1.createBrotliDecompress)();
+        case 'identity':
+            return new node_stream_1.PassThrough();
+        default:
+            throw new TypeError(`Unsupported content-encoding: "${normalizedEncoding}"`);
+    }
+}
 //# sourceMappingURL=streams.js.map

package/dist/streams.js.map

@@ -1 +1 @@
-{"version":3,"file":"streams.js","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":";;;AAAA,mCAMe;AAER,MAAM,mBAAmB,GAAG,CAAC,GAAG,OAAiB,EAAE,EAAE;IAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;IACtD,CAAC;AACH,CAAC,CAAA;AAPY,QAAA,mBAAmB,uBAO/B;AAEM,MAAM,WAAW,GAAG,CAAC,MAAgB,EAAY,EAAE;IACxD,MAAM,WAAW,GAAG,IAAI,oBAAW,EAAE,CAAA;IACrC,IAAA,2BAAmB,EAAC,MAAM,EAAE,WAAW,CAAC,CAAA;IACxC,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;AACjC,CAAC,CAAA;AAJY,QAAA,WAAW,eAIvB;AAEM,MAAM,UAAU,GAAG,KAAK,EAAE,MAAgB,EAAmB,EAAE;IACpE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAA;IACtB,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AANY,QAAA,UAAU,cAMtB;AAEM,MAAM,aAAa,GAAG,KAAK,EAAE,MAAgB,EAAuB,EAAE;IAC3E,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAClB,CAAC;IACD,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;AAC5C,CAAC,CAAA;AANY,QAAA,aAAa,iBAMzB;AAEM,MAAM,oBAAoB,GAAG,CAClC,IAA+B,EACrB,EAAE;IACZ,OAAO,iBAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;AACnD,CAAC,CAAA;AAJY,QAAA,oBAAoB,wBAIhC;AAEM,MAAM,aAAa,GAAG,CAAC,KAAiB,EAAY,EAAE;IAC3D,MAAM,MAAM,GAAG,IAAI,iBAAQ,EAAE,CAAA;IAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjB,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AALY,QAAA,aAAa,iBAKzB;AAED,MAAa,cAAe,SAAQ,kBAAS;IAE3C,YACS,OAAe,EACf,WAAwB;QAE/B,KAAK,EAAE,CAAA;QAHP;;;;mBAAO,OAAO;WAAQ;QACtB;;;;mBAAO,WAAW;WAAa;QAHjC;;;;mBAAY,CAAC;WAAA;IAMb,CAAC;IACD,UAAU,CAAC,KAAiB,EAAE,IAAoB,EAAE,EAAqB;QACvE,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,CAAA;QAC9B,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;CACF;AAhBD,wCAgBC"}
+{"version":3,"file":"streams.js","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":";;;AAAA,6CAQoB;AACpB,yCAA+E;AAExE,MAAM,mBAAmB,GAAG,CAAC,GAAG,OAAiB,EAAE,EAAE;IAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;IACtD,CAAC;AACH,CAAC,CAAA;AAPY,QAAA,mBAAmB,uBAO/B;AAEM,MAAM,WAAW,GAAG,CAAC,MAAgB,EAAY,EAAE;IACxD,MAAM,WAAW,GAAG,IAAI,yBAAW,EAAE,CAAA;IACrC,IAAA,2BAAmB,EAAC,MAAM,EAAE,WAAW,CAAC,CAAA;IACxC,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;AACjC,CAAC,CAAA;AAJY,QAAA,WAAW,eAIvB;AAEM,MAAM,UAAU,GAAG,KAAK,EAAE,MAAgB,EAAmB,EAAE;IACpE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;IAClC,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AANY,QAAA,UAAU,cAMtB;AAEM,MAAM,aAAa,GAAG,KAAK,EAAE,MAAiC,EAAE,EAAE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,iDAAiD;AACjD,IAAI,UAAU,CAAC,MAAM,IAAA,0BAAkB,EAAC,MAAM,CAAC,CAAC,CAAA;AAJrC,QAAA,aAAa,iBAIwB;AAElD,sEAAsE;AAC/D,MAAM,kBAAkB,GAAG,KAAK,EACrC,MAAwD,EACvC,EAAE;IACnB,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA,CAAC,+CAA+C;IACnE,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;AAC3C,CAAC,CAAA;AAdY,QAAA,kBAAkB,sBAc9B;AAEM,MAAM,oBAAoB,GAAG,CAClC,IAA+B,EACrB,EAAE;IACZ,OAAO,sBAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;AACnD,CAAC,CAAA;AAJY,QAAA,oBAAoB,wBAIhC;AAEM,MAAM,aAAa,GAAG,CAAC,KAAiB,EAAY,EAAE;IAC3D,MAAM,MAAM,GAAG,IAAI,sBAAQ,EAAE,CAAA;IAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjB,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AALY,QAAA,aAAa,iBAKzB;AAED,MAAa,cAAe,SAAQ,uBAAS;IAE3C,YACS,OAAe,EACf,WAAwB;QAE/B,KAAK,EAAE,CAAA;QAHP;;;;mBAAO,OAAO;WAAQ;QACtB;;;;mBAAO,WAAW;WAAa;QAHjC;;;;mBAAY,CAAC;WAAA;IAMb,CAAC;IACD,UAAU,CAAC,KAAiB,EAAE,IAAoB,EAAE,EAAqB;QACvE,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,CAAA;QAC9B,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;CACF;AAhBD,wCAgBC;AAUD,SAAgB,YAAY,CAC1B,MAA4C,EAC5C,eAAwB;IAExB,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,CAAC,CAAA;IAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IACxC,OAAO,IAAA,sBAAQ,EAAC,CAAC,MAAkB,EAAE,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAE,CAAC,CAAW,CAAA;AACxE,CAAC;AAPD,oCAOC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAAC,eAAwB;IACrD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC5C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;YACtD,IAAI,kBAAkB,KAAK,UAAU;gBAAE,SAAQ;YAE/C,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,CAAA;QAClD,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,OAAO,EAAE,CAAA;AAC3B,CAAC;AAdD,wCAcC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACzC,yDAAyD;IACzD,sDAAsD;IACtD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;AACtC,CAAC;AAED,SAAS,aAAa,CAAC,kBAA0B;IAC/C,QAAQ,kBAAkB,EAAE,CAAC;QAC3B,0DAA0D;QAC1D,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,IAAA,wBAAY,GAAE,CAAA;QACvB,KAAK,SAAS;YACZ,OAAO,IAAA,yBAAa,GAAE,CAAA;QACxB,KAAK,IAAI;YACP,OAAO,IAAA,kCAAsB,GAAE,CAAA;QACjC,KAAK,UAAU;YACb,OAAO,IAAI,yBAAW,EAAE,CAAA;QAC1B;YACE,MAAM,IAAI,SAAS,CACjB,kCAAkC,kBAAkB,GAAG,CACxD,CAAA;IACL,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/common",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "license": "MIT",
   "description": "Shared web-platform-friendly code for atproto libraries",
   "keywords": [
@@ -20,7 +20,7 @@
     "iso-datestring-validator": "^2.2.2",
     "multiformats": "^9.9.0",
     "pino": "^8.21.0",
-    "@atproto/common-web": "^0.3.0"
+    "@atproto/common-web": "^0.3.1"
   },
   "devDependencies": {
     "jest": "^28.1.2",

package/src/streams.ts

@@ -1,10 +1,13 @@
 import {
-  Stream,
-  Readable,
+  Duplex,
   PassThrough,
+  pipeline,
+  Readable,
+  Stream,
   Transform,
   TransformCallback,
-} from 'stream'
+} from 'node:stream'
+import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'
 
 export const forwardStreamErrors = (...streams: Stream[]) => {
   for (let i = 1; i < streams.length; ++i) {
@@ -24,17 +27,32 @@ export const cloneStream = (stream: Readable): Readable => {
 export const streamSize = async (stream: Readable): Promise<number> => {
   let size = 0
   for await (const chunk of stream) {
-    size += chunk.length
+    size += Buffer.byteLength(chunk)
   }
   return size
 }
 
-export const streamToBytes = async (stream: Readable): Promise<Uint8Array> => {
-  const bufs: Buffer[] = []
-  for await (const bytes of stream) {
-    bufs.push(bytes)
+export const streamToBytes = async (stream: AsyncIterable<Uint8Array>) =>
+  // @NOTE Though Buffer is a sub-class of Uint8Array, we have observed
+  // inconsistencies when using a Buffer in place of Uint8Array. For this
+  // reason, we convert the Buffer to a Uint8Array.
+  new Uint8Array(await streamToNodeBuffer(stream))
+
+// streamToBuffer identifier name already taken by @atproto/common-web
+export const streamToNodeBuffer = async (
+  stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+): Promise<Buffer> => {
+  const chunks: Uint8Array[] = []
+  let totalLength = 0 // keep track of total length for Buffer.concat
+  for await (const chunk of stream) {
+    if (chunk instanceof Uint8Array) {
+      chunks.push(chunk)
+      totalLength += Buffer.byteLength(chunk)
+    } else {
+      throw new TypeError('expected Uint8Array')
+    }
   }
-  return new Uint8Array(Buffer.concat(bufs))
+  return Buffer.concat(chunks, totalLength)
 }
 
 export const byteIterableToStream = (
@@ -67,3 +85,65 @@ export class MaxSizeChecker extends Transform {
     }
   }
 }
+
+export function decodeStream(
+  stream: Readable,
+  contentEncoding?: string,
+): Readable
+export function decodeStream(
+  stream: AsyncIterable<Uint8Array>,
+  contentEncoding?: string,
+): AsyncIterable<Uint8Array> | Readable
+export function decodeStream(
+  stream: Readable | AsyncIterable<Uint8Array>,
+  contentEncoding?: string,
+): Readable | AsyncIterable<Uint8Array> {
+  const decoders = createDecoders(contentEncoding)
+  if (decoders.length === 0) return stream
+  return pipeline([stream as Readable, ...decoders], () => {}) as Duplex
+}
+
+/**
+ * Create a series of decoding streams based on the content-encoding header. The
+ * resulting streams should be piped together to decode the content.
+ */
+export function createDecoders(contentEncoding?: string): Duplex[] {
+  const decoders: Duplex[] = []
+
+  if (contentEncoding) {
+    const encodings = contentEncoding.split(',')
+    for (const encoding of encodings) {
+      const normalizedEncoding = normalizeEncoding(encoding)
+      if (normalizedEncoding === 'identity') continue
+
+      decoders.push(createDecoder(normalizedEncoding))
+    }
+  }
+
+  return decoders.reverse()
+}
+
+function normalizeEncoding(encoding: string) {
+  // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1
+  // > All content-coding values are case-insensitive...
+  return encoding.trim().toLowerCase()
+}
+
+function createDecoder(normalizedEncoding: string): Duplex {
+  switch (normalizedEncoding) {
+    // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.2
+    case 'gzip':
+    case 'x-gzip':
+      return createGunzip()
+    case 'deflate':
+      return createInflate()
+    case 'br':
+      return createBrotliDecompress()
+    case 'identity':
+      return new PassThrough()
+    default:
+      throw new TypeError(
+        `Unsupported content-encoding: "${normalizedEncoding}"`,
+      )
+  }
+}

package/tests/streams.test.ts

@@ -61,14 +61,41 @@ describe('streams', () => {
     })
   })
 
-  describe('streamToBytes', () => {
+  describe('streamToNodeBuffer', () => {
     it('converts stream to byte array', async () => {
       const stream = Readable.from(Buffer.from('foo'))
-      const bytes = await streams.streamToBytes(stream)
+      const bytes = await streams.streamToNodeBuffer(stream)
 
       expect(bytes[0]).toBe('f'.charCodeAt(0))
       expect(bytes[1]).toBe('o'.charCodeAt(0))
       expect(bytes[2]).toBe('o'.charCodeAt(0))
+      expect(bytes.length).toBe(3)
+    })
+
+    it('converts async iterable to byte array', async () => {
+      const iterable = (async function* () {
+        yield Buffer.from('b')
+        yield Buffer.from('a')
+        yield new Uint8Array(['r'.charCodeAt(0)])
+      })()
+      const bytes = await streams.streamToNodeBuffer(iterable)
+
+      expect(bytes[0]).toBe('b'.charCodeAt(0))
+      expect(bytes[1]).toBe('a'.charCodeAt(0))
+      expect(bytes[2]).toBe('r'.charCodeAt(0))
+      expect(bytes.length).toBe(3)
+    })
+
+    it('throws error for non Uint8Array chunks', async () => {
+      const iterable: AsyncIterable<any> = (async function* () {
+        yield Buffer.from('b')
+        yield Buffer.from('a')
+        yield 'r'
+      })()
+
+      await expect(streams.streamToNodeBuffer(iterable)).rejects.toThrow(
+        'expected Uint8Array',
+      )
     })
   })