@atproto/common 0.6.0 → 0.6.2

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @atproto/common
 
+## 0.6.2
+
+### Patch Changes
+
+- [#5080](https://github.com/bluesky-social/atproto/pull/5080) [`b2c098f`](https://github.com/bluesky-social/atproto/commit/b2c098fdfa26137bb6f7d092ef89757e39748353) Thanks [@devinivy](https://github.com/devinivy)! - Make rmIfExists() more robust to partial failures.
+
+- [#5078](https://github.com/bluesky-social/atproto/pull/5078) [`cc329bf`](https://github.com/bluesky-social/atproto/commit/cc329bf55c658752675d561dba792fedbe9c2626) Thanks [@jcalabro](https://github.com/jcalabro)! - Add `coalesceByteStream` utility to coalesce a byte stream into chunks with a minimal size.
+
+## 0.6.1
+
+### Patch Changes
+
+- [#4954](https://github.com/bluesky-social/atproto/pull/4954) [`e6c6343`](https://github.com/bluesky-social/atproto/commit/e6c6343bd3727455bd0da12300bb4929a944e4f1) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Add back support for any `Uint8Array` as input to the `ui8ToArrayBuffer` helper
+
+- Updated dependencies [[`e6c6343`](https://github.com/bluesky-social/atproto/commit/e6c6343bd3727455bd0da12300bb4929a944e4f1)]:
+  - @atproto/lex-data@0.1.1
+
 ## 0.6.0
 
 ### Minor Changes

package/dist/buffers.d.ts

@@ -1,3 +1,3 @@
 export declare function ui8ToBuffer(bytes: Uint8Array): Buffer;
-export declare function ui8ToArrayBuffer(bytes: Uint8Array<ArrayBuffer>): ArrayBuffer;
+export declare function ui8ToArrayBuffer(bytes: Uint8Array): ArrayBuffer;
 //# sourceMappingURL=buffers.d.ts.map

package/dist/buffers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"buffers.d.ts","sourceRoot":"","sources":["../src/buffers.ts"],"names":[],"mappings":"AAAA,wBAAgB,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAErD;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC,GAAG,WAAW,CAK5E"}
+{"version":3,"file":"buffers.d.ts","sourceRoot":"","sources":["../src/buffers.ts"],"names":[],"mappings":"AAAA,wBAAgB,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAErD;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,CAS/D"}

package/dist/buffers.js

@@ -2,6 +2,9 @@ export function ui8ToBuffer(bytes) {
     return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
 }
 export function ui8ToArrayBuffer(bytes) {
-    return bytes.buffer.slice(bytes.byteOffset, bytes.byteLength + bytes.byteOffset);
+    if (bytes.buffer instanceof ArrayBuffer) {
+        return bytes.buffer.slice(bytes.byteOffset, bytes.byteLength + bytes.byteOffset);
+    }
+    return new Uint8Array(bytes).buffer;
 }
 //# sourceMappingURL=buffers.js.map

package/dist/buffers.js.map

@@ -1 +1 @@
-{"version":3,"file":"buffers.js","sourceRoot":"","sources":["../src/buffers.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,WAAW,CAAC,KAAiB;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAA8B;IAC7D,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CACvB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CACpC,CAAA;AACH,CAAC","sourcesContent":["export function ui8ToBuffer(bytes: Uint8Array): Buffer {\n  return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)\n}\n\nexport function ui8ToArrayBuffer(bytes: Uint8Array<ArrayBuffer>): ArrayBuffer {\n  return bytes.buffer.slice(\n    bytes.byteOffset,\n    bytes.byteLength + bytes.byteOffset,\n  )\n}\n"]}
+{"version":3,"file":"buffers.js","sourceRoot":"","sources":["../src/buffers.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,WAAW,CAAC,KAAiB;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAiB;IAChD,IAAI,KAAK,CAAC,MAAM,YAAY,WAAW,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CACvB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CACpC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAA;AACrC,CAAC","sourcesContent":["export function ui8ToBuffer(bytes: Uint8Array): Buffer {\n  return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)\n}\n\nexport function ui8ToArrayBuffer(bytes: Uint8Array): ArrayBuffer {\n  if (bytes.buffer instanceof ArrayBuffer) {\n    return bytes.buffer.slice(\n      bytes.byteOffset,\n      bytes.byteLength + bytes.byteOffset,\n    )\n  }\n\n  return new Uint8Array(bytes).buffer\n}\n"]}

package/dist/fs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,UAAU,GAAU,UAAU,MAAM,KAAG,OAAO,CAAC,OAAO,CAUlE,CAAA;AAED,eAAO,MAAM,YAAY,GACvB,UAAU,MAAM,KACf,OAAO,CAAC,UAAU,GAAG,SAAS,CAShC,CAAA;AAED,eAAO,MAAM,UAAU,GACrB,UAAU,MAAM,EAChB,mBAAiB,KAChB,OAAO,CAAC,IAAI,CASd,CAAA;AAED,eAAO,MAAM,cAAc,GACzB,SAAS,MAAM,EACf,SAAS,MAAM,KACd,OAAO,CAAC,IAAI,CASd,CAAA"}
+{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,UAAU,GAAU,UAAU,MAAM,KAAG,OAAO,CAAC,OAAO,CAUlE,CAAA;AAED,eAAO,MAAM,YAAY,GACvB,UAAU,MAAM,KACf,OAAO,CAAC,UAAU,GAAG,SAAS,CAShC,CAAA;AAED,eAAO,MAAM,UAAU,GACrB,UAAU,MAAM,EAChB,mBAAiB,KAChB,OAAO,CAAC,IAAI,CAOd,CAAA;AAED,eAAO,MAAM,cAAc,GACzB,SAAS,MAAM,EACf,SAAS,MAAM,KACd,OAAO,CAAC,IAAI,CASd,CAAA"}

package/dist/fs.js

@@ -25,15 +25,12 @@ export const readIfExists = async (filepath) => {
     }
 };
 export const rmIfExists = async (filepath, recursive = false) => {
-    try {
-        await fs.rm(filepath, { recursive });
-    }
-    catch (err) {
-        if (isErrnoException(err) && err.code === 'ENOENT') {
-            return;
-        }
-        throw err;
-    }
+    await fs.rm(filepath, {
+        force: true, // ignore errors if the file/directory does not exist
+        recursive,
+        maxRetries: 5,
+        retryDelay: 50,
+    });
 };
 export const renameIfExists = async (oldPath, newPath) => {
     try {

package/dist/fs.js.map

@@ -1 +1 @@
-{"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,MAAM,kBAAkB,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AAEtD,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAoB,EAAE;IACrE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;QACzC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,KAAK,CAAA;QACd,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAC/B,QAAgB,EACiB,EAAE;IACnC,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACpC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAM;QACR,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAC7B,QAAgB,EAChB,SAAS,GAAG,KAAK,EACF,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;IACtC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAM;QACR,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,OAAe,EACf,OAAe,EACA,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAM;QACR,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA","sourcesContent":["import { constants } from 'node:fs'\nimport fs from 'node:fs/promises'\nimport { isErrnoException } from '@atproto/common-web'\n\nexport const fileExists = async (location: string): Promise<boolean> => {\n  try {\n    await fs.access(location, constants.F_OK)\n    return true\n  } catch (err) {\n    if (isErrnoException(err) && err.code === 'ENOENT') {\n      return false\n    }\n    throw err\n  }\n}\n\nexport const readIfExists = async (\n  filepath: string,\n): Promise<Uint8Array | undefined> => {\n  try {\n    return await fs.readFile(filepath)\n  } catch (err) {\n    if (isErrnoException(err) && err.code === 'ENOENT') {\n      return\n    }\n    throw err\n  }\n}\n\nexport const rmIfExists = async (\n  filepath: string,\n  recursive = false,\n): Promise<void> => {\n  try {\n    await fs.rm(filepath, { recursive })\n  } catch (err) {\n    if (isErrnoException(err) && err.code === 'ENOENT') {\n      return\n    }\n    throw err\n  }\n}\n\nexport const renameIfExists = async (\n  oldPath: string,\n  newPath: string,\n): Promise<void> => {\n  try {\n    await fs.rename(oldPath, newPath)\n  } catch (err) {\n    if (isErrnoException(err) && err.code === 'ENOENT') {\n      return\n    }\n    throw err\n  }\n}\n"]}
+{"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,MAAM,kBAAkB,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AAEtD,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAoB,EAAE;IACrE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAA;QACzC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,KAAK,CAAA;QACd,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAC/B,QAAgB,EACiB,EAAE;IACnC,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACpC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAM;QACR,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAC7B,QAAgB,EAChB,SAAS,GAAG,KAAK,EACF,EAAE;IACjB,MAAM,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE;QACpB,KAAK,EAAE,IAAI,EAAE,qDAAqD;QAClE,SAAS;QACT,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,EAAE;KACf,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,OAAe,EACf,OAAe,EACA,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAM;QACR,CAAC;QACD,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA","sourcesContent":["import { constants } from 'node:fs'\nimport fs from 'node:fs/promises'\nimport { isErrnoException } from '@atproto/common-web'\n\nexport const fileExists = async (location: string): Promise<boolean> => {\n  try {\n    await fs.access(location, constants.F_OK)\n    return true\n  } catch (err) {\n    if (isErrnoException(err) && err.code === 'ENOENT') {\n      return false\n    }\n    throw err\n  }\n}\n\nexport const readIfExists = async (\n  filepath: string,\n): Promise<Uint8Array | undefined> => {\n  try {\n    return await fs.readFile(filepath)\n  } catch (err) {\n    if (isErrnoException(err) && err.code === 'ENOENT') {\n      return\n    }\n    throw err\n  }\n}\n\nexport const rmIfExists = async (\n  filepath: string,\n  recursive = false,\n): Promise<void> => {\n  await fs.rm(filepath, {\n    force: true, // ignore errors if the file/directory does not exist\n    recursive,\n    maxRetries: 5,\n    retryDelay: 50,\n  })\n}\n\nexport const renameIfExists = async (\n  oldPath: string,\n  newPath: string,\n): Promise<void> => {\n  try {\n    await fs.rename(oldPath, newPath)\n  } catch (err) {\n    if (isErrnoException(err) && err.code === 'ENOENT') {\n      return\n    }\n    throw err\n  }\n}\n"]}

package/dist/streams.d.ts

@@ -5,6 +5,13 @@ export declare const streamSize: (stream: Readable) => Promise<number>;
 export declare const streamToBytes: (stream: AsyncIterable<Uint8Array>) => Promise<Uint8Array<ArrayBuffer>>;
 export declare const streamToNodeBuffer: (stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) => Promise<Buffer>;
 export declare const byteIterableToStream: (iter: AsyncIterable<Uint8Array>) => Readable;
+/**
+ * Coalesce a stream of Uint8Array chunks into larger chunks of at least the
+ * specified size ({@link minChunkSize}). This is useful for optimizing
+ * downstream processing that benefits from larger chunk sizes, such as
+ * compression or hashing.
+ */
+export declare const coalesceByteStream: (stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>, minChunkSize: number) => Readable;
 export declare const bytesToStream: (bytes: Uint8Array) => Readable;
 export declare class MaxSizeChecker extends Transform {
     maxSize: number;

package/dist/streams.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EAEN,QAAQ,EACR,MAAM,EACN,SAAS,EACT,iBAAiB,EAElB,MAAM,aAAa,CAAA;AAGpB,eAAO,MAAM,mBAAmB,GAAI,GAAG,SAAS,MAAM,EAAE,SAOvD,CAAA;AAED,eAAO,MAAM,WAAW,GAAI,QAAQ,QAAQ,KAAG,QAI9C,CAAA;AAED,eAAO,MAAM,UAAU,GAAU,QAAQ,QAAQ,KAAG,OAAO,CAAC,MAAM,CAMjE,CAAA;AAED,eAAO,MAAM,aAAa,GAAU,QAAQ,aAAa,CAAC,UAAU,CAAC,qCAInB,CAAA;AAGlD,eAAO,MAAM,kBAAkB,GAC7B,QAAQ,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,KACvD,OAAO,CAAC,MAAM,CAYhB,CAAA;AAED,eAAO,MAAM,oBAAoB,GAC/B,MAAM,aAAa,CAAC,UAAU,CAAC,KAC9B,QAEF,CAAA;AAED,eAAO,MAAM,aAAa,GAAI,OAAO,UAAU,KAAG,QAKjD,CAAA;AAED,qBAAa,cAAe,SAAQ,SAAS;IAGlC,OAAO,EAAE,MAAM;IACf,WAAW,EAAE,MAAM,KAAK;IAHjC,SAAS,SAAI;gBAEJ,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,KAAK;IAIjC,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,iBAAiB;CAQ1E;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,QAAQ,EAChB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAClC,QAAQ,CAAA;AACX,wBAAgB,YAAY,CAC1B,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,EACjC,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAClC,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAA;AAUvC;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,CAqB5E"}
+{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EAEN,QAAQ,EACR,MAAM,EACN,SAAS,EACT,iBAAiB,EAElB,MAAM,aAAa,CAAA;AAGpB,eAAO,MAAM,mBAAmB,GAAI,GAAG,SAAS,MAAM,EAAE,SAOvD,CAAA;AAED,eAAO,MAAM,WAAW,GAAI,QAAQ,QAAQ,KAAG,QAI9C,CAAA;AAED,eAAO,MAAM,UAAU,GAAU,QAAQ,QAAQ,KAAG,OAAO,CAAC,MAAM,CAMjE,CAAA;AAED,eAAO,MAAM,aAAa,GAAU,QAAQ,aAAa,CAAC,UAAU,CAAC,qCAInB,CAAA;AAGlD,eAAO,MAAM,kBAAkB,GAC7B,QAAQ,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,KACvD,OAAO,CAAC,MAAM,CAYhB,CAAA;AAED,eAAO,MAAM,oBAAoB,GAC/B,MAAM,aAAa,CAAC,UAAU,CAAC,KAC9B,QAEF,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,GAC7B,QAAQ,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACxD,cAAc,MAAM,KACnB,QAkEF,CAAA;AAED,eAAO,MAAM,aAAa,GAAI,OAAO,UAAU,KAAG,QAKjD,CAAA;AAED,qBAAa,cAAe,SAAQ,SAAS;IAGlC,OAAO,EAAE,MAAM;IACf,WAAW,EAAE,MAAM,KAAK;IAHjC,SAAS,SAAI;gBAEJ,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,KAAK;IAIjC,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,iBAAiB;CAQ1E;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,QAAQ,EAChB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAClC,QAAQ,CAAA;AACX,wBAAgB,YAAY,CAC1B,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,EACjC,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAClC,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAA;AAUvC;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,CAqB5E"}

package/dist/streams.js

@@ -42,6 +42,73 @@ export const streamToNodeBuffer = async (stream) => {
 export const byteIterableToStream = (iter) => {
     return Readable.from(iter, { objectMode: false });
 };
+/**
+ * Coalesce a stream of Uint8Array chunks into larger chunks of at least the
+ * specified size ({@link minChunkSize}). This is useful for optimizing
+ * downstream processing that benefits from larger chunk sizes, such as
+ * compression or hashing.
+ */
+export const coalesceByteStream = (stream, minChunkSize) => {
+    if (!Number.isInteger(minChunkSize) || minChunkSize < 1) {
+        throw new TypeError('minChunkSize must be a positive integer');
+    }
+    // @NOTE On Node 22, this *does* return a PassThrough ("@types/node"
+    // incorrectly types it as Writable).
+    return pipeline(stream, coalesce, (_err) => {
+        // Errors are expected to be handled through the stream
+    });
+    // @NOTE This implementation is not NodeJS specific and could be exported as
+    // utility (from "@atproto/common-web") if needed. We don't do it now to avoid
+    // increasing the API surface of our packages.
+    async function* coalesce(iter) {
+        // @NOTE This implementation avoids as many un-necessary copies as possible
+        // and only buffers incoming chunks when they are smaller than the
+        // coalescing buffer.
+        let buffer = new Uint8Array(minChunkSize);
+        let offset = 0;
+        for await (const chunk of iter) {
+            const freeSpace = buffer.length - offset;
+            if (freeSpace > chunk.length) {
+                // If the incoming chunk is smaller than the free space, we copy the
+                // entire chunk into the coalescing buffer and continue to the next
+                // chunk
+                buffer.set(chunk, offset);
+                offset += chunk.length;
+            }
+            else if (offset === 0 && chunk.length >= minChunkSize) {
+                // If the coalescing buffer is empty and the incoming chunk is larger
+                // than the coalescing buffer, we can skip any copying and yield the
+                // incoming chunk directly
+                yield chunk;
+            }
+            else {
+                // Otherwise, we need to copy as much of the incoming chunk as we can
+                // into the coalescing buffer and yield the full coalescing buffer.
+                buffer.set(chunk.subarray(0, freeSpace), offset);
+                yield buffer;
+                // We create a new coalescing buffer (for future use)
+                buffer = new Uint8Array(minChunkSize);
+                offset = 0;
+                const remainingBytes = chunk.subarray(freeSpace);
+                if (remainingBytes.length > minChunkSize) {
+                    // If the remaining of chunk is still too big to fit in the
+                    // coalescing buffer, we yield it directly without copying it
+                    yield remainingBytes;
+                }
+                else if (remainingBytes.length > 0) {
+                    // Otherwise, we copy the remaining bytes into the coalescing buffer
+                    // and continue to the next chunk
+                    buffer.set(remainingBytes, offset);
+                    offset += remainingBytes.length;
+                }
+            }
+        }
+        // Yield any remaining bytes in the coalescing buffer
+        if (offset > 0) {
+            yield buffer.subarray(0, offset);
+        }
+    }
+};
 export const bytesToStream = (bytes) => {
     const stream = new Readable();
     stream.push(bytes);

package/dist/streams.js.map

@@ -1 +1 @@
-{"version":3,"file":"streams.js","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,WAAW,EACX,QAAQ,EAER,SAAS,EAET,QAAQ,GACT,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AAE/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,OAAiB,EAAE,EAAE;IAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;IACtD,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,MAAgB,EAAY,EAAE;IACxD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IACrC,mBAAmB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;IACxC,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;AACjC,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAgB,EAAmB,EAAE;IACpE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;IAClC,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,MAAiC,EAAE,EAAE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,iDAAiD;AACjD,IAAI,UAAU,CAAC,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAA;AAElD,sEAAsE;AACtE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,MAAwD,EACvC,EAAE;IACnB,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA,CAAC,+CAA+C;IACnE,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;AAC3C,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,IAA+B,EACrB,EAAE;IACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;AACnD,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAAiB,EAAY,EAAE;IAC3D,MAAM,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAA;IAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjB,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AAED,MAAM,OAAO,cAAe,SAAQ,SAAS;IAE3C,YACS,OAAe,EACf,WAAwB;QAE/B,KAAK,EAAE,CAAA;QAHA,YAAO,GAAP,OAAO,CAAQ;QACf,gBAAW,GAAX,WAAW,CAAa;QAHjC,cAAS,GAAG,CAAC,CAAA;IAMb,CAAC;IACD,UAAU,CAAC,KAAiB,EAAE,IAAoB,EAAE,EAAqB;QACvE,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,CAAA;QAC9B,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;CACF;AAUD,MAAM,UAAU,YAAY,CAC1B,MAA4C,EAC5C,eAAmC;IAEnC,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,CAAC,CAAA;IAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IACxC,OAAO,QAAQ,CAAC,CAAC,MAAkB,EAAE,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAE,CAAC,CAAW,CAAA;AACxE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,eAAmC;IAChE,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,IAAI,eAAe,EAAE,MAAM,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAa,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC;YACxD,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,UAAU,CAAC;YACrC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC9B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;YAEtD,QAAQ;YACR,8DAA8D;YAC9D,0DAA0D;YAC1D,6BAA6B;YAC7B,IAAI,kBAAkB,KAAK,UAAU;gBAAE,SAAQ;YAE/C,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,CAAA;QAClD,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,OAAO,EAAE,CAAA;AAC3B,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;AAC1B,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACzC,yDAAyD;IACzD,sDAAsD;IACtD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;AACtC,CAAC;AAED,SAAS,aAAa,CAAC,kBAA0B;IAC/C,QAAQ,kBAAkB,EAAE,CAAC;QAC3B,0DAA0D;QAC1D,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,YAAY,EAAE,CAAA;QACvB,KAAK,SAAS;YACZ,OAAO,aAAa,EAAE,CAAA;QACxB,KAAK,IAAI;YACP,OAAO,sBAAsB,EAAE,CAAA;QACjC,KAAK,UAAU;YACb,OAAO,IAAI,WAAW,EAAE,CAAA;QAC1B;YACE,MAAM,IAAI,SAAS,CACjB,kCAAkC,kBAAkB,GAAG,CACxD,CAAA;IACL,CAAC;AACH,CAAC","sourcesContent":["import {\n  Duplex,\n  PassThrough,\n  Readable,\n  Stream,\n  Transform,\n  TransformCallback,\n  pipeline,\n} from 'node:stream'\nimport { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'\n\nexport const forwardStreamErrors = (...streams: Stream[]) => {\n  for (let i = 1; i < streams.length; ++i) {\n    const prev = streams[i - 1]\n    const next = streams[i]\n\n    prev.once('error', (err) => next.emit('error', err))\n  }\n}\n\nexport const cloneStream = (stream: Readable): Readable => {\n  const passthrough = new PassThrough()\n  forwardStreamErrors(stream, passthrough)\n  return stream.pipe(passthrough)\n}\n\nexport const streamSize = async (stream: Readable): Promise<number> => {\n  let size = 0\n  for await (const chunk of stream) {\n    size += Buffer.byteLength(chunk)\n  }\n  return size\n}\n\nexport const streamToBytes = async (stream: AsyncIterable<Uint8Array>) =>\n  // @NOTE Though Buffer is a sub-class of Uint8Array, we have observed\n  // inconsistencies when using a Buffer in place of Uint8Array. For this\n  // reason, we convert the Buffer to a Uint8Array.\n  new Uint8Array(await streamToNodeBuffer(stream))\n\n// streamToBuffer identifier name already taken by @atproto/common-web\nexport const streamToNodeBuffer = async (\n  stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,\n): Promise<Buffer> => {\n  const chunks: Uint8Array[] = []\n  let totalLength = 0 // keep track of total length for Buffer.concat\n  for await (const chunk of stream) {\n    if (chunk instanceof Uint8Array) {\n      chunks.push(chunk)\n      totalLength += Buffer.byteLength(chunk)\n    } else {\n      throw new TypeError('expected Uint8Array')\n    }\n  }\n  return Buffer.concat(chunks, totalLength)\n}\n\nexport const byteIterableToStream = (\n  iter: AsyncIterable<Uint8Array>,\n): Readable => {\n  return Readable.from(iter, { objectMode: false })\n}\n\nexport const bytesToStream = (bytes: Uint8Array): Readable => {\n  const stream = new Readable()\n  stream.push(bytes)\n  stream.push(null)\n  return stream\n}\n\nexport class MaxSizeChecker extends Transform {\n  totalSize = 0\n  constructor(\n    public maxSize: number,\n    public createError: () => Error,\n  ) {\n    super()\n  }\n  _transform(chunk: Uint8Array, _enc: BufferEncoding, cb: TransformCallback) {\n    this.totalSize += chunk.length\n    if (this.totalSize > this.maxSize) {\n      cb(this.createError())\n    } else {\n      cb(null, chunk)\n    }\n  }\n}\n\nexport function decodeStream(\n  stream: Readable,\n  contentEncoding?: string | string[],\n): Readable\nexport function decodeStream(\n  stream: AsyncIterable<Uint8Array>,\n  contentEncoding?: string | string[],\n): AsyncIterable<Uint8Array> | Readable\nexport function decodeStream(\n  stream: Readable | AsyncIterable<Uint8Array>,\n  contentEncoding?: string | string[],\n): Readable | AsyncIterable<Uint8Array> {\n  const decoders = createDecoders(contentEncoding)\n  if (decoders.length === 0) return stream\n  return pipeline([stream as Readable, ...decoders], () => {}) as Duplex\n}\n\n/**\n * Create a series of decoding streams based on the content-encoding header. The\n * resulting streams should be piped together to decode the content.\n *\n * @see {@link https://datatracker.ietf.org/doc/html/rfc9110#section-8.4.1}\n */\nexport function createDecoders(contentEncoding?: string | string[]): Duplex[] {\n  const decoders: Duplex[] = []\n\n  if (contentEncoding?.length) {\n    const encodings: string[] = Array.isArray(contentEncoding)\n      ? contentEncoding.flatMap(commaSplit)\n      : contentEncoding.split(',')\n    for (const encoding of encodings) {\n      const normalizedEncoding = normalizeEncoding(encoding)\n\n      // @NOTE\n      // > The default (identity) encoding [...] is used only in the\n      // > Accept-Encoding header, and SHOULD NOT be used in the\n      // > Content-Encoding header.\n      if (normalizedEncoding === 'identity') continue\n\n      decoders.push(createDecoder(normalizedEncoding))\n    }\n  }\n\n  return decoders.reverse()\n}\n\nfunction commaSplit(header: string): string[] {\n  return header.split(',')\n}\n\nfunction normalizeEncoding(encoding: string) {\n  // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1\n  // > All content-coding values are case-insensitive...\n  return encoding.trim().toLowerCase()\n}\n\nfunction createDecoder(normalizedEncoding: string): Duplex {\n  switch (normalizedEncoding) {\n    // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.2\n    case 'gzip':\n    case 'x-gzip':\n      return createGunzip()\n    case 'deflate':\n      return createInflate()\n    case 'br':\n      return createBrotliDecompress()\n    case 'identity':\n      return new PassThrough()\n    default:\n      throw new TypeError(\n        `Unsupported content-encoding: \"${normalizedEncoding}\"`,\n      )\n  }\n}\n"]}
+{"version":3,"file":"streams.js","sourceRoot":"","sources":["../src/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,WAAW,EACX,QAAQ,EAER,SAAS,EAET,QAAQ,GACT,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AAE/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,OAAiB,EAAE,EAAE;IAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;IACtD,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,MAAgB,EAAY,EAAE;IACxD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IACrC,mBAAmB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;IACxC,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;AACjC,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAgB,EAAmB,EAAE;IACpE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;IAClC,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,MAAiC,EAAE,EAAE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,iDAAiD;AACjD,IAAI,UAAU,CAAC,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAA;AAElD,sEAAsE;AACtE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,MAAwD,EACvC,EAAE;IACnB,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA,CAAC,+CAA+C;IACnE,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;AAC3C,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,IAA+B,EACrB,EAAE;IACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;AACnD,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,MAAwD,EACxD,YAAoB,EACV,EAAE;IACZ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,SAAS,CAAC,yCAAyC,CAAC,CAAA;IAChE,CAAC;IAED,oEAAoE;IACpE,qCAAqC;IACrC,OAAO,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;QACzC,uDAAuD;IACzD,CAAC,CAAgB,CAAA;IAEjB,4EAA4E;IAC5E,8EAA8E;IAC9E,8CAA8C;IAC9C,KAAK,SAAS,CAAC,CAAC,QAAQ,CACtB,IAAsD;QAEtD,2EAA2E;QAC3E,kEAAkE;QAClE,qBAAqB;QAErB,IAAI,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,CAAC,CAAA;QACzC,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YAC/B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAA;YACxC,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,oEAAoE;gBACpE,mEAAmE;gBACnE,QAAQ;gBACR,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;gBACzB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,CAAC;iBAAM,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;gBACxD,qEAAqE;gBACrE,oEAAoE;gBACpE,0BAA0B;gBAC1B,MAAM,KAAK,CAAA;YACb,CAAC;iBAAM,CAAC;gBACN,qEAAqE;gBACrE,mEAAmE;gBACnE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAA;gBAChD,MAAM,MAAM,CAAA;gBAEZ,qDAAqD;gBACrD,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,CAAC,CAAA;gBACrC,MAAM,GAAG,CAAC,CAAA;gBAEV,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;gBAChD,IAAI,cAAc,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;oBACzC,2DAA2D;oBAC3D,6DAA6D;oBAC7D,MAAM,cAAc,CAAA;gBACtB,CAAC;qBAAM,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrC,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAA;oBAClC,MAAM,IAAI,cAAc,CAAC,MAAM,CAAA;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;QAClC,CAAC;IACH,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAAiB,EAAY,EAAE;IAC3D,MAAM,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAA;IAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjB,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AAED,MAAM,OAAO,cAAe,SAAQ,SAAS;IAE3C,YACS,OAAe,EACf,WAAwB;QAE/B,KAAK,EAAE,CAAA;QAHA,YAAO,GAAP,OAAO,CAAQ;QACf,gBAAW,GAAX,WAAW,CAAa;QAHjC,cAAS,GAAG,CAAC,CAAA;IAMb,CAAC;IACD,UAAU,CAAC,KAAiB,EAAE,IAAoB,EAAE,EAAqB;QACvE,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,CAAA;QAC9B,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;CACF;AAUD,MAAM,UAAU,YAAY,CAC1B,MAA4C,EAC5C,eAAmC;IAEnC,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,CAAC,CAAA;IAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IACxC,OAAO,QAAQ,CAAC,CAAC,MAAkB,EAAE,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,GAAE,CAAC,CAAW,CAAA;AACxE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,eAAmC;IAChE,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,IAAI,eAAe,EAAE,MAAM,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAa,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC;YACxD,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,UAAU,CAAC;YACrC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC9B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;YAEtD,QAAQ;YACR,8DAA8D;YAC9D,0DAA0D;YAC1D,6BAA6B;YAC7B,IAAI,kBAAkB,KAAK,UAAU;gBAAE,SAAQ;YAE/C,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,CAAA;QAClD,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,OAAO,EAAE,CAAA;AAC3B,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;AAC1B,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACzC,yDAAyD;IACzD,sDAAsD;IACtD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;AACtC,CAAC;AAED,SAAS,aAAa,CAAC,kBAA0B;IAC/C,QAAQ,kBAAkB,EAAE,CAAC;QAC3B,0DAA0D;QAC1D,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,YAAY,EAAE,CAAA;QACvB,KAAK,SAAS;YACZ,OAAO,aAAa,EAAE,CAAA;QACxB,KAAK,IAAI;YACP,OAAO,sBAAsB,EAAE,CAAA;QACjC,KAAK,UAAU;YACb,OAAO,IAAI,WAAW,EAAE,CAAA;QAC1B;YACE,MAAM,IAAI,SAAS,CACjB,kCAAkC,kBAAkB,GAAG,CACxD,CAAA;IACL,CAAC;AACH,CAAC","sourcesContent":["import {\n  Duplex,\n  PassThrough,\n  Readable,\n  Stream,\n  Transform,\n  TransformCallback,\n  pipeline,\n} from 'node:stream'\nimport { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'\n\nexport const forwardStreamErrors = (...streams: Stream[]) => {\n  for (let i = 1; i < streams.length; ++i) {\n    const prev = streams[i - 1]\n    const next = streams[i]\n\n    prev.once('error', (err) => next.emit('error', err))\n  }\n}\n\nexport const cloneStream = (stream: Readable): Readable => {\n  const passthrough = new PassThrough()\n  forwardStreamErrors(stream, passthrough)\n  return stream.pipe(passthrough)\n}\n\nexport const streamSize = async (stream: Readable): Promise<number> => {\n  let size = 0\n  for await (const chunk of stream) {\n    size += Buffer.byteLength(chunk)\n  }\n  return size\n}\n\nexport const streamToBytes = async (stream: AsyncIterable<Uint8Array>) =>\n  // @NOTE Though Buffer is a sub-class of Uint8Array, we have observed\n  // inconsistencies when using a Buffer in place of Uint8Array. For this\n  // reason, we convert the Buffer to a Uint8Array.\n  new Uint8Array(await streamToNodeBuffer(stream))\n\n// streamToBuffer identifier name already taken by @atproto/common-web\nexport const streamToNodeBuffer = async (\n  stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,\n): Promise<Buffer> => {\n  const chunks: Uint8Array[] = []\n  let totalLength = 0 // keep track of total length for Buffer.concat\n  for await (const chunk of stream) {\n    if (chunk instanceof Uint8Array) {\n      chunks.push(chunk)\n      totalLength += Buffer.byteLength(chunk)\n    } else {\n      throw new TypeError('expected Uint8Array')\n    }\n  }\n  return Buffer.concat(chunks, totalLength)\n}\n\nexport const byteIterableToStream = (\n  iter: AsyncIterable<Uint8Array>,\n): Readable => {\n  return Readable.from(iter, { objectMode: false })\n}\n\n/**\n * Coalesce a stream of Uint8Array chunks into larger chunks of at least the\n * specified size ({@link minChunkSize}). This is useful for optimizing\n * downstream processing that benefits from larger chunk sizes, such as\n * compression or hashing.\n */\nexport const coalesceByteStream = (\n  stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,\n  minChunkSize: number,\n): Readable => {\n  if (!Number.isInteger(minChunkSize) || minChunkSize < 1) {\n    throw new TypeError('minChunkSize must be a positive integer')\n  }\n\n  // @NOTE On Node 22, this *does* return a PassThrough (\"@types/node\"\n  // incorrectly types it as Writable).\n  return pipeline(stream, coalesce, (_err) => {\n    // Errors are expected to be handled through the stream\n  }) as PassThrough\n\n  // @NOTE This implementation is not NodeJS specific and could be exported as\n  // utility (from \"@atproto/common-web\") if needed. We don't do it now to avoid\n  // increasing the API surface of our packages.\n  async function* coalesce(\n    iter: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,\n  ): AsyncGenerator<Uint8Array> {\n    // @NOTE This implementation avoids as many un-necessary copies as possible\n    // and only buffers incoming chunks when they are smaller than the\n    // coalescing buffer.\n\n    let buffer = new Uint8Array(minChunkSize)\n    let offset = 0\n\n    for await (const chunk of iter) {\n      const freeSpace = buffer.length - offset\n      if (freeSpace > chunk.length) {\n        // If the incoming chunk is smaller than the free space, we copy the\n        // entire chunk into the coalescing buffer and continue to the next\n        // chunk\n        buffer.set(chunk, offset)\n        offset += chunk.length\n      } else if (offset === 0 && chunk.length >= minChunkSize) {\n        // If the coalescing buffer is empty and the incoming chunk is larger\n        // than the coalescing buffer, we can skip any copying and yield the\n        // incoming chunk directly\n        yield chunk\n      } else {\n        // Otherwise, we need to copy as much of the incoming chunk as we can\n        // into the coalescing buffer and yield the full coalescing buffer.\n        buffer.set(chunk.subarray(0, freeSpace), offset)\n        yield buffer\n\n        // We create a new coalescing buffer (for future use)\n        buffer = new Uint8Array(minChunkSize)\n        offset = 0\n\n        const remainingBytes = chunk.subarray(freeSpace)\n        if (remainingBytes.length > minChunkSize) {\n          // If the remaining of chunk is still too big to fit in the\n          // coalescing buffer, we yield it directly without copying it\n          yield remainingBytes\n        } else if (remainingBytes.length > 0) {\n          // Otherwise, we copy the remaining bytes into the coalescing buffer\n          // and continue to the next chunk\n          buffer.set(remainingBytes, offset)\n          offset += remainingBytes.length\n        }\n      }\n    }\n\n    // Yield any remaining bytes in the coalescing buffer\n    if (offset > 0) {\n      yield buffer.subarray(0, offset)\n    }\n  }\n}\n\nexport const bytesToStream = (bytes: Uint8Array): Readable => {\n  const stream = new Readable()\n  stream.push(bytes)\n  stream.push(null)\n  return stream\n}\n\nexport class MaxSizeChecker extends Transform {\n  totalSize = 0\n  constructor(\n    public maxSize: number,\n    public createError: () => Error,\n  ) {\n    super()\n  }\n  _transform(chunk: Uint8Array, _enc: BufferEncoding, cb: TransformCallback) {\n    this.totalSize += chunk.length\n    if (this.totalSize > this.maxSize) {\n      cb(this.createError())\n    } else {\n      cb(null, chunk)\n    }\n  }\n}\n\nexport function decodeStream(\n  stream: Readable,\n  contentEncoding?: string | string[],\n): Readable\nexport function decodeStream(\n  stream: AsyncIterable<Uint8Array>,\n  contentEncoding?: string | string[],\n): AsyncIterable<Uint8Array> | Readable\nexport function decodeStream(\n  stream: Readable | AsyncIterable<Uint8Array>,\n  contentEncoding?: string | string[],\n): Readable | AsyncIterable<Uint8Array> {\n  const decoders = createDecoders(contentEncoding)\n  if (decoders.length === 0) return stream\n  return pipeline([stream as Readable, ...decoders], () => {}) as Duplex\n}\n\n/**\n * Create a series of decoding streams based on the content-encoding header. The\n * resulting streams should be piped together to decode the content.\n *\n * @see {@link https://datatracker.ietf.org/doc/html/rfc9110#section-8.4.1}\n */\nexport function createDecoders(contentEncoding?: string | string[]): Duplex[] {\n  const decoders: Duplex[] = []\n\n  if (contentEncoding?.length) {\n    const encodings: string[] = Array.isArray(contentEncoding)\n      ? contentEncoding.flatMap(commaSplit)\n      : contentEncoding.split(',')\n    for (const encoding of encodings) {\n      const normalizedEncoding = normalizeEncoding(encoding)\n\n      // @NOTE\n      // > The default (identity) encoding [...] is used only in the\n      // > Accept-Encoding header, and SHOULD NOT be used in the\n      // > Content-Encoding header.\n      if (normalizedEncoding === 'identity') continue\n\n      decoders.push(createDecoder(normalizedEncoding))\n    }\n  }\n\n  return decoders.reverse()\n}\n\nfunction commaSplit(header: string): string[] {\n  return header.split(',')\n}\n\nfunction normalizeEncoding(encoding: string) {\n  // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1\n  // > All content-coding values are case-insensitive...\n  return encoding.trim().toLowerCase()\n}\n\nfunction createDecoder(normalizedEncoding: string): Duplex {\n  switch (normalizedEncoding) {\n    // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.2\n    case 'gzip':\n    case 'x-gzip':\n      return createGunzip()\n    case 'deflate':\n      return createInflate()\n    case 'br':\n      return createBrotliDecompress()\n    case 'identity':\n      return new PassThrough()\n    default:\n      throw new TypeError(\n        `Unsupported content-encoding: \"${normalizedEncoding}\"`,\n      )\n  }\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/common",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "license": "MIT",
   "description": "Shared web-platform-friendly code for atproto libraries",
   "keywords": [
@@ -19,8 +19,8 @@
     "multiformats": "^13.0.0",
     "pino": "^8.21.0",
     "@atproto/common-web": "^0.5.0",
-    "@atproto/lex-cbor": "^0.1.0",
-    "@atproto/lex-data": "^0.1.0"
+    "@atproto/lex-data": "^0.1.1",
+    "@atproto/lex-cbor": "^0.1.0"
   },
   "devDependencies": {
     "jest": "^30.0.0",

package/src/buffers.ts

@@ -2,9 +2,13 @@ export function ui8ToBuffer(bytes: Uint8Array): Buffer {
   return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
 }
 
-export function ui8ToArrayBuffer(bytes: Uint8Array<ArrayBuffer>): ArrayBuffer {
-  return bytes.buffer.slice(
-    bytes.byteOffset,
-    bytes.byteLength + bytes.byteOffset,
-  )
+export function ui8ToArrayBuffer(bytes: Uint8Array): ArrayBuffer {
+  if (bytes.buffer instanceof ArrayBuffer) {
+    return bytes.buffer.slice(
+      bytes.byteOffset,
+      bytes.byteLength + bytes.byteOffset,
+    )
+  }
+
+  return new Uint8Array(bytes).buffer
 }

package/src/fs.ts

@@ -31,14 +31,12 @@ export const rmIfExists = async (
   filepath: string,
   recursive = false,
 ): Promise<void> => {
-  try {
-    await fs.rm(filepath, { recursive })
-  } catch (err) {
-    if (isErrnoException(err) && err.code === 'ENOENT') {
-      return
-    }
-    throw err
-  }
+  await fs.rm(filepath, {
+    force: true, // ignore errors if the file/directory does not exist
+    recursive,
+    maxRetries: 5,
+    retryDelay: 50,
+  })
 }
 
 export const renameIfExists = async (

package/src/streams.ts

@@ -61,6 +61,83 @@ export const byteIterableToStream = (
   return Readable.from(iter, { objectMode: false })
 }
 
+/**
+ * Coalesce a stream of Uint8Array chunks into larger chunks of at least the
+ * specified size ({@link minChunkSize}). This is useful for optimizing
+ * downstream processing that benefits from larger chunk sizes, such as
+ * compression or hashing.
+ */
+export const coalesceByteStream = (
+  stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+  minChunkSize: number,
+): Readable => {
+  if (!Number.isInteger(minChunkSize) || minChunkSize < 1) {
+    throw new TypeError('minChunkSize must be a positive integer')
+  }
+
+  // @NOTE On Node 22, this *does* return a PassThrough ("@types/node"
+  // incorrectly types it as Writable).
+  return pipeline(stream, coalesce, (_err) => {
+    // Errors are expected to be handled through the stream
+  }) as PassThrough
+
+  // @NOTE This implementation is not NodeJS specific and could be exported as
+  // utility (from "@atproto/common-web") if needed. We don't do it now to avoid
+  // increasing the API surface of our packages.
+  async function* coalesce(
+    iter: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+  ): AsyncGenerator<Uint8Array> {
+    // @NOTE This implementation avoids as many un-necessary copies as possible
+    // and only buffers incoming chunks when they are smaller than the
+    // coalescing buffer.
+
+    let buffer = new Uint8Array(minChunkSize)
+    let offset = 0
+
+    for await (const chunk of iter) {
+      const freeSpace = buffer.length - offset
+      if (freeSpace > chunk.length) {
+        // If the incoming chunk is smaller than the free space, we copy the
+        // entire chunk into the coalescing buffer and continue to the next
+        // chunk
+        buffer.set(chunk, offset)
+        offset += chunk.length
+      } else if (offset === 0 && chunk.length >= minChunkSize) {
+        // If the coalescing buffer is empty and the incoming chunk is larger
+        // than the coalescing buffer, we can skip any copying and yield the
+        // incoming chunk directly
+        yield chunk
+      } else {
+        // Otherwise, we need to copy as much of the incoming chunk as we can
+        // into the coalescing buffer and yield the full coalescing buffer.
+        buffer.set(chunk.subarray(0, freeSpace), offset)
+        yield buffer
+
+        // We create a new coalescing buffer (for future use)
+        buffer = new Uint8Array(minChunkSize)
+        offset = 0
+
+        const remainingBytes = chunk.subarray(freeSpace)
+        if (remainingBytes.length > minChunkSize) {
+          // If the remaining of chunk is still too big to fit in the
+          // coalescing buffer, we yield it directly without copying it
+          yield remainingBytes
+        } else if (remainingBytes.length > 0) {
+          // Otherwise, we copy the remaining bytes into the coalescing buffer
+          // and continue to the next chunk
+          buffer.set(remainingBytes, offset)
+          offset += remainingBytes.length
+        }
+      }
+    }
+
+    // Yield any remaining bytes in the coalescing buffer
+    if (offset > 0) {
+      yield buffer.subarray(0, offset)
+    }
+  }
+}
+
 export const bytesToStream = (bytes: Uint8Array): Readable => {
   const stream = new Readable()
   stream.push(bytes)

package/tests/streams.test.ts

@@ -1,3 +1,4 @@
+import events from 'node:events'
 import { PassThrough, Readable } from 'node:stream'
 import * as streams from '../src/streams.js'
 
@@ -116,6 +117,105 @@ describe('streams', () => {
     })
   })
 
+  describe('coalesceByteStream', () => {
+    it('coalesces chunks without changing bytes', async () => {
+      const stream = streams.coalesceByteStream(
+        Readable.from([
+          new Uint8Array([0x1]),
+          new Uint8Array([0x2, 0x3]),
+          new Uint8Array([0x4, 0x5, 0x6]),
+          new Uint8Array([0x7]),
+          new Uint8Array([0x8]),
+          new Uint8Array([0x9]),
+        ]),
+        4,
+      )
+
+      const chunks = await stream.toArray()
+
+      expect(Buffer.concat(chunks)).toEqual(
+        Buffer.from([0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9]),
+      )
+
+      for (let i = 0; i < chunks.length; i++) {
+        expect(chunks[i]).toBeInstanceOf(Uint8Array)
+        if (i < chunks.length - 1) {
+          expect(chunks[i].length).toBeGreaterThanOrEqual(4)
+        }
+      }
+    })
+
+    it('yields chunks as they come when target size is 1', async () => {
+      const stream = streams.coalesceByteStream(
+        Readable.from([
+          new Uint8Array([0x1]),
+          new Uint8Array([0x2, 0x3]),
+          new Uint8Array([0x4, 0x5, 0x6]),
+        ]),
+        1,
+      )
+
+      const chunks = await stream.toArray()
+
+      expect(chunks).toEqual([
+        new Uint8Array([0x1]),
+        new Uint8Array([0x2, 0x3]),
+        new Uint8Array([0x4, 0x5, 0x6]),
+      ])
+    })
+
+    it('coalesces into a single chunk when target size exceeds total', async () => {
+      const stream = streams.coalesceByteStream(
+        Readable.from([
+          new Uint8Array([0x1]),
+          new Uint8Array([0x2, 0x3]),
+          new Uint8Array([0x4, 0x5, 0x6]),
+        ]),
+        1000,
+      )
+
+      const chunks = await stream.toArray()
+
+      expect(chunks.length).toBe(1)
+      expect(Buffer.concat(chunks)).toEqual(
+        Buffer.from([0x1, 0x2, 0x3, 0x4, 0x5, 0x6]),
+      )
+    })
+
+    it('forwards source stream errors', async () => {
+      const source = new PassThrough()
+      const stream = streams.coalesceByteStream(source, 4)
+      const err = new Error('source failed')
+
+      const gotError = events.once(stream, 'error')
+      source.emit('error', err)
+
+      expect(await gotError).toEqual([err])
+    })
+
+    it('destroying the coalesced stream destroys the source', async () => {
+      let finalized = false
+      async function* gen() {
+        try {
+          while (true) {
+            yield new Uint8Array(1024)
+          }
+        } finally {
+          finalized = true
+        }
+      }
+      const source = Readable.from(gen(), { objectMode: false })
+      const stream = streams.coalesceByteStream(source, 4096)
+
+      await events.once(stream, 'data')
+      stream.destroy()
+      await new Promise((resolve) => source.once('close', resolve))
+
+      expect(source.destroyed).toBe(true)
+      expect(finalized).toBe(true)
+    })
+  })
+
   describe('bytesToStream', () => {
     it('converts byte array to readable stream', async () => {
       const bytes = new Uint8Array([0xa, 0xb])

package/tsconfig.build.json

@@ -2,7 +2,7 @@
   "extends": "../../tsconfig/node.json",
   "compilerOptions": {
     "rootDir": "./src",
-    "outDir": "./dist"
+    "outDir": "./dist",
   },
-  "include": ["./src"]
+  "include": ["./src"],
 }

package/tsconfig.json

@@ -2,6 +2,6 @@
   "include": [],
   "references": [
     { "path": "./tsconfig.build.json" },
-    { "path": "./tsconfig.tests.json" }
-  ]
+    { "path": "./tsconfig.tests.json" },
+  ],
 }

package/tsconfig.tests.json

@@ -1,7 +1,7 @@
 {
   "extends": "../../tsconfig/tests.json",
   "compilerOptions": {
-    "rootDir": "."
+    "rootDir": ".",
   },
-  "include": ["./tests"]
+  "include": ["./tests"],
 }