@atproto/repo 0.8.12 → 0.9.0

package/dist/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAwB,MAAM,kBAAkB,CAAA;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EACL,MAAM,EACN,cAAc,EACd,oBAAoB,EAEpB,UAAU,EAEV,mBAAmB,EACnB,cAAc,EAEf,MAAM,SAAS,CAAA;AAEhB,eAAO,MAAM,oBAAoB,GAC/B,MAAM,QAAQ,KACb,OAAO,CAAC,mBAAmB,EAAE,CA+B/B,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,WAAW,mBAAmB,EAAE,KAC/B,oBAAoB,EAUtB,CAAA;AAED,eAAO,MAAM,YAAY,GAAI,KAAK,MAAM,KAAG,UAI1C,CAAA;AAED,eAAO,MAAM,aAAa,GAAI,YAAY,MAAM,EAAE,MAAM,MAAM,KAAG,MAEhE,CAAA;AAED,eAAO,MAAM,SAAS,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,OAEhD,CAAA;AAED,eAAO,MAAM,UAAU,GACrB,UAAU,cAAc,EACxB,SAAS,OAAO,KACf,OAAO,CAAC,MAAM,CAOhB,CAAA;AAED,eAAO,MAAM,eAAe,GAC1B,QAAQ,MAAM,EACd,QAAQ,MAAM,KACb,OAAO,CAAC,OAAO,CAIjB,CAAA;AAED,eAAO,MAAM,SAAS,GAAI,KAAK,UAAU,KAAG,QAE3C,CAAA;AAED,eAAO,MAAM,eAAe,GAAI,KAAK,UAAU,KAAG,UAMjD,CAAA;AAED,eAAO,MAAM,YAAY,GAAU,KAAK,QAAQ,4CAE/C,CAAA;AAED,eAAO,MAAM,cAAc,GAAI,QAAQ,cAAc,GAAG,MAAM,KAAG,MAUhE,CAAA"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAiB,MAAM,mBAAmB,CAAA;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EACL,MAAM,EACN,cAAc,EACd,oBAAoB,EAEpB,UAAU,EAEV,mBAAmB,EACnB,cAAc,EAEf,MAAM,SAAS,CAAA;AAEhB,eAAO,MAAM,oBAAoB,GAC/B,MAAM,QAAQ,KACb,OAAO,CAAC,mBAAmB,EAAE,CA+B/B,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,WAAW,mBAAmB,EAAE,KAC/B,oBAAoB,EAUtB,CAAA;AAED,eAAO,MAAM,YAAY,GAAI,KAAK,MAAM,KAAG,UAI1C,CAAA;AAED,eAAO,MAAM,aAAa,GAAI,YAAY,MAAM,EAAE,MAAM,MAAM,KAAG,MAEhE,CAAA;AAED,eAAO,MAAM,SAAS,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,OAEhD,CAAA;AAED,eAAO,MAAM,UAAU,GACrB,UAAU,cAAc,EACxB,SAAS,OAAO,KACf,OAAO,CAAC,MAAM,CAOhB,CAAA;AAED,eAAO,MAAM,eAAe,GAC1B,QAAQ,MAAM,EACd,QAAQ,MAAM,KACb,OAAO,CAAC,OAAO,CAIjB,CAAA;AAED,eAAO,MAAM,SAAS,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,QAAsB,CAAA;AAEnE,eAAO,MAAM,eAAe,GAAI,KAAK,UAAU,KAAG,MAMjD,CAAA;AAED,eAAO,MAAM,YAAY,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,OAAO,CAAC,GAAG,CAAkB,CAAA;AAE3E,eAAO,MAAM,cAAc,GAAI,QAAQ,cAAc,GAAG,MAAM,KAAG,MAUhE,CAAA;AAED,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,wCAIzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,GAAG,UAAU,CAUrE"}

package/dist/util.js

@@ -1,5 +1,4 @@
 "use strict";
-/* eslint-disable import/no-deprecated */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -35,10 +34,12 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ensureV3Commit = exports.cidForRecord = exports.cborToLexRecord = exports.cborToLex = exports.verifyCommitSig = exports.signCommit = exports.metaEqual = exports.formatDataKey = exports.parseDataKey = exports.ensureCreates = exports.diffToWriteDescripts = void 0;
-const cbor = __importStar(require("@ipld/dag-cbor"));
-const common_1 = require("@atproto/common");
+exports.concatBytesAsync = concatBytesAsync;
+exports.concatBytes = concatBytes;
+const common_web_1 = require("@atproto/common-web");
 const crypto = __importStar(require("@atproto/crypto"));
-const lexicon_1 = require("@atproto/lexicon");
+const cbor = __importStar(require("@atproto/lex-cbor"));
+const lex_data_1 = require("@atproto/lex-data");
 const types_1 = require("./types");
 const diffToWriteDescripts = (diff) => {
     return Promise.all([
@@ -87,10 +88,10 @@ const ensureCreates = (descripts) => {
 };
 exports.ensureCreates = ensureCreates;
 const parseDataKey = (key) => {
-    const parts = key.split('/');
-    if (parts.length !== 2)
+    const { length, 0: collection, 1: rkey } = key.split('/');
+    if (length !== 2)
         throw new Error(`Invalid record key: ${key}`);
-    return { collection: parts[0], rkey: parts[1] };
+    return { collection, rkey };
 };
 exports.parseDataKey = parseDataKey;
 const formatDataKey = (collection, rkey) => {
@@ -116,22 +117,16 @@ const verifyCommitSig = async (commit, didKey) => {
     return crypto.verifySignature(didKey, encoded, sig);
 };
 exports.verifyCommitSig = verifyCommitSig;
-const cborToLex = (val) => {
-    return (0, lexicon_1.ipldToLex)((0, common_1.cborDecode)(val));
-};
-exports.cborToLex = cborToLex;
+exports.cborToLex = cbor.decode;
 const cborToLexRecord = (val) => {
     const parsed = (0, exports.cborToLex)(val);
-    if (!common_1.check.is(parsed, common_1.schema.map)) {
+    if (!(0, lex_data_1.isPlainObject)(parsed)) {
         throw new Error('lexicon records be a json object');
     }
     return parsed;
 };
 exports.cborToLexRecord = cborToLexRecord;
-const cidForRecord = async (val) => {
-    return (0, common_1.cidForCbor)((0, lexicon_1.lexToIpld)(val));
-};
-exports.cidForRecord = cidForRecord;
+exports.cidForRecord = cbor.cidForLex;
 const ensureV3Commit = (commit) => {
     if (commit.version === 3) {
         return commit;
@@ -140,9 +135,30 @@ const ensureV3Commit = (commit) => {
         return {
             ...commit,
             version: 3,
-            rev: commit.rev ?? common_1.TID.nextStr(),
+            rev: commit.rev ?? common_web_1.TID.nextStr(),
         };
     }
 };
 exports.ensureV3Commit = ensureV3Commit;
+async function concatBytesAsync(iterable) {
+    const chunks = [];
+    for await (const chunk of iterable)
+        chunks.push(chunk);
+    return concatBytes(chunks);
+}
+/**
+ * This is the same as {@link Buffer.concat}, without the `totalLength` argument.
+ */
+function concatBytes(chunks) {
+    let totalLength = 0;
+    for (const chunk of chunks)
+        totalLength += chunk.byteLength;
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return result;
+}
 //# sourceMappingURL=util.js.map

package/dist/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEzC,qDAAsC;AACtC,4CAA4E;AAC5E,wDAAyC;AAEzC,8CAA6E;AAE7E,mCAUgB;AAET,MAAM,oBAAoB,GAAG,CAClC,IAAc,EACkB,EAAE;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC;QACjB,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAClC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACrC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,IAAI;aACS,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/B,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAA;AAjCY,QAAA,oBAAoB,wBAiChC;AAEM,MAAM,aAAa,GAAG,CAC3B,SAAgC,EACR,EAAE;IAC1B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,KAAK,qBAAa,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAZY,QAAA,aAAa,iBAYzB;AAEM,MAAM,YAAY,GAAG,CAAC,GAAW,EAAc,EAAE;IACtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAA;IACrE,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;AACjD,CAAC,CAAA;AAJY,QAAA,YAAY,gBAIxB;AAEM,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,IAAY,EAAU,EAAE;IACxE,OAAO,UAAU,GAAG,GAAG,GAAG,IAAI,CAAA;AAChC,CAAC,CAAA;AAFY,QAAA,aAAa,iBAEzB;AAEM,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS,EAAW,EAAE;IACzD,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAA;AACnD,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,UAAU,GAAG,KAAK,EAC7B,QAAwB,EACxB,OAAgB,EACC,EAAE;IACnB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACvC,OAAO;QACL,GAAG,QAAQ;QACX,GAAG;KACJ,CAAA;AACH,CAAC,CAAA;AAVY,QAAA,UAAU,cAUtB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,MAAc,EACd,MAAc,EACI,EAAE;IACpB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAA;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACjC,OAAO,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;AACrD,CAAC,CAAA;AAPY,QAAA,eAAe,mBAO3B;AAEM,MAAM,SAAS,GAAG,CAAC,GAAe,EAAY,EAAE;IACrD,OAAO,IAAA,mBAAS,EAAC,IAAA,mBAAU,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,eAAe,GAAG,CAAC,GAAe,EAAc,EAAE;IAC7D,MAAM,MAAM,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,CAAA;IAC7B,IAAI,CAAC,cAAK,CAAC,EAAE,CAAC,MAAM,EAAE,eAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,YAAY,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;IAClD,OAAO,IAAA,mBAAU,EAAC,IAAA,mBAAS,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,YAAY,gBAExB;AAEM,MAAM,cAAc,GAAG,CAAC,MAA+B,EAAU,EAAE;IACxE,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,MAAM,CAAA;IACf,CAAC;SAAM,CAAC;QACN,OAAO;YACL,GAAG,MAAM;YACT,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,YAAG,CAAC,OAAO,EAAE;SACjC,CAAA;IACH,CAAC;AACH,CAAC,CAAA;AAVY,QAAA,cAAc,kBAU1B","sourcesContent":["/* eslint-disable import/no-deprecated */\n\nimport * as cbor from '@ipld/dag-cbor'\nimport { TID, cborDecode, check, cidForCbor, schema } from '@atproto/common'\nimport * as crypto from '@atproto/crypto'\nimport { Keypair } from '@atproto/crypto'\nimport { LexValue, RepoRecord, ipldToLex, lexToIpld } from '@atproto/lexicon'\nimport { DataDiff } from './data-diff'\nimport {\n  Commit,\n  LegacyV2Commit,\n  RecordCreateDescript,\n  RecordDeleteDescript,\n  RecordPath,\n  RecordUpdateDescript,\n  RecordWriteDescript,\n  UnsignedCommit,\n  WriteOpAction,\n} from './types'\n\nexport const diffToWriteDescripts = (\n  diff: DataDiff,\n): Promise<RecordWriteDescript[]> => {\n  return Promise.all([\n    ...diff.addList().map(async (add) => {\n      const { collection, rkey } = parseDataKey(add.key)\n      return {\n        action: WriteOpAction.Create,\n        collection,\n        rkey,\n        cid: add.cid,\n      } as RecordCreateDescript\n    }),\n    ...diff.updateList().map(async (upd) => {\n      const { collection, rkey } = parseDataKey(upd.key)\n      return {\n        action: WriteOpAction.Update,\n        collection,\n        rkey,\n        cid: upd.cid,\n        prev: upd.prev,\n      } as RecordUpdateDescript\n    }),\n    ...diff.deleteList().map((del) => {\n      const { collection, rkey } = parseDataKey(del.key)\n      return {\n        action: WriteOpAction.Delete,\n        collection,\n        rkey,\n        cid: del.cid,\n      } as RecordDeleteDescript\n    }),\n  ])\n}\n\nexport const ensureCreates = (\n  descripts: RecordWriteDescript[],\n): RecordCreateDescript[] => {\n  const creates: RecordCreateDescript[] = []\n  for (const descript of descripts) {\n    if (descript.action !== WriteOpAction.Create) {\n      throw new Error(`Unexpected action: ${descript.action}`)\n    } else {\n      creates.push(descript)\n    }\n  }\n  return creates\n}\n\nexport const parseDataKey = (key: string): RecordPath => {\n  const parts = key.split('/')\n  if (parts.length !== 2) throw new Error(`Invalid record key: ${key}`)\n  return { collection: parts[0], rkey: parts[1] }\n}\n\nexport const formatDataKey = (collection: string, rkey: string): string => {\n  return collection + '/' + rkey\n}\n\nexport const metaEqual = (a: Commit, b: Commit): boolean => {\n  return a.did === b.did && a.version === b.version\n}\n\nexport const signCommit = async (\n  unsigned: UnsignedCommit,\n  keypair: Keypair,\n): Promise<Commit> => {\n  const encoded = cbor.encode(unsigned)\n  const sig = await keypair.sign(encoded)\n  return {\n    ...unsigned,\n    sig,\n  }\n}\n\nexport const verifyCommitSig = async (\n  commit: Commit,\n  didKey: string,\n): Promise<boolean> => {\n  const { sig, ...rest } = commit\n  const encoded = cbor.encode(rest)\n  return crypto.verifySignature(didKey, encoded, sig)\n}\n\nexport const cborToLex = (val: Uint8Array): LexValue => {\n  return ipldToLex(cborDecode(val))\n}\n\nexport const cborToLexRecord = (val: Uint8Array): RepoRecord => {\n  const parsed = cborToLex(val)\n  if (!check.is(parsed, schema.map)) {\n    throw new Error('lexicon records be a json object')\n  }\n  return parsed\n}\n\nexport const cidForRecord = async (val: LexValue) => {\n  return cidForCbor(lexToIpld(val))\n}\n\nexport const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {\n  if (commit.version === 3) {\n    return commit\n  } else {\n    return {\n      ...commit,\n      version: 3,\n      rev: commit.rev ?? TID.nextStr(),\n    }\n  }\n}\n"]}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8HA,4CAIC;AAKD,kCAUC;AAjJD,oDAAyC;AACzC,wDAAyC;AAEzC,wDAAyC;AACzC,gDAAwE;AAExE,mCAUgB;AAET,MAAM,oBAAoB,GAAG,CAClC,IAAc,EACkB,EAAE;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC;QACjB,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAClC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACrC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,IAAI;aACS,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/B,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAA;AAjCY,QAAA,oBAAoB,wBAiChC;AAEM,MAAM,aAAa,GAAG,CAC3B,SAAgC,EACR,EAAE;IAC1B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,KAAK,qBAAa,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAZY,QAAA,aAAa,iBAYzB;AAEM,MAAM,YAAY,GAAG,CAAC,GAAW,EAAc,EAAE;IACtD,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACzD,IAAI,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAA;IAC/D,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AAC7B,CAAC,CAAA;AAJY,QAAA,YAAY,gBAIxB;AAEM,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,IAAY,EAAU,EAAE;IACxE,OAAO,UAAU,GAAG,GAAG,GAAG,IAAI,CAAA;AAChC,CAAC,CAAA;AAFY,QAAA,aAAa,iBAEzB;AAEM,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS,EAAW,EAAE;IACzD,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAA;AACnD,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,UAAU,GAAG,KAAK,EAC7B,QAAwB,EACxB,OAAgB,EACC,EAAE;IACnB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACvC,OAAO;QACL,GAAG,QAAQ;QACX,GAAG;KACJ,CAAA;AACH,CAAC,CAAA;AAVY,QAAA,UAAU,cAUtB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,MAAc,EACd,MAAc,EACI,EAAE;IACpB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAA;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACjC,OAAO,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;AACrD,CAAC,CAAA;AAPY,QAAA,eAAe,mBAO3B;AAEY,QAAA,SAAS,GAAkC,IAAI,CAAC,MAAM,CAAA;AAE5D,MAAM,eAAe,GAAG,CAAC,GAAe,EAAU,EAAE;IACzD,MAAM,MAAM,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,CAAA;IAC7B,IAAI,CAAC,IAAA,wBAAa,EAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEY,QAAA,YAAY,GAAoC,IAAI,CAAC,SAAS,CAAA;AAEpE,MAAM,cAAc,GAAG,CAAC,MAA+B,EAAU,EAAE;IACxE,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,MAAM,CAAA;IACf,CAAC;SAAM,CAAC;QACN,OAAO;YACL,GAAG,MAAM;YACT,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,gBAAG,CAAC,OAAO,EAAE;SACjC,CAAA;IACH,CAAC;AACH,CAAC,CAAA;AAVY,QAAA,cAAc,kBAU1B;AAEM,KAAK,UAAU,gBAAgB,CAAC,QAAmC;IACxE,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACtD,OAAO,WAAW,CAAC,MAAM,CAAC,CAAA;AAC5B,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW,CAAC,MAA6B;IACvD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,WAAW,IAAI,KAAK,CAAC,UAAU,CAAA;IAC3D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAA;IAC1C,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACzB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAA;IAC5B,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC","sourcesContent":["import { TID } from '@atproto/common-web'\nimport * as crypto from '@atproto/crypto'\nimport { Keypair } from '@atproto/crypto'\nimport * as cbor from '@atproto/lex-cbor'\nimport { Cid, LexMap, LexValue, isPlainObject } from '@atproto/lex-data'\nimport { DataDiff } from './data-diff'\nimport {\n  Commit,\n  LegacyV2Commit,\n  RecordCreateDescript,\n  RecordDeleteDescript,\n  RecordPath,\n  RecordUpdateDescript,\n  RecordWriteDescript,\n  UnsignedCommit,\n  WriteOpAction,\n} from './types'\n\nexport const diffToWriteDescripts = (\n  diff: DataDiff,\n): Promise<RecordWriteDescript[]> => {\n  return Promise.all([\n    ...diff.addList().map(async (add) => {\n      const { collection, rkey } = parseDataKey(add.key)\n      return {\n        action: WriteOpAction.Create,\n        collection,\n        rkey,\n        cid: add.cid,\n      } as RecordCreateDescript\n    }),\n    ...diff.updateList().map(async (upd) => {\n      const { collection, rkey } = parseDataKey(upd.key)\n      return {\n        action: WriteOpAction.Update,\n        collection,\n        rkey,\n        cid: upd.cid,\n        prev: upd.prev,\n      } as RecordUpdateDescript\n    }),\n    ...diff.deleteList().map((del) => {\n      const { collection, rkey } = parseDataKey(del.key)\n      return {\n        action: WriteOpAction.Delete,\n        collection,\n        rkey,\n        cid: del.cid,\n      } as RecordDeleteDescript\n    }),\n  ])\n}\n\nexport const ensureCreates = (\n  descripts: RecordWriteDescript[],\n): RecordCreateDescript[] => {\n  const creates: RecordCreateDescript[] = []\n  for (const descript of descripts) {\n    if (descript.action !== WriteOpAction.Create) {\n      throw new Error(`Unexpected action: ${descript.action}`)\n    } else {\n      creates.push(descript)\n    }\n  }\n  return creates\n}\n\nexport const parseDataKey = (key: string): RecordPath => {\n  const { length, 0: collection, 1: rkey } = key.split('/')\n  if (length !== 2) throw new Error(`Invalid record key: ${key}`)\n  return { collection, rkey }\n}\n\nexport const formatDataKey = (collection: string, rkey: string): string => {\n  return collection + '/' + rkey\n}\n\nexport const metaEqual = (a: Commit, b: Commit): boolean => {\n  return a.did === b.did && a.version === b.version\n}\n\nexport const signCommit = async (\n  unsigned: UnsignedCommit,\n  keypair: Keypair,\n): Promise<Commit> => {\n  const encoded = cbor.encode(unsigned)\n  const sig = await keypair.sign(encoded)\n  return {\n    ...unsigned,\n    sig,\n  }\n}\n\nexport const verifyCommitSig = async (\n  commit: Commit,\n  didKey: string,\n): Promise<boolean> => {\n  const { sig, ...rest } = commit\n  const encoded = cbor.encode(rest)\n  return crypto.verifySignature(didKey, encoded, sig)\n}\n\nexport const cborToLex: (val: Uint8Array) => LexValue = cbor.decode\n\nexport const cborToLexRecord = (val: Uint8Array): LexMap => {\n  const parsed = cborToLex(val)\n  if (!isPlainObject(parsed)) {\n    throw new Error('lexicon records be a json object')\n  }\n  return parsed\n}\n\nexport const cidForRecord: (val: LexValue) => Promise<Cid> = cbor.cidForLex\n\nexport const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {\n  if (commit.version === 3) {\n    return commit\n  } else {\n    return {\n      ...commit,\n      version: 3,\n      rev: commit.rev ?? TID.nextStr(),\n    }\n  }\n}\n\nexport async function concatBytesAsync(iterable: AsyncIterable<Uint8Array>) {\n  const chunks: Uint8Array[] = []\n  for await (const chunk of iterable) chunks.push(chunk)\n  return concatBytes(chunks)\n}\n\n/**\n * This is the same as {@link Buffer.concat}, without the `totalLength` argument.\n */\nexport function concatBytes(chunks: readonly Uint8Array[]): Uint8Array {\n  let totalLength = 0\n  for (const chunk of chunks) totalLength += chunk.byteLength\n  const result = new Uint8Array(totalLength)\n  let offset = 0\n  for (const chunk of chunks) {\n    result.set(chunk, offset)\n    offset += chunk.byteLength\n  }\n  return result\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/repo",
-  "version": "0.8.12",
+  "version": "0.9.0",
   "license": "MIT",
   "description": "atproto repo and MST implementation",
   "keywords": [
@@ -19,15 +19,14 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
-    "@ipld/dag-cbor": "^7.0.0",
-    "multiformats": "^9.9.0",
-    "uint8arrays": "3.0.0",
     "varint": "^6.0.0",
     "zod": "^3.23.8",
-    "@atproto/common": "^0.5.3",
-    "@atproto/common-web": "^0.4.7",
+    "@atproto/common": "^0.5.15",
+    "@atproto/common-web": "^0.4.19",
     "@atproto/crypto": "^0.4.5",
-    "@atproto/lexicon": "^0.6.0"
+    "@atproto/lex-cbor": "^0.0.15",
+    "@atproto/lex-data": "^0.0.14",
+    "@atproto/syntax": "^0.5.2"
   },
   "devDependencies": {
     "jest": "^28.1.2",

package/src/block-map.ts

@@ -1,14 +1,16 @@
-/* eslint-disable import/no-deprecated */
-
-import { CID } from 'multiformats/cid'
-import * as uint8arrays from 'uint8arrays'
-import { dataToCborBlock } from '@atproto/common'
-import { LexValue, lexToIpld } from '@atproto/lexicon'
-
-export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
+import { encode } from '@atproto/lex-cbor'
+import {
+  Cid,
+  LexValue,
+  cidForCbor,
+  parseCid,
+  ui8Equals,
+} from '@atproto/lex-data'
+
+export class BlockMap implements Iterable<[cid: Cid, bytes: Uint8Array]> {
   private map: Map<string, Uint8Array> = new Map()
 
-  constructor(entries?: Iterable<readonly [cid: CID, bytes: Uint8Array]>) {
+  constructor(entries?: Iterable<readonly [cid: Cid, bytes: Uint8Array]>) {
     if (entries) {
       for (const [cid, bytes] of entries) {
         this.set(cid, bytes)
@@ -16,28 +18,29 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
     }
   }
 
-  async add(value: LexValue): Promise<CID> {
-    const block = await dataToCborBlock(lexToIpld(value))
-    this.set(block.cid, block.bytes)
-    return block.cid
+  async add(value: LexValue): Promise<Cid> {
+    const bytes = encode(value)
+    const cid = await cidForCbor(bytes)
+    this.set(cid, bytes)
+    return cid
   }
 
-  set(cid: CID, bytes: Uint8Array): BlockMap {
+  set(cid: Cid, bytes: Uint8Array): BlockMap {
     this.map.set(cid.toString(), bytes)
     return this
   }
 
-  get(cid: CID): Uint8Array | undefined {
+  get(cid: Cid): Uint8Array | undefined {
     return this.map.get(cid.toString())
   }
 
-  delete(cid: CID): BlockMap {
+  delete(cid: Cid): BlockMap {
     this.map.delete(cid.toString())
     return this
   }
 
-  getMany(cids: CID[]): { blocks: BlockMap; missing: CID[] } {
-    const missing: CID[] = []
+  getMany(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } {
+    const missing: Cid[] = []
     const blocks = new BlockMap()
     for (const cid of cids) {
       const got = this.map.get(cid.toString())
@@ -50,7 +53,7 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
     return { blocks, missing }
   }
 
-  has(cid: CID): boolean {
+  has(cid: Cid): boolean {
     return this.map.has(cid.toString())
   }
 
@@ -58,7 +61,7 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
     this.map.clear()
   }
 
-  forEach(cb: (bytes: Uint8Array, cid: CID) => void): void {
+  forEach(cb: (bytes: Uint8Array, cid: Cid) => void): void {
     for (const [cid, bytes] of this) cb(bytes, cid)
   }
 
@@ -66,7 +69,7 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
     return Array.from(this, toEntry)
   }
 
-  cids(): CID[] {
+  cids(): Cid[] {
     return Array.from(this.keys())
   }
 
@@ -92,16 +95,16 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
     for (const [cid, bytes] of this) {
       const otherBytes = other.get(cid)
       if (!otherBytes) return false
-      if (!uint8arrays.equals(bytes, otherBytes)) {
+      if (!ui8Equals(bytes, otherBytes)) {
         return false
       }
     }
     return true
   }
 
-  *keys(): Generator<CID, void, unknown> {
+  *keys(): Generator<Cid, void, unknown> {
     for (const key of this.map.keys()) {
-      yield CID.parse(key)
+      yield parseCid(key)
     }
   }
 
@@ -109,19 +112,19 @@ export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
     yield* this.map.values()
   }
 
-  *[Symbol.iterator](): Generator<[CID, Uint8Array], void, unknown> {
+  *[Symbol.iterator](): Generator<[Cid, Uint8Array], void, unknown> {
     for (const [key, value] of this.map) {
-      yield [CID.parse(key), value]
+      yield [parseCid(key), value]
     }
   }
 }
 
-function toEntry([cid, bytes]: readonly [CID, Uint8Array]): Entry {
+function toEntry([cid, bytes]: readonly [Cid, Uint8Array]): Entry {
   return { cid, bytes }
 }
 
 type Entry = {
-  cid: CID
+  cid: Cid
   bytes: Uint8Array
 }
 

package/src/car.ts

@@ -1,22 +1,13 @@
-/* eslint-disable import/no-deprecated */
-
 import { setImmediate } from 'node:timers/promises'
-import * as cbor from '@ipld/dag-cbor'
-import { CID } from 'multiformats/cid'
-import * as ui8 from 'uint8arrays'
 import * as varint from 'varint'
-import {
-  check,
-  parseCidFromBytes,
-  schema,
-  streamToBuffer,
-  verifyCidForBytes,
-} from '@atproto/common'
+import * as cbor from '@atproto/lex-cbor'
+import { Cid, decodeCid, isCidForBytes } from '@atproto/lex-data'
 import { BlockMap } from './block-map'
-import { CarBlock } from './types'
+import { CarBlock, schema } from './types'
+import { concatBytesAsync } from './util'
 
 export async function* writeCarStream(
-  root: CID | null,
+  root: Cid | null,
   blocks: AsyncIterable<CarBlock>,
 ): AsyncIterable<Uint8Array> {
   const header = new Uint8Array(
@@ -36,16 +27,15 @@ export async function* writeCarStream(
   }
 }
 
-export const blocksToCarFile = (
-  root: CID | null,
+export async function blocksToCarFile(
+  root: Cid | null,
   blocks: BlockMap,
-): Promise<Uint8Array> => {
-  const carStream = blocksToCarStream(root, blocks)
-  return streamToBuffer(carStream)
+): Promise<Uint8Array> {
+  return concatBytesAsync(blocksToCarStream(root, blocks))
 }
 
 export const blocksToCarStream = (
-  root: CID | null,
+  root: Cid | null,
   blocks: BlockMap,
 ): AsyncIterable<Uint8Array> => {
   return writeCarStream(root, iterateBlocks(blocks))
@@ -67,7 +57,7 @@ export type ReadCarOptions = {
 export const readCar = async (
   bytes: Uint8Array,
   opts?: ReadCarOptions,
-): Promise<{ roots: CID[]; blocks: BlockMap }> => {
+): Promise<{ roots: Cid[]; blocks: BlockMap }> => {
   const { roots, blocks } = await readCarReader(new Ui8Reader(bytes), opts)
   const blockMap = new BlockMap()
   for await (const block of blocks) {
@@ -79,7 +69,7 @@ export const readCar = async (
 export const readCarWithRoot = async (
   bytes: Uint8Array,
   opts?: ReadCarOptions,
-): Promise<{ root: CID; blocks: BlockMap }> => {
+): Promise<{ root: Cid; blocks: BlockMap }> => {
   const { roots, blocks } = await readCar(bytes, opts)
   if (roots.length !== 1) {
     throw new Error(`Expected one root, got ${roots.length}`)
@@ -98,7 +88,7 @@ export const readCarStream = async (
   car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
   opts?: ReadCarOptions,
 ): Promise<{
-  roots: CID[]
+  roots: Cid[]
   blocks: CarBlockIterable
 }> => {
   return readCarReader(new BufferedReader(car), opts)
@@ -108,7 +98,7 @@ export const readCarReader = async (
   reader: BytesReader,
   opts?: ReadCarOptions,
 ): Promise<{
-  roots: CID[]
+  roots: Cid[]
   blocks: CarBlockIterable
 }> => {
   try {
@@ -118,11 +108,12 @@ export const readCarReader = async (
     }
     const headerBytes = await reader.read(headerSize)
     const header = cbor.decode(headerBytes)
-    if (!check.is(header, schema.carHeader)) {
-      throw new Error('Could not parse CAR header')
+    const result = schema.carHeader.safeParse(header)
+    if (!result.success) {
+      throw new Error('Could not parse CAR header', { cause: result.error })
     }
     return {
-      roots: header.roots,
+      roots: result.data.roots,
       blocks: readCarBlocksIter(reader, opts),
     }
   } catch (err) {
@@ -165,7 +156,7 @@ async function* readCarBlocksIterGenerator(
         break
       }
       const blockBytes = await reader.read(blockSize)
-      const cid = parseCidFromBytes(blockBytes.subarray(0, 36))
+      const cid = decodeCid(blockBytes.subarray(0, 36))
       const bytes = blockBytes.subarray(36)
       yield { cid, bytes }
 
@@ -185,7 +176,9 @@ export async function* verifyIncomingCarBlocks(
   car: AsyncIterable<CarBlock>,
 ): AsyncGenerator<CarBlock, void, unknown> {
   for await (const block of car) {
-    await verifyCidForBytes(block.cid, block.bytes)
+    if (!(await isCidForBytes(block.cid, block.bytes))) {
+      throw new Error(`Not a valid CID for bytes (${block.cid.toString()})`)
+    }
     yield block
   }
 }
@@ -207,7 +200,7 @@ const readVarint = async (reader: BytesReader): Promise<number | null> => {
       done = true
     }
   }
-  const concatted = ui8.concat(bytes)
+  const concatted = Buffer.concat(bytes)
   return varint.decode(concatted)
 }
 
@@ -235,11 +228,18 @@ class Ui8Reader implements BytesReader {
   async close(): Promise<void> {}
 }
 
+/**
+ * This code was optimized for performance. See
+ * {@link https://github.com/bluesky-social/atproto/pull/4729 #4729} for more details
+ * and benchmarks.
+ */
 class BufferedReader implements BytesReader {
-  buffer: Uint8Array = new Uint8Array()
   iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>
   isDone = false
 
+  /** fifo list of chunks to consume */
+  private chunks: Uint8Array[] = []
+
   constructor(stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) {
     this.iterator =
       Symbol.asyncIterator in stream
@@ -247,32 +247,110 @@ class BufferedReader implements BytesReader {
         : stream[Symbol.iterator]()
   }
 
-  async read(bytesToRead: number): Promise<Uint8Array> {
-    await this.readUntilBuffered(bytesToRead)
-    const value = this.buffer.subarray(0, bytesToRead)
-    this.buffer = this.buffer.subarray(bytesToRead)
-    return value
+  /** Number of bytes currently buffered and available for reading */
+  get bufferedByteLength() {
+    let total = 0
+    for (let i = 0; i < this.chunks.length; i++) {
+      total += this.chunks[i].byteLength
+    }
+    return total
   }
 
-  private async readUntilBuffered(bytesToRead: number) {
-    if (this.isDone) {
-      return
+  /**
+   * @note concurrent reads are **NOT** supported by the current implementation
+   * and would require call to readUntilBuffered to be using a fifo lock for
+   * read()s to be processed in fifo order.
+   */
+  async read(bytesToRead: number): Promise<Uint8Array> {
+    const bytesNeeded = bytesToRead - this.bufferedByteLength
+    if (bytesNeeded > 0 && !this.isDone) {
+      await this.readUntilBuffered(bytesNeeded)
     }
-    while (this.buffer.length < bytesToRead) {
+
+    const resultLength = Math.min(bytesToRead, this.bufferedByteLength)
+    if (resultLength <= 0) return new Uint8Array()
+
+    const firstChunk = this.consumeChunk(resultLength)
+    if (firstChunk.byteLength === resultLength) {
+      // If the data consumed from the first chunk contains all we need, return
+      // it as-is. This allows to avoid any copy operation.
+      return firstChunk
+    }
+
+    // The first chunk does not have all the data we need. We have to copy
+    // multiple chunks into a larger buffer
+    const result = new Uint8Array(resultLength)
+    let resultWriteIndex = 0
+
+    // Copy the first chunk into the result buffer
+    result.set(firstChunk, resultWriteIndex)
+    resultWriteIndex += firstChunk.byteLength
+
+    // Copy more chunks as needed (we use do-while because we *know* we need
+    // more than one chunk)
+    do {
+      const missingLength = resultLength - resultWriteIndex
+      const currentChunk = this.consumeChunk(missingLength)
+
+      result.set(currentChunk, resultWriteIndex)
+      resultWriteIndex += currentChunk.byteLength
+    } while (resultWriteIndex < resultLength)
+
+    return result
+  }
+
+  private async readUntilBuffered(bytesNeeded: number) {
+    let bytesRead = 0
+    while (bytesRead < bytesNeeded) {
       const next = await this.iterator.next()
       if (next.done) {
         this.isDone = true
-        return
+        break
+      } else {
+        this.chunks.push(next.value)
+        bytesRead += next.value.byteLength
       }
-      this.buffer = ui8.concat([this.buffer, next.value])
+    }
+    return bytesRead
+  }
+
+  private consumeChunk(bytesToConsume: number) {
+    const firstChunk = this.chunks[0]!
+    if (bytesToConsume < firstChunk.byteLength) {
+      // return a sub-view of the data being read and replace the first chunk
+      // with a sub-view that does not contain that data.
+
+      // @NOTE for some reason, subarray() revealed to be 7-8% slower in NodeJS
+      // benchmarks.
+
+      // this.chunks[0] = firstChunk.subarray(bytesToConsume)
+      // return firstChunk.subarray(0, bytesToConsume)
+
+      this.chunks[0] = new Uint8Array(
+        firstChunk.buffer,
+        firstChunk.byteOffset + bytesToConsume,
+        firstChunk.byteLength - bytesToConsume,
+      )
+      return new Uint8Array(
+        firstChunk.buffer,
+        firstChunk.byteOffset,
+        bytesToConsume,
+      )
+    } else {
+      // First chunk is being read in full, discard it
+      this.chunks.shift()
+      return firstChunk
     }
   }
 
   async close(): Promise<void> {
-    if (!this.isDone && this.iterator.return) {
-      await this.iterator.return()
+    try {
+      if (!this.isDone && this.iterator.return) {
+        await this.iterator.return()
+      }
+    } finally {
+      this.isDone = true
+      this.chunks.length = 0
     }
-    this.isDone = true
-    this.buffer = new Uint8Array()
   }
 }

package/src/cid-set.ts

@@ -1,34 +1,34 @@
-import { CID } from 'multiformats'
+import { Cid, parseCid } from '@atproto/lex-data'
 
-export class CidSet {
+export class CidSet implements Iterable<Cid> {
   private set: Set<string>
 
-  constructor(arr: CID[] = []) {
+  constructor(arr: Cid[] = []) {
     const strArr = arr.map((c) => c.toString())
     this.set = new Set(strArr)
   }
 
-  add(cid: CID): CidSet {
+  add(cid: Cid): CidSet {
     this.set.add(cid.toString())
     return this
   }
 
   addSet(toMerge: CidSet): CidSet {
-    toMerge.toList().map((c) => this.add(c))
+    for (const c of toMerge.set) this.set.add(c)
     return this
   }
 
   subtractSet(toSubtract: CidSet): CidSet {
-    toSubtract.toList().map((c) => this.delete(c))
+    for (const c of toSubtract.set) this.set.delete(c)
     return this
   }
 
-  delete(cid: CID) {
+  delete(cid: Cid) {
     this.set.delete(cid.toString())
     return this
   }
 
-  has(cid: CID): boolean {
+  has(cid: Cid): boolean {
     return this.set.has(cid.toString())
   }
 
@@ -41,8 +41,14 @@ export class CidSet {
     return this
   }
 
-  toList(): CID[] {
-    return [...this.set].map((c) => CID.parse(c))
+  toList(): Cid[] {
+    return Array.from(this)
+  }
+
+  *[Symbol.iterator](): Generator<Cid, void, unknown> {
+    for (const c of this.set) {
+      yield parseCid(c)
+    }
   }
 }