@teambit/objects 0.0.333 → 0.0.335

package/dist/objects/object-list.d.ts

@@ -17,6 +17,12 @@ export declare class ObjectList {
     objects: ObjectItem[];
     constructor(objects?: ObjectItem[]);
     count(): number;
+    /**
+     * Generates a SHA1 hash from all object buffers.
+     * Used to identify duplicate export requests during retries.
+     * Performance wise, it's not too bad. ~98ms for 150 components on CircleCI.
+     */
+    getSha1Hash(): string;
     static mergeMultipleInstances(objectLists: ObjectList[]): ObjectList;
     mergeObjectList(objectList: ObjectList): void;
     static fromJsonString(jsonStr: string): ObjectList;

package/dist/objects/object-list.js

@@ -32,6 +32,13 @@ function _stream() {
   };
   return data;
 }
+function _crypto() {
+  const data = _interopRequireDefault(require("crypto"));
+  _crypto = function () {
+    return data;
+  };
+  return data;
+}
 function _object() {
   const data = _interopRequireDefault(require("./object"));
   _object = function () {
@@ -111,6 +118,19 @@ class ObjectList {
   count() {
     return this.objects.length;
   }
+
+  /**
+   * Generates a SHA1 hash from all object buffers.
+   * Used to identify duplicate export requests during retries.
+   * Performance wise, it's not too bad. ~98ms for 150 components on CircleCI.
+   */
+  getSha1Hash() {
+    const hash = _crypto().default.createHash('sha1');
+    for (const obj of this.objects) {
+      hash.update(new Uint8Array(obj.buffer));
+    }
+    return hash.digest('hex');
+  }
   static mergeMultipleInstances(objectLists) {
     const objectList = new ObjectList();
     objectLists.forEach(objList => objectList.mergeObjectList(objList));

package/dist/objects/object-list.js.map

@@ -1 +1 @@
-{"version":3,"names":["_tarStream","data","_interopRequireDefault","require","_toolboxPromise","_lodash","_stream","_object","_bitObjectList","_ref","_legacy","_harmonyModules","_models","_legacy2","e","__esModule","default","ownKeys","r","t","Object","keys","getOwnPropertySymbols","o","filter","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","arguments","length","forEach","_defineProperty","getOwnPropertyDescriptors","defineProperties","defineProperty","_toPropertyKey","value","configurable","writable","i","_toPrimitive","Symbol","toPrimitive","call","TypeError","String","Number","TAR_STREAM_ERROR_FILENAME","OBJECT_LIST_CURRENT_SCHEMA","TAR_STREAM_START_FILENAME","TAR_STREAM_END_FILENAME","FETCH_FORMAT_OBJECT_LIST","exports","ObjectList","constructor","objects","count","mergeMultipleInstances","objectLists","objectList","objList","mergeObjectList","addIfNotExist","fromJsonString","jsonStr","jsonParsed","JSON","parse","Array","isArray","Error","obj","ref","Ref","hash","buffer","Buffer","from","toJsonString","stringify","toTar","pack","tarStream","entry","name","combineScopeAndHash","finalize","toReadableStream","Readable","fromTar","packStream","extract","objectItems","Promise","resolve","reject","on","header","stream","next","chunk","extractScopeAndHash","concat","err","resume","pipe","fromTarToObjectStream","passThrough","PassThrough","objectMode","startData","endData","allData","emit","toString","logger","debug","write","schema","end","pipeline","error","fromObjectStreamToTar","readable","scopeName","startFile","numOfFiles","endFile","errorMessage","message","fromReadableStream","nameSplit","split","hasScope","scope","undefined","objectItem","exists","find","object","isEqual","toBitObjects","throwForUnknownTypes","concurrency","concurrentIOLimit","bitObjects","pMapPool","BitObject","parseObject","UnknownObjectType","type","BitObjectList","compact","fromBitObjects","compress","getType","addScopeName","splitByScopeName","objectListPerScope","ExportMetadata","toConsoleLog","console","log","map","join"],"sources":["object-list.ts"],"sourcesContent":["import tarStream from 'tar-stream';\nimport { pMapPool } from '@teambit/toolbox.promise.map-pool';\nimport { compact } from 'lodash';\nimport { Readable, PassThrough, pipeline } from 'stream';\nimport BitObject from './object';\nimport { BitObjectList } from './bit-object-list';\nimport Ref from './ref';\nimport { logger } from '@teambit/legacy.logger';\nimport { concurrentIOLimit } from '@teambit/harmony.modules.concurrency';\nimport { ExportMetadata } from '../models';\nimport { UnknownObjectType } from '@teambit/legacy.scope';\n\n/**\n * when error occurred during streaming between HTTP server and client, there is no good way to\n * indicate this other than sending a new file with a special name and the error message.\n */\nconst TAR_STREAM_ERROR_FILENAME = '.BIT.ERROR';\n/**\n * schema 1.0.0 - added the start and end file with basic info\n */\nconst OBJECT_LIST_CURRENT_SCHEMA = '1.0.0';\nconst TAR_STREAM_START_FILENAME = '.BIT.START';\nconst TAR_STREAM_END_FILENAME = '.BIT.END';\n\ntype StartFile = {\n  schema: string;\n  scopeName: string;\n};\ntype EndFile = {\n  numOfFiles: number;\n  scopeName: string;\n};\n\nexport type ObjectItem = {\n  ref: Ref;\n  buffer: Buffer; // zlib deflated BitObject\n  type?: string; // for future use. e.g. to be able to export only Component/Version types but not Source/Artifact, etc.\n  scope?: string; // used for the export process\n};\n\nexport const FETCH_FORMAT_OBJECT_LIST = 'ObjectList';\n\n/**\n * Stream.Readable that operates with objectMode, while each 'data' event emits one ObjectItem object.\n */\nexport type ObjectItemsStream = Readable;\n\nexport class ObjectList {\n  constructor(public objects: ObjectItem[] = []) {}\n\n  count() {\n    return this.objects.length;\n  }\n\n  static mergeMultipleInstances(objectLists: ObjectList[]): ObjectList {\n    const objectList = new ObjectList();\n    objectLists.forEach((objList) => objectList.mergeObjectList(objList));\n    return objectList;\n  }\n  mergeObjectList(objectList: ObjectList) {\n    this.addIfNotExist(objectList.objects);\n  }\n  static fromJsonString(jsonStr: string): ObjectList {\n    const jsonParsed = JSON.parse(jsonStr);\n    if (!Array.isArray(jsonParsed)) {\n      throw new Error(`fromJsonString expect an array, got ${typeof jsonParsed}`);\n    }\n    jsonParsed.forEach((obj) => {\n      obj.ref = new Ref(obj.ref.hash);\n      obj.buffer = Buffer.from(obj.buffer);\n    });\n    return new ObjectList(jsonParsed);\n  }\n  toJsonString(): string {\n    return JSON.stringify(this.objects);\n  }\n  toTar(): NodeJS.ReadableStream {\n    const pack = tarStream.pack();\n    this.objects.forEach((obj) => {\n      pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);\n    });\n    pack.finalize();\n    return pack;\n  }\n  toReadableStream(): ObjectItemsStream {\n    return Readable.from(this.objects);\n  }\n  static async fromTar(packStream: NodeJS.ReadableStream): Promise<ObjectList> {\n    const extract = tarStream.extract();\n    const objectItems: ObjectItem[] = await new Promise((resolve, reject) => {\n      const objects: ObjectItem[] = [];\n      extract.on('entry', (header, stream, next) => {\n        const data: Buffer[] = [];\n        stream.on('data', (chunk) => {\n          data.push(chunk);\n        });\n        stream.on('end', () => {\n          objects.push({\n            ...ObjectList.extractScopeAndHash(header.name),\n            buffer: Buffer.concat(data as unknown as Uint8Array[]),\n          });\n          next(); // ready for next entry\n        });\n        stream.on('error', (err) => reject(err));\n\n        stream.resume(); // just auto drain the stream\n      });\n\n      extract.on('finish', () => {\n        resolve(objects);\n      });\n\n      packStream.pipe(extract);\n    });\n    return new ObjectList(objectItems);\n  }\n\n  static fromTarToObjectStream(packStream: NodeJS.ReadableStream): ObjectItemsStream {\n    const passThrough = new PassThrough({ objectMode: true });\n    const extract = tarStream.extract();\n    let startData: StartFile | undefined;\n    let endData: EndFile | undefined;\n    extract.on('entry', (header, stream, next) => {\n      const data: Buffer[] = [];\n      stream.on('data', (chunk) => {\n        data.push(chunk);\n      });\n      stream.on('end', () => {\n        const allData = Buffer.concat(data as unknown as Uint8Array[]);\n        if (header.name === TAR_STREAM_ERROR_FILENAME) {\n          passThrough.emit('error', new Error(allData.toString()));\n          return;\n        }\n        if (header.name === TAR_STREAM_START_FILENAME) {\n          startData = JSON.parse(allData.toString());\n          logger.debug('fromTarToObjectStream, start getting data', startData);\n          next();\n          return;\n        }\n        if (header.name === TAR_STREAM_END_FILENAME) {\n          endData = JSON.parse(allData.toString());\n          logger.debug('fromTarToObjectStream, finished getting data', endData);\n          next();\n          return;\n        }\n        passThrough.write({ ...ObjectList.extractScopeAndHash(header.name), buffer: allData });\n        next(); // ready for next entry\n      });\n      stream.on('error', (err) => {\n        passThrough.emit('error', err);\n      });\n\n      stream.resume(); // just auto drain the stream\n    });\n\n    // not sure if needed\n    extract.on('error', (err) => {\n      passThrough.emit('error', err);\n    });\n\n    extract.on('finish', () => {\n      if (startData?.schema === OBJECT_LIST_CURRENT_SCHEMA && !endData) {\n        // wasn't able to find a better way to indicate whether the server aborted the request\n        // see https://github.com/node-fetch/node-fetch/issues/1117\n        passThrough.emit(\n          'error',\n          new Error(`server terminated the stream unexpectedly (metadata: ${JSON.stringify(startData)})`)\n        );\n      }\n      passThrough.end();\n    });\n    pipeline(packStream, extract, (err) => {\n      if (err) {\n        logger.error('fromTarToObjectStream, pipeline', err);\n        passThrough.emit('error', err);\n      } else {\n        logger.debug('fromTarToObjectStream, pipeline is completed');\n      }\n    });\n\n    return passThrough;\n  }\n\n  static fromObjectStreamToTar(readable: Readable, scopeName: string) {\n    const pack = tarStream.pack();\n    const startFile: StartFile = { schema: OBJECT_LIST_CURRENT_SCHEMA, scopeName };\n    logger.debug('fromObjectStreamToTar, start sending data', startFile);\n    pack.entry({ name: TAR_STREAM_START_FILENAME }, JSON.stringify(startFile));\n    let numOfFiles = 0;\n    readable.on('data', (obj: ObjectItem) => {\n      numOfFiles += 1;\n      pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);\n    });\n    readable.on('end', () => {\n      const endFile: EndFile = { numOfFiles, scopeName };\n      logger.debug('fromObjectStreamToTar, finished sending data', endFile);\n      pack.entry({ name: TAR_STREAM_END_FILENAME }, JSON.stringify(endFile));\n      pack.finalize();\n    });\n    readable.on('error', (err) => {\n      const errorMessage = err.message || `unexpected error (${err.name})`;\n      logger.error(`ObjectList.fromObjectStreamToTar, streaming an error as a file`, err);\n      pack.entry({ name: TAR_STREAM_ERROR_FILENAME }, errorMessage);\n      pack.finalize();\n    });\n    return pack;\n  }\n\n  static async fromReadableStream(readable: ObjectItemsStream): Promise<ObjectList> {\n    const objectItems: ObjectItem[] = [];\n    for await (const obj of readable) {\n      objectItems.push(obj);\n    }\n    return new ObjectList(objectItems);\n  }\n\n  /**\n   * the opposite of this.combineScopeAndHash\n   */\n  static extractScopeAndHash(name: string): { scope?: string; ref: Ref } {\n    const nameSplit = name.split('/');\n    const hasScope = nameSplit.length > 1;\n    return {\n      scope: hasScope ? nameSplit[0] : undefined,\n      ref: new Ref(hasScope ? nameSplit[1] : nameSplit[0]),\n    };\n  }\n  /**\n   * the opposite of this.extractScopeAndHash\n   */\n  static combineScopeAndHash(objectItem: ObjectItem): string {\n    const scope = objectItem.scope ? `${objectItem.scope}/` : '';\n    return `${scope}${objectItem.ref.hash}`;\n  }\n\n  addIfNotExist(objectItems: ObjectItem[]) {\n    objectItems.forEach((objectItem) => {\n      const exists = this.objects.find(\n        (object) => object.ref.isEqual(objectItem.ref) && object.scope === objectItem.scope\n      );\n      if (!exists) {\n        this.objects.push(objectItem);\n      }\n    });\n  }\n\n  async toBitObjects(throwForUnknownTypes = false): Promise<BitObjectList> {\n    const concurrency = concurrentIOLimit();\n    const bitObjects = await pMapPool(\n      this.objects,\n      async (object) => {\n        try {\n          return await BitObject.parseObject(object.buffer);\n        } catch (err) {\n          if (throwForUnknownTypes || !(err instanceof UnknownObjectType)) {\n            throw err;\n          }\n          logger.error(\n            `toBitObjects, unable to parse object of type ${err.type}, ignoring it. please update your bit version`\n          );\n          return null;\n        }\n      },\n      { concurrency }\n    );\n    return new BitObjectList(compact(bitObjects));\n  }\n\n  static async fromBitObjects(bitObjects: BitObject[]): Promise<ObjectList> {\n    const concurrency = concurrentIOLimit();\n    const objectItems = await pMapPool(\n      bitObjects,\n      async (obj) => ({\n        ref: obj.hash(),\n        buffer: await obj.compress(),\n        type: obj.getType(),\n      }),\n      { concurrency }\n    );\n    return new ObjectList(objectItems);\n  }\n\n  addScopeName(scopeName: string) {\n    this.objects.forEach((object) => {\n      object.scope = scopeName;\n    });\n  }\n\n  splitByScopeName(): { [scopeName: string]: ObjectList } {\n    const objectListPerScope: { [scopeName: string]: ObjectList } = {};\n    this.objects.forEach((obj) => {\n      if (obj.type === ExportMetadata.name) {\n        return; // no scope for this type. it's general for all export data from all scopes\n      }\n      if (!obj.scope) {\n        throw new Error(`ObjectList: unable to split by scopeName, the scopeName is missing for ${obj.ref.hash}`);\n      }\n      if (objectListPerScope[obj.scope]) {\n        objectListPerScope[obj.scope].addIfNotExist([obj]);\n      } else {\n        objectListPerScope[obj.scope] = new ObjectList([obj]);\n      }\n    });\n    return objectListPerScope;\n  }\n\n  /**\n   * helps debugging\n   */\n  toConsoleLog() {\n    console.log(this.objects.map((o) => o.ref.hash).join('\\n')); // eslint-disable-line no-console\n  }\n}\n"],"mappings":";;;;;;AAAA,SAAAA,WAAA;EAAA,MAAAC,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAH,UAAA,YAAAA,CAAA;IAAA,OAAAC,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAG,gBAAA;EAAA,MAAAH,IAAA,GAAAE,OAAA;EAAAC,eAAA,YAAAA,CAAA;IAAA,OAAAH,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAI,QAAA;EAAA,MAAAJ,IAAA,GAAAE,OAAA;EAAAE,OAAA,YAAAA,CAAA;IAAA,OAAAJ,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAK,QAAA;EAAA,MAAAL,IAAA,GAAAE,OAAA;EAAAG,OAAA,YAAAA,CAAA;IAAA,OAAAL,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAM,QAAA;EAAA,MAAAN,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAI,OAAA,YAAAA,CAAA;IAAA,OAAAN,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAO,eAAA;EAAA,MAAAP,IAAA,GAAAE,OAAA;EAAAK,cAAA,YAAAA,CAAA;IAAA,OAAAP,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAQ,KAAA;EAAA,MAAAR,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAM,IAAA,YAAAA,CAAA;IAAA,OAAAR,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAS,QAAA;EAAA,MAAAT,IAAA,GAAAE,OAAA;EAAAO,OAAA,YAAAA,CAAA;IAAA,OAAAT,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAU,gBAAA;EAAA,MAAAV,IAAA,GAAAE,OAAA;EAAAQ,eAAA,YAAAA,CAAA;IAAA,OAAAV,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAW,QAAA;EAAA,MAAAX,IAAA,GAAAE,OAAA;EAAAS,OAAA,YAAAA,CAAA;IAAA,OAAAX,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAY,SAAA;EAAA,MAAAZ,IAAA,GAAAE,OAAA;EAAAU,QAAA,YAAAA,CAAA;IAAA,OAAAZ,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AAA0D,SAAAC,uBAAAY,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAAA,SAAAG,QAAAH,CAAA,EAAAI,CAAA,QAAAC,CAAA,GAAAC,MAAA,CAAAC,IAAA,CAAAP,CAAA,OAAAM,MAAA,CAAAE,qBAAA,QAAAC,CAAA,GAAAH,MAAA,CAAAE,qBAAA,CAAAR,CAAA,GAAAI,CAAA,KAAAK,CAAA,GAAAA,CAAA,CAAAC,MAAA,WAAAN,CAAA,WAAAE,MAAA,CAAAK,wBAAA,CAAAX,CAAA,EAAAI,CAAA,EAAAQ,UAAA,OAAAP,CAAA,CAAAQ,IAAA,CAAAC,KAAA,CAAAT,CAAA,EAAAI,CAAA,YAAAJ,CAAA;AAAA,SAAAU,cAAAf,CAAA,aAAAI,CAAA,MAAAA,CAAA,GAAAY,SAAA,CAAAC,MAAA,EAAAb,CAAA,UAAAC,CAAA,WAAAW,SAAA,CAAAZ,CAAA,IAAAY,SAAA,CAAAZ,CAAA,QAAAA,CAAA,OAAAD,OAAA,CAAAG,MAAA,CAAAD,CAAA,OAAAa,OAAA,WAAAd,CAAA,IAAAe,eAAA,CAAAnB,CAAA,EAAAI,CAAA,EAAAC,CAAA,CAAAD,CAAA,SAAAE,MAAA,CAAAc,yBAAA,GAAAd,MAAA,CAAAe,gBAAA,CAAArB,CAAA,EAAAM,MAAA,CAAAc,yBAAA,CAAAf,CAAA,KAAAF,OAAA,CAAAG,MAAA,CAAAD,CAAA,GAAAa,OAAA,WAAAd,CAAA,IAAAE,MAAA,CAAAgB,cAAA,CAAAtB,CAAA,EAAAI,CAAA,EAAAE,MAAA,CAAAK,wBAAA,CAAAN,CAAA,EAAAD,CAAA,iBAAAJ,CAAA;AAAA,SAAAmB,gBAAAnB,CAAA,EAAAI,CAAA,EAAAC,CAAA,YAAAD,CAAA,GAAAmB,cAAA,CAAAnB,CAAA,MAAAJ,CAAA,GAAAM,MAAA,CAAAgB,cAAA,CAAAtB,CAAA,EAAAI,CAAA,IAAAoB,KAAA,EAAAnB,CAAA,EAAAO,UAAA,MAAAa,YAAA,MAAAC,QAAA,UAAA1B,CAAA,CAAAI,CAAA,IAAAC,CAAA,EAAAL,CAAA;AAAA,SAAAuB,eAAAlB,CAAA,QAAAsB,CAAA,GAAAC,YAAA,CAAAvB,CAAA,uCAAAsB,CAAA,GAAAA,CAAA,GAAAA,CAAA;AAAA,SAAAC,aAAAvB,CAAA,EAAAD,CAAA,2BAAAC,CAAA,KAAAA,CAAA,SAAAA,CAAA,MAAAL,CAAA,GAAAK,CAAA,CAAAwB,MAAA,CAAAC,WAAA,kBAAA9B,CAAA,QAAA2B,CAAA,GAAA3B,CAAA,CAAA+B,IAAA,CAAA1B,CAAA,EAAAD,CAAA,uCAAAuB,CAAA,SAAAA,CAAA,YAAAK,SAAA,yEAAA5B,CAAA,GAAA6B,MAAA,GAAAC,MAAA,EAAA7B,CAAA;AAE1D;AACA;AACA;AACA;AACA,MAAM8B,yBAAyB,GAAG,YAAY;AAC9C;AACA;AACA;AACA,MAAMC,0BAA0B,GAAG,OAAO;AAC1C,MAAMC,yBAAyB,GAAG,YAAY;AAC9C,MAAMC,uBAAuB,GAAG,UAAU;AAkBnC,MAAMC,wBAAwB,GAAAC,OAAA,CAAAD,wBAAA,GAAG,YAAY;;AAEpD;AACA;AACA;;AAGO,MAAME,UAAU,CAAC;EACtBC,WAAWA,CAAQC,OAAqB,GAAG,EAAE,EAAE;IAAA,KAA5BA,OAAqB,GAArBA,OAAqB;EAAQ;EAEhDC,KAAKA,CAAA,EAAG;IACN,OAAO,IAAI,CAACD,OAAO,CAAC1B,MAAM;EAC5B;EAEA,OAAO4B,sBAAsBA,CAACC,WAAyB,EAAc;IACnE,MAAMC,UAAU,GAAG,IAAIN,UAAU,CAAC,CAAC;IACnCK,WAAW,CAAC5B,OAAO,CAAE8B,OAAO,IAAKD,UAAU,CAACE,eAAe,CAACD,OAAO,CAAC,CAAC;IACrE,OAAOD,UAAU;EACnB;EACAE,eAAeA,CAACF,UAAsB,EAAE;IACtC,IAAI,CAACG,aAAa,CAACH,UAAU,CAACJ,OAAO,CAAC;EACxC;EACA,OAAOQ,cAAcA,CAACC,OAAe,EAAc;IACjD,MAAMC,UAAU,GAAGC,IAAI,CAACC,KAAK,CAACH,OAAO,CAAC;IACtC,IAAI,CAACI,KAAK,CAACC,OAAO,CAACJ,UAAU,CAAC,EAAE;MAC9B,MAAM,IAAIK,KAAK,CAAC,uCAAuC,OAAOL,UAAU,EAAE,CAAC;IAC7E;IACAA,UAAU,CAACnC,OAAO,CAAEyC,GAAG,IAAK;MAC1BA,GAAG,CAACC,GAAG,GAAG,KAAIC,cAAG,EAACF,GAAG,CAACC,GAAG,CAACE,IAAI,CAAC;MAC/BH,GAAG,CAACI,MAAM,GAAGC,MAAM,CAACC,IAAI,CAACN,GAAG,CAACI,MAAM,CAAC;IACtC,CAAC,CAAC;IACF,OAAO,IAAItB,UAAU,CAACY,UAAU,CAAC;EACnC;EACAa,YAAYA,CAAA,EAAW;IACrB,OAAOZ,IAAI,CAACa,SAAS,CAAC,IAAI,CAACxB,OAAO,CAAC;EACrC;EACAyB,KAAKA,CAAA,EAA0B;IAC7B,MAAMC,IAAI,GAAGC,oBAAS,CAACD,IAAI,CAAC,CAAC;IAC7B,IAAI,CAAC1B,OAAO,CAACzB,OAAO,CAAEyC,GAAG,IAAK;MAC5BU,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAE/B,UAAU,CAACgC,mBAAmB,CAACd,GAAG;MAAE,CAAC,EAAEA,GAAG,CAACI,MAAM,CAAC;IACvE,CAAC,CAAC;IACFM,IAAI,CAACK,QAAQ,CAAC,CAAC;IACf,OAAOL,IAAI;EACb;EACAM,gBAAgBA,CAAA,EAAsB;IACpC,OAAOC,kBAAQ,CAACX,IAAI,CAAC,IAAI,CAACtB,OAAO,CAAC;EACpC;EACA,aAAakC,OAAOA,CAACC,UAAiC,EAAuB;IAC3E,MAAMC,OAAO,GAAGT,oBAAS,CAACS,OAAO,CAAC,CAAC;IACnC,MAAMC,WAAyB,GAAG,MAAM,IAAIC,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;MACvE,MAAMxC,OAAqB,GAAG,EAAE;MAChCoC,OAAO,CAACK,EAAE,CAAC,OAAO,EAAE,CAACC,MAAM,EAAEC,MAAM,EAAEC,IAAI,KAAK;QAC5C,MAAMpG,IAAc,GAAG,EAAE;QACzBmG,MAAM,CAACF,EAAE,CAAC,MAAM,EAAGI,KAAK,IAAK;UAC3BrG,IAAI,CAAC0B,IAAI,CAAC2E,KAAK,CAAC;QAClB,CAAC,CAAC;QACFF,MAAM,CAACF,EAAE,CAAC,KAAK,EAAE,MAAM;UACrBzC,OAAO,CAAC9B,IAAI,CAAAE,aAAA,CAAAA,aAAA,KACP0B,UAAU,CAACgD,mBAAmB,CAACJ,MAAM,CAACb,IAAI,CAAC;YAC9CT,MAAM,EAAEC,MAAM,CAAC0B,MAAM,CAACvG,IAA+B;UAAC,EACvD,CAAC;UACFoG,IAAI,CAAC,CAAC,CAAC,CAAC;QACV,CAAC,CAAC;QACFD,MAAM,CAACF,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAKR,MAAM,CAACQ,GAAG,CAAC,CAAC;QAExCL,MAAM,CAACM,MAAM,CAAC,CAAC,CAAC,CAAC;MACnB,CAAC,CAAC;MAEFb,OAAO,CAACK,EAAE,CAAC,QAAQ,EAAE,MAAM;QACzBF,OAAO,CAACvC,OAAO,CAAC;MAClB,CAAC,CAAC;MAEFmC,UAAU,CAACe,IAAI,CAACd,OAAO,CAAC;IAC1B,CAAC,CAAC;IACF,OAAO,IAAItC,UAAU,CAACuC,WAAW,CAAC;EACpC;EAEA,OAAOc,qBAAqBA,CAAChB,UAAiC,EAAqB;IACjF,MAAMiB,WAAW,GAAG,KAAIC,qBAAW,EAAC;MAAEC,UAAU,EAAE;IAAK,CAAC,CAAC;IACzD,MAAMlB,OAAO,GAAGT,oBAAS,CAACS,OAAO,CAAC,CAAC;IACnC,IAAImB,SAAgC;IACpC,IAAIC,OAA4B;IAChCpB,OAAO,CAACK,EAAE,CAAC,OAAO,EAAE,CAACC,MAAM,EAAEC,MAAM,EAAEC,IAAI,KAAK;MAC5C,MAAMpG,IAAc,GAAG,EAAE;MACzBmG,MAAM,CAACF,EAAE,CAAC,MAAM,EAAGI,KAAK,IAAK;QAC3BrG,IAAI,CAAC0B,IAAI,CAAC2E,KAAK,CAAC;MAClB,CAAC,CAAC;MACFF,MAAM,CAACF,EAAE,CAAC,KAAK,EAAE,MAAM;QACrB,MAAMgB,OAAO,GAAGpC,MAAM,CAAC0B,MAAM,CAACvG,IAA+B,CAAC;QAC9D,IAAIkG,MAAM,CAACb,IAAI,KAAKrC,yBAAyB,EAAE;UAC7C4D,WAAW,CAACM,IAAI,CAAC,OAAO,EAAE,IAAI3C,KAAK,CAAC0C,OAAO,CAACE,QAAQ,CAAC,CAAC,CAAC,CAAC;UACxD;QACF;QACA,IAAIjB,MAAM,CAACb,IAAI,KAAKnC,yBAAyB,EAAE;UAC7C6D,SAAS,GAAG5C,IAAI,CAACC,KAAK,CAAC6C,OAAO,CAACE,QAAQ,CAAC,CAAC,CAAC;UAC1CC,gBAAM,CAACC,KAAK,CAAC,2CAA2C,EAAEN,SAAS,CAAC;UACpEX,IAAI,CAAC,CAAC;UACN;QACF;QACA,IAAIF,MAAM,CAACb,IAAI,KAAKlC,uBAAuB,EAAE;UAC3C6D,OAAO,GAAG7C,IAAI,CAACC,KAAK,CAAC6C,OAAO,CAACE,QAAQ,CAAC,CAAC,CAAC;UACxCC,gBAAM,CAACC,KAAK,CAAC,8CAA8C,EAAEL,OAAO,CAAC;UACrEZ,IAAI,CAAC,CAAC;UACN;QACF;QACAQ,WAAW,CAACU,KAAK,CAAA1F,aAAA,CAAAA,aAAA,KAAM0B,UAAU,CAACgD,mBAAmB,CAACJ,MAAM,CAACb,IAAI,CAAC;UAAET,MAAM,EAAEqC;QAAO,EAAE,CAAC;QACtFb,IAAI,CAAC,CAAC,CAAC,CAAC;MACV,CAAC,CAAC;MACFD,MAAM,CAACF,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAK;QAC1BI,WAAW,CAACM,IAAI,CAAC,OAAO,EAAEV,GAAG,CAAC;MAChC,CAAC,CAAC;MAEFL,MAAM,CAACM,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC,CAAC;;IAEF;IACAb,OAAO,CAACK,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAK;MAC3BI,WAAW,CAACM,IAAI,CAAC,OAAO,EAAEV,GAAG,CAAC;IAChC,CAAC,CAAC;IAEFZ,OAAO,CAACK,EAAE,CAAC,QAAQ,EAAE,MAAM;MACzB,IAAIc,SAAS,EAAEQ,MAAM,KAAKtE,0BAA0B,IAAI,CAAC+D,OAAO,EAAE;QAChE;QACA;QACAJ,WAAW,CAACM,IAAI,CACd,OAAO,EACP,IAAI3C,KAAK,CAAC,wDAAwDJ,IAAI,CAACa,SAAS,CAAC+B,SAAS,CAAC,GAAG,CAChG,CAAC;MACH;MACAH,WAAW,CAACY,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC;IACF,IAAAC,kBAAQ,EAAC9B,UAAU,EAAEC,OAAO,EAAGY,GAAG,IAAK;MACrC,IAAIA,GAAG,EAAE;QACPY,gBAAM,CAACM,KAAK,CAAC,iCAAiC,EAAElB,GAAG,CAAC;QACpDI,WAAW,CAACM,IAAI,CAAC,OAAO,EAAEV,GAAG,CAAC;MAChC,CAAC,MAAM;QACLY,gBAAM,CAACC,KAAK,CAAC,8CAA8C,CAAC;MAC9D;IACF,CAAC,CAAC;IAEF,OAAOT,WAAW;EACpB;EAEA,OAAOe,qBAAqBA,CAACC,QAAkB,EAAEC,SAAiB,EAAE;IAClE,MAAM3C,IAAI,GAAGC,oBAAS,CAACD,IAAI,CAAC,CAAC;IAC7B,MAAM4C,SAAoB,GAAG;MAAEP,MAAM,EAAEtE,0BAA0B;MAAE4E;IAAU,CAAC;IAC9ET,gBAAM,CAACC,KAAK,CAAC,2CAA2C,EAAES,SAAS,CAAC;IACpE5C,IAAI,CAACE,KAAK,CAAC;MAAEC,IAAI,EAAEnC;IAA0B,CAAC,EAAEiB,IAAI,CAACa,SAAS,CAAC8C,SAAS,CAAC,CAAC;IAC1E,IAAIC,UAAU,GAAG,CAAC;IAClBH,QAAQ,CAAC3B,EAAE,CAAC,MAAM,EAAGzB,GAAe,IAAK;MACvCuD,UAAU,IAAI,CAAC;MACf7C,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAE/B,UAAU,CAACgC,mBAAmB,CAACd,GAAG;MAAE,CAAC,EAAEA,GAAG,CAACI,MAAM,CAAC;IACvE,CAAC,CAAC;IACFgD,QAAQ,CAAC3B,EAAE,CAAC,KAAK,EAAE,MAAM;MACvB,MAAM+B,OAAgB,GAAG;QAAED,UAAU;QAAEF;MAAU,CAAC;MAClDT,gBAAM,CAACC,KAAK,CAAC,8CAA8C,EAAEW,OAAO,CAAC;MACrE9C,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAElC;MAAwB,CAAC,EAAEgB,IAAI,CAACa,SAAS,CAACgD,OAAO,CAAC,CAAC;MACtE9C,IAAI,CAACK,QAAQ,CAAC,CAAC;IACjB,CAAC,CAAC;IACFqC,QAAQ,CAAC3B,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAK;MAC5B,MAAMyB,YAAY,GAAGzB,GAAG,CAAC0B,OAAO,IAAI,qBAAqB1B,GAAG,CAACnB,IAAI,GAAG;MACpE+B,gBAAM,CAACM,KAAK,CAAC,gEAAgE,EAAElB,GAAG,CAAC;MACnFtB,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAErC;MAA0B,CAAC,EAAEiF,YAAY,CAAC;MAC7D/C,IAAI,CAACK,QAAQ,CAAC,CAAC;IACjB,CAAC,CAAC;IACF,OAAOL,IAAI;EACb;EAEA,aAAaiD,kBAAkBA,CAACP,QAA2B,EAAuB;IAChF,MAAM/B,WAAyB,GAAG,EAAE;IACpC,WAAW,MAAMrB,GAAG,IAAIoD,QAAQ,EAAE;MAChC/B,WAAW,CAACnE,IAAI,CAAC8C,GAAG,CAAC;IACvB;IACA,OAAO,IAAIlB,UAAU,CAACuC,WAAW,CAAC;EACpC;;EAEA;AACF;AACA;EACE,OAAOS,mBAAmBA,CAACjB,IAAY,EAAgC;IACrE,MAAM+C,SAAS,GAAG/C,IAAI,CAACgD,KAAK,CAAC,GAAG,CAAC;IACjC,MAAMC,QAAQ,GAAGF,SAAS,CAACtG,MAAM,GAAG,CAAC;IACrC,OAAO;MACLyG,KAAK,EAAED,QAAQ,GAAGF,SAAS,CAAC,CAAC,CAAC,GAAGI,SAAS;MAC1C/D,GAAG,EAAE,KAAIC,cAAG,EAAC4D,QAAQ,GAAGF,SAAS,CAAC,CAAC,CAAC,GAAGA,SAAS,CAAC,CAAC,CAAC;IACrD,CAAC;EACH;EACA;AACF;AACA;EACE,OAAO9C,mBAAmBA,CAACmD,UAAsB,EAAU;IACzD,MAAMF,KAAK,GAAGE,UAAU,CAACF,KAAK,GAAG,GAAGE,UAAU,CAACF,KAAK,GAAG,GAAG,EAAE;IAC5D,OAAO,GAAGA,KAAK,GAAGE,UAAU,CAAChE,GAAG,CAACE,IAAI,EAAE;EACzC;EAEAZ,aAAaA,CAAC8B,WAAyB,EAAE;IACvCA,WAAW,CAAC9D,OAAO,CAAE0G,UAAU,IAAK;MAClC,MAAMC,MAAM,GAAG,IAAI,CAAClF,OAAO,CAACmF,IAAI,CAC7BC,MAAM,IAAKA,MAAM,CAACnE,GAAG,CAACoE,OAAO,CAACJ,UAAU,CAAChE,GAAG,CAAC,IAAImE,MAAM,CAACL,KAAK,KAAKE,UAAU,CAACF,KAChF,CAAC;MACD,IAAI,CAACG,MAAM,EAAE;QACX,IAAI,CAAClF,OAAO,CAAC9B,IAAI,CAAC+G,UAAU,CAAC;MAC/B;IACF,CAAC,CAAC;EACJ;EAEA,MAAMK,YAAYA,CAACC,oBAAoB,GAAG,KAAK,EAA0B;IACvE,MAAMC,WAAW,GAAG,IAAAC,mCAAiB,EAAC,CAAC;IACvC,MAAMC,UAAU,GAAG,MAAM,IAAAC,0BAAQ,EAC/B,IAAI,CAAC3F,OAAO,EACZ,MAAOoF,MAAM,IAAK;MAChB,IAAI;QACF,OAAO,MAAMQ,iBAAS,CAACC,WAAW,CAACT,MAAM,CAAChE,MAAM,CAAC;MACnD,CAAC,CAAC,OAAO4B,GAAG,EAAE;QACZ,IAAIuC,oBAAoB,IAAI,EAAEvC,GAAG,YAAY8C,4BAAiB,CAAC,EAAE;UAC/D,MAAM9C,GAAG;QACX;QACAY,gBAAM,CAACM,KAAK,CACV,gDAAgDlB,GAAG,CAAC+C,IAAI,+CAC1D,CAAC;QACD,OAAO,IAAI;MACb;IACF,CAAC,EACD;MAAEP;IAAY,CAChB,CAAC;IACD,OAAO,KAAIQ,8BAAa,EAAC,IAAAC,iBAAO,EAACP,UAAU,CAAC,CAAC;EAC/C;EAEA,aAAaQ,cAAcA,CAACR,UAAuB,EAAuB;IACxE,MAAMF,WAAW,GAAG,IAAAC,mCAAiB,EAAC,CAAC;IACvC,MAAMpD,WAAW,GAAG,MAAM,IAAAsD,0BAAQ,EAChCD,UAAU,EACV,MAAO1E,GAAG,KAAM;MACdC,GAAG,EAAED,GAAG,CAACG,IAAI,CAAC,CAAC;MACfC,MAAM,EAAE,MAAMJ,GAAG,CAACmF,QAAQ,CAAC,CAAC;MAC5BJ,IAAI,EAAE/E,GAAG,CAACoF,OAAO,CAAC;IACpB,CAAC,CAAC,EACF;MAAEZ;IAAY,CAChB,CAAC;IACD,OAAO,IAAI1F,UAAU,CAACuC,WAAW,CAAC;EACpC;EAEAgE,YAAYA,CAAChC,SAAiB,EAAE;IAC9B,IAAI,CAACrE,OAAO,CAACzB,OAAO,CAAE6G,MAAM,IAAK;MAC/BA,MAAM,CAACL,KAAK,GAAGV,SAAS;IAC1B,CAAC,CAAC;EACJ;EAEAiC,gBAAgBA,CAAA,EAAwC;IACtD,MAAMC,kBAAuD,GAAG,CAAC,CAAC;IAClE,IAAI,CAACvG,OAAO,CAACzB,OAAO,CAAEyC,GAAG,IAAK;MAC5B,IAAIA,GAAG,CAAC+E,IAAI,KAAKS,wBAAc,CAAC3E,IAAI,EAAE;QACpC,OAAO,CAAC;MACV;MACA,IAAI,CAACb,GAAG,CAAC+D,KAAK,EAAE;QACd,MAAM,IAAIhE,KAAK,CAAC,0EAA0EC,GAAG,CAACC,GAAG,CAACE,IAAI,EAAE,CAAC;MAC3G;MACA,IAAIoF,kBAAkB,CAACvF,GAAG,CAAC+D,KAAK,CAAC,EAAE;QACjCwB,kBAAkB,CAACvF,GAAG,CAAC+D,KAAK,CAAC,CAACxE,aAAa,CAAC,CAACS,GAAG,CAAC,CAAC;MACpD,CAAC,MAAM;QACLuF,kBAAkB,CAACvF,GAAG,CAAC+D,KAAK,CAAC,GAAG,IAAIjF,UAAU,CAAC,CAACkB,GAAG,CAAC,CAAC;MACvD;IACF,CAAC,CAAC;IACF,OAAOuF,kBAAkB;EAC3B;;EAEA;AACF;AACA;EACEE,YAAYA,CAAA,EAAG;IACbC,OAAO,CAACC,GAAG,CAAC,IAAI,CAAC3G,OAAO,CAAC4G,GAAG,CAAE9I,CAAC,IAAKA,CAAC,CAACmD,GAAG,CAACE,IAAI,CAAC,CAAC0F,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;EAC/D;AACF;AAAChH,OAAA,CAAAC,UAAA,GAAAA,UAAA","ignoreList":[]}
+{"version":3,"names":["_tarStream","data","_interopRequireDefault","require","_toolboxPromise","_lodash","_stream","_crypto","_object","_bitObjectList","_ref","_legacy","_harmonyModules","_models","_legacy2","e","__esModule","default","ownKeys","r","t","Object","keys","getOwnPropertySymbols","o","filter","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","arguments","length","forEach","_defineProperty","getOwnPropertyDescriptors","defineProperties","defineProperty","_toPropertyKey","value","configurable","writable","i","_toPrimitive","Symbol","toPrimitive","call","TypeError","String","Number","TAR_STREAM_ERROR_FILENAME","OBJECT_LIST_CURRENT_SCHEMA","TAR_STREAM_START_FILENAME","TAR_STREAM_END_FILENAME","FETCH_FORMAT_OBJECT_LIST","exports","ObjectList","constructor","objects","count","getSha1Hash","hash","crypto","createHash","obj","update","Uint8Array","buffer","digest","mergeMultipleInstances","objectLists","objectList","objList","mergeObjectList","addIfNotExist","fromJsonString","jsonStr","jsonParsed","JSON","parse","Array","isArray","Error","ref","Ref","Buffer","from","toJsonString","stringify","toTar","pack","tarStream","entry","name","combineScopeAndHash","finalize","toReadableStream","Readable","fromTar","packStream","extract","objectItems","Promise","resolve","reject","on","header","stream","next","chunk","extractScopeAndHash","concat","err","resume","pipe","fromTarToObjectStream","passThrough","PassThrough","objectMode","startData","endData","allData","emit","toString","logger","debug","write","schema","end","pipeline","error","fromObjectStreamToTar","readable","scopeName","startFile","numOfFiles","endFile","errorMessage","message","fromReadableStream","nameSplit","split","hasScope","scope","undefined","objectItem","exists","find","object","isEqual","toBitObjects","throwForUnknownTypes","concurrency","concurrentIOLimit","bitObjects","pMapPool","BitObject","parseObject","UnknownObjectType","type","BitObjectList","compact","fromBitObjects","compress","getType","addScopeName","splitByScopeName","objectListPerScope","ExportMetadata","toConsoleLog","console","log","map","join"],"sources":["object-list.ts"],"sourcesContent":["import tarStream from 'tar-stream';\nimport { pMapPool } from '@teambit/toolbox.promise.map-pool';\nimport { compact } from 'lodash';\nimport { Readable, PassThrough, pipeline } from 'stream';\nimport crypto from 'crypto';\nimport BitObject from './object';\nimport { BitObjectList } from './bit-object-list';\nimport Ref from './ref';\nimport { logger } from '@teambit/legacy.logger';\nimport { concurrentIOLimit } from '@teambit/harmony.modules.concurrency';\nimport { ExportMetadata } from '../models';\nimport { UnknownObjectType } from '@teambit/legacy.scope';\n\n/**\n * when error occurred during streaming between HTTP server and client, there is no good way to\n * indicate this other than sending a new file with a special name and the error message.\n */\nconst TAR_STREAM_ERROR_FILENAME = '.BIT.ERROR';\n/**\n * schema 1.0.0 - added the start and end file with basic info\n */\nconst OBJECT_LIST_CURRENT_SCHEMA = '1.0.0';\nconst TAR_STREAM_START_FILENAME = '.BIT.START';\nconst TAR_STREAM_END_FILENAME = '.BIT.END';\n\ntype StartFile = {\n  schema: string;\n  scopeName: string;\n};\ntype EndFile = {\n  numOfFiles: number;\n  scopeName: string;\n};\n\nexport type ObjectItem = {\n  ref: Ref;\n  buffer: Buffer; // zlib deflated BitObject\n  type?: string; // for future use. e.g. to be able to export only Component/Version types but not Source/Artifact, etc.\n  scope?: string; // used for the export process\n};\n\nexport const FETCH_FORMAT_OBJECT_LIST = 'ObjectList';\n\n/**\n * Stream.Readable that operates with objectMode, while each 'data' event emits one ObjectItem object.\n */\nexport type ObjectItemsStream = Readable;\n\nexport class ObjectList {\n  constructor(public objects: ObjectItem[] = []) {}\n\n  count() {\n    return this.objects.length;\n  }\n\n  /**\n   * Generates a SHA1 hash from all object buffers.\n   * Used to identify duplicate export requests during retries.\n   * Performance wise, it's not too bad. ~98ms for 150 components on CircleCI.\n   */\n  getSha1Hash(): string {\n    const hash = crypto.createHash('sha1');\n    for (const obj of this.objects) {\n      hash.update(new Uint8Array(obj.buffer));\n    }\n    return hash.digest('hex');\n  }\n\n  static mergeMultipleInstances(objectLists: ObjectList[]): ObjectList {\n    const objectList = new ObjectList();\n    objectLists.forEach((objList) => objectList.mergeObjectList(objList));\n    return objectList;\n  }\n  mergeObjectList(objectList: ObjectList) {\n    this.addIfNotExist(objectList.objects);\n  }\n  static fromJsonString(jsonStr: string): ObjectList {\n    const jsonParsed = JSON.parse(jsonStr);\n    if (!Array.isArray(jsonParsed)) {\n      throw new Error(`fromJsonString expect an array, got ${typeof jsonParsed}`);\n    }\n    jsonParsed.forEach((obj) => {\n      obj.ref = new Ref(obj.ref.hash);\n      obj.buffer = Buffer.from(obj.buffer);\n    });\n    return new ObjectList(jsonParsed);\n  }\n  toJsonString(): string {\n    return JSON.stringify(this.objects);\n  }\n  toTar(): NodeJS.ReadableStream {\n    const pack = tarStream.pack();\n    this.objects.forEach((obj) => {\n      pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);\n    });\n    pack.finalize();\n    return pack;\n  }\n  toReadableStream(): ObjectItemsStream {\n    return Readable.from(this.objects);\n  }\n  static async fromTar(packStream: NodeJS.ReadableStream): Promise<ObjectList> {\n    const extract = tarStream.extract();\n    const objectItems: ObjectItem[] = await new Promise((resolve, reject) => {\n      const objects: ObjectItem[] = [];\n      extract.on('entry', (header, stream, next) => {\n        const data: Buffer[] = [];\n        stream.on('data', (chunk) => {\n          data.push(chunk);\n        });\n        stream.on('end', () => {\n          objects.push({\n            ...ObjectList.extractScopeAndHash(header.name),\n            buffer: Buffer.concat(data as unknown as Uint8Array[]),\n          });\n          next(); // ready for next entry\n        });\n        stream.on('error', (err) => reject(err));\n\n        stream.resume(); // just auto drain the stream\n      });\n\n      extract.on('finish', () => {\n        resolve(objects);\n      });\n\n      packStream.pipe(extract);\n    });\n    return new ObjectList(objectItems);\n  }\n\n  static fromTarToObjectStream(packStream: NodeJS.ReadableStream): ObjectItemsStream {\n    const passThrough = new PassThrough({ objectMode: true });\n    const extract = tarStream.extract();\n    let startData: StartFile | undefined;\n    let endData: EndFile | undefined;\n    extract.on('entry', (header, stream, next) => {\n      const data: Buffer[] = [];\n      stream.on('data', (chunk) => {\n        data.push(chunk);\n      });\n      stream.on('end', () => {\n        const allData = Buffer.concat(data as unknown as Uint8Array[]);\n        if (header.name === TAR_STREAM_ERROR_FILENAME) {\n          passThrough.emit('error', new Error(allData.toString()));\n          return;\n        }\n        if (header.name === TAR_STREAM_START_FILENAME) {\n          startData = JSON.parse(allData.toString());\n          logger.debug('fromTarToObjectStream, start getting data', startData);\n          next();\n          return;\n        }\n        if (header.name === TAR_STREAM_END_FILENAME) {\n          endData = JSON.parse(allData.toString());\n          logger.debug('fromTarToObjectStream, finished getting data', endData);\n          next();\n          return;\n        }\n        passThrough.write({ ...ObjectList.extractScopeAndHash(header.name), buffer: allData });\n        next(); // ready for next entry\n      });\n      stream.on('error', (err) => {\n        passThrough.emit('error', err);\n      });\n\n      stream.resume(); // just auto drain the stream\n    });\n\n    // not sure if needed\n    extract.on('error', (err) => {\n      passThrough.emit('error', err);\n    });\n\n    extract.on('finish', () => {\n      if (startData?.schema === OBJECT_LIST_CURRENT_SCHEMA && !endData) {\n        // wasn't able to find a better way to indicate whether the server aborted the request\n        // see https://github.com/node-fetch/node-fetch/issues/1117\n        passThrough.emit(\n          'error',\n          new Error(`server terminated the stream unexpectedly (metadata: ${JSON.stringify(startData)})`)\n        );\n      }\n      passThrough.end();\n    });\n    pipeline(packStream, extract, (err) => {\n      if (err) {\n        logger.error('fromTarToObjectStream, pipeline', err);\n        passThrough.emit('error', err);\n      } else {\n        logger.debug('fromTarToObjectStream, pipeline is completed');\n      }\n    });\n\n    return passThrough;\n  }\n\n  static fromObjectStreamToTar(readable: Readable, scopeName: string) {\n    const pack = tarStream.pack();\n    const startFile: StartFile = { schema: OBJECT_LIST_CURRENT_SCHEMA, scopeName };\n    logger.debug('fromObjectStreamToTar, start sending data', startFile);\n    pack.entry({ name: TAR_STREAM_START_FILENAME }, JSON.stringify(startFile));\n    let numOfFiles = 0;\n    readable.on('data', (obj: ObjectItem) => {\n      numOfFiles += 1;\n      pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);\n    });\n    readable.on('end', () => {\n      const endFile: EndFile = { numOfFiles, scopeName };\n      logger.debug('fromObjectStreamToTar, finished sending data', endFile);\n      pack.entry({ name: TAR_STREAM_END_FILENAME }, JSON.stringify(endFile));\n      pack.finalize();\n    });\n    readable.on('error', (err) => {\n      const errorMessage = err.message || `unexpected error (${err.name})`;\n      logger.error(`ObjectList.fromObjectStreamToTar, streaming an error as a file`, err);\n      pack.entry({ name: TAR_STREAM_ERROR_FILENAME }, errorMessage);\n      pack.finalize();\n    });\n    return pack;\n  }\n\n  static async fromReadableStream(readable: ObjectItemsStream): Promise<ObjectList> {\n    const objectItems: ObjectItem[] = [];\n    for await (const obj of readable) {\n      objectItems.push(obj);\n    }\n    return new ObjectList(objectItems);\n  }\n\n  /**\n   * the opposite of this.combineScopeAndHash\n   */\n  static extractScopeAndHash(name: string): { scope?: string; ref: Ref } {\n    const nameSplit = name.split('/');\n    const hasScope = nameSplit.length > 1;\n    return {\n      scope: hasScope ? nameSplit[0] : undefined,\n      ref: new Ref(hasScope ? nameSplit[1] : nameSplit[0]),\n    };\n  }\n  /**\n   * the opposite of this.extractScopeAndHash\n   */\n  static combineScopeAndHash(objectItem: ObjectItem): string {\n    const scope = objectItem.scope ? `${objectItem.scope}/` : '';\n    return `${scope}${objectItem.ref.hash}`;\n  }\n\n  addIfNotExist(objectItems: ObjectItem[]) {\n    objectItems.forEach((objectItem) => {\n      const exists = this.objects.find(\n        (object) => object.ref.isEqual(objectItem.ref) && object.scope === objectItem.scope\n      );\n      if (!exists) {\n        this.objects.push(objectItem);\n      }\n    });\n  }\n\n  async toBitObjects(throwForUnknownTypes = false): Promise<BitObjectList> {\n    const concurrency = concurrentIOLimit();\n    const bitObjects = await pMapPool(\n      this.objects,\n      async (object) => {\n        try {\n          return await BitObject.parseObject(object.buffer);\n        } catch (err) {\n          if (throwForUnknownTypes || !(err instanceof UnknownObjectType)) {\n            throw err;\n          }\n          logger.error(\n            `toBitObjects, unable to parse object of type ${err.type}, ignoring it. please update your bit version`\n          );\n          return null;\n        }\n      },\n      { concurrency }\n    );\n    return new BitObjectList(compact(bitObjects));\n  }\n\n  static async fromBitObjects(bitObjects: BitObject[]): Promise<ObjectList> {\n    const concurrency = concurrentIOLimit();\n    const objectItems = await pMapPool(\n      bitObjects,\n      async (obj) => ({\n        ref: obj.hash(),\n        buffer: await obj.compress(),\n        type: obj.getType(),\n      }),\n      { concurrency }\n    );\n    return new ObjectList(objectItems);\n  }\n\n  addScopeName(scopeName: string) {\n    this.objects.forEach((object) => {\n      object.scope = scopeName;\n    });\n  }\n\n  splitByScopeName(): { [scopeName: string]: ObjectList } {\n    const objectListPerScope: { [scopeName: string]: ObjectList } = {};\n    this.objects.forEach((obj) => {\n      if (obj.type === ExportMetadata.name) {\n        return; // no scope for this type. it's general for all export data from all scopes\n      }\n      if (!obj.scope) {\n        throw new Error(`ObjectList: unable to split by scopeName, the scopeName is missing for ${obj.ref.hash}`);\n      }\n      if (objectListPerScope[obj.scope]) {\n        objectListPerScope[obj.scope].addIfNotExist([obj]);\n      } else {\n        objectListPerScope[obj.scope] = new ObjectList([obj]);\n      }\n    });\n    return objectListPerScope;\n  }\n\n  /**\n   * helps debugging\n   */\n  toConsoleLog() {\n    console.log(this.objects.map((o) => o.ref.hash).join('\\n')); // eslint-disable-line no-console\n  }\n}\n"],"mappings":";;;;;;AAAA,SAAAA,WAAA;EAAA,MAAAC,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAH,UAAA,YAAAA,CAAA;IAAA,OAAAC,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAG,gBAAA;EAAA,MAAAH,IAAA,GAAAE,OAAA;EAAAC,eAAA,YAAAA,CAAA;IAAA,OAAAH,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAI,QAAA;EAAA,MAAAJ,IAAA,GAAAE,OAAA;EAAAE,OAAA,YAAAA,CAAA;IAAA,OAAAJ,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAK,QAAA;EAAA,MAAAL,IAAA,GAAAE,OAAA;EAAAG,OAAA,YAAAA,CAAA;IAAA,OAAAL,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAM,QAAA;EAAA,MAAAN,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAI,OAAA,YAAAA,CAAA;IAAA,OAAAN,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAO,QAAA;EAAA,MAAAP,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAK,OAAA,YAAAA,CAAA;IAAA,OAAAP,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAQ,eAAA;EAAA,MAAAR,IAAA,GAAAE,OAAA;EAAAM,cAAA,YAAAA,CAAA;IAAA,OAAAR,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAS,KAAA;EAAA,MAAAT,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAO,IAAA,YAAAA,CAAA;IAAA,OAAAT,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAU,QAAA;EAAA,MAAAV,IAAA,GAAAE,OAAA;EAAAQ,OAAA,YAAAA,CAAA;IAAA,OAAAV,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAW,gBAAA;EAAA,MAAAX,IAAA,GAAAE,OAAA;EAAAS,eAAA,YAAAA,CAAA;IAAA,OAAAX,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAY,QAAA;EAAA,MAAAZ,IAAA,GAAAE,OAAA;EAAAU,OAAA,YAAAA,CAAA;IAAA,OAAAZ,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAa,SAAA;EAAA,MAAAb,IAAA,GAAAE,OAAA;EAAAW,QAAA,YAAAA,CAAA;IAAA,OAAAb,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AAA0D,SAAAC,uBAAAa,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAAA,SAAAG,QAAAH,CAAA,EAAAI,CAAA,QAAAC,CAAA,GAAAC,MAAA,CAAAC,IAAA,CAAAP,CAAA,OAAAM,MAAA,CAAAE,qBAAA,QAAAC,CAAA,GAAAH,MAAA,CAAAE,qBAAA,CAAAR,CAAA,GAAAI,CAAA,KAAAK,CAAA,GAAAA,CAAA,CAAAC,MAAA,WAAAN,CAAA,WAAAE,MAAA,CAAAK,wBAAA,CAAAX,CAAA,EAAAI,CAAA,EAAAQ,UAAA,OAAAP,CAAA,CAAAQ,IAAA,CAAAC,KAAA,CAAAT,CAAA,EAAAI,CAAA,YAAAJ,CAAA;AAAA,SAAAU,cAAAf,CAAA,aAAAI,CAAA,MAAAA,CAAA,GAAAY,SAAA,CAAAC,MAAA,EAAAb,CAAA,UAAAC,CAAA,WAAAW,SAAA,CAAAZ,CAAA,IAAAY,SAAA,CAAAZ,CAAA,QAAAA,CAAA,OAAAD,OAAA,CAAAG,MAAA,CAAAD,CAAA,OAAAa,OAAA,WAAAd,CAAA,IAAAe,eAAA,CAAAnB,CAAA,EAAAI,CAAA,EAAAC,CAAA,CAAAD,CAAA,SAAAE,MAAA,CAAAc,yBAAA,GAAAd,MAAA,CAAAe,gBAAA,CAAArB,CAAA,EAAAM,MAAA,CAAAc,yBAAA,CAAAf,CAAA,KAAAF,OAAA,CAAAG,MAAA,CAAAD,CAAA,GAAAa,OAAA,WAAAd,CAAA,IAAAE,MAAA,CAAAgB,cAAA,CAAAtB,CAAA,EAAAI,CAAA,EAAAE,MAAA,CAAAK,wBAAA,CAAAN,CAAA,EAAAD,CAAA,iBAAAJ,CAAA;AAAA,SAAAmB,gBAAAnB,CAAA,EAAAI,CAAA,EAAAC,CAAA,YAAAD,CAAA,GAAAmB,cAAA,CAAAnB,CAAA,MAAAJ,CAAA,GAAAM,MAAA,CAAAgB,cAAA,CAAAtB,CAAA,EAAAI,CAAA,IAAAoB,KAAA,EAAAnB,CAAA,EAAAO,UAAA,MAAAa,YAAA,MAAAC,QAAA,UAAA1B,CAAA,CAAAI,CAAA,IAAAC,CAAA,EAAAL,CAAA;AAAA,SAAAuB,eAAAlB,CAAA,QAAAsB,CAAA,GAAAC,YAAA,CAAAvB,CAAA,uCAAAsB,CAAA,GAAAA,CAAA,GAAAA,CAAA;AAAA,SAAAC,aAAAvB,CAAA,EAAAD,CAAA,2BAAAC,CAAA,KAAAA,CAAA,SAAAA,CAAA,MAAAL,CAAA,GAAAK,CAAA,CAAAwB,MAAA,CAAAC,WAAA,kBAAA9B,CAAA,QAAA2B,CAAA,GAAA3B,CAAA,CAAA+B,IAAA,CAAA1B,CAAA,EAAAD,CAAA,uCAAAuB,CAAA,SAAAA,CAAA,YAAAK,SAAA,yEAAA5B,CAAA,GAAA6B,MAAA,GAAAC,MAAA,EAAA7B,CAAA;AAE1D;AACA;AACA;AACA;AACA,MAAM8B,yBAAyB,GAAG,YAAY;AAC9C;AACA;AACA;AACA,MAAMC,0BAA0B,GAAG,OAAO;AAC1C,MAAMC,yBAAyB,GAAG,YAAY;AAC9C,MAAMC,uBAAuB,GAAG,UAAU;AAkBnC,MAAMC,wBAAwB,GAAAC,OAAA,CAAAD,wBAAA,GAAG,YAAY;;AAEpD;AACA;AACA;;AAGO,MAAME,UAAU,CAAC;EACtBC,WAAWA,CAAQC,OAAqB,GAAG,EAAE,EAAE;IAAA,KAA5BA,OAAqB,GAArBA,OAAqB;EAAQ;EAEhDC,KAAKA,CAAA,EAAG;IACN,OAAO,IAAI,CAACD,OAAO,CAAC1B,MAAM;EAC5B;;EAEA;AACF;AACA;AACA;AACA;EACE4B,WAAWA,CAAA,EAAW;IACpB,MAAMC,IAAI,GAAGC,iBAAM,CAACC,UAAU,CAAC,MAAM,CAAC;IACtC,KAAK,MAAMC,GAAG,IAAI,IAAI,CAACN,OAAO,EAAE;MAC9BG,IAAI,CAACI,MAAM,CAAC,IAAIC,UAAU,CAACF,GAAG,CAACG,MAAM,CAAC,CAAC;IACzC;IACA,OAAON,IAAI,CAACO,MAAM,CAAC,KAAK,CAAC;EAC3B;EAEA,OAAOC,sBAAsBA,CAACC,WAAyB,EAAc;IACnE,MAAMC,UAAU,GAAG,IAAIf,UAAU,CAAC,CAAC;IACnCc,WAAW,CAACrC,OAAO,CAAEuC,OAAO,IAAKD,UAAU,CAACE,eAAe,CAACD,OAAO,CAAC,CAAC;IACrE,OAAOD,UAAU;EACnB;EACAE,eAAeA,CAACF,UAAsB,EAAE;IACtC,IAAI,CAACG,aAAa,CAACH,UAAU,CAACb,OAAO,CAAC;EACxC;EACA,OAAOiB,cAAcA,CAACC,OAAe,EAAc;IACjD,MAAMC,UAAU,GAAGC,IAAI,CAACC,KAAK,CAACH,OAAO,CAAC;IACtC,IAAI,CAACI,KAAK,CAACC,OAAO,CAACJ,UAAU,CAAC,EAAE;MAC9B,MAAM,IAAIK,KAAK,CAAC,uCAAuC,OAAOL,UAAU,EAAE,CAAC;IAC7E;IACAA,UAAU,CAAC5C,OAAO,CAAE+B,GAAG,IAAK;MAC1BA,GAAG,CAACmB,GAAG,GAAG,KAAIC,cAAG,EAACpB,GAAG,CAACmB,GAAG,CAACtB,IAAI,CAAC;MAC/BG,GAAG,CAACG,MAAM,GAAGkB,MAAM,CAACC,IAAI,CAACtB,GAAG,CAACG,MAAM,CAAC;IACtC,CAAC,CAAC;IACF,OAAO,IAAIX,UAAU,CAACqB,UAAU,CAAC;EACnC;EACAU,YAAYA,CAAA,EAAW;IACrB,OAAOT,IAAI,CAACU,SAAS,CAAC,IAAI,CAAC9B,OAAO,CAAC;EACrC;EACA+B,KAAKA,CAAA,EAA0B;IAC7B,MAAMC,IAAI,GAAGC,oBAAS,CAACD,IAAI,CAAC,CAAC;IAC7B,IAAI,CAAChC,OAAO,CAACzB,OAAO,CAAE+B,GAAG,IAAK;MAC5B0B,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAErC,UAAU,CAACsC,mBAAmB,CAAC9B,GAAG;MAAE,CAAC,EAAEA,GAAG,CAACG,MAAM,CAAC;IACvE,CAAC,CAAC;IACFuB,IAAI,CAACK,QAAQ,CAAC,CAAC;IACf,OAAOL,IAAI;EACb;EACAM,gBAAgBA,CAAA,EAAsB;IACpC,OAAOC,kBAAQ,CAACX,IAAI,CAAC,IAAI,CAAC5B,OAAO,CAAC;EACpC;EACA,aAAawC,OAAOA,CAACC,UAAiC,EAAuB;IAC3E,MAAMC,OAAO,GAAGT,oBAAS,CAACS,OAAO,CAAC,CAAC;IACnC,MAAMC,WAAyB,GAAG,MAAM,IAAIC,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;MACvE,MAAM9C,OAAqB,GAAG,EAAE;MAChC0C,OAAO,CAACK,EAAE,CAAC,OAAO,EAAE,CAACC,MAAM,EAAEC,MAAM,EAAEC,IAAI,KAAK;QAC5C,MAAM3G,IAAc,GAAG,EAAE;QACzB0G,MAAM,CAACF,EAAE,CAAC,MAAM,EAAGI,KAAK,IAAK;UAC3B5G,IAAI,CAAC2B,IAAI,CAACiF,KAAK,CAAC;QAClB,CAAC,CAAC;QACFF,MAAM,CAACF,EAAE,CAAC,KAAK,EAAE,MAAM;UACrB/C,OAAO,CAAC9B,IAAI,CAAAE,aAAA,CAAAA,aAAA,KACP0B,UAAU,CAACsD,mBAAmB,CAACJ,MAAM,CAACb,IAAI,CAAC;YAC9C1B,MAAM,EAAEkB,MAAM,CAAC0B,MAAM,CAAC9G,IAA+B;UAAC,EACvD,CAAC;UACF2G,IAAI,CAAC,CAAC,CAAC,CAAC;QACV,CAAC,CAAC;QACFD,MAAM,CAACF,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAKR,MAAM,CAACQ,GAAG,CAAC,CAAC;QAExCL,MAAM,CAACM,MAAM,CAAC,CAAC,CAAC,CAAC;MACnB,CAAC,CAAC;MAEFb,OAAO,CAACK,EAAE,CAAC,QAAQ,EAAE,MAAM;QACzBF,OAAO,CAAC7C,OAAO,CAAC;MAClB,CAAC,CAAC;MAEFyC,UAAU,CAACe,IAAI,CAACd,OAAO,CAAC;IAC1B,CAAC,CAAC;IACF,OAAO,IAAI5C,UAAU,CAAC6C,WAAW,CAAC;EACpC;EAEA,OAAOc,qBAAqBA,CAAChB,UAAiC,EAAqB;IACjF,MAAMiB,WAAW,GAAG,KAAIC,qBAAW,EAAC;MAAEC,UAAU,EAAE;IAAK,CAAC,CAAC;IACzD,MAAMlB,OAAO,GAAGT,oBAAS,CAACS,OAAO,CAAC,CAAC;IACnC,IAAImB,SAAgC;IACpC,IAAIC,OAA4B;IAChCpB,OAAO,CAACK,EAAE,CAAC,OAAO,EAAE,CAACC,MAAM,EAAEC,MAAM,EAAEC,IAAI,KAAK;MAC5C,MAAM3G,IAAc,GAAG,EAAE;MACzB0G,MAAM,CAACF,EAAE,CAAC,MAAM,EAAGI,KAAK,IAAK;QAC3B5G,IAAI,CAAC2B,IAAI,CAACiF,KAAK,CAAC;MAClB,CAAC,CAAC;MACFF,MAAM,CAACF,EAAE,CAAC,KAAK,EAAE,MAAM;QACrB,MAAMgB,OAAO,GAAGpC,MAAM,CAAC0B,MAAM,CAAC9G,IAA+B,CAAC;QAC9D,IAAIyG,MAAM,CAACb,IAAI,KAAK3C,yBAAyB,EAAE;UAC7CkE,WAAW,CAACM,IAAI,CAAC,OAAO,EAAE,IAAIxC,KAAK,CAACuC,OAAO,CAACE,QAAQ,CAAC,CAAC,CAAC,CAAC;UACxD;QACF;QACA,IAAIjB,MAAM,CAACb,IAAI,KAAKzC,yBAAyB,EAAE;UAC7CmE,SAAS,GAAGzC,IAAI,CAACC,KAAK,CAAC0C,OAAO,CAACE,QAAQ,CAAC,CAAC,CAAC;UAC1CC,gBAAM,CAACC,KAAK,CAAC,2CAA2C,EAAEN,SAAS,CAAC;UACpEX,IAAI,CAAC,CAAC;UACN;QACF;QACA,IAAIF,MAAM,CAACb,IAAI,KAAKxC,uBAAuB,EAAE;UAC3CmE,OAAO,GAAG1C,IAAI,CAACC,KAAK,CAAC0C,OAAO,CAACE,QAAQ,CAAC,CAAC,CAAC;UACxCC,gBAAM,CAACC,KAAK,CAAC,8CAA8C,EAAEL,OAAO,CAAC;UACrEZ,IAAI,CAAC,CAAC;UACN;QACF;QACAQ,WAAW,CAACU,KAAK,CAAAhG,aAAA,CAAAA,aAAA,KAAM0B,UAAU,CAACsD,mBAAmB,CAACJ,MAAM,CAACb,IAAI,CAAC;UAAE1B,MAAM,EAAEsD;QAAO,EAAE,CAAC;QACtFb,IAAI,CAAC,CAAC,CAAC,CAAC;MACV,CAAC,CAAC;MACFD,MAAM,CAACF,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAK;QAC1BI,WAAW,CAACM,IAAI,CAAC,OAAO,EAAEV,GAAG,CAAC;MAChC,CAAC,CAAC;MAEFL,MAAM,CAACM,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC,CAAC;;IAEF;IACAb,OAAO,CAACK,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAK;MAC3BI,WAAW,CAACM,IAAI,CAAC,OAAO,EAAEV,GAAG,CAAC;IAChC,CAAC,CAAC;IAEFZ,OAAO,CAACK,EAAE,CAAC,QAAQ,EAAE,MAAM;MACzB,IAAIc,SAAS,EAAEQ,MAAM,KAAK5E,0BAA0B,IAAI,CAACqE,OAAO,EAAE;QAChE;QACA;QACAJ,WAAW,CAACM,IAAI,CACd,OAAO,EACP,IAAIxC,KAAK,CAAC,wDAAwDJ,IAAI,CAACU,SAAS,CAAC+B,SAAS,CAAC,GAAG,CAChG,CAAC;MACH;MACAH,WAAW,CAACY,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC;IACF,IAAAC,kBAAQ,EAAC9B,UAAU,EAAEC,OAAO,EAAGY,GAAG,IAAK;MACrC,IAAIA,GAAG,EAAE;QACPY,gBAAM,CAACM,KAAK,CAAC,iCAAiC,EAAElB,GAAG,CAAC;QACpDI,WAAW,CAACM,IAAI,CAAC,OAAO,EAAEV,GAAG,CAAC;MAChC,CAAC,MAAM;QACLY,gBAAM,CAACC,KAAK,CAAC,8CAA8C,CAAC;MAC9D;IACF,CAAC,CAAC;IAEF,OAAOT,WAAW;EACpB;EAEA,OAAOe,qBAAqBA,CAACC,QAAkB,EAAEC,SAAiB,EAAE;IAClE,MAAM3C,IAAI,GAAGC,oBAAS,CAACD,IAAI,CAAC,CAAC;IAC7B,MAAM4C,SAAoB,GAAG;MAAEP,MAAM,EAAE5E,0BAA0B;MAAEkF;IAAU,CAAC;IAC9ET,gBAAM,CAACC,KAAK,CAAC,2CAA2C,EAAES,SAAS,CAAC;IACpE5C,IAAI,CAACE,KAAK,CAAC;MAAEC,IAAI,EAAEzC;IAA0B,CAAC,EAAE0B,IAAI,CAACU,SAAS,CAAC8C,SAAS,CAAC,CAAC;IAC1E,IAAIC,UAAU,GAAG,CAAC;IAClBH,QAAQ,CAAC3B,EAAE,CAAC,MAAM,EAAGzC,GAAe,IAAK;MACvCuE,UAAU,IAAI,CAAC;MACf7C,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAErC,UAAU,CAACsC,mBAAmB,CAAC9B,GAAG;MAAE,CAAC,EAAEA,GAAG,CAACG,MAAM,CAAC;IACvE,CAAC,CAAC;IACFiE,QAAQ,CAAC3B,EAAE,CAAC,KAAK,EAAE,MAAM;MACvB,MAAM+B,OAAgB,GAAG;QAAED,UAAU;QAAEF;MAAU,CAAC;MAClDT,gBAAM,CAACC,KAAK,CAAC,8CAA8C,EAAEW,OAAO,CAAC;MACrE9C,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAExC;MAAwB,CAAC,EAAEyB,IAAI,CAACU,SAAS,CAACgD,OAAO,CAAC,CAAC;MACtE9C,IAAI,CAACK,QAAQ,CAAC,CAAC;IACjB,CAAC,CAAC;IACFqC,QAAQ,CAAC3B,EAAE,CAAC,OAAO,EAAGO,GAAG,IAAK;MAC5B,MAAMyB,YAAY,GAAGzB,GAAG,CAAC0B,OAAO,IAAI,qBAAqB1B,GAAG,CAACnB,IAAI,GAAG;MACpE+B,gBAAM,CAACM,KAAK,CAAC,gEAAgE,EAAElB,GAAG,CAAC;MACnFtB,IAAI,CAACE,KAAK,CAAC;QAAEC,IAAI,EAAE3C;MAA0B,CAAC,EAAEuF,YAAY,CAAC;MAC7D/C,IAAI,CAACK,QAAQ,CAAC,CAAC;IACjB,CAAC,CAAC;IACF,OAAOL,IAAI;EACb;EAEA,aAAaiD,kBAAkBA,CAACP,QAA2B,EAAuB;IAChF,MAAM/B,WAAyB,GAAG,EAAE;IACpC,WAAW,MAAMrC,GAAG,IAAIoE,QAAQ,EAAE;MAChC/B,WAAW,CAACzE,IAAI,CAACoC,GAAG,CAAC;IACvB;IACA,OAAO,IAAIR,UAAU,CAAC6C,WAAW,CAAC;EACpC;;EAEA;AACF;AACA;EACE,OAAOS,mBAAmBA,CAACjB,IAAY,EAAgC;IACrE,MAAM+C,SAAS,GAAG/C,IAAI,CAACgD,KAAK,CAAC,GAAG,CAAC;IACjC,MAAMC,QAAQ,GAAGF,SAAS,CAAC5G,MAAM,GAAG,CAAC;IACrC,OAAO;MACL+G,KAAK,EAAED,QAAQ,GAAGF,SAAS,CAAC,CAAC,CAAC,GAAGI,SAAS;MAC1C7D,GAAG,EAAE,KAAIC,cAAG,EAAC0D,QAAQ,GAAGF,SAAS,CAAC,CAAC,CAAC,GAAGA,SAAS,CAAC,CAAC,CAAC;IACrD,CAAC;EACH;EACA;AACF;AACA;EACE,OAAO9C,mBAAmBA,CAACmD,UAAsB,EAAU;IACzD,MAAMF,KAAK,GAAGE,UAAU,CAACF,KAAK,GAAG,GAAGE,UAAU,CAACF,KAAK,GAAG,GAAG,EAAE;IAC5D,OAAO,GAAGA,KAAK,GAAGE,UAAU,CAAC9D,GAAG,CAACtB,IAAI,EAAE;EACzC;EAEAa,aAAaA,CAAC2B,WAAyB,EAAE;IACvCA,WAAW,CAACpE,OAAO,CAAEgH,UAAU,IAAK;MAClC,MAAMC,MAAM,GAAG,IAAI,CAACxF,OAAO,CAACyF,IAAI,CAC7BC,MAAM,IAAKA,MAAM,CAACjE,GAAG,CAACkE,OAAO,CAACJ,UAAU,CAAC9D,GAAG,CAAC,IAAIiE,MAAM,CAACL,KAAK,KAAKE,UAAU,CAACF,KAChF,CAAC;MACD,IAAI,CAACG,MAAM,EAAE;QACX,IAAI,CAACxF,OAAO,CAAC9B,IAAI,CAACqH,UAAU,CAAC;MAC/B;IACF,CAAC,CAAC;EACJ;EAEA,MAAMK,YAAYA,CAACC,oBAAoB,GAAG,KAAK,EAA0B;IACvE,MAAMC,WAAW,GAAG,IAAAC,mCAAiB,EAAC,CAAC;IACvC,MAAMC,UAAU,GAAG,MAAM,IAAAC,0BAAQ,EAC/B,IAAI,CAACjG,OAAO,EACZ,MAAO0F,MAAM,IAAK;MAChB,IAAI;QACF,OAAO,MAAMQ,iBAAS,CAACC,WAAW,CAACT,MAAM,CAACjF,MAAM,CAAC;MACnD,CAAC,CAAC,OAAO6C,GAAG,EAAE;QACZ,IAAIuC,oBAAoB,IAAI,EAAEvC,GAAG,YAAY8C,4BAAiB,CAAC,EAAE;UAC/D,MAAM9C,GAAG;QACX;QACAY,gBAAM,CAACM,KAAK,CACV,gDAAgDlB,GAAG,CAAC+C,IAAI,+CAC1D,CAAC;QACD,OAAO,IAAI;MACb;IACF,CAAC,EACD;MAAEP;IAAY,CAChB,CAAC;IACD,OAAO,KAAIQ,8BAAa,EAAC,IAAAC,iBAAO,EAACP,UAAU,CAAC,CAAC;EAC/C;EAEA,aAAaQ,cAAcA,CAACR,UAAuB,EAAuB;IACxE,MAAMF,WAAW,GAAG,IAAAC,mCAAiB,EAAC,CAAC;IACvC,MAAMpD,WAAW,GAAG,MAAM,IAAAsD,0BAAQ,EAChCD,UAAU,EACV,MAAO1F,GAAG,KAAM;MACdmB,GAAG,EAAEnB,GAAG,CAACH,IAAI,CAAC,CAAC;MACfM,MAAM,EAAE,MAAMH,GAAG,CAACmG,QAAQ,CAAC,CAAC;MAC5BJ,IAAI,EAAE/F,GAAG,CAACoG,OAAO,CAAC;IACpB,CAAC,CAAC,EACF;MAAEZ;IAAY,CAChB,CAAC;IACD,OAAO,IAAIhG,UAAU,CAAC6C,WAAW,CAAC;EACpC;EAEAgE,YAAYA,CAAChC,SAAiB,EAAE;IAC9B,IAAI,CAAC3E,OAAO,CAACzB,OAAO,CAAEmH,MAAM,IAAK;MAC/BA,MAAM,CAACL,KAAK,GAAGV,SAAS;IAC1B,CAAC,CAAC;EACJ;EAEAiC,gBAAgBA,CAAA,EAAwC;IACtD,MAAMC,kBAAuD,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC7G,OAAO,CAACzB,OAAO,CAAE+B,GAAG,IAAK;MAC5B,IAAIA,GAAG,CAAC+F,IAAI,KAAKS,wBAAc,CAAC3E,IAAI,EAAE;QACpC,OAAO,CAAC;MACV;MACA,IAAI,CAAC7B,GAAG,CAAC+E,KAAK,EAAE;QACd,MAAM,IAAI7D,KAAK,CAAC,0EAA0ElB,GAAG,CAACmB,GAAG,CAACtB,IAAI,EAAE,CAAC;MAC3G;MACA,IAAI0G,kBAAkB,CAACvG,GAAG,CAAC+E,KAAK,CAAC,EAAE;QACjCwB,kBAAkB,CAACvG,GAAG,CAAC+E,KAAK,CAAC,CAACrE,aAAa,CAAC,CAACV,GAAG,CAAC,CAAC;MACpD,CAAC,MAAM;QACLuG,kBAAkB,CAACvG,GAAG,CAAC+E,KAAK,CAAC,GAAG,IAAIvF,UAAU,CAAC,CAACQ,GAAG,CAAC,CAAC;MACvD;IACF,CAAC,CAAC;IACF,OAAOuG,kBAAkB;EAC3B;;EAEA;AACF;AACA;EACEE,YAAYA,CAAA,EAAG;IACbC,OAAO,CAACC,GAAG,CAAC,IAAI,CAACjH,OAAO,CAACkH,GAAG,CAAEpJ,CAAC,IAAKA,CAAC,CAAC2D,GAAG,CAACtB,IAAI,CAAC,CAACgH,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;EAC/D;AACF;AAACtH,OAAA,CAAAC,UAAA,GAAAA,UAAA","ignoreList":[]}

package/objects/object-list.ts

@@ -2,6 +2,7 @@ import tarStream from 'tar-stream';
 import { pMapPool } from '@teambit/toolbox.promise.map-pool';
 import { compact } from 'lodash';
 import { Readable, PassThrough, pipeline } from 'stream';
+import crypto from 'crypto';
 import BitObject from './object';
 import { BitObjectList } from './bit-object-list';
 import Ref from './ref';
@@ -52,6 +53,19 @@ export class ObjectList {
     return this.objects.length;
   }
 
+  /**
+   * Generates a SHA1 hash from all object buffers.
+   * Used to identify duplicate export requests during retries.
+   * Performance wise, it's not too bad. ~98ms for 150 components on CircleCI.
+   */
+  getSha1Hash(): string {
+    const hash = crypto.createHash('sha1');
+    for (const obj of this.objects) {
+      hash.update(new Uint8Array(obj.buffer));
+    }
+    return hash.digest('hex');
+  }
+
   static mergeMultipleInstances(objectLists: ObjectList[]): ObjectList {
     const objectList = new ObjectList();
     objectLists.forEach((objList) => objectList.mergeObjectList(objList));

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@teambit/objects",
-  "version": "0.0.333",
+  "version": "0.0.335",
   "homepage": "https://bit.cloud/teambit/scope/objects",
   "main": "dist/index.js",
   "componentId": {
     "scope": "teambit.scope",
     "name": "objects",
-    "version": "0.0.333"
+    "version": "0.0.335"
   },
   "dependencies": {
     "lodash": "4.17.21",
@@ -20,13 +20,13 @@
     "semver": "7.7.1",
     "uuid": "8.3.2",
     "@teambit/harmony": "0.4.7",
-    "@teambit/cli": "0.0.1288",
+    "@teambit/cli": "0.0.1289",
     "@teambit/component-id": "1.2.4",
-    "@teambit/component.snap-distance": "0.0.89",
+    "@teambit/component.snap-distance": "0.0.90",
     "@teambit/graph.cleargraph": "0.0.11",
     "@teambit/harmony.modules.concurrency": "0.0.21",
     "@teambit/legacy.logger": "0.0.31",
-    "@teambit/legacy.scope": "0.0.88",
+    "@teambit/legacy.scope": "0.0.89",
     "@teambit/toolbox.promise.map-pool": "0.0.10",
     "@teambit/legacy.constants": "0.0.20",
     "@teambit/legacy.utils": "0.0.29",
@@ -36,20 +36,20 @@
     "@teambit/harmony.modules.in-memory-cache": "0.0.24",
     "@teambit/toolbox.fs.remove-empty-dir": "0.0.9",
     "@teambit/lane-id": "0.0.312",
-    "@teambit/harmony.modules.get-basic-log": "0.0.89",
-    "@teambit/config-store": "0.0.168",
+    "@teambit/harmony.modules.get-basic-log": "0.0.90",
+    "@teambit/config-store": "0.0.169",
     "@teambit/legacy.cli.error": "0.0.31",
-    "@teambit/component.sources": "0.0.140",
+    "@teambit/component.sources": "0.0.141",
     "@teambit/legacy-bit-id": "1.1.3",
     "@teambit/legacy-component-log": "0.0.413",
-    "@teambit/legacy.consumer-component": "0.0.89",
-    "@teambit/legacy.consumer-config": "0.0.88",
-    "@teambit/legacy.extension-data": "0.0.90",
+    "@teambit/legacy.consumer-component": "0.0.90",
+    "@teambit/legacy.consumer-config": "0.0.89",
+    "@teambit/legacy.extension-data": "0.0.91",
     "@teambit/pkg.modules.semver-helper": "0.0.18",
     "@teambit/toolbox.array.duplications-finder": "0.0.3",
     "@teambit/bit.get-bit-version": "0.0.11",
-    "@teambit/semantics.doc-parser": "0.0.96",
-    "@teambit/graph": "1.0.826"
+    "@teambit/semantics.doc-parser": "0.0.97",
+    "@teambit/graph": "1.0.828"
   },
   "devDependencies": {
     "@types/lodash": "4.14.165",