@teambit/objects 0.0.0-85e7bed641b09ff1080a0cd55e300cbb76607d83 → 0.0.0-87722eae60f4d4be4fea8a9a6ebacae673140534

package/objects/object-list.ts

@@ -2,6 +2,7 @@ import tarStream from 'tar-stream';
 import { pMapPool } from '@teambit/toolbox.promise.map-pool';
 import { compact } from 'lodash';
 import { Readable, PassThrough, pipeline } from 'stream';
+import crypto from 'crypto';
 import BitObject from './object';
 import { BitObjectList } from './bit-object-list';
 import Ref from './ref';
@@ -52,6 +53,19 @@ export class ObjectList {
     return this.objects.length;
   }
 
+  /**
+   * Generates a SHA1 hash from all object buffers.
+   * Used to identify duplicate export requests during retries.
+   * Performance wise, it's not too bad. ~98ms for 150 components on CircleCI.
+   */
+  getSha1Hash(): string {
+    const hash = crypto.createHash('sha1');
+    for (const obj of this.objects) {
+      hash.update(new Uint8Array(obj.buffer));
+    }
+    return hash.digest('hex');
+  }
+
   static mergeMultipleInstances(objectLists: ObjectList[]): ObjectList {
     const objectList = new ObjectList();
     objectLists.forEach((objList) => objectList.mergeObjectList(objList));
@@ -120,7 +134,10 @@ export class ObjectList {
     const extract = tarStream.extract();
     let startData: StartFile | undefined;
     let endData: EndFile | undefined;
+    let entriesReceived = 0;
+
     extract.on('entry', (header, stream, next) => {
+      entriesReceived += 1;
       const data: Buffer[] = [];
       stream.on('data', (chunk) => {
         data.push(chunk);
@@ -154,24 +171,40 @@ export class ObjectList {
     });
 
     // not sure if needed
-    extract.on('error', (err) => {
-      passThrough.emit('error', err);
+    extract.on('error', (err: any) => {
+      logger.error(`fromTarToObjectStream tar extraction error after receiving ${entriesReceived} entries`, err);
+
+      // Enhanced error message for tar corruption with entry count
+      if (err.message?.includes('invalid tar header') || err.message?.includes('Invalid tar header')) {
+        const enhancedError = new Error(
+          `Invalid tar header after receiving ${entriesReceived} entries. Possible causes: 1) Server aborted request mid-stream, 2) Network corruption during transfer. Original error: ${err.message}`
+        );
+        enhancedError.stack = err.stack;
+        passThrough.emit('error', enhancedError);
+      } else {
+        passThrough.emit('error', err);
+      }
     });
 
     extract.on('finish', () => {
       if (startData?.schema === OBJECT_LIST_CURRENT_SCHEMA && !endData) {
         // wasn't able to find a better way to indicate whether the server aborted the request
         // see https://github.com/node-fetch/node-fetch/issues/1117
+        logger.error(
+          `fromTarToObjectStream, server terminated the stream unexpectedly. Start: ${JSON.stringify(startData)}, entries received: ${entriesReceived}`
+        );
         passThrough.emit(
           'error',
-          new Error(`server terminated the stream unexpectedly (metadata: ${JSON.stringify(startData)})`)
+          new Error(
+            `server terminated the stream unexpectedly (metadata: ${JSON.stringify(startData)}, entries received: ${entriesReceived})`
+          )
         );
       }
       passThrough.end();
     });
     pipeline(packStream, extract, (err) => {
       if (err) {
-        logger.error('fromTarToObjectStream, pipeline', err);
+        logger.error(`fromTarToObjectStream, pipeline error after ${entriesReceived} entries`, err);
         passThrough.emit('error', err);
       } else {
         logger.debug('fromTarToObjectStream, pipeline is completed');
@@ -187,19 +220,34 @@ export class ObjectList {
     logger.debug('fromObjectStreamToTar, start sending data', startFile);
     pack.entry({ name: TAR_STREAM_START_FILENAME }, JSON.stringify(startFile));
     let numOfFiles = 0;
+    let isFinalized = false;
+
     readable.on('data', (obj: ObjectItem) => {
       numOfFiles += 1;
       pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);
     });
     readable.on('end', () => {
+      if (isFinalized) {
+        logger.warn(`fromObjectStreamToTar (${scopeName}), received 'end' event but pack already finalized`);
+        return;
+      }
+      isFinalized = true;
       const endFile: EndFile = { numOfFiles, scopeName };
       logger.debug('fromObjectStreamToTar, finished sending data', endFile);
       pack.entry({ name: TAR_STREAM_END_FILENAME }, JSON.stringify(endFile));
       pack.finalize();
     });
     readable.on('error', (err) => {
+      if (isFinalized) {
+        logger.error(
+          `ObjectList.fromObjectStreamToTar, received error but pack already finalized. Files sent: ${numOfFiles}`,
+          err
+        );
+        return;
+      }
+      isFinalized = true;
       const errorMessage = err.message || `unexpected error (${err.name})`;
-      logger.error(`ObjectList.fromObjectStreamToTar, streaming an error as a file`, err);
+      logger.error(`ObjectList.fromObjectStreamToTar, streaming an error as a file after ${numOfFiles} files`, err);
       pack.entry({ name: TAR_STREAM_ERROR_FILENAME }, errorMessage);
       pack.finalize();
     });

package/objects/repository.ts

@@ -10,7 +10,8 @@ import { pMapPool } from '@teambit/toolbox.promise.map-pool';
 import { OBJECTS_DIR } from '@teambit/legacy.constants';
 import { logger } from '@teambit/legacy.logger';
 import type { ChownOptions, PathOsBasedAbsolute } from '@teambit/legacy.utils';
-import { glob, writeFile } from '@teambit/legacy.utils';
+import { writeFile } from '@teambit/legacy.utils';
+import { glob } from 'glob';
 import { removeEmptyDir } from '@teambit/toolbox.fs.remove-empty-dir';
 import { concurrentIOLimit } from '@teambit/harmony.modules.concurrency';
 import type { Types, ScopeJson } from '@teambit/legacy.scope';

package/package.json

@@ -1,16 +1,16 @@
 {
   "name": "@teambit/objects",
-  "version": "0.0.0-85e7bed641b09ff1080a0cd55e300cbb76607d83",
+  "version": "0.0.0-87722eae60f4d4be4fea8a9a6ebacae673140534",
   "homepage": "https://bit.cloud/teambit/scope/objects",
   "main": "dist/index.js",
   "componentId": {
     "scope": "teambit.scope",
     "name": "objects",
-    "version": "85e7bed641b09ff1080a0cd55e300cbb76607d83"
+    "version": "87722eae60f4d4be4fea8a9a6ebacae673140534"
   },
   "dependencies": {
-    "@pnpm/dependency-path": "1001.1.0",
-    "@pnpm/lockfile.types": "^1002.0.0",
+    "@pnpm/dependency-path": "1001.1.6",
+    "@pnpm/lockfile.types": "^1002.0.5",
     "semver": "7.7.1",
     "lodash": "4.17.21",
     "uuid": "8.3.2",
@@ -18,45 +18,46 @@
     "p-map-series": "2.1.0",
     "tar-stream": "2.2.0",
     "fs-extra": "10.0.0",
+    "glob": "13.0.0",
     "uid-number": "0.0.6",
     "@teambit/harmony": "0.4.7",
-    "@teambit/cli": "0.0.1273",
+    "@teambit/cli": "0.0.1290",
     "@teambit/component-id": "1.2.4",
-    "@teambit/legacy.utils": "0.0.27",
-    "@teambit/harmony.modules.get-basic-log": "0.0.75",
+    "@teambit/legacy.utils": "0.0.30",
+    "@teambit/harmony.modules.get-basic-log": "0.0.91",
     "@teambit/bit-error": "0.0.404",
     "@teambit/component-version": "1.0.4",
-    "@teambit/component.snap-distance": "0.0.75",
-    "@teambit/config-store": "0.0.153",
+    "@teambit/component.snap-distance": "0.0.91",
+    "@teambit/config-store": "0.0.170",
     "@teambit/lane-id": "0.0.312",
-    "@teambit/legacy.cli.error": "0.0.28",
-    "@teambit/legacy.constants": "0.0.18",
-    "@teambit/legacy.logger": "0.0.28",
-    "@teambit/legacy.scope": "0.0.74",
-    "@teambit/toolbox.crypto.sha1": "0.0.9",
-    "@teambit/component.sources": "0.0.126",
+    "@teambit/legacy.cli.error": "0.0.31",
+    "@teambit/legacy.constants": "0.0.20",
+    "@teambit/legacy.logger": "0.0.31",
+    "@teambit/legacy.scope": "0.0.90",
+    "@teambit/toolbox.crypto.sha1": "0.0.11",
+    "@teambit/component.sources": "0.0.142",
     "@teambit/legacy-bit-id": "1.1.3",
-    "@teambit/legacy-component-log": "0.0.411",
-    "@teambit/legacy.consumer-component": "0.0.75",
-    "@teambit/legacy.consumer-config": "0.0.74",
-    "@teambit/legacy.extension-data": "0.0.76",
-    "@teambit/pkg.modules.semver-helper": "0.0.16",
+    "@teambit/legacy-component-log": "0.0.413",
+    "@teambit/legacy.consumer-component": "0.0.91",
+    "@teambit/legacy.consumer-config": "0.0.90",
+    "@teambit/legacy.extension-data": "0.0.92",
+    "@teambit/pkg.modules.semver-helper": "0.0.18",
     "@teambit/toolbox.array.duplications-finder": "0.0.3",
     "@teambit/graph.cleargraph": "0.0.11",
-    "@teambit/bit.get-bit-version": "0.0.9",
-    "@teambit/semantics.doc-parser": "0.0.82",
-    "@teambit/harmony.modules.concurrency": "0.0.19",
-    "@teambit/toolbox.promise.map-pool": "0.0.8",
-    "@teambit/harmony.modules.in-memory-cache": "0.0.21",
-    "@teambit/toolbox.fs.remove-empty-dir": "0.0.7",
-    "@teambit/graph": "0.0.0-6647e691eb0ac93aae9ba2bc4fbbea4962976a21"
+    "@teambit/bit.get-bit-version": "0.0.11",
+    "@teambit/semantics.doc-parser": "0.0.98",
+    "@teambit/harmony.modules.concurrency": "0.0.21",
+    "@teambit/toolbox.promise.map-pool": "0.0.10",
+    "@teambit/harmony.modules.in-memory-cache": "0.0.24",
+    "@teambit/toolbox.fs.remove-empty-dir": "0.0.9",
+    "@teambit/graph": "0.0.0-bd99e08f4df12bb87883e301c2d192c94b2c1d1e"
   },
   "devDependencies": {
     "@types/semver": "7.5.8",
     "@types/lodash": "4.14.165",
     "@types/uuid": "8.3.4",
     "@types/fs-extra": "9.0.7",
-    "@teambit/harmony.envs.core-aspect-env": "0.0.78"
+    "@teambit/harmony.envs.core-aspect-env": "0.0.80"
   },
   "peerDependencies": {
     "chai": "5.2.1",