@teambit/objects 0.0.0-859acf6dda2dd3d32072c1bf3fd75796ba45dc62 → 0.0.0-87722eae60f4d4be4fea8a9a6ebacae673140534

package/objects/object-list.ts

@@ -2,6 +2,7 @@ import tarStream from 'tar-stream';
 import { pMapPool } from '@teambit/toolbox.promise.map-pool';
 import { compact } from 'lodash';
 import { Readable, PassThrough, pipeline } from 'stream';
+import crypto from 'crypto';
 import BitObject from './object';
 import { BitObjectList } from './bit-object-list';
 import Ref from './ref';
@@ -52,6 +53,19 @@ export class ObjectList {
     return this.objects.length;
   }
 
+  /**
+   * Generates a SHA1 hash from all object buffers.
+   * Used to identify duplicate export requests during retries.
+   * Performance wise, it's not too bad. ~98ms for 150 components on CircleCI.
+   */
+  getSha1Hash(): string {
+    const hash = crypto.createHash('sha1');
+    for (const obj of this.objects) {
+      hash.update(new Uint8Array(obj.buffer));
+    }
+    return hash.digest('hex');
+  }
+
   static mergeMultipleInstances(objectLists: ObjectList[]): ObjectList {
     const objectList = new ObjectList();
     objectLists.forEach((objList) => objectList.mergeObjectList(objList));
@@ -120,7 +134,10 @@ export class ObjectList {
     const extract = tarStream.extract();
     let startData: StartFile | undefined;
     let endData: EndFile | undefined;
+    let entriesReceived = 0;
+
     extract.on('entry', (header, stream, next) => {
+      entriesReceived += 1;
       const data: Buffer[] = [];
       stream.on('data', (chunk) => {
         data.push(chunk);
@@ -154,24 +171,40 @@ export class ObjectList {
     });
 
     // not sure if needed
-    extract.on('error', (err) => {
-      passThrough.emit('error', err);
+    extract.on('error', (err: any) => {
+      logger.error(`fromTarToObjectStream tar extraction error after receiving ${entriesReceived} entries`, err);
+
+      // Enhanced error message for tar corruption with entry count
+      if (err.message?.includes('invalid tar header') || err.message?.includes('Invalid tar header')) {
+        const enhancedError = new Error(
+          `Invalid tar header after receiving ${entriesReceived} entries. Possible causes: 1) Server aborted request mid-stream, 2) Network corruption during transfer. Original error: ${err.message}`
+        );
+        enhancedError.stack = err.stack;
+        passThrough.emit('error', enhancedError);
+      } else {
+        passThrough.emit('error', err);
+      }
     });
 
     extract.on('finish', () => {
       if (startData?.schema === OBJECT_LIST_CURRENT_SCHEMA && !endData) {
         // wasn't able to find a better way to indicate whether the server aborted the request
         // see https://github.com/node-fetch/node-fetch/issues/1117
+        logger.error(
+          `fromTarToObjectStream, server terminated the stream unexpectedly. Start: ${JSON.stringify(startData)}, entries received: ${entriesReceived}`
+        );
         passThrough.emit(
           'error',
-          new Error(`server terminated the stream unexpectedly (metadata: ${JSON.stringify(startData)})`)
+          new Error(
+            `server terminated the stream unexpectedly (metadata: ${JSON.stringify(startData)}, entries received: ${entriesReceived})`
+          )
         );
       }
       passThrough.end();
     });
     pipeline(packStream, extract, (err) => {
       if (err) {
-        logger.error('fromTarToObjectStream, pipeline', err);
+        logger.error(`fromTarToObjectStream, pipeline error after ${entriesReceived} entries`, err);
         passThrough.emit('error', err);
       } else {
         logger.debug('fromTarToObjectStream, pipeline is completed');
@@ -187,19 +220,34 @@ export class ObjectList {
     logger.debug('fromObjectStreamToTar, start sending data', startFile);
     pack.entry({ name: TAR_STREAM_START_FILENAME }, JSON.stringify(startFile));
     let numOfFiles = 0;
+    let isFinalized = false;
+
     readable.on('data', (obj: ObjectItem) => {
       numOfFiles += 1;
       pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);
     });
     readable.on('end', () => {
+      if (isFinalized) {
+        logger.warn(`fromObjectStreamToTar (${scopeName}), received 'end' event but pack already finalized`);
+        return;
+      }
+      isFinalized = true;
       const endFile: EndFile = { numOfFiles, scopeName };
       logger.debug('fromObjectStreamToTar, finished sending data', endFile);
       pack.entry({ name: TAR_STREAM_END_FILENAME }, JSON.stringify(endFile));
       pack.finalize();
     });
     readable.on('error', (err) => {
+      if (isFinalized) {
+        logger.error(
+          `ObjectList.fromObjectStreamToTar, received error but pack already finalized. Files sent: ${numOfFiles}`,
+          err
+        );
+        return;
+      }
+      isFinalized = true;
       const errorMessage = err.message || `unexpected error (${err.name})`;
-      logger.error(`ObjectList.fromObjectStreamToTar, streaming an error as a file`, err);
+      logger.error(`ObjectList.fromObjectStreamToTar, streaming an error as a file after ${numOfFiles} files`, err);
       pack.entry({ name: TAR_STREAM_ERROR_FILENAME }, errorMessage);
       pack.finalize();
     });

package/objects/object.ts

@@ -147,7 +147,6 @@ path: ${err.path}`);
   }
 
   static makeHash(str: string | Buffer): string {
-    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
     return sha1(str);
   }
 }

package/objects/raw-object.ts

@@ -8,7 +8,6 @@ export default class BitRawObject {
   headers: string[];
   type: string;
   content: Buffer;
-  // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   parsedContent: any;
   _ref: string;
 
@@ -17,7 +16,6 @@ export default class BitRawObject {
     ref: string | null | undefined,
     type: string | null | undefined,
     content: Buffer | null | undefined,
-    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
     parsedContent: any | null | undefined
   ) {
     let headers;
@@ -72,17 +70,14 @@ export default class BitRawObject {
     }
   }
 
-  // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   set ref(ref: string) {
     this._ref = ref;
   }
 
-  // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   get ref(): string {
     return this._ref;
   }
 
-  // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   get id(): string {
     switch (this.type) {
       case 'Version':
@@ -114,11 +109,7 @@ export default class BitRawObject {
     return [];
   }
 
-  static async fromDeflatedBuffer(
-    fileContents: Buffer,
-    ref: string | null | undefined
-    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
-  ): Promise<BitObject> {
+  static async fromDeflatedBuffer(fileContents: Buffer, ref: string | null | undefined): Promise<BitObject> {
     // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
     return inflate(fileContents).then((buffer) => new BitRawObject(buffer, ref));
   }
@@ -129,7 +120,6 @@ export default class BitRawObject {
    * @param {Any} parsedContent
    */
   toRealObject() {
-    // @ts-ignore
     return types[this.type].from(this.parsedContent || this.getParsedContent(), this.headers[1]);
   }
 

package/objects/repository.ts

@@ -10,7 +10,8 @@ import { pMapPool } from '@teambit/toolbox.promise.map-pool';
 import { OBJECTS_DIR } from '@teambit/legacy.constants';
 import { logger } from '@teambit/legacy.logger';
 import type { ChownOptions, PathOsBasedAbsolute } from '@teambit/legacy.utils';
-import { glob, writeFile } from '@teambit/legacy.utils';
+import { writeFile } from '@teambit/legacy.utils';
+import { glob } from 'glob';
 import { removeEmptyDir } from '@teambit/toolbox.fs.remove-empty-dir';
 import { concurrentIOLimit } from '@teambit/harmony.modules.concurrency';
 import type { Types, ScopeJson } from '@teambit/legacy.scope';
@@ -22,17 +23,15 @@ import type { ObjectItem } from './object-list';
 import { ObjectList } from './object-list';
 import BitRawObject from './raw-object';
 import Ref from './ref';
-import type { ContentTransformer } from './repository-hooks';
-import { onPersist, onRead } from './repository-hooks';
 import type { InMemoryCache } from '@teambit/harmony.modules.in-memory-cache';
 import { getMaxSizeForObjects, createInMemoryCache } from '@teambit/harmony.modules.in-memory-cache';
 import { ScopeMeta, Lane, ModelComponent } from '../models';
 
+type ContentTransformer = (content: Buffer) => Buffer;
 const OBJECTS_BACKUP_DIR = `${OBJECTS_DIR}.bak`;
 const TRASH_DIR = 'trash';
 
 export default class Repository {
-  // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   objects: { [key: string]: BitObject } = {};
   objectsToRemove: Ref[] = [];
   scopeJson: ScopeJson;
@@ -47,8 +46,8 @@ export default class Repository {
   constructor(scopePath: string, scopeJson: ScopeJson) {
     this.scopePath = scopePath;
     this.scopeJson = scopeJson;
-    this.onRead = onRead(scopePath, scopeJson);
-    this.onPersist = onPersist(scopePath, scopeJson);
+    this.onRead = (content: Buffer) => Repository.onPostObjectRead?.(content) || content;
+    this.onPersist = (content: Buffer) => Repository.onPreObjectPersist?.(content) || content;
     this.cache = createInMemoryCache({ maxSize: getMaxSizeForObjects() });
   }
 
@@ -91,6 +90,20 @@ export default class Repository {
 
   static onPostObjectsPersist: () => Promise<void>;
 
+  /**
+   * Hook for transforming content before objects are persisted to the filesystem.
+   * Note: This function cannot be async because it's used by the synchronous `loadSync` method
+   * which needs to maintain sync behavior for compatibility with existing code.
+   */
+  static onPreObjectPersist: (content: Buffer) => Buffer;
+
+  /**
+   * Hook for transforming content after objects are read from the filesystem.
+   * Note: This function cannot be async because it's used by the synchronous `loadSync` method
+   * which needs to maintain sync behavior for compatibility with existing code.
+   */
+  static onPostObjectRead: (content: Buffer) => Buffer;
+
   async reLoadScopeIndex() {
     this.scopeIndex = await this.loadOptionallyCreateScopeIndex();
   }

package/objects/scope-index.ts

@@ -111,7 +111,6 @@ export class ScopeIndex {
   }
 
   getHashes(indexType: IndexType): string[] {
-    // @ts-ignore how to tell TS that all this.index.prop are array?
     return this.index[indexType].map((indexItem: IndexItem) => indexItem.hash);
   }
   getHashesByQuery(indexType: IndexType, filter: Function): string[] {

package/package.json

@@ -1,63 +1,63 @@
 {
   "name": "@teambit/objects",
-  "version": "0.0.0-859acf6dda2dd3d32072c1bf3fd75796ba45dc62",
+  "version": "0.0.0-87722eae60f4d4be4fea8a9a6ebacae673140534",
   "homepage": "https://bit.cloud/teambit/scope/objects",
   "main": "dist/index.js",
   "componentId": {
     "scope": "teambit.scope",
     "name": "objects",
-    "version": "859acf6dda2dd3d32072c1bf3fd75796ba45dc62"
+    "version": "87722eae60f4d4be4fea8a9a6ebacae673140534"
   },
   "dependencies": {
-    "@pnpm/dependency-path": "1001.1.0",
-    "@pnpm/lockfile.types": "^1002.0.0",
+    "@pnpm/dependency-path": "1001.1.6",
+    "@pnpm/lockfile.types": "^1002.0.5",
     "semver": "7.7.1",
     "lodash": "4.17.21",
     "uuid": "8.3.2",
     "async-mutex": "0.3.1",
     "p-map-series": "2.1.0",
     "tar-stream": "2.2.0",
-    "is-relative-path": "2.0.0",
     "fs-extra": "10.0.0",
+    "glob": "13.0.0",
     "uid-number": "0.0.6",
     "@teambit/harmony": "0.4.7",
-    "@teambit/cli": "0.0.1250",
+    "@teambit/cli": "0.0.1290",
     "@teambit/component-id": "1.2.4",
-    "@teambit/legacy.utils": "0.0.23",
-    "@teambit/harmony.modules.get-basic-log": "0.0.61",
+    "@teambit/legacy.utils": "0.0.30",
+    "@teambit/harmony.modules.get-basic-log": "0.0.91",
     "@teambit/bit-error": "0.0.404",
     "@teambit/component-version": "1.0.4",
-    "@teambit/component.snap-distance": "0.0.61",
-    "@teambit/config-store": "0.0.130",
+    "@teambit/component.snap-distance": "0.0.91",
+    "@teambit/config-store": "0.0.170",
     "@teambit/lane-id": "0.0.312",
-    "@teambit/legacy.cli.error": "0.0.24",
-    "@teambit/legacy.constants": "0.0.14",
-    "@teambit/legacy.logger": "0.0.24",
-    "@teambit/legacy.scope": "0.0.60",
-    "@teambit/toolbox.crypto.sha1": "0.0.7",
-    "@teambit/component.sources": "0.0.112",
+    "@teambit/legacy.cli.error": "0.0.31",
+    "@teambit/legacy.constants": "0.0.20",
+    "@teambit/legacy.logger": "0.0.31",
+    "@teambit/legacy.scope": "0.0.90",
+    "@teambit/toolbox.crypto.sha1": "0.0.11",
+    "@teambit/component.sources": "0.0.142",
     "@teambit/legacy-bit-id": "1.1.3",
-    "@teambit/legacy-component-log": "0.0.408",
-    "@teambit/legacy.consumer-component": "0.0.61",
-    "@teambit/legacy.consumer-config": "0.0.60",
-    "@teambit/legacy.extension-data": "0.0.62",
-    "@teambit/pkg.modules.semver-helper": "0.0.13",
+    "@teambit/legacy-component-log": "0.0.413",
+    "@teambit/legacy.consumer-component": "0.0.91",
+    "@teambit/legacy.consumer-config": "0.0.90",
+    "@teambit/legacy.extension-data": "0.0.92",
+    "@teambit/pkg.modules.semver-helper": "0.0.18",
     "@teambit/toolbox.array.duplications-finder": "0.0.3",
     "@teambit/graph.cleargraph": "0.0.11",
-    "@teambit/bit.get-bit-version": "0.0.6",
-    "@teambit/semantics.doc-parser": "0.0.68",
-    "@teambit/harmony.modules.concurrency": "0.0.15",
-    "@teambit/toolbox.promise.map-pool": "0.0.6",
-    "@teambit/harmony.modules.in-memory-cache": "0.0.17",
-    "@teambit/toolbox.fs.remove-empty-dir": "0.0.5",
-    "@teambit/graph": "0.0.0-13b63e7b77d701d0d7c1db840e3dfbb1c79ea2f7"
+    "@teambit/bit.get-bit-version": "0.0.11",
+    "@teambit/semantics.doc-parser": "0.0.98",
+    "@teambit/harmony.modules.concurrency": "0.0.21",
+    "@teambit/toolbox.promise.map-pool": "0.0.10",
+    "@teambit/harmony.modules.in-memory-cache": "0.0.24",
+    "@teambit/toolbox.fs.remove-empty-dir": "0.0.9",
+    "@teambit/graph": "0.0.0-bd99e08f4df12bb87883e301c2d192c94b2c1d1e"
   },
   "devDependencies": {
     "@types/semver": "7.5.8",
     "@types/lodash": "4.14.165",
     "@types/uuid": "8.3.4",
     "@types/fs-extra": "9.0.7",
-    "@teambit/harmony.envs.core-aspect-env": "0.0.72"
+    "@teambit/harmony.envs.core-aspect-env": "0.0.80"
   },
   "peerDependencies": {
     "chai": "5.2.1",

package/dist/objects/repository-hooks.d.ts

@@ -1,4 +0,0 @@
-import type { ScopeJson } from '@teambit/legacy.scope';
-export type ContentTransformer = (content: Buffer) => Buffer;
-export declare function onPersist(scopePath: string, scopeJson: ScopeJson): ContentTransformer;
-export declare function onRead(scopePath: string, scopeJson: ScopeJson): ContentTransformer;

package/dist/objects/repository-hooks.js

@@ -1,56 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.onPersist = onPersist;
-exports.onRead = onRead;
-function _isRelativePath() {
-  const data = _interopRequireDefault(require("is-relative-path"));
-  _isRelativePath = function () {
-    return data;
-  };
-  return data;
-}
-function _path() {
-  const data = _interopRequireDefault(require("path"));
-  _path = function () {
-    return data;
-  };
-  return data;
-}
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
-function loadHooks(scopePath, scopeJson) {
-  const hooksPath = scopeJson.hooksPath;
-  if (hooksPath) {
-    const hooksFinalPath = (0, _isRelativePath().default)(hooksPath) ? _path().default.join(scopePath, hooksPath) : hooksPath;
-    // eslint-disable-next-line global-require, import/no-dynamic-require
-    const hooks = require(hooksFinalPath);
-    return hooks;
-  }
-  return undefined;
-}
-function onPersist(scopePath, scopeJson) {
-  const defaultFunc = content => content;
-  const hooks = loadHooks(scopePath, scopeJson);
-  if (hooks) {
-    const onReadFunction = hooks.onPersist;
-    if (onReadFunction) {
-      return onReadFunction;
-    }
-  }
-  return defaultFunc;
-}
-function onRead(scopePath, scopeJson) {
-  const defaultFunc = content => content;
-  const hooks = loadHooks(scopePath, scopeJson);
-  if (hooks) {
-    const onReadFunction = hooks.onRead;
-    if (onReadFunction) {
-      return onReadFunction;
-    }
-  }
-  return defaultFunc;
-}
-
-//# sourceMappingURL=repository-hooks.js.map

package/dist/objects/repository-hooks.js.map

@@ -1 +0,0 @@
-{"version":3,"names":["_isRelativePath","data","_interopRequireDefault","require","_path","e","__esModule","default","loadHooks","scopePath","scopeJson","hooksPath","hooksFinalPath","isRelative","path","join","hooks","undefined","onPersist","defaultFunc","content","onReadFunction","onRead"],"sources":["repository-hooks.ts"],"sourcesContent":["import isRelative from 'is-relative-path';\nimport path from 'path';\n\nimport type { ScopeJson } from '@teambit/legacy.scope';\n\nexport type ContentTransformer = (content: Buffer) => Buffer;\n\nfunction loadHooks(scopePath: string, scopeJson: ScopeJson): any | undefined {\n  const hooksPath = scopeJson.hooksPath;\n  if (hooksPath) {\n    const hooksFinalPath = isRelative(hooksPath) ? path.join(scopePath, hooksPath) : hooksPath;\n    // eslint-disable-next-line global-require, import/no-dynamic-require\n    const hooks = require(hooksFinalPath);\n    return hooks;\n  }\n  return undefined;\n}\n\nexport function onPersist(scopePath: string, scopeJson: ScopeJson): ContentTransformer {\n  const defaultFunc = (content) => content;\n  const hooks = loadHooks(scopePath, scopeJson);\n  if (hooks) {\n    const onReadFunction = hooks.onPersist;\n    if (onReadFunction) {\n      return onReadFunction;\n    }\n  }\n  return defaultFunc;\n}\n\nexport function onRead(scopePath: string, scopeJson: ScopeJson): ContentTransformer {\n  const defaultFunc = (content) => content;\n\n  const hooks = loadHooks(scopePath, scopeJson);\n  if (hooks) {\n    const onReadFunction = hooks.onRead;\n    if (onReadFunction) {\n      return onReadFunction;\n    }\n  }\n  return defaultFunc;\n}\n"],"mappings":";;;;;;;AAAA,SAAAA,gBAAA;EAAA,MAAAC,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAH,eAAA,YAAAA,CAAA;IAAA,OAAAC,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AACA,SAAAG,MAAA;EAAA,MAAAH,IAAA,GAAAC,sBAAA,CAAAC,OAAA;EAAAC,KAAA,YAAAA,CAAA;IAAA,OAAAH,IAAA;EAAA;EAAA,OAAAA,IAAA;AAAA;AAAwB,SAAAC,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAMxB,SAASG,SAASA,CAACC,SAAiB,EAAEC,SAAoB,EAAmB;EAC3E,MAAMC,SAAS,GAAGD,SAAS,CAACC,SAAS;EACrC,IAAIA,SAAS,EAAE;IACb,MAAMC,cAAc,GAAG,IAAAC,yBAAU,EAACF,SAAS,CAAC,GAAGG,eAAI,CAACC,IAAI,CAACN,SAAS,EAAEE,SAAS,CAAC,GAAGA,SAAS;IAC1F;IACA,MAAMK,KAAK,GAAGb,OAAO,CAACS,cAAc,CAAC;IACrC,OAAOI,KAAK;EACd;EACA,OAAOC,SAAS;AAClB;AAEO,SAASC,SAASA,CAACT,SAAiB,EAAEC,SAAoB,EAAsB;EACrF,MAAMS,WAAW,GAAIC,OAAO,IAAKA,OAAO;EACxC,MAAMJ,KAAK,GAAGR,SAAS,CAACC,SAAS,EAAEC,SAAS,CAAC;EAC7C,IAAIM,KAAK,EAAE;IACT,MAAMK,cAAc,GAAGL,KAAK,CAACE,SAAS;IACtC,IAAIG,cAAc,EAAE;MAClB,OAAOA,cAAc;IACvB;EACF;EACA,OAAOF,WAAW;AACpB;AAEO,SAASG,MAAMA,CAACb,SAAiB,EAAEC,SAAoB,EAAsB;EAClF,MAAMS,WAAW,GAAIC,OAAO,IAAKA,OAAO;EAExC,MAAMJ,KAAK,GAAGR,SAAS,CAACC,SAAS,EAAEC,SAAS,CAAC;EAC7C,IAAIM,KAAK,EAAE;IACT,MAAMK,cAAc,GAAGL,KAAK,CAACM,MAAM;IACnC,IAAID,cAAc,EAAE;MAClB,OAAOA,cAAc;IACvB;EACF;EACA,OAAOF,WAAW;AACpB","ignoreList":[]}

package/objects/repository-hooks.ts

@@ -1,42 +0,0 @@
-import isRelative from 'is-relative-path';
-import path from 'path';
-
-import type { ScopeJson } from '@teambit/legacy.scope';
-
-export type ContentTransformer = (content: Buffer) => Buffer;
-
-function loadHooks(scopePath: string, scopeJson: ScopeJson): any | undefined {
-  const hooksPath = scopeJson.hooksPath;
-  if (hooksPath) {
-    const hooksFinalPath = isRelative(hooksPath) ? path.join(scopePath, hooksPath) : hooksPath;
-    // eslint-disable-next-line global-require, import/no-dynamic-require
-    const hooks = require(hooksFinalPath);
-    return hooks;
-  }
-  return undefined;
-}
-
-export function onPersist(scopePath: string, scopeJson: ScopeJson): ContentTransformer {
-  const defaultFunc = (content) => content;
-  const hooks = loadHooks(scopePath, scopeJson);
-  if (hooks) {
-    const onReadFunction = hooks.onPersist;
-    if (onReadFunction) {
-      return onReadFunction;
-    }
-  }
-  return defaultFunc;
-}
-
-export function onRead(scopePath: string, scopeJson: ScopeJson): ContentTransformer {
-  const defaultFunc = (content) => content;
-
-  const hooks = loadHooks(scopePath, scopeJson);
-  if (hooks) {
-    const onReadFunction = hooks.onRead;
-    if (onReadFunction) {
-      return onReadFunction;
-    }
-  }
-  return defaultFunc;
-}