@teambit/objects 0.0.0-00042bc99514ea12d897f81bcd0181e2b034c615

package/objects/objects-readable-generator.ts

@@ -0,0 +1,174 @@
+import pMapSeries from 'p-map-series';
+import { BitError } from '@teambit/bit-error';
+import { Readable } from 'stream';
+import { compact } from 'lodash';
+import type BitObject from './object';
+import type Repository from './repository';
+import type Ref from './ref';
+import type { Scope } from '@teambit/legacy.scope';
+import { logger } from '@teambit/legacy.logger';
+import { getAllVersionHashesMemoized } from '@teambit/component.snap-distance';
+import type { Lane, LaneHistory, ModelComponent, Version } from '../models';
+import type { ObjectItem } from './object-list';
+
+export type ComponentWithCollectOptions = {
+  component: ModelComponent;
+  version: string;
+  collectParents: boolean;
+  collectParentsUntil?: Ref | null; // stop traversing when this hash found. helps to import only the delta.
+  collectArtifacts: boolean;
+  includeVersionHistory?: boolean; // send VersionHistory object if exists rather than collecting parents
+};
+
+export class ObjectsReadableGenerator {
+  public readable: Readable;
+  private pushed: string[] = [];
+  constructor(
+    private repo: Repository,
+    private callbackOnceDone: Function
+  ) {
+    this.readable = new Readable({ objectMode: true, read() {} });
+  }
+  async pushObjectsToReadable(componentsWithOptions: ComponentWithCollectOptions[]) {
+    try {
+      await this.pushScopeMeta();
+      await pMapSeries(componentsWithOptions, async (componentWithOptions) =>
+        this.pushComponentObjects(componentWithOptions)
+      );
+      this.closeReadableSuccessfully();
+    } catch (err: any) {
+      this.closeReadableFailure(err);
+    }
+  }
+
+  async pushLanes(lanesToFetch: Lane[], lanesHistory: LaneHistory[] = []) {
+    try {
+      const lanesItems = await Promise.all(lanesToFetch.map((laneToFetch) => this.bitObjectToItem(laneToFetch)));
+      const laneHistoryItems = await Promise.all(lanesHistory.map((laneHistory) => this.bitObjectToItem(laneHistory)));
+      this.pushManyObjects([...lanesItems, ...laneHistoryItems]);
+      this.closeReadableSuccessfully();
+    } catch (err: any) {
+      this.closeReadableFailure(err);
+    }
+  }
+
+  async pushObjects(refs: Ref[], scope: Scope) {
+    try {
+      await pMapSeries(refs, async (ref) => {
+        const objectItem = await this.getObjectGracefully(ref, scope);
+        if (objectItem) this.push(objectItem);
+      });
+      this.closeReadableSuccessfully();
+    } catch (err: any) {
+      this.closeReadableFailure(err);
+    }
+  }
+
+  private closeReadableSuccessfully() {
+    logger.debug(`ObjectsReadableGenerator, pushed ${this.pushed.length} objects`);
+    this.callbackOnceDone();
+    this.readable.push(null);
+  }
+
+  private closeReadableFailure(err: Error) {
+    logger.debug(`ObjectsReadableGenerator, pushed ${this.pushed.length} objects`);
+    logger.error(`ObjectsReadableGenerator, got an error`, err);
+    this.callbackOnceDone(err);
+    this.readable.destroy(err);
+  }
+
+  private async bitObjectToItem(obj: BitObject): Promise<ObjectItem> {
+    return {
+      ref: obj.hash(),
+      buffer: await obj.compress(),
+      type: obj.getType(),
+    };
+  }
+
+  private async getObjectGracefully(ref: Ref, scope: Scope) {
+    try {
+      return await scope.getObjectItem(ref);
+    } catch (err: any) {
+      logger.warn(
+        `getObjectGracefully: failed retrieving an object ${ref.toString()} from the filesystem.\n${err.message}`
+      );
+      return null;
+    }
+  }
+
+  private async pushScopeMeta() {
+    const scopeMeta = await this.repo.getScopeMetaObject();
+    this.push(scopeMeta);
+  }
+
+  private push(obj: ObjectItem) {
+    const hashStr = obj.ref.toString();
+    if (this.pushed.includes(hashStr)) {
+      return;
+    }
+    logger.trace('ObjectsReadableGenerator.push', hashStr);
+    this.readable.push(obj);
+    this.pushed.push(hashStr);
+  }
+  private pushManyObjects(objects: ObjectItem[]) {
+    objects.map((obj) => this.push(obj));
+  }
+
+  private async pushComponentObjects(componentWithOptions: ComponentWithCollectOptions): Promise<void> {
+    const { component, collectParents, collectArtifacts, collectParentsUntil, includeVersionHistory } =
+      componentWithOptions;
+    const version = await component.loadVersion(componentWithOptions.version, this.repo, false);
+    if (!version) throw new BitError(`failed loading version ${componentWithOptions.version} of ${component.id()}`);
+    if (collectParentsUntil && version.hash().isEqual(collectParentsUntil)) {
+      return;
+    }
+    const collectVersionObjects = async (ver: Version): Promise<ObjectItem[]> => {
+      const versionRefs = ver.refsWithOptions(false, collectArtifacts);
+      const missingVersionRefs = versionRefs.filter((ref) => !this.pushed.includes(ref.toString()));
+      const versionObjects = await ver.collectManyObjects(this.repo, missingVersionRefs);
+      const versionData = { ref: ver.hash(), buffer: await ver.asRaw(this.repo), type: ver.getType() };
+      return [...versionObjects, versionData];
+    };
+    if (!this.pushed.includes(component.hash().toString())) {
+      const componentData = {
+        ref: component.hash(),
+        buffer: await component.asRaw(this.repo),
+        type: component.getType(),
+      };
+      this.push(componentData);
+    }
+    const allVersions: Version[] = [];
+    if (includeVersionHistory) {
+      const versionHistory = await component.getAndPopulateVersionHistory(this.repo, version.hash());
+      const versionHistoryData = {
+        ref: versionHistory.hash(),
+        buffer: await versionHistory.asRaw(this.repo),
+        type: versionHistory.getType(),
+      };
+      this.push(versionHistoryData);
+    }
+    if (collectParents) {
+      const allParentsHashes = await getAllVersionHashesMemoized({
+        modelComponent: component,
+        repo: this.repo,
+        startFrom: version.hash(),
+        stopAt: collectParentsUntil ? [collectParentsUntil] : undefined,
+      });
+      const missingParentsHashes = allParentsHashes.filter((h) => !h.isEqual(version.hash()));
+      const parentVersions = await pMapSeries(missingParentsHashes, (parentHash) => parentHash.load(this.repo));
+      const nullParentHashes = missingParentsHashes.filter((_, i) => !parentVersions[i]);
+      if (nullParentHashes.length) {
+        logger.warn(
+          `ObjectsReadableGenerator.pushComponentObjects: failed loading ${nullParentHashes.length} parent version(s) for ${component.id()} (version: ${componentWithOptions.version}), missing: ${nullParentHashes.join(', ')}`
+        );
+      }
+      allVersions.push(...(compact(parentVersions) as Version[]));
+      // note: don't bring the head. otherwise, component-delta of the head won't bring all history of this comp.
+    }
+    allVersions.push(version);
+    await pMapSeries(allVersions, async (ver) => {
+      const versionObjects = await collectVersionObjects(ver);
+      this.pushManyObjects(versionObjects);
+    });
+  }
+}

package/objects/raw-object.ts

@@ -0,0 +1,132 @@
+import { NULL_BYTE, SPACE_DELIMITER } from '@teambit/legacy.constants';
+import { getStringifyArgs, inflate } from '@teambit/legacy.utils';
+import { typesObj as types } from '@teambit/legacy.scope';
+import type BitObject from './object';
+import { clone } from 'lodash';
+
+export default class BitRawObject {
+  headers: string[];
+  type: string;
+  content: Buffer;
+  parsedContent: any;
+  _ref: string;
+
+  constructor(
+    buffer: Buffer,
+    ref: string | null | undefined,
+    type: string | null | undefined,
+    content: Buffer | null | undefined,
+    parsedContent: any | null | undefined
+  ) {
+    let headers;
+    let contentFromBuffer;
+    if (buffer) {
+      const firstNullByteLocation = buffer.indexOf(NULL_BYTE);
+      headers = buffer.slice(0, firstNullByteLocation).toString();
+      contentFromBuffer = buffer.slice(firstNullByteLocation + 1, buffer.length);
+    }
+    this.content = content || contentFromBuffer;
+    this.headers = headers ? headers.split(SPACE_DELIMITER) : undefined;
+    const typeFromHeader = this.headers ? this.headers[0] : undefined;
+    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
+    this.type = type || typeFromHeader;
+    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
+    this._ref = ref;
+    this.parsedContent = parsedContent || this.getParsedContent();
+  }
+
+  getParsedContent() {
+    if (this.parsedContent) return this.parsedContent;
+    let parsedContent;
+    switch (this.type) {
+      case 'Version':
+      case 'Component':
+      case 'Symlink':
+      case 'ScopeMeta':
+      case 'VersionHistory':
+        parsedContent = JSON.parse(this.content.toString());
+        break;
+
+      default:
+        parsedContent = this.content;
+    }
+    return parsedContent;
+  }
+
+  getString(pretty: boolean) {
+    const args = getStringifyArgs(pretty);
+    switch (this.type) {
+      case 'Version':
+      case 'Component':
+      case 'Symlink':
+      case 'ScopeMeta':
+      case 'VersionHistory':
+        return JSON.stringify(this.parsedContent, ...args);
+      case 'Source':
+        return pretty ? JSON.stringify(JSON.parse(this.content.toString()), ...args) : this.content;
+
+      default:
+        return this.content;
+    }
+  }
+
+  set ref(ref: string) {
+    this._ref = ref;
+  }
+
+  get ref(): string {
+    return this._ref;
+  }
+
+  get id(): string {
+    switch (this.type) {
+      case 'Version':
+        return 'component version';
+      case 'Component':
+        return this.parsedContent.scope
+          ? [this.parsedContent.scope, this.parsedContent.name].join('/')
+          : this.parsedContent.name;
+      case 'Symlink':
+        return this.parsedContent.name;
+      case 'ScopeMeta':
+        return this.parsedContent.name;
+
+      default:
+        return 'component source file';
+    }
+  }
+
+  refs(): string[] {
+    if (this.type === 'Component') {
+      return Object.values(this.parsedContent.versions);
+    }
+    if (this.type === 'Version') {
+      const files = this.parsedContent.files ? this.parsedContent.files.map((file) => file.file) : [];
+      const dists = this.parsedContent.dists ? this.parsedContent.dists.map((dist) => dist.file) : [];
+      return [...dists, ...files].filter((ref) => ref);
+    }
+
+    return [];
+  }
+
+  static async fromDeflatedBuffer(fileContents: Buffer, ref: string | null | undefined): Promise<BitObject> {
+    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
+    return inflate(fileContents).then((buffer) => new BitRawObject(buffer, ref));
+  }
+
+  /**
+   * Build a real object (model) from a parsed content (can be the original parsed conents or a provided one)
+   * We use the provided version during the migration process when we change the parsed content outside
+   * @param {Any} parsedContent
+   */
+  toRealObject() {
+    return types[this.type].from(this.parsedContent || this.getParsedContent(), this.headers[1]);
+  }
+
+  clone() {
+    const parsedContent = this.parsedContent ? clone(this.parsedContent) : undefined;
+    // TODO: Should also clone the buffers (content)
+    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
+    return new BitRawObject(undefined, this._ref, this.type, this.content, parsedContent);
+  }
+}

package/objects/ref.ts

@@ -0,0 +1,45 @@
+import type BitObject from './object';
+import type Repository from './repository';
+
+export default class Ref {
+  hash: string;
+
+  constructor(hash: string) {
+    if (!hash) throw new Error('failed creating a Ref object, the hash argument is empty');
+    this.hash = hash;
+  }
+
+  toString() {
+    // reason for hash.toString() is when working with short hash, it's possible that all characters are numbers
+    // so it's needed to convert it to string
+    return this.hash.toString();
+  }
+
+  toShortString(numOfChars = 9) {
+    return this.hash.substring(0, numOfChars).toString();
+  }
+
+  load(repository: Repository, throws = false): Promise<BitObject> {
+    return repository.load(this, throws);
+  }
+
+  loadSync(repo: Repository, throws = true): BitObject {
+    return repo.loadSync(this, throws);
+  }
+
+  loadRaw(repo: Repository): Promise<Buffer> {
+    return repo.loadRaw(this);
+  }
+
+  isEqual(ref: Ref): boolean {
+    return this.toString() === ref.toString();
+  }
+
+  clone() {
+    return new Ref(this.hash);
+  }
+
+  static from(hash: string): Ref {
+    return new Ref(hash);
+  }
+}