@teambit/objects 0.0.19

package/objects/bit-object-list.ts

@@ -0,0 +1,59 @@
+import BitObject from './object';
+import { ExportMetadata, Lane, LaneHistory, ModelComponent, Version, VersionHistory } from '../models';
+
+export class BitObjectList {
+  constructor(private objects: BitObject[]) {}
+
+  getComponents(): ModelComponent[] {
+    return this.objects.filter((object) => object instanceof ModelComponent) as ModelComponent[];
+  }
+
+  getVersions(): Version[] {
+    return this.objects.filter((object) => object instanceof Version) as Version[];
+  }
+
+  getLanes(): Lane[] {
+    return this.objects.filter((object) => object instanceof Lane) as Lane[];
+  }
+
+  getVersionHistories(): VersionHistory[] {
+    return this.objects.filter((object) => object instanceof VersionHistory) as VersionHistory[];
+  }
+
+  getLaneHistories(): LaneHistory[] {
+    return this.objects.filter((object) => object instanceof LaneHistory) as LaneHistory[];
+  }
+
+  getAll(): BitObject[] {
+    return this.objects;
+  }
+
+  excludeTypes(types: string[]): BitObject[] {
+    return this.objects.filter((object) => !types.includes(object.getType()));
+  }
+
+  getExportMetadata(): ExportMetadata | undefined {
+    return this.objects.find((object) => object instanceof ExportMetadata) as ExportMetadata | undefined;
+  }
+
+  /**
+   * object that needs merge operation before saving them into the scope, such as ModelComponent
+   */
+  getObjectsRequireMerge() {
+    const typeRequireMerge = this.objectTypesRequireMerge();
+    return this.objects.filter((object) => typeRequireMerge.some((ObjClass) => object instanceof ObjClass));
+  }
+
+  /**
+   * object that don't need merge operation and can be saved immediately into the scope.
+   * such as Source or Version
+   */
+  getObjectsNotRequireMerge() {
+    const typeRequireMerge = this.objectTypesRequireMerge();
+    return this.objects.filter((object) => typeRequireMerge.every((ObjClass) => !(object instanceof ObjClass)));
+  }
+
+  private objectTypesRequireMerge() {
+    return [ModelComponent, Lane, VersionHistory, LaneHistory];
+  }
+}

package/objects/index.ts

@@ -0,0 +1,6 @@
+import BitObject from './object';
+import BitRawObject from './raw-object';
+import Ref from './ref';
+import Repository from './repository';
+
+export { BitObject, BitRawObject, Ref, Repository };

package/objects/object-list-to-graph.ts

@@ -0,0 +1,69 @@
+import { Graph, Node, Edge } from '@teambit/graph.cleargraph';
+import { uniqBy } from 'lodash';
+import { ComponentID } from '@teambit/component-id';
+import type { ObjectList } from './object-list';
+import { BitObjectList } from './bit-object-list';
+import { getAllVersionsInfo } from '@teambit/component.snap-distance';
+import { Dependency } from '@teambit/graph';
+
+type BitIdNode = Node<ComponentID>;
+type DependencyEdge = Edge<Dependency>;
+
+export class IdGraph extends Graph<ComponentID, Dependency> {
+  constructor(nodes: BitIdNode[] = [], edges: DependencyEdge[] = []) {
+    super(nodes, edges);
+  }
+}
+
+export async function objectListToGraph(objectList: ObjectList): Promise<IdGraph> {
+  const bitObjectsList = await objectList.toBitObjects();
+
+  return bitObjectListToGraph(bitObjectsList);
+}
+
+export async function bitObjectListToGraph(bitObjectsList: BitObjectList): Promise<IdGraph> {
+  const exportMetadata = bitObjectsList.getExportMetadata();
+  const components = bitObjectsList.getComponents();
+  const versions = bitObjectsList.getVersions();
+  const nodes: BitIdNode[] = [];
+  const edges: DependencyEdge[] = [];
+  await Promise.all(
+    components.map(async (component) => {
+      const compFromMetadata = exportMetadata?.exportVersions.find((c) =>
+        c.id.isEqualWithoutVersion(component.toComponentId())
+      );
+      const startFrom = compFromMetadata?.head;
+      const versionsInfo = await getAllVersionsInfo({
+        modelComponent: component,
+        versionObjects: versions,
+        startFrom,
+        throws: false,
+      });
+      versionsInfo.forEach((versionInfo) => {
+        const id = component.toComponentId().changeVersion(versionInfo.tag || versionInfo.ref.toString());
+        const idStr = id.toString();
+        nodes.push(new Node(idStr, id));
+        if (!versionInfo.version) {
+          return;
+        }
+        const { dependencies, devDependencies, peerDependencies, extensionDependencies } =
+          versionInfo.version.depsIdsGroupedByType;
+        const addDep = (depId: ComponentID, edge: Dependency) => {
+          const depIdStr = depId.toString();
+          nodes.push(new Node(depIdStr, depId));
+          edges.push(new Edge(idStr, depIdStr, edge));
+        };
+        const runTime = new Dependency('runtime');
+        const dev = new Dependency('dev');
+        const peer = new Dependency('peer');
+        dependencies.forEach((depId) => addDep(depId, runTime));
+        [...devDependencies, ...extensionDependencies].forEach((depId) => addDep(depId, dev));
+        peerDependencies.forEach((depId) => addDep(depId, peer));
+      });
+    })
+  );
+  const uniqNodes = uniqBy(nodes, 'id');
+  const idGraph = new IdGraph(uniqNodes, edges);
+
+  return idGraph;
+}

package/objects/object-list.ts

@@ -0,0 +1,313 @@
+import tarStream from 'tar-stream';
+import { pMapPool } from '@teambit/toolbox.promise.map-pool';
+import { compact } from 'lodash';
+import { Readable, PassThrough, pipeline } from 'stream';
+import BitObject from './object';
+import { BitObjectList } from './bit-object-list';
+import Ref from './ref';
+import { logger } from '@teambit/legacy.logger';
+import { concurrentIOLimit } from '@teambit/harmony.modules.concurrency';
+import { ExportMetadata } from '../models';
+import { UnknownObjectType } from '@teambit/legacy.scope';
+
+/**
+ * when error occurred during streaming between HTTP server and client, there is no good way to
+ * indicate this other than sending a new file with a special name and the error message.
+ */
+const TAR_STREAM_ERROR_FILENAME = '.BIT.ERROR';
+/**
+ * schema 1.0.0 - added the start and end file with basic info
+ */
+const OBJECT_LIST_CURRENT_SCHEMA = '1.0.0';
+const TAR_STREAM_START_FILENAME = '.BIT.START';
+const TAR_STREAM_END_FILENAME = '.BIT.END';
+
+type StartFile = {
+  schema: string;
+  scopeName: string;
+};
+type EndFile = {
+  numOfFiles: number;
+  scopeName: string;
+};
+
+export type ObjectItem = {
+  ref: Ref;
+  buffer: Buffer; // zlib deflated BitObject
+  type?: string; // for future use. e.g. to be able to export only Component/Version types but not Source/Artifact, etc.
+  scope?: string; // used for the export process
+};
+
+export const FETCH_FORMAT_OBJECT_LIST = 'ObjectList';
+
+/**
+ * Stream.Readable that operates with objectMode, while each 'data' event emits one ObjectItem object.
+ */
+export type ObjectItemsStream = Readable;
+
+export class ObjectList {
+  constructor(public objects: ObjectItem[] = []) {}
+
+  count() {
+    return this.objects.length;
+  }
+
+  static mergeMultipleInstances(objectLists: ObjectList[]): ObjectList {
+    const objectList = new ObjectList();
+    objectLists.forEach((objList) => objectList.mergeObjectList(objList));
+    return objectList;
+  }
+  mergeObjectList(objectList: ObjectList) {
+    this.addIfNotExist(objectList.objects);
+  }
+  static fromJsonString(jsonStr: string): ObjectList {
+    const jsonParsed = JSON.parse(jsonStr);
+    if (!Array.isArray(jsonParsed)) {
+      throw new Error(`fromJsonString expect an array, got ${typeof jsonParsed}`);
+    }
+    jsonParsed.forEach((obj) => {
+      obj.ref = new Ref(obj.ref.hash);
+      obj.buffer = Buffer.from(obj.buffer);
+    });
+    return new ObjectList(jsonParsed);
+  }
+  toJsonString(): string {
+    return JSON.stringify(this.objects);
+  }
+  toTar(): NodeJS.ReadableStream {
+    const pack = tarStream.pack();
+    this.objects.forEach((obj) => {
+      pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);
+    });
+    pack.finalize();
+    return pack;
+  }
+  toReadableStream(): ObjectItemsStream {
+    return Readable.from(this.objects);
+  }
+  static async fromTar(packStream: NodeJS.ReadableStream): Promise<ObjectList> {
+    const extract = tarStream.extract();
+    const objectItems: ObjectItem[] = await new Promise((resolve, reject) => {
+      const objects: ObjectItem[] = [];
+      extract.on('entry', (header, stream, next) => {
+        const data: Buffer[] = [];
+        stream.on('data', (chunk) => {
+          data.push(chunk);
+        });
+        stream.on('end', () => {
+          objects.push({
+            ...ObjectList.extractScopeAndHash(header.name),
+            buffer: Buffer.concat(data as unknown as Uint8Array[]),
+          });
+          next(); // ready for next entry
+        });
+        stream.on('error', (err) => reject(err));
+
+        stream.resume(); // just auto drain the stream
+      });
+
+      extract.on('finish', () => {
+        resolve(objects);
+      });
+
+      packStream.pipe(extract);
+    });
+    return new ObjectList(objectItems);
+  }
+
+  static fromTarToObjectStream(packStream: NodeJS.ReadableStream): ObjectItemsStream {
+    const passThrough = new PassThrough({ objectMode: true });
+    const extract = tarStream.extract();
+    let startData: StartFile | undefined;
+    let endData: EndFile | undefined;
+    extract.on('entry', (header, stream, next) => {
+      const data: Buffer[] = [];
+      stream.on('data', (chunk) => {
+        data.push(chunk);
+      });
+      stream.on('end', () => {
+        const allData = Buffer.concat(data as unknown as Uint8Array[]);
+        if (header.name === TAR_STREAM_ERROR_FILENAME) {
+          passThrough.emit('error', new Error(allData.toString()));
+          return;
+        }
+        if (header.name === TAR_STREAM_START_FILENAME) {
+          startData = JSON.parse(allData.toString());
+          logger.debug('fromTarToObjectStream, start getting data', startData);
+          next();
+          return;
+        }
+        if (header.name === TAR_STREAM_END_FILENAME) {
+          endData = JSON.parse(allData.toString());
+          logger.debug('fromTarToObjectStream, finished getting data', endData);
+          next();
+          return;
+        }
+        passThrough.write({ ...ObjectList.extractScopeAndHash(header.name), buffer: allData });
+        next(); // ready for next entry
+      });
+      stream.on('error', (err) => {
+        passThrough.emit('error', err);
+      });
+
+      stream.resume(); // just auto drain the stream
+    });
+
+    // not sure if needed
+    extract.on('error', (err) => {
+      passThrough.emit('error', err);
+    });
+
+    extract.on('finish', () => {
+      if (startData?.schema === OBJECT_LIST_CURRENT_SCHEMA && !endData) {
+        // wasn't able to find a better way to indicate whether the server aborted the request
+        // see https://github.com/node-fetch/node-fetch/issues/1117
+        passThrough.emit(
+          'error',
+          new Error(`server terminated the stream unexpectedly (metadata: ${JSON.stringify(startData)})`)
+        );
+      }
+      passThrough.end();
+    });
+    pipeline(packStream, extract, (err) => {
+      if (err) {
+        logger.error('fromTarToObjectStream, pipeline', err);
+        passThrough.emit('error', err);
+      } else {
+        logger.debug('fromTarToObjectStream, pipeline is completed');
+      }
+    });
+
+    return passThrough;
+  }
+
+  static fromObjectStreamToTar(readable: Readable, scopeName: string) {
+    const pack = tarStream.pack();
+    const startFile: StartFile = { schema: OBJECT_LIST_CURRENT_SCHEMA, scopeName };
+    logger.debug('fromObjectStreamToTar, start sending data', startFile);
+    pack.entry({ name: TAR_STREAM_START_FILENAME }, JSON.stringify(startFile));
+    let numOfFiles = 0;
+    readable.on('data', (obj: ObjectItem) => {
+      numOfFiles += 1;
+      pack.entry({ name: ObjectList.combineScopeAndHash(obj) }, obj.buffer);
+    });
+    readable.on('end', () => {
+      const endFile: EndFile = { numOfFiles, scopeName };
+      logger.debug('fromObjectStreamToTar, finished sending data', endFile);
+      pack.entry({ name: TAR_STREAM_END_FILENAME }, JSON.stringify(endFile));
+      pack.finalize();
+    });
+    readable.on('error', (err) => {
+      const errorMessage = err.message || `unexpected error (${err.name})`;
+      logger.error(`ObjectList.fromObjectStreamToTar, streaming an error as a file`, err);
+      pack.entry({ name: TAR_STREAM_ERROR_FILENAME }, errorMessage);
+      pack.finalize();
+    });
+    return pack;
+  }
+
+  static async fromReadableStream(readable: ObjectItemsStream): Promise<ObjectList> {
+    const objectItems: ObjectItem[] = [];
+    for await (const obj of readable) {
+      objectItems.push(obj);
+    }
+    return new ObjectList(objectItems);
+  }
+
+  /**
+   * the opposite of this.combineScopeAndHash
+   */
+  static extractScopeAndHash(name: string): { scope?: string; ref: Ref } {
+    const nameSplit = name.split('/');
+    const hasScope = nameSplit.length > 1;
+    return {
+      scope: hasScope ? nameSplit[0] : undefined,
+      ref: new Ref(hasScope ? nameSplit[1] : nameSplit[0]),
+    };
+  }
+  /**
+   * the opposite of this.extractScopeAndHash
+   */
+  static combineScopeAndHash(objectItem: ObjectItem): string {
+    const scope = objectItem.scope ? `${objectItem.scope}/` : '';
+    return `${scope}${objectItem.ref.hash}`;
+  }
+
+  addIfNotExist(objectItems: ObjectItem[]) {
+    objectItems.forEach((objectItem) => {
+      const exists = this.objects.find(
+        (object) => object.ref.isEqual(objectItem.ref) && object.scope === objectItem.scope
+      );
+      if (!exists) {
+        this.objects.push(objectItem);
+      }
+    });
+  }
+
+  async toBitObjects(throwForUnknownTypes = false): Promise<BitObjectList> {
+    const concurrency = concurrentIOLimit();
+    const bitObjects = await pMapPool(
+      this.objects,
+      async (object) => {
+        try {
+          return await BitObject.parseObject(object.buffer);
+        } catch (err) {
+          if (throwForUnknownTypes || !(err instanceof UnknownObjectType)) {
+            throw err;
+          }
+          logger.error(
+            `toBitObjects, unable to parse object of type ${err.type}, ignoring it. please update your bit version`
+          );
+          return null;
+        }
+      },
+      { concurrency }
+    );
+    return new BitObjectList(compact(bitObjects));
+  }
+
+  static async fromBitObjects(bitObjects: BitObject[]): Promise<ObjectList> {
+    const concurrency = concurrentIOLimit();
+    const objectItems = await pMapPool(
+      bitObjects,
+      async (obj) => ({
+        ref: obj.hash(),
+        buffer: await obj.compress(),
+        type: obj.getType(),
+      }),
+      { concurrency }
+    );
+    return new ObjectList(objectItems);
+  }
+
+  addScopeName(scopeName: string) {
+    this.objects.forEach((object) => {
+      object.scope = scopeName;
+    });
+  }
+
+  splitByScopeName(): { [scopeName: string]: ObjectList } {
+    const objectListPerScope: { [scopeName: string]: ObjectList } = {};
+    this.objects.forEach((obj) => {
+      if (obj.type === ExportMetadata.name) {
+        return; // no scope for this type. it's general for all export data from all scopes
+      }
+      if (!obj.scope) {
+        throw new Error(`ObjectList: unable to split by scopeName, the scopeName is missing for ${obj.ref.hash}`);
+      }
+      if (objectListPerScope[obj.scope]) {
+        objectListPerScope[obj.scope].addIfNotExist([obj]);
+      } else {
+        objectListPerScope[obj.scope] = new ObjectList([obj]);
+      }
+    });
+    return objectListPerScope;
+  }
+
+  /**
+   * helps debugging
+   */
+  toConsoleLog() {
+    console.log(this.objects.map((o) => o.ref.hash).join('\n')); // eslint-disable-line no-console
+  }
+}

package/objects/object.ts

@@ -0,0 +1,153 @@
+import { inflateSync } from 'zlib';
+
+import { NULL_BYTE, SPACE_DELIMITER } from '@teambit/legacy.constants';
+import { deflate, inflate } from '@teambit/legacy.utils';
+import { sha1 } from '@teambit/toolbox.crypto.sha1';
+import { UnknownObjectType, typesObj as types } from '@teambit/legacy.scope';
+import { ObjectItem } from './object-list';
+import Ref from './ref';
+import Repository from './repository';
+
+function parse(buffer: Buffer): BitObject {
+  const { type, hash, contents } = extractHeaderAndContent(buffer);
+  if (!types[type]) throw new UnknownObjectType(type);
+  return types[type].parse(contents, hash);
+}
+
+function extractHeaderAndContent(buffer: Buffer): { type: string; hash: string; contents: Buffer } {
+  const firstNullByteLocation = buffer.indexOf(NULL_BYTE);
+  const headers = buffer.slice(0, firstNullByteLocation).toString();
+  const [type, hash] = headers.split(SPACE_DELIMITER);
+  const contents = buffer.slice(firstNullByteLocation + 1, buffer.length);
+  return { type, hash, contents };
+}
+
+export default class BitObject {
+  validateBeforePersist = true; // validate the object before persisting
+  id(): string | Buffer {
+    throw new Error('id() was not implemented...');
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  toBuffer(pretty?: boolean): Buffer {
+    throw new Error('toBuffer() was not implemented...');
+  }
+
+  refs(): Ref[] {
+    return [];
+  }
+
+  getType(): string {
+    return this.constructor.name;
+  }
+
+  getHeader(buffer: Buffer): string {
+    return `${this.getType()} ${this.hash().toString()} ${buffer.toString().length}${NULL_BYTE}`;
+  }
+
+  async collectRefs(repo: Repository): Promise<Ref[]> {
+    const refsCollection = [];
+    const objectType = this.getType();
+    const objectId = objectType === 'Component' ? `Component ${this.id()}` : objectType;
+
+    async function addRefs(object: BitObject) {
+      const refs = object.refs();
+      let objs;
+      try {
+        objs = await Promise.all(refs.map((ref) => ref.load(repo, true)));
+      } catch (err: any) {
+        if (err.code === 'ENOENT') {
+          throw new Error(`failed finding an object file required by ${object.constructor.name} object, originated from ${objectId}
+path: ${err.path}`);
+        }
+        throw err;
+      }
+
+      // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
+      refsCollection.push(...refs);
+      await Promise.all(objs.map((obj) => addRefs(obj)));
+    }
+
+    await addRefs(this);
+    return refsCollection;
+  }
+
+  async collectRaw(repo: Repository): Promise<ObjectItem[]> {
+    const refs = await this.collectRefs(repo);
+    return repo.loadManyRaw(refs);
+  }
+
+  asRaw(repo: Repository): Promise<Buffer> {
+    return repo.loadRaw(this.hash());
+  }
+
+  collect(repo: Repository): BitObject[] {
+    const objects: BitObject[] = [];
+
+    function addRefs(object: BitObject) {
+      const objs = object.refs().map((ref) => {
+        return ref.loadSync(repo);
+      });
+
+      objects.push(...objs);
+      objs.forEach((obj) => addRefs(obj));
+    }
+
+    addRefs(this);
+    return objects;
+  }
+
+  /**
+   * indexing method
+   */
+  hash(): Ref {
+    // console.log(`sha ${sha1(this.id())}, id ${this.id()}`); // uncomment when debugging hash issues
+    return new Ref(BitObject.makeHash(this.id()));
+  }
+
+  compress(): Promise<Buffer> {
+    return deflate(this.serialize());
+  }
+
+  serialize(): Buffer {
+    const buffer = this.toBuffer();
+    return Buffer.concat([Buffer.from(this.getHeader(buffer)), buffer] as unknown as Uint8Array[]);
+  }
+
+  /**
+   * see `this.parseSync` for the sync version
+   */
+  static async parseObject(fileContents: Buffer, filePath?: string): Promise<BitObject> {
+    const buffer = await inflate(fileContents, filePath);
+    return parse(buffer);
+  }
+
+  /**
+   * same as `parseObject`, however, if the type is not one of the given "typeNames", it returns null.
+   * the performance improvement is huge compare to "parseObject", as it doesn't parse the object if not needed.
+   */
+  static async parseObjectOnlyIfType(
+    fileContents: Buffer,
+    typeNames: string[],
+    filePath?: string
+  ): Promise<BitObject | null> {
+    const buffer = await inflate(fileContents, filePath);
+    const { type } = extractHeaderAndContent(buffer);
+    if (typeNames.includes(type)) return parse(buffer);
+    return null;
+  }
+
+  /**
+   * prefer using `this.parseObject()`, unless it must be sync.
+   */
+  static parseSync(fileContents: Buffer): BitObject {
+    // @ts-ignore todo: fix after merging #9359
+    const buffer = inflateSync(fileContents);
+    return parse(buffer);
+  }
+
+  static makeHash(str: string | Buffer): string {
+    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
+    return sha1(str);
+  }
+}