@teambit/workspace 0.0.534 → 0.0.539

package/build-graph-from-fs.ts

@@ -0,0 +1,160 @@
+import mapSeries from 'p-map-series';
+import { flatten } from 'lodash';
+import { Consumer } from '@teambit/legacy/dist/consumer';
+import BitIds from '@teambit/legacy/dist/bit-id/bit-ids';
+import Component from '@teambit/legacy/dist/consumer/component/consumer-component';
+import LegacyGraph from '@teambit/legacy/dist/scope/graph/graph';
+import ScopeComponentsImporter from '@teambit/legacy/dist/scope/component-ops/scope-components-importer';
+import compact from 'lodash.compact';
+import { BitId } from '@teambit/legacy-bit-id';
+import { Logger } from '@teambit/logger';
+import { ComponentNotFound } from '@teambit/scope';
+import { BitError } from '@teambit/bit-error';
+import { Workspace } from './workspace';
+
+export type ShouldIgnoreFunc = (bitId: BitId) => Promise<boolean>;
+
+export class GraphFromFsBuilder {
+  private graph = new LegacyGraph();
+  private completed: string[] = [];
+  private depth = 1;
+  private shouldThrowOnMissingDep = true;
+  private consumer: Consumer;
+  constructor(
+    private workspace: Workspace,
+    private logger: Logger,
+    private ignoreIds = new BitIds(),
+    private shouldLoadItsDeps?: ShouldIgnoreFunc
+  ) {
+    this.consumer = this.workspace.consumer;
+  }
+
+  /**
+   * create a graph with all dependencies and flattened dependencies of the given components.
+   * the nodes are components and the edges has a label of the dependency type.
+   *
+   * the way how it is done is iterations by depths. each depth we gather all the dependencies of
+   * that depths, make sure all objects exist and then check their dependencies for the next depth.
+   * once there is no dependency left, we're on the last depth level and the graph is ready.
+   *
+   * for example, imagine the following graph:
+   * A1 -> A2 -> A3
+   * B1 -> B2 -> B3
+   * C1 -> C2 -> C3
+   *
+   * where the buildGraph is given [A1, B1, C1].
+   * first, it saves all these components as nodes in the graph. then, it finds the dependencies of
+   * the next level, in this case they're [A2, B2, C2]. it runs `importMany` in case some objects
+   * are missing. then, it loads them all (some from FS, some from the model) and sets the edges
+   * between the component and the dependencies.
+   * once done, it finds all their dependencies, which are [A3, B3, C3] and repeat the process
+   * above. since there are no more dependencies, the graph is completed.
+   * in this case, the total depth levels are 3.
+   *
+   * even with a huge project, there are not many depth levels. by iterating through depth levels
+   * we keep performance sane as the importMany doesn't run multiple time and therefore the round
+   * trips to the remotes are minimal.
+   *
+   * normally, one importMany of the seeders is enough as importMany knows to fetch all flattened.
+   * however, since this buildGraph is performed on the workspace, a dependency may be new or
+   * modified and as such, we don't know its flattened yet.
+   */
+  async buildGraph(ids: BitId[]): Promise<LegacyGraph> {
+    this.logger.debug(`GraphFromFsBuilder, buildGraph with ${ids.length} seeders`);
+    const start = Date.now();
+    const components = await this.loadManyComponents(ids, '<none>');
+    await this.processManyComponents(components);
+    this.logger.debug(
+      `GraphFromFsBuilder, buildGraph with ${ids.length} seeders completed (${(Date.now() - start) / 1000} sec)`
+    );
+    return this.graph;
+  }
+
+  private getAllDepsUnfiltered(component: Component) {
+    return component.getAllDependenciesIds().difference(this.ignoreIds);
+  }
+
+  private async getAllDepsFiltered(component: Component): Promise<BitIds> {
+    const depsWithoutIgnore = this.getAllDepsUnfiltered(component);
+    const shouldLoadFunc = this.shouldLoadItsDeps;
+    if (!shouldLoadFunc) return depsWithoutIgnore;
+    const deps = await mapSeries(depsWithoutIgnore, async (depId) => {
+      const shouldLoad = await shouldLoadFunc(depId);
+      if (!shouldLoad) this.ignoreIds.push(depId);
+      return shouldLoad ? depId : null;
+    });
+    return BitIds.fromArray(compact(deps));
+  }
+
+  private async processManyComponents(components: Component[]) {
+    this.logger.debug(`GraphFromFsBuilder.processManyComponents depth ${this.depth}, ${components.length} components`);
+    this.depth += 1;
+    await this.importObjects(components);
+    const allDependencies = await mapSeries(components, (component) => this.processOneComponent(component));
+    const allDependenciesFlattened = flatten(allDependencies);
+    if (allDependenciesFlattened.length) await this.processManyComponents(allDependenciesFlattened);
+  }
+
+  private async importObjects(components: Component[]) {
+    const allDeps = components.map((c) => this.getAllDepsUnfiltered(c)).flat();
+    const allDepsWithScope = allDeps.filter((dep) => dep.hasScope());
+    const scopeComponentsImporter = new ScopeComponentsImporter(this.consumer.scope);
+    await scopeComponentsImporter.importMany(
+      BitIds.uniqFromArray(allDepsWithScope),
+      undefined,
+      this.shouldThrowOnMissingDep
+    );
+  }
+
+  private async processOneComponent(component: Component) {
+    const idStr = component.id.toString();
+    if (this.completed.includes(idStr)) return [];
+    const allIds = await this.getAllDepsFiltered(component);
+
+    const allDependencies = await this.loadManyComponents(allIds, idStr);
+    Object.entries(component.depsIdsGroupedByType).forEach(([depType, depsIds]) => {
+      depsIds.forEach((depId) => {
+        if (this.ignoreIds.has(depId)) return;
+        if (!this.graph.hasNode(depId.toString())) {
+          throw new Error(`buildOneComponent: missing node of ${depId.toString()}`);
+        }
+        this.graph.setEdge(idStr, depId.toString(), depType);
+      });
+    });
+    this.completed.push(idStr);
+    return allDependencies;
+  }
+
+  private async loadManyComponents(componentsIds: BitId[], dependenciesOf: string): Promise<Component[]> {
+    const components = await mapSeries(componentsIds, async (comp: BitId) => {
+      const idStr = comp.toString();
+      const fromGraph = this.graph.node(idStr);
+      if (fromGraph) return fromGraph;
+      try {
+        const component = await this.loadComponent(comp);
+        this.graph.setNode(idStr, component);
+        return component;
+      } catch (err: any) {
+        if (err instanceof ComponentNotFound) {
+          throw new BitError(
+            `error: component "${idStr}" was not found.\nthis component is a dependency of "${dependenciesOf}" and is needed as part of the graph generation`
+          );
+        }
+        this.logger.error(`failed loading dependencies of ${dependenciesOf}`);
+        throw err;
+      }
+    });
+    return components;
+  }
+  private async loadComponent(componentId: BitId): Promise<Component> {
+    const componentMap = this.consumer.bitMap.getComponentIfExist(componentId);
+    const isOnWorkspace = Boolean(componentMap);
+    if (isOnWorkspace) {
+      return this.consumer.loadComponentForCapsule(componentId);
+    }
+    // a dependency might have been installed as a package in the workspace, and as such doesn't
+    // have a componentMap.
+    const componentFromModel = await this.consumer.loadComponentFromModel(componentId);
+    return componentFromModel.clone();
+  }
+}

package/dist/build-graph-from-fs.d.ts

@@ -0,0 +1,26 @@
+import BitIds from '@teambit/legacy/dist/bit-id/bit-ids';
+import LegacyGraph from '@teambit/legacy/dist/scope/graph/graph';
+import { BitId } from '@teambit/legacy-bit-id';
+import { Logger } from '@teambit/logger';
+import { Workspace } from './workspace';
+export declare type ShouldIgnoreFunc = (bitId: BitId) => Promise<boolean>;
+export declare class GraphFromFsBuilder {
+    private workspace;
+    private logger;
+    private ignoreIds;
+    private shouldLoadItsDeps?;
+    private graph;
+    private completed;
+    private depth;
+    private shouldThrowOnMissingDep;
+    private consumer;
+    constructor(workspace: Workspace, logger: Logger, ignoreIds?: BitIds, shouldLoadItsDeps?: ShouldIgnoreFunc | undefined);
+    buildGraph(ids: BitId[]): Promise<LegacyGraph>;
+    private getAllDepsUnfiltered;
+    private getAllDepsFiltered;
+    private processManyComponents;
+    private importObjects;
+    private processOneComponent;
+    private loadManyComponents;
+    private loadComponent;
+}

package/dist/build-graph-from-fs.js

@@ -0,0 +1,254 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+require("core-js/modules/es.array.flat.js");
+
+require("core-js/modules/es.array.iterator.js");
+
+require("core-js/modules/es.array.unscopables.flat.js");
+
+require("core-js/modules/es.promise.js");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.GraphFromFsBuilder = void 0;
+
+function _defineProperty2() {
+  const data = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+  _defineProperty2 = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _pMapSeries() {
+  const data = _interopRequireDefault(require("p-map-series"));
+
+  _pMapSeries = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _lodash() {
+  const data = require("lodash");
+
+  _lodash = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _bitIds() {
+  const data = _interopRequireDefault(require("@teambit/legacy/dist/bit-id/bit-ids"));
+
+  _bitIds = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _graph() {
+  const data = _interopRequireDefault(require("@teambit/legacy/dist/scope/graph/graph"));
+
+  _graph = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _scopeComponentsImporter() {
+  const data = _interopRequireDefault(require("@teambit/legacy/dist/scope/component-ops/scope-components-importer"));
+
+  _scopeComponentsImporter = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _lodash2() {
+  const data = _interopRequireDefault(require("lodash.compact"));
+
+  _lodash2 = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _scope() {
+  const data = require("@teambit/scope");
+
+  _scope = function () {
+    return data;
+  };
+
+  return data;
+}
+
+function _bitError() {
+  const data = require("@teambit/bit-error");
+
+  _bitError = function () {
+    return data;
+  };
+
+  return data;
+}
+
+class GraphFromFsBuilder {
+  constructor(workspace, logger, ignoreIds = new (_bitIds().default)(), shouldLoadItsDeps) {
+    this.workspace = workspace;
+    this.logger = logger;
+    this.ignoreIds = ignoreIds;
+    this.shouldLoadItsDeps = shouldLoadItsDeps;
+    (0, _defineProperty2().default)(this, "graph", new (_graph().default)());
+    (0, _defineProperty2().default)(this, "completed", []);
+    (0, _defineProperty2().default)(this, "depth", 1);
+    (0, _defineProperty2().default)(this, "shouldThrowOnMissingDep", true);
+    (0, _defineProperty2().default)(this, "consumer", void 0);
+    this.consumer = this.workspace.consumer;
+  }
+  /**
+   * create a graph with all dependencies and flattened dependencies of the given components.
+   * the nodes are components and the edges has a label of the dependency type.
+   *
+   * the way how it is done is iterations by depths. each depth we gather all the dependencies of
+   * that depths, make sure all objects exist and then check their dependencies for the next depth.
+   * once there is no dependency left, we're on the last depth level and the graph is ready.
+   *
+   * for example, imagine the following graph:
+   * A1 -> A2 -> A3
+   * B1 -> B2 -> B3
+   * C1 -> C2 -> C3
+   *
+   * where the buildGraph is given [A1, B1, C1].
+   * first, it saves all these components as nodes in the graph. then, it finds the dependencies of
+   * the next level, in this case they're [A2, B2, C2]. it runs `importMany` in case some objects
+   * are missing. then, it loads them all (some from FS, some from the model) and sets the edges
+   * between the component and the dependencies.
+   * once done, it finds all their dependencies, which are [A3, B3, C3] and repeat the process
+   * above. since there are no more dependencies, the graph is completed.
+   * in this case, the total depth levels are 3.
+   *
+   * even with a huge project, there are not many depth levels. by iterating through depth levels
+   * we keep performance sane as the importMany doesn't run multiple time and therefore the round
+   * trips to the remotes are minimal.
+   *
+   * normally, one importMany of the seeders is enough as importMany knows to fetch all flattened.
+   * however, since this buildGraph is performed on the workspace, a dependency may be new or
+   * modified and as such, we don't know its flattened yet.
+   */
+
+
+  async buildGraph(ids) {
+    this.logger.debug(`GraphFromFsBuilder, buildGraph with ${ids.length} seeders`);
+    const start = Date.now();
+    const components = await this.loadManyComponents(ids, '<none>');
+    await this.processManyComponents(components);
+    this.logger.debug(`GraphFromFsBuilder, buildGraph with ${ids.length} seeders completed (${(Date.now() - start) / 1000} sec)`);
+    return this.graph;
+  }
+
+  getAllDepsUnfiltered(component) {
+    return component.getAllDependenciesIds().difference(this.ignoreIds);
+  }
+
+  async getAllDepsFiltered(component) {
+    const depsWithoutIgnore = this.getAllDepsUnfiltered(component);
+    const shouldLoadFunc = this.shouldLoadItsDeps;
+    if (!shouldLoadFunc) return depsWithoutIgnore;
+    const deps = await (0, _pMapSeries().default)(depsWithoutIgnore, async depId => {
+      const shouldLoad = await shouldLoadFunc(depId);
+      if (!shouldLoad) this.ignoreIds.push(depId);
+      return shouldLoad ? depId : null;
+    });
+    return _bitIds().default.fromArray((0, _lodash2().default)(deps));
+  }
+
+  async processManyComponents(components) {
+    this.logger.debug(`GraphFromFsBuilder.processManyComponents depth ${this.depth}, ${components.length} components`);
+    this.depth += 1;
+    await this.importObjects(components);
+    const allDependencies = await (0, _pMapSeries().default)(components, component => this.processOneComponent(component));
+    const allDependenciesFlattened = (0, _lodash().flatten)(allDependencies);
+    if (allDependenciesFlattened.length) await this.processManyComponents(allDependenciesFlattened);
+  }
+
+  async importObjects(components) {
+    const allDeps = components.map(c => this.getAllDepsUnfiltered(c)).flat();
+    const allDepsWithScope = allDeps.filter(dep => dep.hasScope());
+    const scopeComponentsImporter = new (_scopeComponentsImporter().default)(this.consumer.scope);
+    await scopeComponentsImporter.importMany(_bitIds().default.uniqFromArray(allDepsWithScope), undefined, this.shouldThrowOnMissingDep);
+  }
+
+  async processOneComponent(component) {
+    const idStr = component.id.toString();
+    if (this.completed.includes(idStr)) return [];
+    const allIds = await this.getAllDepsFiltered(component);
+    const allDependencies = await this.loadManyComponents(allIds, idStr);
+    Object.entries(component.depsIdsGroupedByType).forEach(([depType, depsIds]) => {
+      depsIds.forEach(depId => {
+        if (this.ignoreIds.has(depId)) return;
+
+        if (!this.graph.hasNode(depId.toString())) {
+          throw new Error(`buildOneComponent: missing node of ${depId.toString()}`);
+        }
+
+        this.graph.setEdge(idStr, depId.toString(), depType);
+      });
+    });
+    this.completed.push(idStr);
+    return allDependencies;
+  }
+
+  async loadManyComponents(componentsIds, dependenciesOf) {
+    const components = await (0, _pMapSeries().default)(componentsIds, async comp => {
+      const idStr = comp.toString();
+      const fromGraph = this.graph.node(idStr);
+      if (fromGraph) return fromGraph;
+
+      try {
+        const component = await this.loadComponent(comp);
+        this.graph.setNode(idStr, component);
+        return component;
+      } catch (err) {
+        if (err instanceof _scope().ComponentNotFound) {
+          throw new (_bitError().BitError)(`error: component "${idStr}" was not found.\nthis component is a dependency of "${dependenciesOf}" and is needed as part of the graph generation`);
+        }
+
+        this.logger.error(`failed loading dependencies of ${dependenciesOf}`);
+        throw err;
+      }
+    });
+    return components;
+  }
+
+  async loadComponent(componentId) {
+    const componentMap = this.consumer.bitMap.getComponentIfExist(componentId);
+    const isOnWorkspace = Boolean(componentMap);
+
+    if (isOnWorkspace) {
+      return this.consumer.loadComponentForCapsule(componentId);
+    } // a dependency might have been installed as a package in the workspace, and as such doesn't
+    // have a componentMap.
+
+
+    const componentFromModel = await this.consumer.loadComponentFromModel(componentId);
+    return componentFromModel.clone();
+  }
+
+}
+
+exports.GraphFromFsBuilder = GraphFromFsBuilder;
+
+//# sourceMappingURL=build-graph-from-fs.js.map

package/dist/build-graph-from-fs.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["build-graph-from-fs.ts"],"names":["GraphFromFsBuilder","constructor","workspace","logger","ignoreIds","BitIds","shouldLoadItsDeps","LegacyGraph","consumer","buildGraph","ids","debug","length","start","Date","now","components","loadManyComponents","processManyComponents","graph","getAllDepsUnfiltered","component","getAllDependenciesIds","difference","getAllDepsFiltered","depsWithoutIgnore","shouldLoadFunc","deps","depId","shouldLoad","push","fromArray","depth","importObjects","allDependencies","processOneComponent","allDependenciesFlattened","allDeps","map","c","flat","allDepsWithScope","filter","dep","hasScope","scopeComponentsImporter","ScopeComponentsImporter","scope","importMany","uniqFromArray","undefined","shouldThrowOnMissingDep","idStr","id","toString","completed","includes","allIds","Object","entries","depsIdsGroupedByType","forEach","depType","depsIds","has","hasNode","Error","setEdge","componentsIds","dependenciesOf","comp","fromGraph","node","loadComponent","setNode","err","ComponentNotFound","BitError","error","componentId","componentMap","bitMap","getComponentIfExist","isOnWorkspace","Boolean","loadComponentForCapsule","componentFromModel","loadComponentFromModel","clone"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AACA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAEA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAEA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AACA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AACA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAGA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AACA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAKO,MAAMA,kBAAN,CAAyB;AAM9BC,EAAAA,WAAW,CACDC,SADC,EAEDC,MAFC,EAGDC,SAAS,GAAG,KAAIC,iBAAJ,GAHX,EAIDC,iBAJC,EAKT;AAAA,SAJQJ,SAIR,GAJQA,SAIR;AAAA,SAHQC,MAGR,GAHQA,MAGR;AAAA,SAFQC,SAER,GAFQA,SAER;AAAA,SADQE,iBACR,GADQA,iBACR;AAAA,mDAVc,KAAIC,gBAAJ,GAUd;AAAA,uDAT4B,EAS5B;AAAA,mDARc,CAQd;AAAA,qEAPgC,IAOhC;AAAA;AACA,SAAKC,QAAL,GAAgB,KAAKN,SAAL,CAAeM,QAA/B;AACD;AAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;;AACkB,QAAVC,UAAU,CAACC,GAAD,EAAqC;AACnD,SAAKP,MAAL,CAAYQ,KAAZ,CAAmB,uCAAsCD,GAAG,CAACE,MAAO,UAApE;AACA,UAAMC,KAAK,GAAGC,IAAI,CAACC,GAAL,EAAd;AACA,UAAMC,UAAU,GAAG,MAAM,KAAKC,kBAAL,CAAwBP,GAAxB,EAA6B,QAA7B,CAAzB;AACA,UAAM,KAAKQ,qBAAL,CAA2BF,UAA3B,CAAN;AACA,SAAKb,MAAL,CAAYQ,KAAZ,CACG,uCAAsCD,GAAG,CAACE,MAAO,uBAAsB,CAACE,IAAI,CAACC,GAAL,KAAaF,KAAd,IAAuB,IAAK,OADtG;AAGA,WAAO,KAAKM,KAAZ;AACD;;AAEOC,EAAAA,oBAAoB,CAACC,SAAD,EAAuB;AACjD,WAAOA,SAAS,CAACC,qBAAV,GAAkCC,UAAlC,CAA6C,KAAKnB,SAAlD,CAAP;AACD;;AAE+B,QAAlBoB,kBAAkB,CAACH,SAAD,EAAwC;AACtE,UAAMI,iBAAiB,GAAG,KAAKL,oBAAL,CAA0BC,SAA1B,CAA1B;AACA,UAAMK,cAAc,GAAG,KAAKpB,iBAA5B;AACA,QAAI,CAACoB,cAAL,EAAqB,OAAOD,iBAAP;AACrB,UAAME,IAAI,GAAG,MAAM,2BAAUF,iBAAV,EAA6B,MAAOG,KAAP,IAAiB;AAC/D,YAAMC,UAAU,GAAG,MAAMH,cAAc,CAACE,KAAD,CAAvC;AACA,UAAI,CAACC,UAAL,EAAiB,KAAKzB,SAAL,CAAe0B,IAAf,CAAoBF,KAApB;AACjB,aAAOC,UAAU,GAAGD,KAAH,GAAW,IAA5B;AACD,KAJkB,CAAnB;AAKA,WAAOvB,kBAAO0B,SAAP,CAAiB,wBAAQJ,IAAR,CAAjB,CAAP;AACD;;AAEkC,QAArBT,qBAAqB,CAACF,UAAD,EAA0B;AAC3D,SAAKb,MAAL,CAAYQ,KAAZ,CAAmB,kDAAiD,KAAKqB,KAAM,KAAIhB,UAAU,CAACJ,MAAO,aAArG;AACA,SAAKoB,KAAL,IAAc,CAAd;AACA,UAAM,KAAKC,aAAL,CAAmBjB,UAAnB,CAAN;AACA,UAAMkB,eAAe,GAAG,MAAM,2BAAUlB,UAAV,EAAuBK,SAAD,IAAe,KAAKc,mBAAL,CAAyBd,SAAzB,CAArC,CAA9B;AACA,UAAMe,wBAAwB,GAAG,uBAAQF,eAAR,CAAjC;AACA,QAAIE,wBAAwB,CAACxB,MAA7B,EAAqC,MAAM,KAAKM,qBAAL,CAA2BkB,wBAA3B,CAAN;AACtC;;AAE0B,QAAbH,aAAa,CAACjB,UAAD,EAA0B;AACnD,UAAMqB,OAAO,GAAGrB,UAAU,CAACsB,GAAX,CAAgBC,CAAD,IAAO,KAAKnB,oBAAL,CAA0BmB,CAA1B,CAAtB,EAAoDC,IAApD,EAAhB;AACA,UAAMC,gBAAgB,GAAGJ,OAAO,CAACK,MAAR,CAAgBC,GAAD,IAASA,GAAG,CAACC,QAAJ,EAAxB,CAAzB;AACA,UAAMC,uBAAuB,GAAG,KAAIC,kCAAJ,EAA4B,KAAKtC,QAAL,CAAcuC,KAA1C,CAAhC;AACA,UAAMF,uBAAuB,CAACG,UAAxB,CACJ3C,kBAAO4C,aAAP,CAAqBR,gBAArB,CADI,EAEJS,SAFI,EAGJ,KAAKC,uBAHD,CAAN;AAKD;;AAEgC,QAAnBhB,mBAAmB,CAACd,SAAD,EAAuB;AACtD,UAAM+B,KAAK,GAAG/B,SAAS,CAACgC,EAAV,CAAaC,QAAb,EAAd;AACA,QAAI,KAAKC,SAAL,CAAeC,QAAf,CAAwBJ,KAAxB,CAAJ,EAAoC,OAAO,EAAP;AACpC,UAAMK,MAAM,GAAG,MAAM,KAAKjC,kBAAL,CAAwBH,SAAxB,CAArB;AAEA,UAAMa,eAAe,GAAG,MAAM,KAAKjB,kBAAL,CAAwBwC,MAAxB,EAAgCL,KAAhC,CAA9B;AACAM,IAAAA,MAAM,CAACC,OAAP,CAAetC,SAAS,CAACuC,oBAAzB,EAA+CC,OAA/C,CAAuD,CAAC,CAACC,OAAD,EAAUC,OAAV,CAAD,KAAwB;AAC7EA,MAAAA,OAAO,CAACF,OAAR,CAAiBjC,KAAD,IAAW;AACzB,YAAI,KAAKxB,SAAL,CAAe4D,GAAf,CAAmBpC,KAAnB,CAAJ,EAA+B;;AAC/B,YAAI,CAAC,KAAKT,KAAL,CAAW8C,OAAX,CAAmBrC,KAAK,CAAC0B,QAAN,EAAnB,CAAL,EAA2C;AACzC,gBAAM,IAAIY,KAAJ,CAAW,sCAAqCtC,KAAK,CAAC0B,QAAN,EAAiB,EAAjE,CAAN;AACD;;AACD,aAAKnC,KAAL,CAAWgD,OAAX,CAAmBf,KAAnB,EAA0BxB,KAAK,CAAC0B,QAAN,EAA1B,EAA4CQ,OAA5C;AACD,OAND;AAOD,KARD;AASA,SAAKP,SAAL,CAAezB,IAAf,CAAoBsB,KAApB;AACA,WAAOlB,eAAP;AACD;;AAE+B,QAAlBjB,kBAAkB,CAACmD,aAAD,EAAyBC,cAAzB,EAAuE;AACrG,UAAMrD,UAAU,GAAG,MAAM,2BAAUoD,aAAV,EAAyB,MAAOE,IAAP,IAAuB;AACvE,YAAMlB,KAAK,GAAGkB,IAAI,CAAChB,QAAL,EAAd;AACA,YAAMiB,SAAS,GAAG,KAAKpD,KAAL,CAAWqD,IAAX,CAAgBpB,KAAhB,CAAlB;AACA,UAAImB,SAAJ,EAAe,OAAOA,SAAP;;AACf,UAAI;AACF,cAAMlD,SAAS,GAAG,MAAM,KAAKoD,aAAL,CAAmBH,IAAnB,CAAxB;AACA,aAAKnD,KAAL,CAAWuD,OAAX,CAAmBtB,KAAnB,EAA0B/B,SAA1B;AACA,eAAOA,SAAP;AACD,OAJD,CAIE,OAAOsD,GAAP,EAAiB;AACjB,YAAIA,GAAG,YAAYC,0BAAnB,EAAsC;AACpC,gBAAM,KAAIC,oBAAJ,EACH,qBAAoBzB,KAAM,wDAAuDiB,cAAe,iDAD7F,CAAN;AAGD;;AACD,aAAKlE,MAAL,CAAY2E,KAAZ,CAAmB,kCAAiCT,cAAe,EAAnE;AACA,cAAMM,GAAN;AACD;AACF,KAjBwB,CAAzB;AAkBA,WAAO3D,UAAP;AACD;;AAC0B,QAAbyD,aAAa,CAACM,WAAD,EAAyC;AAClE,UAAMC,YAAY,GAAG,KAAKxE,QAAL,CAAcyE,MAAd,CAAqBC,mBAArB,CAAyCH,WAAzC,CAArB;AACA,UAAMI,aAAa,GAAGC,OAAO,CAACJ,YAAD,CAA7B;;AACA,QAAIG,aAAJ,EAAmB;AACjB,aAAO,KAAK3E,QAAL,CAAc6E,uBAAd,CAAsCN,WAAtC,CAAP;AACD,KALiE,CAMlE;AACA;;;AACA,UAAMO,kBAAkB,GAAG,MAAM,KAAK9E,QAAL,CAAc+E,sBAAd,CAAqCR,WAArC,CAAjC;AACA,WAAOO,kBAAkB,CAACE,KAAnB,EAAP;AACD;;AA9I6B","sourcesContent":["import mapSeries from 'p-map-series';\nimport { flatten } from 'lodash';\nimport { Consumer } from '@teambit/legacy/dist/consumer';\nimport BitIds from '@teambit/legacy/dist/bit-id/bit-ids';\nimport Component from '@teambit/legacy/dist/consumer/component/consumer-component';\nimport LegacyGraph from '@teambit/legacy/dist/scope/graph/graph';\nimport ScopeComponentsImporter from '@teambit/legacy/dist/scope/component-ops/scope-components-importer';\nimport compact from 'lodash.compact';\nimport { BitId } from '@teambit/legacy-bit-id';\nimport { Logger } from '@teambit/logger';\nimport { ComponentNotFound } from '@teambit/scope';\nimport { BitError } from '@teambit/bit-error';\nimport { Workspace } from './workspace';\n\nexport type ShouldIgnoreFunc = (bitId: BitId) => Promise<boolean>;\n\nexport class GraphFromFsBuilder {\n  private graph = new LegacyGraph();\n  private completed: string[] = [];\n  private depth = 1;\n  private shouldThrowOnMissingDep = true;\n  private consumer: Consumer;\n  constructor(\n    private workspace: Workspace,\n    private logger: Logger,\n    private ignoreIds = new BitIds(),\n    private shouldLoadItsDeps?: ShouldIgnoreFunc\n  ) {\n    this.consumer = this.workspace.consumer;\n  }\n\n  /**\n   * create a graph with all dependencies and flattened dependencies of the given components.\n   * the nodes are components and the edges has a label of the dependency type.\n   *\n   * the way how it is done is iterations by depths. each depth we gather all the dependencies of\n   * that depths, make sure all objects exist and then check their dependencies for the next depth.\n   * once there is no dependency left, we're on the last depth level and the graph is ready.\n   *\n   * for example, imagine the following graph:\n   * A1 -> A2 -> A3\n   * B1 -> B2 -> B3\n   * C1 -> C2 -> C3\n   *\n   * where the buildGraph is given [A1, B1, C1].\n   * first, it saves all these components as nodes in the graph. then, it finds the dependencies of\n   * the next level, in this case they're [A2, B2, C2]. it runs `importMany` in case some objects\n   * are missing. then, it loads them all (some from FS, some from the model) and sets the edges\n   * between the component and the dependencies.\n   * once done, it finds all their dependencies, which are [A3, B3, C3] and repeat the process\n   * above. since there are no more dependencies, the graph is completed.\n   * in this case, the total depth levels are 3.\n   *\n   * even with a huge project, there are not many depth levels. by iterating through depth levels\n   * we keep performance sane as the importMany doesn't run multiple time and therefore the round\n   * trips to the remotes are minimal.\n   *\n   * normally, one importMany of the seeders is enough as importMany knows to fetch all flattened.\n   * however, since this buildGraph is performed on the workspace, a dependency may be new or\n   * modified and as such, we don't know its flattened yet.\n   */\n  async buildGraph(ids: BitId[]): Promise<LegacyGraph> {\n    this.logger.debug(`GraphFromFsBuilder, buildGraph with ${ids.length} seeders`);\n    const start = Date.now();\n    const components = await this.loadManyComponents(ids, '<none>');\n    await this.processManyComponents(components);\n    this.logger.debug(\n      `GraphFromFsBuilder, buildGraph with ${ids.length} seeders completed (${(Date.now() - start) / 1000} sec)`\n    );\n    return this.graph;\n  }\n\n  private getAllDepsUnfiltered(component: Component) {\n    return component.getAllDependenciesIds().difference(this.ignoreIds);\n  }\n\n  private async getAllDepsFiltered(component: Component): Promise<BitIds> {\n    const depsWithoutIgnore = this.getAllDepsUnfiltered(component);\n    const shouldLoadFunc = this.shouldLoadItsDeps;\n    if (!shouldLoadFunc) return depsWithoutIgnore;\n    const deps = await mapSeries(depsWithoutIgnore, async (depId) => {\n      const shouldLoad = await shouldLoadFunc(depId);\n      if (!shouldLoad) this.ignoreIds.push(depId);\n      return shouldLoad ? depId : null;\n    });\n    return BitIds.fromArray(compact(deps));\n  }\n\n  private async processManyComponents(components: Component[]) {\n    this.logger.debug(`GraphFromFsBuilder.processManyComponents depth ${this.depth}, ${components.length} components`);\n    this.depth += 1;\n    await this.importObjects(components);\n    const allDependencies = await mapSeries(components, (component) => this.processOneComponent(component));\n    const allDependenciesFlattened = flatten(allDependencies);\n    if (allDependenciesFlattened.length) await this.processManyComponents(allDependenciesFlattened);\n  }\n\n  private async importObjects(components: Component[]) {\n    const allDeps = components.map((c) => this.getAllDepsUnfiltered(c)).flat();\n    const allDepsWithScope = allDeps.filter((dep) => dep.hasScope());\n    const scopeComponentsImporter = new ScopeComponentsImporter(this.consumer.scope);\n    await scopeComponentsImporter.importMany(\n      BitIds.uniqFromArray(allDepsWithScope),\n      undefined,\n      this.shouldThrowOnMissingDep\n    );\n  }\n\n  private async processOneComponent(component: Component) {\n    const idStr = component.id.toString();\n    if (this.completed.includes(idStr)) return [];\n    const allIds = await this.getAllDepsFiltered(component);\n\n    const allDependencies = await this.loadManyComponents(allIds, idStr);\n    Object.entries(component.depsIdsGroupedByType).forEach(([depType, depsIds]) => {\n      depsIds.forEach((depId) => {\n        if (this.ignoreIds.has(depId)) return;\n        if (!this.graph.hasNode(depId.toString())) {\n          throw new Error(`buildOneComponent: missing node of ${depId.toString()}`);\n        }\n        this.graph.setEdge(idStr, depId.toString(), depType);\n      });\n    });\n    this.completed.push(idStr);\n    return allDependencies;\n  }\n\n  private async loadManyComponents(componentsIds: BitId[], dependenciesOf: string): Promise<Component[]> {\n    const components = await mapSeries(componentsIds, async (comp: BitId) => {\n      const idStr = comp.toString();\n      const fromGraph = this.graph.node(idStr);\n      if (fromGraph) return fromGraph;\n      try {\n        const component = await this.loadComponent(comp);\n        this.graph.setNode(idStr, component);\n        return component;\n      } catch (err: any) {\n        if (err instanceof ComponentNotFound) {\n          throw new BitError(\n            `error: component \"${idStr}\" was not found.\\nthis component is a dependency of \"${dependenciesOf}\" and is needed as part of the graph generation`\n          );\n        }\n        this.logger.error(`failed loading dependencies of ${dependenciesOf}`);\n        throw err;\n      }\n    });\n    return components;\n  }\n  private async loadComponent(componentId: BitId): Promise<Component> {\n    const componentMap = this.consumer.bitMap.getComponentIfExist(componentId);\n    const isOnWorkspace = Boolean(componentMap);\n    if (isOnWorkspace) {\n      return this.consumer.loadComponentForCapsule(componentId);\n    }\n    // a dependency might have been installed as a package in the workspace, and as such doesn't\n    // have a componentMap.\n    const componentFromModel = await this.consumer.loadComponentFromModel(componentId);\n    return componentFromModel.clone();\n  }\n}\n"]}

package/dist/workspace.d.ts

@@ -14,6 +14,7 @@ import { RequireableComponent } from '@teambit/harmony.modules.requireable-compo
 import { ResolvedComponent } from '@teambit/harmony.modules.resolved-component';
 import type { VariantsMain } from '@teambit/variants';
 import LegacyGraph from '@teambit/legacy/dist/scope/graph/graph';
+import { BitIds } from '@teambit/legacy/dist/bit-id';
 import { BitId } from '@teambit/legacy-bit-id';
 import { LocalLaneId } from '@teambit/legacy/dist/lane-id/lane-id';
 import { Consumer } from '@teambit/legacy/dist/consumer';
@@ -31,6 +32,7 @@ import { Watcher, WatchOptions } from './watch/watcher';
 import { ComponentStatus } from './workspace-component/component-status';
 import { OnComponentAddSlot, OnComponentChangeSlot, OnComponentLoadSlot, OnComponentRemoveSlot, OnPreWatch, OnPreWatchSlot } from './workspace.provider';
 import { WorkspaceComponentLoader } from './workspace-component/workspace-component-loader';
+import { ShouldIgnoreFunc } from './build-graph-from-fs';
 export declare type EjectConfResult = {
     configPath: string;
 };
@@ -159,8 +161,9 @@ export declare class Workspace implements ComponentFactory {
     private isVendorComponentByComponentDir;
     private componentConfigFile;
     private componentConfigFileFromComponentDirAndName;
-    getGraphWithoutCore(components: Component[]): Promise<LegacyGraph>;
+    getAspectsGraphWithoutCore(components: Component[], isAspect?: ShouldIgnoreFunc): Promise<LegacyGraph>;
     loadAspects(ids?: string[], throwOnError?: boolean): Promise<void>;
+    buildOneGraphForComponents(ids: BitId[], ignoreIds?: BitIds, shouldIgnoreFunc?: ShouldIgnoreFunc): Promise<LegacyGraph>;
     resolveAspects(runtimeName?: string, componentIds?: ComponentID[]): Promise<AspectDefinition[]>;
     private groupIdsByWorkspaceAndScope;
     private groupComponentsByWorkspaceAndScope;

package/dist/workspace.js

@@ -225,16 +225,6 @@ function _extensionData() {
   return data;
 }
 
-function _componentsGraph() {
-  const data = require("@teambit/legacy/dist/scope/graph/components-graph");
-
-  _componentsGraph = function () {
-    return data;
-  };
-
-  return data;
-}
-
 function _utils() {
   const data = require("@teambit/legacy/dist/utils");
 
@@ -365,6 +355,16 @@ function _incorrectEnvAspect() {
   return data;
 }
 
+function _buildGraphFromFs() {
+  const data = require("./build-graph-from-fs");
+
+  _buildGraphFromFs = function () {
+    return data;
+  };
+
+  return data;
+}
+
 const ComponentAdded = 'componentAdded';
 exports.ComponentAdded = ComponentAdded;
 const ComponentChanged = 'componentChanged';
@@ -640,7 +640,7 @@ class Workspace {
   async getLegacyGraph(ids) {
     if (!ids || ids.length < 1) ids = await this.listIds();
     const legacyIds = ids.map(id => id._legacy);
-    const legacyGraph = await (0, _componentsGraph().buildOneGraphForComponents)(legacyIds, this.consumer);
+    const legacyGraph = await this.buildOneGraphForComponents(legacyIds);
     return legacyGraph;
   }
 
@@ -875,7 +875,9 @@ class Workspace {
 
 
   async importAndGetMany(ids, forCapsule = false) {
-    await this.scope.import(ids);
+    await this.scope.import(ids, {
+      reFetchUnBuiltVersion: shouldReFetchUnBuiltVersion()
+    });
     return this.componentLoader.getMany(ids, forCapsule);
   }
   /**
@@ -901,7 +903,7 @@ class Workspace {
     const addResults = await addComponent.add(); // @todo: the legacy commands have `consumer.onDestroy()` on command completion, it writes the
     //  .bitmap file. workspace needs a similar mechanism. once done, remove the next line.
 
-    await this.consumer.bitMap.write(this.consumer.componentFsCache);
+    await this.writeBitMap();
     return addResults;
   }
   /**
@@ -1192,7 +1194,7 @@ class Workspace {
     return componentConfigFile;
   }
 
-  async getGraphWithoutCore(components) {
+  async getAspectsGraphWithoutCore(components, isAspect) {
     const ids = components.map(component => component.id._legacy);
     const coreAspectsStringIds = this.aspectLoader.getCoreAspectIds();
     const coreAspectsComponentIds = coreAspectsStringIds.map(id => _legacyBitId().BitId.parse(id, true));
@@ -1216,7 +1218,7 @@ class Workspace {
 
 
     const ignoredIds = coreAspectsBitIds.concat([]);
-    return (0, _componentsGraph().buildOneGraphForComponents)(ids, this.consumer, undefined, _bitId().BitIds.fromArray(ignoredIds));
+    return this.buildOneGraphForComponents(ids, _bitId().BitIds.fromArray(ignoredIds), isAspect);
   }
   /**
    * load aspects from the workspace and if not exists in the workspace, load from the scope.
@@ -1232,15 +1234,12 @@ class Workspace {
     const idsWithoutCore = (0, _lodash().difference)(notLoadedIds, coreAspectsStringIds);
     const componentIds = await this.resolveMultipleComponentIds(idsWithoutCore);
     const components = await this.importAndGetAspects(componentIds);
-    const graph = await this.getGraphWithoutCore(components);
-    const allIdsP = graph.nodes().map(async id => {
-      return this.resolveComponentId(id);
-    });
-    const allIds = await Promise.all(allIdsP);
-    const allComponents = await this.getMany(allIds);
-    const aspects = allComponents.filter(component => {
+
+    const isAspect = async bitId => {
       var _component$config$ext;
 
+      const id = await this.resolveComponentId(bitId);
+      const component = await this.get(id);
       let data = (_component$config$ext = component.config.extensions.findExtension(_envs().EnvsAspect.id)) === null || _component$config$ext === void 0 ? void 0 : _component$config$ext.data;
 
       if (!data) {
@@ -1264,14 +1263,16 @@ class Workspace {
       }
 
       return data.type === 'aspect';
-    }); // no need to filter core aspects as they are not included in the graph
-    // here we are trying to load extensions from the workspace.
+    };
 
+    const graph = await this.getAspectsGraphWithoutCore(components, isAspect);
+    const idsStr = graph.nodes();
+    const compIds = await this.resolveMultipleComponentIds(idsStr);
+    const aspects = await this.getMany(compIds);
     const {
       workspaceComps,
       scopeComps
-    } = await this.groupComponentsByWorkspaceAndScope(aspects); // load the scope first because we might need it for custom envs that extend external aspects
-
+    } = await this.groupComponentsByWorkspaceAndScope(aspects);
     const scopeIds = scopeComps.map(aspect => aspect.id.toString());
     const workspaceAspects = await this.requireComponents(workspaceComps);
     const workspaceManifests = await this.aspectLoader.getManifestsFromRequireableExtensions(workspaceAspects, throwOnError);
@@ -1280,6 +1281,17 @@ class Workspace {
     const allManifests = [...scopeManifests, ...workspaceManifests];
     await this.aspectLoader.loadExtensionsByManifests(allManifests, throwOnError);
   }
+  /**
+   * Note - this gets called from Harmony only.
+   * returns one graph that includes all dependencies types. each edge has a label of the dependency
+   * type. the nodes content is the Component object.
+   */
+
+
+  async buildOneGraphForComponents(ids, ignoreIds, shouldIgnoreFunc) {
+    const graphFromFsBuilder = new (_buildGraphFromFs().GraphFromFsBuilder)(this, this.logger, ignoreIds, shouldIgnoreFunc);
+    return graphFromFsBuilder.buildGraph(ids);
+  }
 
   async resolveAspects(runtimeName, componentIds) {
     let missingPaths = false;
@@ -1608,21 +1620,16 @@ your workspace.jsonc has this component-id set. you might want to remove/change
       verbose: false,
       merge: false,
       objectsOnly: true,
-      withEnvironments: false,
       override: false,
       writeDists: false,
       writeConfig: false,
       installNpmPackages: false,
-      writePackageJson: false,
       importDependenciesDirectly: false,
       importDependents: false
     };
 
     try {
-      const res = await (0, _consumer().importAction)({
-        tester: false,
-        compiler: false
-      }, importOptions, []);
+      const res = await (0, _consumer().importAction)(importOptions, []);
       return res;
     } catch (err) {
       // TODO: this is a hack since the legacy throw an error, we should provide a way to not throw this error from the legacy
@@ -1808,8 +1815,23 @@ your workspace.jsonc has this component-id set. you might want to remove/change
   }
 
 }
+/**
+ * this is a super hacky way to do it. problem is that loadAspect is running as onStart hook, where we don't
+ * have the CLI fully loaded yet, so we can't get the command from the CLI aspect, we have to retrieve it from
+ * process.argv.
+ * in general, we don't want every command to try again and again fetching un-built versions. otherwise, every time
+ * Bit loads (even bit --help), it'll fetch them and slow down everything.
+ * instead, long-running commands and those that need the artifacts from the Version objects, should try to re-fetch.
+ */
+
 
 exports.Workspace = Workspace;
+
+function shouldReFetchUnBuiltVersion() {
+  const commandsToReFetch = ['build', 'show', 'start', 'tag', 'install', 'link', 'import'];
+  return commandsToReFetch.includes(process.argv[2]);
+}
+
 var _default = Workspace;
 exports.default = _default;