@teambit/objects 0.0.710 → 0.0.711

package/models/version.ts

@@ -29,6 +29,57 @@ import Source from './source';
 import { DependenciesGraph } from './dependencies-graph';
 import { getBitVersion } from '@teambit/bit.get-bit-version';
 
+// the same flattened dependencies repeat across versions. in a typical scope, hundreds of thousands of entries are a
+// few thousand unique ids. ids are immutable, so the versions share them instead of each holding its own copies.
+// an id object is looked up by its parts (scope, then name, then version). building a key string for each entry instead
+// is slower than creating a new id.
+const flattenedIdsCache = new Map<string, Map<string, Map<string, ComponentID>>>();
+const flattenedIdStrsCache = new Map<string, ComponentID>();
+const MAX_CACHED_FLATTENED_IDS = 100_000;
+let cachedFlattenedIds = 0;
+
+function countCachedFlattenedId() {
+  if (cachedFlattenedIds >= MAX_CACHED_FLATTENED_IDS) {
+    flattenedIdsCache.clear();
+    flattenedIdStrsCache.clear();
+    cachedFlattenedIds = 0;
+  }
+  cachedFlattenedIds += 1;
+}
+
+function getFlattenedId(dep: string | Record<string, any>): ComponentID {
+  if (typeof dep === 'string') {
+    let id = flattenedIdStrsCache.get(dep);
+    if (!id) {
+      countCachedFlattenedId();
+      id = ComponentID.fromString(dep);
+      flattenedIdStrsCache.set(dep, id);
+    }
+    return id;
+  }
+  if (dep.box) return ComponentID.fromObject(dep as any); // legacy ids, rare. not worth another level.
+  let byName = flattenedIdsCache.get(dep.scope);
+  if (!byName) {
+    byName = new Map();
+    flattenedIdsCache.set(dep.scope, byName);
+  }
+  let byVersion = byName.get(dep.name);
+  if (!byVersion) {
+    byVersion = new Map();
+    byName.set(dep.name, byVersion);
+  }
+  let id = byVersion.get(dep.version);
+  if (!id) {
+    countCachedFlattenedId();
+    id = ComponentID.fromObject(dep as any);
+    byVersion.set(dep.version, id);
+  }
+  return id;
+}
+
+// the deprecated `Version.flattenedDependencies` getter warns only once per process.
+let warnedFlattenedDependenciesGetter = false;
+
 export type SourceFileModel = {
   name: string;
   relativePath: PathLinux;
@@ -50,6 +101,9 @@ export type Log = {
   email: string | undefined;
 };
 
+// anything objects can be loaded from by their ref: the scope repository, or the objects received from a remote.
+export type ObjectsLoader = { load(ref: Ref, throws?: boolean): Promise<BitObject | undefined> };
+
 export type DepEdgeType = 'prod' | 'dev' | 'peer' | 'ext';
 export type DepEdge = { source: ComponentID; target: ComponentID; type: DepEdgeType };
 
@@ -66,6 +120,7 @@ export type VersionProps = {
   devDependencies?: Dependency[];
   peerDependencies?: Dependency[];
   flattenedDependencies?: ComponentIdList;
+  flattenedDependenciesRef?: Ref;
   _flattenedEdges?: DepEdge[];
   flattenedEdges?: DepEdge[];
   flattenedEdgesRef?: Ref;
@@ -102,23 +157,23 @@ export default class Version extends BitObject {
   dependencies: Dependencies;
   devDependencies: Dependencies;
   peerDependencies: Dependencies;
-  flattenedDependencies: ComponentIdList;
+  private _flattenedDependencies?: ComponentIdList;
+  // the flattened dependencies of a parsed Version. the list is built on first access, most commands never read it.
+  private flattenedDependencyIds?: ComponentID[];
+  /**
+   * ref to a Source object with the flattened dependencies, as a JSON array of id strings. when it's set, the Version
+   * itself has no flattened dependencies, and they have to be loaded with `loadFlattenedDependencies()`.
+   */
+  flattenedDependenciesRef?: Ref;
   dependenciesGraphRef?: Ref;
   _dependenciesGraph?: DependenciesGraph; // caching for the dependencies graph
   flattenedEdgesRef?: Ref; // ref to a BitObject Source file, which is a JSON object containing the flattened edge
   _flattenedEdges?: DepEdge[]; // caching for the flattenedEdges
   /**
    * @deprecated
-   * to get the flattenedEdges, please use `this.getFlattenedEdges()`.
-   * this function handles the backward compatibility and provides the flattened edges regardless whether it was saved
-   * the `flattenedEdgesRef` introduced or after.
-   *
-   * the reason this is left here is not for backward compatibility, but for forward compatibility. meaning, if a
-   * Version object created by the new version is parsed by an old version that doesn't support the flattenedEdgesRef,
-   * then, it'll be able to still get the flattenedEdges by this prop.
-   * this is causing duplication currently. the data is kept in both, `this.flattenedEdges` and the file stored in `flattenedEdgesRef`.
-   * so it'll be best to delete this prop as soon as all scopes are deployed with the new version.
-   * (around August 2023 should be safe)
+   * to get the flattenedEdges, please use `this.getFlattenedEdges()`, which supports both formats.
+   * this holds the edges of old Versions, which were saved inside the Version, before `flattenedEdgesRef` was
+   * introduced. Versions that have `flattenedEdgesRef` don't save the edges here (see `toObject()`).
    */
   private flattenedEdges: DepEdge[];
   packageDependencies: { [key: string]: string };
@@ -154,7 +209,8 @@ export default class Version extends BitObject {
     this.devDependencies = new Dependencies(props.devDependencies);
     this.peerDependencies = new Dependencies(props.peerDependencies);
     this.docs = props.docs;
-    this.flattenedDependencies = props.flattenedDependencies || new ComponentIdList();
+    if (props.flattenedDependencies) this.flattenedDependencies = props.flattenedDependencies;
+    this.flattenedDependenciesRef = props.flattenedDependenciesRef;
     this.flattenedEdges = props.flattenedEdges || [];
     this.flattenedEdgesRef = props.flattenedEdgesRef;
     this.dependenciesGraphRef = props.dependenciesGraphRef;
@@ -306,8 +362,101 @@ export default class Version extends BitObject {
     return this.modified[this.modified.length - 1].date;
   }
 
-  getAllFlattenedDependencies(): ComponentIdList {
-    return ComponentIdList.fromArray([...this.flattenedDependencies]);
+  /**
+   * @deprecated use `await version.loadFlattenedDependencies(repo)`, which supports both formats. this getter throws
+   * when the flattened dependencies are stored in a separate object (`flattenedDependenciesRef`) and weren't loaded.
+   * bit doesn't write this format yet, so until it does (around Oct 2027), the getter keeps working.
+   */
+  get flattenedDependencies(): ComponentIdList {
+    if (!warnedFlattenedDependenciesGetter) {
+      warnedFlattenedDependenciesGetter = true;
+      logger.warn(
+        `Version.flattenedDependencies is deprecated and will throw for newer versions, use "await version.loadFlattenedDependencies(repo)" instead`,
+        new Error().stack
+      );
+    }
+    return this.getLoadedFlattenedDependencies();
+  }
+
+  /**
+   * the flattened dependencies, when they're stored in the Version itself or were loaded already. throws otherwise,
+   * when they're stored in a separate object (`flattenedDependenciesRef`), use `loadFlattenedDependencies()`.
+   */
+  getLoadedFlattenedDependencies(): ComponentIdList {
+    if (!this._flattenedDependencies) {
+      if (this.flattenedDependenciesRef) {
+        throw new Error(
+          `the flattened dependencies of version ${this._hash} are stored in the object ${this.flattenedDependenciesRef.toString()}, load them with loadFlattenedDependencies()`
+        );
+      }
+      this._flattenedDependencies = ComponentIdList.fromArray(this.flattenedDependencyIds || []);
+      this.flattenedDependencyIds = undefined;
+    }
+    return this._flattenedDependencies;
+  }
+
+  /**
+   * a new list is saved in the Version itself, so a ref to a separate object, if there was one, is dropped.
+   */
+  set flattenedDependencies(flattenedDependencies: ComponentIdList) {
+    this._flattenedDependencies = flattenedDependencies;
+    this.flattenedDependencyIds = undefined;
+    this.flattenedDependenciesRef = undefined;
+  }
+
+  /**
+   * the flattened dependencies, if they're available without loading a separate object. otherwise, undefined.
+   */
+  getFlattenedDependenciesIfLoaded(): ComponentIdList | undefined {
+    if (this.flattenedDependenciesRef && !this._flattenedDependencies) return undefined;
+    return this.getLoadedFlattenedDependencies();
+  }
+
+  /**
+   * use this to get the flattened dependencies. it supports both formats: stored in the Version itself, or stored in
+   * a separate Source object (`flattenedDependenciesRef`). `objects` is usually the scope repository, but can be any
+   * container of the objects, such as the objects received from a remote.
+   */
+  async loadFlattenedDependencies(objects: ObjectsLoader): Promise<ComponentIdList> {
+    if (this._flattenedDependencies || !this.flattenedDependenciesRef) return this.getLoadedFlattenedDependencies();
+    const source = (await objects.load(this.flattenedDependenciesRef, false)) as Source | undefined;
+    return this.setFlattenedDependenciesFromSource(source);
+  }
+
+  /**
+   * same as `loadFlattenedDependencies()`, for code that can't be async.
+   */
+  loadFlattenedDependenciesSync(repo: Repository): ComponentIdList {
+    if (this._flattenedDependencies || !this.flattenedDependenciesRef) return this.getLoadedFlattenedDependencies();
+    // objects added during tag, or fetched from a remote without being imported, are only in memory, not on the disk.
+    const source = (repo.getFromMemory(this.flattenedDependenciesRef) ||
+      repo.loadSync(this.flattenedDependenciesRef, false)) as Source | undefined;
+    return this.setFlattenedDependenciesFromSource(source);
+  }
+
+  private setFlattenedDependenciesFromSource(source?: Source): ComponentIdList {
+    if (!source) {
+      // unlike the flattened edges, there is no fallback here. an empty list would be wrong, not just less efficient.
+      throw new Error(
+        `unable to find the object ${this.flattenedDependenciesRef?.toString()} with the flattened dependencies of version ${this._hash}`
+      );
+    }
+    const ids: string[] = JSON.parse(source.contents.toString());
+    // not by the setter, which would drop the ref.
+    this._flattenedDependencies = ComponentIdList.fromArray(ids.map(getFlattenedId));
+    this.flattenedDependencyIds = undefined;
+    return this._flattenedDependencies;
+  }
+
+  /**
+   * moves the flattened dependencies out of this Version into a new Source object, which the caller has to persist.
+   */
+  moveFlattenedDependenciesToSource(): Source | undefined {
+    const flattenedDependencies = this.getLoadedFlattenedDependencies();
+    if (!flattenedDependencies.length) return undefined;
+    const source = Source.from(Buffer.from(JSON.stringify(flattenedDependencies.map((id) => id.toString()))));
+    this.flattenedDependenciesRef = source.hash();
+    return source;
   }
 
   getAllDependencies(): Dependency[] {
@@ -346,17 +495,6 @@ export default class Version extends BitObject {
     return ComponentIdList.fromArray([...this.dependencies.getAllIds(), ...this.devDependencies.getAllIds()]);
   }
 
-  updateFlattenedDependency(currentId: ComponentID, newId: ComponentID) {
-    const getUpdated = (flattenedDependencies: ComponentIdList): ComponentIdList => {
-      const updatedIds = flattenedDependencies.map((depId) => {
-        if (depId.isEqual(currentId)) return newId;
-        return depId;
-      });
-      return ComponentIdList.fromArray(updatedIds);
-    };
-    this.flattenedDependencies = getUpdated(this.flattenedDependencies);
-  }
-
   refs(): Ref[] {
     return this.refsWithOptions();
   }
@@ -378,6 +516,7 @@ export default class Version extends BitObject {
     }
     if (this.flattenedEdgesRef) allRefs.push(this.flattenedEdgesRef);
     if (this.dependenciesGraphRef) allRefs.push(this.dependenciesGraphRef);
+    if (this.flattenedDependenciesRef) allRefs.push(this.flattenedDependenciesRef);
     return allRefs;
   }
 
@@ -467,9 +606,10 @@ export default class Version extends BitObject {
         docs: this.docs,
         dependencies: this.dependencies.cloneAsObject(),
         devDependencies: this.devDependencies.cloneAsObject(),
-        flattenedDependencies: this.flattenedDependencies.map((dep) => dep.toObject()),
-        // @todo: uncomment this in the future, once all remotes are updated to support the backward compatibility.
-        // flattenedDependencies: this.flattenedDependencies.map((dep) => dep.toString()),
+        flattenedDependencies: this.flattenedDependenciesRef
+          ? undefined
+          : this.getLoadedFlattenedDependencies().map((dep) => dep.toObject()),
+        flattenedDependenciesRef: this.flattenedDependenciesRef?.toString(),
         flattenedEdges: this.flattenedEdgesRef ? undefined : this.flattenedEdges.map((f) => Version.depEdgeToObject(f)),
         flattenedEdgesRef: this.flattenedEdgesRef?.toString(),
         dependenciesGraphRef: this.dependenciesGraphRef?.toString(),
@@ -529,6 +669,7 @@ export default class Version extends BitObject {
       devDependencies,
       flattenedDependencies,
       flattenedDevDependencies,
+      flattenedDependenciesRef,
       flattenedEdges,
       flattenedEdgesRef,
       dependenciesGraphRef,
@@ -577,21 +718,6 @@ export default class Version extends BitObject {
       });
     };
 
-    // Accept both string[] and object[] for backward compatibility
-    const parseFlattenedDeps = (deps = []): ComponentID[] => {
-      if (!deps.length) return [];
-      if (typeof deps[0] === 'string') return deps.map((dep) => ComponentID.fromString(dep));
-      return deps.map((dep) => ComponentID.fromObject(dep));
-    };
-
-    const _groupFlattenedDependencies = () => {
-      // support backward compatibility. until v15, there was both flattenedDependencies and
-      // flattenedDevDependencies. since then, these both were grouped to one flattenedDependencies
-      const flattenedDeps = parseFlattenedDeps(flattenedDependencies);
-      const flattenedDevDeps = parseFlattenedDeps(flattenedDevDependencies);
-      return ComponentIdList.fromArray([...flattenedDeps, ...flattenedDevDeps]);
-    };
-
     const parseFile = (file) => {
       return {
         file: Ref.from(file.file),
@@ -622,7 +748,7 @@ export default class Version extends BitObject {
       return new ExtensionDataList();
     };
 
-    return new Version({
+    const version = new Version({
       mainFile,
       files: files.map(parseFile),
       bindingPrefix,
@@ -638,11 +764,11 @@ export default class Version extends BitObject {
       docs,
       dependencies: _getDependencies(dependencies),
       devDependencies: _getDependencies(devDependencies),
-      flattenedDependencies: _groupFlattenedDependencies(),
       // backward compatibility. before introducing `flattenedEdgesRef`, we only had `flattenedEdges`. see getFlattenedEdges() for more info.
       flattenedEdges: flattenedEdgesRef ? [] : flattenedEdges?.map((f) => Version.depEdgeFromObject(f)) || [],
       flattenedEdgesRef: flattenedEdgesRef ? Ref.from(flattenedEdgesRef) : undefined,
       dependenciesGraphRef: dependenciesGraphRef ? Ref.from(dependenciesGraphRef) : undefined,
+      flattenedDependenciesRef: flattenedDependenciesRef ? Ref.from(flattenedDependenciesRef) : undefined,
       devPackageDependencies,
       peerPackageDependencies,
       packageDependencies,
@@ -662,6 +788,12 @@ export default class Version extends BitObject {
       hidden,
       batchId,
     });
+    // until v15, there were both flattenedDependencies and flattenedDevDependencies. since then, they're grouped into
+    // flattenedDependencies. an entry is either an id string or an id object.
+    version.flattenedDependencyIds = [...(flattenedDependencies || []), ...(flattenedDevDependencies || [])].map(
+      getFlattenedId
+    );
+    return version;
   }
 
   /**
@@ -709,7 +841,7 @@ export default class Version extends BitObject {
       peerPackageDependencies: component.peerPackageDependencies,
       dependenciesGraphRef: dependenciesGraph?.hash(),
       flattenedDependencies: component.flattenedDependencies,
-      // it's safe to remove this line once the version.flattenedEdges prop is deleted
+      // not saved in the Version when `flattenedEdgesRef` is set, see `toObject()`.
       flattenedEdges: component.flattenedEdges,
       flattenedEdgesRef: flattenedEdges?.hash(),
       schema: component.schema,

package/objects/bit-object-list.ts

@@ -1,8 +1,15 @@
 import type BitObject from './object';
+import type Ref from './ref';
 import { ExportMetadata, Lane, LaneHistory, ModelComponent, Version, VersionHistory } from '../models';
 
 export class BitObjectList {
-  constructor(private objects: BitObject[]) {}
+  /**
+   * `objectsByRef` maps the refs the objects were received with, so finding an object doesn't need hashing them all.
+   */
+  constructor(
+    private objects: BitObject[],
+    private objectsByRef = new Map<string, BitObject>()
+  ) {}
 
   getComponents(): ModelComponent[] {
     return this.objects.filter((object) => object instanceof ModelComponent) as ModelComponent[];
@@ -28,6 +35,16 @@ export class BitObjectList {
     return this.objects;
   }
 
+  /**
+   * finds an object of this list by its ref. it has the same signature as `Repository.load()`, so these objects can be
+   * used where a repository is expected, e.g. `Version.loadFlattenedDependencies()`.
+   */
+  async load(ref: Ref): Promise<BitObject | undefined> {
+    const object = this.objectsByRef.get(ref.toString());
+    // a received ref is not trusted, the found object is hashed to verify it.
+    return object?.hash().isEqual(ref) ? object : undefined;
+  }
+
   excludeTypes(types: string[]): BitObject[] {
     return this.objects.filter((object) => !types.includes(object.getType()));
   }

package/objects/object-list.ts

@@ -290,11 +290,13 @@ export class ObjectList {
   ): Promise<{ bitObjectList: BitObjectList; rawSources: RawObjectsMap }> {
     const concurrency = concurrentIOLimit();
     const rawSources: RawObjectsMap = new Map();
+    const objectsByRef = new Map<string, BitObject>();
     const bitObjects = await pMapPool(
       this.objects,
       async (object) => {
         try {
           const { object: bitObject, inflatedSize } = await BitObject.parseObjectWithSize(object.buffer);
+          objectsByRef.set(object.ref.toString(), bitObject);
           if (collectRawSources && bitObject.getType() === 'Source') {
             if (bitObject.hash().isEqual(object.ref)) {
               rawSources.set(bitObject, { buffer: object.buffer, inflatedSize, ref: object.ref });
@@ -313,7 +315,7 @@ export class ObjectList {
       },
       { concurrency }
     );
-    return { bitObjectList: new BitObjectList(compact(bitObjects)), rawSources };
+    return { bitObjectList: new BitObjectList(compact(bitObjects), objectsByRef), rawSources };
   }
 
   static async fromBitObjects(bitObjects: BitObject[]): Promise<ObjectList> {

package/objects/repository.ts

@@ -579,6 +579,13 @@ export default class Repository {
     return this;
   }
 
+  /**
+   * the object, if it's in memory: added during tag but not persisted yet, or cached. same lookup as `load()`.
+   */
+  getFromMemory(ref: Ref): BitObject | undefined {
+    return this.objects[ref.hash.toString()] || this.getCache(ref);
+  }
+
   /**
    * an object evicted from the cache is still returned as long as it's in use elsewhere (see `LiveObjects`).
    */

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@teambit/objects",
-  "version": "0.0.710",
+  "version": "0.0.711",
   "homepage": "https://bit.cloud/teambit/scope/objects",
   "main": "dist/index.js",
   "componentId": {
     "scope": "teambit.scope",
     "name": "objects",
-    "version": "0.0.710"
+    "version": "0.0.711"
   },
   "dependencies": {
     "@pnpm/lockfile.types": "1100.0.19",
@@ -21,33 +21,33 @@
     "@teambit/harmony": "0.4.12",
     "@teambit/harmony.modules.runtimes": "0.0.4",
     "@teambit/component-id": "1.2.5",
+    "@teambit/legacy.utils": "0.0.56",
     "@teambit/bit-error": "0.0.404",
     "@teambit/component-version": "1.0.4",
     "@teambit/lane-id": "0.0.312",
+    "@teambit/legacy.cli.error": "0.0.67",
     "@teambit/legacy.constants": "0.0.49",
+    "@teambit/legacy.logger": "0.0.67",
     "@teambit/toolbox.crypto.sha1": "0.0.24",
     "@teambit/harmony.modules.concurrency": "0.0.50",
     "@teambit/legacy-bit-id": "1.1.4",
     "@teambit/legacy-component-log": "0.0.434",
     "@teambit/pkg.modules.semver-helper": "0.0.38",
+    "@teambit/toolbox.array.duplications-finder": "0.0.19",
+    "@teambit/toolbox.promise.map-pool": "0.0.24",
     "@teambit/graph.cleargraph": "0.0.11",
     "@teambit/bit.get-bit-version": "0.0.31",
     "@teambit/harmony.modules.in-memory-cache": "0.0.54",
     "@teambit/toolbox.fs.remove-empty-dir": "0.0.22",
-    "@teambit/legacy.utils": "0.0.56",
-    "@teambit/harmony.modules.get-basic-log": "0.0.180",
-    "@teambit/component.snap-distance": "0.0.180",
-    "@teambit/config-store": "0.0.291",
-    "@teambit/legacy.cli.error": "0.0.67",
-    "@teambit/legacy.logger": "0.0.67",
-    "@teambit/legacy.scope": "0.0.179",
-    "@teambit/component.sources": "0.0.231",
-    "@teambit/legacy.consumer-component": "0.0.180",
-    "@teambit/legacy.consumer-config": "0.0.179",
-    "@teambit/legacy.extension-data": "0.0.181",
-    "@teambit/toolbox.array.duplications-finder": "0.0.19",
-    "@teambit/toolbox.promise.map-pool": "0.0.24",
-    "@teambit/semantics.doc-parser": "0.0.187"
+    "@teambit/harmony.modules.get-basic-log": "0.0.181",
+    "@teambit/component.snap-distance": "0.0.181",
+    "@teambit/config-store": "0.0.292",
+    "@teambit/legacy.scope": "0.0.180",
+    "@teambit/component.sources": "0.0.232",
+    "@teambit/legacy.consumer-component": "0.0.181",
+    "@teambit/legacy.consumer-config": "0.0.180",
+    "@teambit/legacy.extension-data": "0.0.182",
+    "@teambit/semantics.doc-parser": "0.0.188"
   },
   "devDependencies": {
     "@types/semver": "7.5.8",