@teambit/objects 0.0.710 → 0.0.711

package/dist/models/version.d.ts

@@ -30,6 +30,9 @@ export type Log = {
     username: string | undefined;
     email: string | undefined;
 };
+export type ObjectsLoader = {
+    load(ref: Ref, throws?: boolean): Promise<BitObject | undefined>;
+};
 export type DepEdgeType = 'prod' | 'dev' | 'peer' | 'ext';
 export type DepEdge = {
     source: ComponentID;
@@ -64,6 +67,7 @@ export type VersionProps = {
     devDependencies?: Dependency[];
     peerDependencies?: Dependency[];
     flattenedDependencies?: ComponentIdList;
+    flattenedDependenciesRef?: Ref;
     _flattenedEdges?: DepEdge[];
     flattenedEdges?: DepEdge[];
     flattenedEdgesRef?: Ref;
@@ -105,23 +109,22 @@ export default class Version extends BitObject {
     dependencies: Dependencies;
     devDependencies: Dependencies;
     peerDependencies: Dependencies;
-    flattenedDependencies: ComponentIdList;
+    private _flattenedDependencies?;
+    private flattenedDependencyIds?;
+    /**
+     * ref to a Source object with the flattened dependencies, as a JSON array of id strings. when it's set, the Version
+     * itself has no flattened dependencies, and they have to be loaded with `loadFlattenedDependencies()`.
+     */
+    flattenedDependenciesRef?: Ref;
     dependenciesGraphRef?: Ref;
     _dependenciesGraph?: DependenciesGraph;
     flattenedEdgesRef?: Ref;
     _flattenedEdges?: DepEdge[];
     /**
      * @deprecated
-     * to get the flattenedEdges, please use `this.getFlattenedEdges()`.
-     * this function handles the backward compatibility and provides the flattened edges regardless whether it was saved
-     * the `flattenedEdgesRef` introduced or after.
-     *
-     * the reason this is left here is not for backward compatibility, but for forward compatibility. meaning, if a
-     * Version object created by the new version is parsed by an old version that doesn't support the flattenedEdgesRef,
-     * then, it'll be able to still get the flattenedEdges by this prop.
-     * this is causing duplication currently. the data is kept in both, `this.flattenedEdges` and the file stored in `flattenedEdgesRef`.
-     * so it'll be best to delete this prop as soon as all scopes are deployed with the new version.
-     * (around August 2023 should be safe)
+     * to get the flattenedEdges, please use `this.getFlattenedEdges()`, which supports both formats.
+     * this holds the edges of old Versions, which were saved inside the Version, before `flattenedEdgesRef` was
+     * introduced. Versions that have `flattenedEdgesRef` don't save the edges here (see `toObject()`).
      */
     private flattenedEdges;
     packageDependencies: {
@@ -162,7 +165,40 @@ export default class Version extends BitObject {
     hash(): Ref;
     get extensionDependencies(): Dependencies;
     lastModified(): string;
-    getAllFlattenedDependencies(): ComponentIdList;
+    /**
+     * @deprecated use `await version.loadFlattenedDependencies(repo)`, which supports both formats. this getter throws
+     * when the flattened dependencies are stored in a separate object (`flattenedDependenciesRef`) and weren't loaded.
+     * bit doesn't write this format yet, so until it does (around Oct 2027), the getter keeps working.
+     */
+    get flattenedDependencies(): ComponentIdList;
+    /**
+     * the flattened dependencies, when they're stored in the Version itself or were loaded already. throws otherwise,
+     * when they're stored in a separate object (`flattenedDependenciesRef`), use `loadFlattenedDependencies()`.
+     */
+    getLoadedFlattenedDependencies(): ComponentIdList;
+    /**
+     * a new list is saved in the Version itself, so a ref to a separate object, if there was one, is dropped.
+     */
+    set flattenedDependencies(flattenedDependencies: ComponentIdList);
+    /**
+     * the flattened dependencies, if they're available without loading a separate object. otherwise, undefined.
+     */
+    getFlattenedDependenciesIfLoaded(): ComponentIdList | undefined;
+    /**
+     * use this to get the flattened dependencies. it supports both formats: stored in the Version itself, or stored in
+     * a separate Source object (`flattenedDependenciesRef`). `objects` is usually the scope repository, but can be any
+     * container of the objects, such as the objects received from a remote.
+     */
+    loadFlattenedDependencies(objects: ObjectsLoader): Promise<ComponentIdList>;
+    /**
+     * same as `loadFlattenedDependencies()`, for code that can't be async.
+     */
+    loadFlattenedDependenciesSync(repo: Repository): ComponentIdList;
+    private setFlattenedDependenciesFromSource;
+    /**
+     * moves the flattened dependencies out of this Version into a new Source object, which the caller has to persist.
+     */
+    moveFlattenedDependenciesToSource(): Source | undefined;
     getAllDependencies(): Dependency[];
     get depsIdsGroupedByType(): {
         dependencies: ComponentIdList;
@@ -173,7 +209,6 @@ export default class Version extends BitObject {
     getAllDependenciesCloned(): Dependencies;
     getAllDependenciesIds(): ComponentIdList;
     getDependenciesIdsExcludeExtensions(): ComponentIdList;
-    updateFlattenedDependency(currentId: ComponentID, newId: ComponentID): void;
     refs(): Ref[];
     refsWithOptions(includeParents?: boolean, includeArtifacts?: boolean): Ref[];
     refsWithoutParents(): Ref[];

package/dist/models/version.js

@@ -127,6 +127,57 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
 function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
 function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == typeof i ? i : i + ""; }
 function _toPrimitive(t, r) { if ("object" != typeof t || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != typeof i) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
+// the same flattened dependencies repeat across versions. in a typical scope, hundreds of thousands of entries are a
+// few thousand unique ids. ids are immutable, so the versions share them instead of each holding its own copies.
+// an id object is looked up by its parts (scope, then name, then version). building a key string for each entry instead
+// is slower than creating a new id.
+const flattenedIdsCache = new Map();
+const flattenedIdStrsCache = new Map();
+const MAX_CACHED_FLATTENED_IDS = 100_000;
+let cachedFlattenedIds = 0;
+function countCachedFlattenedId() {
+  if (cachedFlattenedIds >= MAX_CACHED_FLATTENED_IDS) {
+    flattenedIdsCache.clear();
+    flattenedIdStrsCache.clear();
+    cachedFlattenedIds = 0;
+  }
+  cachedFlattenedIds += 1;
+}
+function getFlattenedId(dep) {
+  if (typeof dep === 'string') {
+    let id = flattenedIdStrsCache.get(dep);
+    if (!id) {
+      countCachedFlattenedId();
+      id = _componentId().ComponentID.fromString(dep);
+      flattenedIdStrsCache.set(dep, id);
+    }
+    return id;
+  }
+  if (dep.box) return _componentId().ComponentID.fromObject(dep); // legacy ids, rare. not worth another level.
+  let byName = flattenedIdsCache.get(dep.scope);
+  if (!byName) {
+    byName = new Map();
+    flattenedIdsCache.set(dep.scope, byName);
+  }
+  let byVersion = byName.get(dep.name);
+  if (!byVersion) {
+    byVersion = new Map();
+    byName.set(dep.name, byVersion);
+  }
+  let id = byVersion.get(dep.version);
+  if (!id) {
+    countCachedFlattenedId();
+    id = _componentId().ComponentID.fromObject(dep);
+    byVersion.set(dep.version, id);
+  }
+  return id;
+}
+
+// the deprecated `Version.flattenedDependencies` getter warns only once per process.
+let warnedFlattenedDependenciesGetter = false;
+
+// anything objects can be loaded from by their ref: the scope repository, or the objects received from a remote.
+
 /**
  * Represent a version model in the scope
  */
@@ -142,7 +193,14 @@ class Version extends _objects().BitObject {
     _defineProperty(this, "dependencies", void 0);
     _defineProperty(this, "devDependencies", void 0);
     _defineProperty(this, "peerDependencies", void 0);
-    _defineProperty(this, "flattenedDependencies", void 0);
+    _defineProperty(this, "_flattenedDependencies", void 0);
+    // the flattened dependencies of a parsed Version. the list is built on first access, most commands never read it.
+    _defineProperty(this, "flattenedDependencyIds", void 0);
+    /**
+     * ref to a Source object with the flattened dependencies, as a JSON array of id strings. when it's set, the Version
+     * itself has no flattened dependencies, and they have to be loaded with `loadFlattenedDependencies()`.
+     */
+    _defineProperty(this, "flattenedDependenciesRef", void 0);
     _defineProperty(this, "dependenciesGraphRef", void 0);
     _defineProperty(this, "_dependenciesGraph", void 0);
     // caching for the dependencies graph
@@ -152,16 +210,9 @@ class Version extends _objects().BitObject {
     // caching for the flattenedEdges
     /**
      * @deprecated
-     * to get the flattenedEdges, please use `this.getFlattenedEdges()`.
-     * this function handles the backward compatibility and provides the flattened edges regardless whether it was saved
-     * the `flattenedEdgesRef` introduced or after.
-     *
-     * the reason this is left here is not for backward compatibility, but for forward compatibility. meaning, if a
-     * Version object created by the new version is parsed by an old version that doesn't support the flattenedEdgesRef,
-     * then, it'll be able to still get the flattenedEdges by this prop.
-     * this is causing duplication currently. the data is kept in both, `this.flattenedEdges` and the file stored in `flattenedEdgesRef`.
-     * so it'll be best to delete this prop as soon as all scopes are deployed with the new version.
-     * (around August 2023 should be safe)
+     * to get the flattenedEdges, please use `this.getFlattenedEdges()`, which supports both formats.
+     * this holds the edges of old Versions, which were saved inside the Version, before `flattenedEdgesRef` was
+     * introduced. Versions that have `flattenedEdgesRef` don't save the edges here (see `toObject()`).
      */
     _defineProperty(this, "flattenedEdges", void 0);
     _defineProperty(this, "packageDependencies", void 0);
@@ -200,7 +251,8 @@ class Version extends _objects().BitObject {
     this.devDependencies = new (_legacy2().Dependencies)(props.devDependencies);
     this.peerDependencies = new (_legacy2().Dependencies)(props.peerDependencies);
     this.docs = props.docs;
-    this.flattenedDependencies = props.flattenedDependencies || new (_componentId().ComponentIdList)();
+    if (props.flattenedDependencies) this.flattenedDependencies = props.flattenedDependencies;
+    this.flattenedDependenciesRef = props.flattenedDependenciesRef;
     this.flattenedEdges = props.flattenedEdges || [];
     this.flattenedEdgesRef = props.flattenedEdgesRef;
     this.dependenciesGraphRef = props.dependenciesGraphRef;
@@ -325,8 +377,93 @@ class Version extends _objects().BitObject {
     if (!this.modified || !this.modified.length) return this.log.date;
     return this.modified[this.modified.length - 1].date;
   }
-  getAllFlattenedDependencies() {
-    return _componentId().ComponentIdList.fromArray([...this.flattenedDependencies]);
+
+  /**
+   * @deprecated use `await version.loadFlattenedDependencies(repo)`, which supports both formats. this getter throws
+   * when the flattened dependencies are stored in a separate object (`flattenedDependenciesRef`) and weren't loaded.
+   * bit doesn't write this format yet, so until it does (around Oct 2027), the getter keeps working.
+   */
+  get flattenedDependencies() {
+    if (!warnedFlattenedDependenciesGetter) {
+      warnedFlattenedDependenciesGetter = true;
+      _legacy4().logger.warn(`Version.flattenedDependencies is deprecated and will throw for newer versions, use "await version.loadFlattenedDependencies(repo)" instead`, new Error().stack);
+    }
+    return this.getLoadedFlattenedDependencies();
+  }
+
+  /**
+   * the flattened dependencies, when they're stored in the Version itself or were loaded already. throws otherwise,
+   * when they're stored in a separate object (`flattenedDependenciesRef`), use `loadFlattenedDependencies()`.
+   */
+  getLoadedFlattenedDependencies() {
+    if (!this._flattenedDependencies) {
+      if (this.flattenedDependenciesRef) {
+        throw new Error(`the flattened dependencies of version ${this._hash} are stored in the object ${this.flattenedDependenciesRef.toString()}, load them with loadFlattenedDependencies()`);
+      }
+      this._flattenedDependencies = _componentId().ComponentIdList.fromArray(this.flattenedDependencyIds || []);
+      this.flattenedDependencyIds = undefined;
+    }
+    return this._flattenedDependencies;
+  }
+
+  /**
+   * a new list is saved in the Version itself, so a ref to a separate object, if there was one, is dropped.
+   */
+  set flattenedDependencies(flattenedDependencies) {
+    this._flattenedDependencies = flattenedDependencies;
+    this.flattenedDependencyIds = undefined;
+    this.flattenedDependenciesRef = undefined;
+  }
+
+  /**
+   * the flattened dependencies, if they're available without loading a separate object. otherwise, undefined.
+   */
+  getFlattenedDependenciesIfLoaded() {
+    if (this.flattenedDependenciesRef && !this._flattenedDependencies) return undefined;
+    return this.getLoadedFlattenedDependencies();
+  }
+
+  /**
+   * use this to get the flattened dependencies. it supports both formats: stored in the Version itself, or stored in
+   * a separate Source object (`flattenedDependenciesRef`). `objects` is usually the scope repository, but can be any
+   * container of the objects, such as the objects received from a remote.
+   */
+  async loadFlattenedDependencies(objects) {
+    if (this._flattenedDependencies || !this.flattenedDependenciesRef) return this.getLoadedFlattenedDependencies();
+    const source = await objects.load(this.flattenedDependenciesRef, false);
+    return this.setFlattenedDependenciesFromSource(source);
+  }
+
+  /**
+   * same as `loadFlattenedDependencies()`, for code that can't be async.
+   */
+  loadFlattenedDependenciesSync(repo) {
+    if (this._flattenedDependencies || !this.flattenedDependenciesRef) return this.getLoadedFlattenedDependencies();
+    // objects added during tag, or fetched from a remote without being imported, are only in memory, not on the disk.
+    const source = repo.getFromMemory(this.flattenedDependenciesRef) || repo.loadSync(this.flattenedDependenciesRef, false);
+    return this.setFlattenedDependenciesFromSource(source);
+  }
+  setFlattenedDependenciesFromSource(source) {
+    if (!source) {
+      // unlike the flattened edges, there is no fallback here. an empty list would be wrong, not just less efficient.
+      throw new Error(`unable to find the object ${this.flattenedDependenciesRef?.toString()} with the flattened dependencies of version ${this._hash}`);
+    }
+    const ids = JSON.parse(source.contents.toString());
+    // not by the setter, which would drop the ref.
+    this._flattenedDependencies = _componentId().ComponentIdList.fromArray(ids.map(getFlattenedId));
+    this.flattenedDependencyIds = undefined;
+    return this._flattenedDependencies;
+  }
+
+  /**
+   * moves the flattened dependencies out of this Version into a new Source object, which the caller has to persist.
+   */
+  moveFlattenedDependenciesToSource() {
+    const flattenedDependencies = this.getLoadedFlattenedDependencies();
+    if (!flattenedDependencies.length) return undefined;
+    const source = _source().default.from(Buffer.from(JSON.stringify(flattenedDependencies.map(id => id.toString()))));
+    this.flattenedDependenciesRef = source.hash();
+    return source;
   }
   getAllDependencies() {
     return [...this.dependencies.dependencies, ...this.devDependencies.dependencies, ...this.extensionDependencies.dependencies];
@@ -350,16 +487,6 @@ class Version extends _objects().BitObject {
   getDependenciesIdsExcludeExtensions() {
     return _componentId().ComponentIdList.fromArray([...this.dependencies.getAllIds(), ...this.devDependencies.getAllIds()]);
   }
-  updateFlattenedDependency(currentId, newId) {
-    const getUpdated = flattenedDependencies => {
-      const updatedIds = flattenedDependencies.map(depId => {
-        if (depId.isEqual(currentId)) return newId;
-        return depId;
-      });
-      return _componentId().ComponentIdList.fromArray(updatedIds);
-    };
-    this.flattenedDependencies = getUpdated(this.flattenedDependencies);
-  }
   refs() {
     return this.refsWithOptions();
   }
@@ -380,6 +507,7 @@ class Version extends _objects().BitObject {
     }
     if (this.flattenedEdgesRef) allRefs.push(this.flattenedEdgesRef);
     if (this.dependenciesGraphRef) allRefs.push(this.dependenciesGraphRef);
+    if (this.flattenedDependenciesRef) allRefs.push(this.flattenedDependenciesRef);
     return allRefs;
   }
   refsWithoutParents() {
@@ -462,9 +590,8 @@ class Version extends _objects().BitObject {
       docs: this.docs,
       dependencies: this.dependencies.cloneAsObject(),
       devDependencies: this.devDependencies.cloneAsObject(),
-      flattenedDependencies: this.flattenedDependencies.map(dep => dep.toObject()),
-      // @todo: uncomment this in the future, once all remotes are updated to support the backward compatibility.
-      // flattenedDependencies: this.flattenedDependencies.map((dep) => dep.toString()),
+      flattenedDependencies: this.flattenedDependenciesRef ? undefined : this.getLoadedFlattenedDependencies().map(dep => dep.toObject()),
+      flattenedDependenciesRef: this.flattenedDependenciesRef?.toString(),
       flattenedEdges: this.flattenedEdgesRef ? undefined : this.flattenedEdges.map(f => Version.depEdgeToObject(f)),
       flattenedEdgesRef: this.flattenedEdgesRef?.toString(),
       dependenciesGraphRef: this.dependenciesGraphRef?.toString(),
@@ -519,6 +646,7 @@ class Version extends _objects().BitObject {
       devDependencies,
       flattenedDependencies,
       flattenedDevDependencies,
+      flattenedDependenciesRef,
       flattenedEdges,
       flattenedEdgesRef,
       dependenciesGraphRef,
@@ -556,20 +684,6 @@ class Version extends _objects().BitObject {
         return new (_legacy2().Dependency)(_componentId().ComponentID.fromObject(dependency.id), Array.isArray(dependency.relativePaths) ? dependency.relativePaths.map(getRelativePath) : dependency.relativePaths);
       });
     };
-
-    // Accept both string[] and object[] for backward compatibility
-    const parseFlattenedDeps = (deps = []) => {
-      if (!deps.length) return [];
-      if (typeof deps[0] === 'string') return deps.map(dep => _componentId().ComponentID.fromString(dep));
-      return deps.map(dep => _componentId().ComponentID.fromObject(dep));
-    };
-    const _groupFlattenedDependencies = () => {
-      // support backward compatibility. until v15, there was both flattenedDependencies and
-      // flattenedDevDependencies. since then, these both were grouped to one flattenedDependencies
-      const flattenedDeps = parseFlattenedDeps(flattenedDependencies);
-      const flattenedDevDeps = parseFlattenedDeps(flattenedDevDependencies);
-      return _componentId().ComponentIdList.fromArray([...flattenedDeps, ...flattenedDevDeps]);
-    };
     const parseFile = file => {
       return {
         file: _objects().Ref.from(file.file),
@@ -593,7 +707,7 @@ class Version extends _objects().BitObject {
       }
       return new (_legacy3().ExtensionDataList)();
     };
-    return new Version({
+    const version = new Version({
       mainFile,
       files: files.map(parseFile),
       bindingPrefix,
@@ -609,11 +723,11 @@ class Version extends _objects().BitObject {
       docs,
       dependencies: _getDependencies(dependencies),
       devDependencies: _getDependencies(devDependencies),
-      flattenedDependencies: _groupFlattenedDependencies(),
       // backward compatibility. before introducing `flattenedEdgesRef`, we only had `flattenedEdges`. see getFlattenedEdges() for more info.
       flattenedEdges: flattenedEdgesRef ? [] : flattenedEdges?.map(f => Version.depEdgeFromObject(f)) || [],
       flattenedEdgesRef: flattenedEdgesRef ? _objects().Ref.from(flattenedEdgesRef) : undefined,
       dependenciesGraphRef: dependenciesGraphRef ? _objects().Ref.from(dependenciesGraphRef) : undefined,
+      flattenedDependenciesRef: flattenedDependenciesRef ? _objects().Ref.from(flattenedDependenciesRef) : undefined,
       devPackageDependencies,
       peerPackageDependencies,
       packageDependencies,
@@ -637,6 +751,10 @@ class Version extends _objects().BitObject {
       hidden,
       batchId
     });
+    // until v15, there were both flattenedDependencies and flattenedDevDependencies. since then, they're grouped into
+    // flattenedDependencies. an entry is either an id string or an id object.
+    version.flattenedDependencyIds = [...(flattenedDependencies || []), ...(flattenedDevDependencies || [])].map(getFlattenedId);
+    return version;
   }
 
   /**
@@ -678,7 +796,7 @@ class Version extends _objects().BitObject {
       peerPackageDependencies: component.peerPackageDependencies,
       dependenciesGraphRef: dependenciesGraph?.hash(),
       flattenedDependencies: component.flattenedDependencies,
-      // it's safe to remove this line once the version.flattenedEdges prop is deleted
+      // not saved in the Version when `flattenedEdgesRef` is set, see `toObject()`.
       flattenedEdges: component.flattenedEdges,
       flattenedEdgesRef: flattenedEdges?.hash(),
       schema: component.schema,