@teambit/objects 0.0.683 → 0.0.685

package/objects/repository.spec.ts

@@ -0,0 +1,92 @@
+import { expect } from 'chai';
+import fs from 'fs-extra';
+import os from 'os';
+import path from 'path';
+import zlib from 'zlib';
+import { ScopeJson } from '@teambit/legacy.scope';
+import { ModelComponent, Source } from '../models';
+import Ref from './ref';
+import Repository from './repository';
+import { IndexType } from './scope-index';
+
+const SCOPE_NAME = 'my-scope';
+const SOURCES_COUNT = 5;
+
+/**
+ * `Repository.list(types)` is what rebuilds a missing `index.json`, and it's asked for a handful of
+ * object types in a scope that is mostly file contents. the type is in each file's header, so the
+ * rest of a file should only be read when its type is one that was asked for.
+ */
+describe('Repository.list', () => {
+  let scopePath: string;
+  let scopeJson: ScopeJson;
+  let repository: Repository;
+  let contentReads: number;
+  const original = {
+    onPostObjectRead: Repository.onPostObjectRead,
+    hasPostObjectReadTransformer: Repository.hasPostObjectReadTransformer,
+  };
+
+  beforeEach(async () => {
+    scopePath = await fs.mkdtemp(path.join(os.tmpdir(), 'bit-repository-spec-'));
+    scopeJson = new ScopeJson(
+      { name: SCOPE_NAME, version: '1.0.0', groupName: null },
+      path.join(scopePath, 'scope.json')
+    );
+    repository = await Repository.create({ scopePath, scopeJson });
+    const modelComponent = ModelComponent.from({
+      name: 'bar/foo',
+      scope: SCOPE_NAME,
+      lang: 'javascript',
+      deprecated: false,
+      bindingPrefix: '@bit',
+      versions: {},
+    });
+    const sources = Array.from({ length: SOURCES_COUNT }, (_, i) => Source.from(Buffer.from(`file ${i}`)));
+    const objects = [modelComponent, ...sources];
+    objects.forEach((object) => {
+      object.validateBeforePersist = false;
+    });
+    await repository.writeObjectsToTheFS(objects);
+
+    contentReads = 0;
+    // every full read of an object file goes through this hook, while a header read doesn't
+    Repository.onPostObjectRead = (content) => {
+      contentReads += 1;
+      return content;
+    };
+  });
+
+  afterEach(async () => {
+    Repository.onPostObjectRead = original.onPostObjectRead;
+    Repository.hasPostObjectReadTransformer = original.hasPostObjectReadTransformer;
+    await fs.remove(scopePath);
+  });
+
+  it('should rebuild a missing index.json reading the contents of the wanted objects only', async () => {
+    Repository.hasPostObjectReadTransformer = () => false;
+    await Repository.reset(scopePath);
+    const loaded = await Repository.load({ scopePath, scopeJson });
+    expect(loaded.scopeIndex.getHashes(IndexType.components)).to.have.lengthOf(1);
+    expect(contentReads).to.equal(1);
+  });
+
+  it('should read every object once when a transformer is registered, as the header is transformed too', async () => {
+    Repository.hasPostObjectReadTransformer = () => true;
+    const objects = await repository.list([ModelComponent]);
+    expect(objects).to.have.lengthOf(1);
+    expect(contentReads).to.equal(SOURCES_COUNT + 1);
+  });
+
+  it('should skip an object whose header names a type nothing is registered under, as the full read does', async () => {
+    Repository.hasPostObjectReadTransformer = () => false;
+    const hash = 'a'.repeat(40);
+    const content = 'whatever';
+    const unknownObject = zlib.deflateSync(Buffer.from(`NotARealType ${hash} ${content.length}\0${content}`));
+    const objectPath = repository.objectPath(Ref.from(hash));
+    await fs.outputFile(objectPath, unknownObject);
+    const objects = await repository.list([ModelComponent]);
+    expect(objects).to.have.lengthOf(1);
+    expect(objects[0]).to.be.instanceOf(ModelComponent);
+  });
+});

package/objects/repository.ts

@@ -14,7 +14,13 @@ import { glob } from 'glob';
 import { removeEmptyDir } from '@teambit/toolbox.fs.remove-empty-dir';
 import { concurrentIOLimit } from '@teambit/harmony.modules.concurrency';
 import type { Types, ScopeJson } from '@teambit/legacy.scope';
-import { HashNotFound, OutdatedIndexJson, UnmergedComponents, RemoteLanes } from '@teambit/legacy.scope';
+import {
+  HashNotFound,
+  OutdatedIndexJson,
+  UnknownObjectType,
+  UnmergedComponents,
+  RemoteLanes,
+} from '@teambit/legacy.scope';
 import type { IndexType, IndexItem } from './scope-index';
 import { ScopeIndex } from './scope-index';
 import BitObject from './object';
@@ -29,6 +35,18 @@ import { ScopeMeta, Lane, ModelComponent } from '../models';
 type ContentTransformer = (content: Buffer) => Buffer;
 const OBJECTS_BACKUP_DIR = `${OBJECTS_DIR}.bak`;
 const TRASH_DIR = 'trash';
+/**
+ * how much of an object file to read when all that's needed is its header. the header holds the
+ * type and is a few dozen bytes, which 512 compressed bytes always cover - and every byte past it
+ * is inflated for nothing (4KB inflates ten times as much, across every object in the scope). a
+ * chunk that falls short anyway is read in full.
+ */
+const OBJECT_HEADER_CHUNK_SIZE = 512;
+
+/** `mtimeMs` lets a caller tell an object that has been here a while from one just written */
+export type ObjectWithType = { ref: Ref; type: string; size: number; mtimeMs: number };
+/** `unreadable` holds the objects that couldn't be classified, so a caller can refuse to act on a partial inventory */
+export type ObjectsWithType = { objects: ObjectWithType[]; unreadable: Ref[] };
 
 export default class Repository {
   objects: { [key: string]: BitObject } = {};
@@ -94,14 +112,23 @@ export default class Repository {
    * Note: This function cannot be async because it's used by the synchronous `loadSync` method
    * which needs to maintain sync behavior for compatibility with existing code.
    */
-  static onPreObjectPersist: (content: Buffer) => Buffer;
+  static onPreObjectPersist?: (content: Buffer) => Buffer;
 
   /**
    * Hook for transforming content after objects are read from the filesystem.
    * Note: This function cannot be async because it's used by the synchronous `loadSync` method
    * which needs to maintain sync behavior for compatibility with existing code.
    */
-  static onPostObjectRead: (content: Buffer) => Buffer;
+  static onPostObjectRead?: (content: Buffer) => Buffer;
+
+  /**
+   * whether `onPostObjectRead` actually transforms anything.
+   *
+   * the scope aspect installs the hook above unconditionally - it's a reduce over a slot that is
+   * usually empty - so the hook being set says nothing about whether object content is transformed.
+   * only a caller that can skip reading content altogether needs to tell the two apart.
+   */
+  static hasPostObjectReadTransformer?: () => boolean;
 
   async reLoadScopeIndex() {
     this.scopeIndex = await this.loadOptionallyCreateScopeIndex();
@@ -259,6 +286,79 @@ export default class Repository {
     return objects;
   }
 
+  /**
+   * classify every object in the scope by its type, along with its size on the filesystem.
+   *
+   * needed by the garbage collector, which must know the type of each object but has no use for
+   * its content. inflating every object would mean inflating gigabytes of file-content (`Source`)
+   * objects, so only the first chunk of each file is read - enough to hold the header, which is
+   * where the type is. when a content transformer is registered (`onPostObjectRead`), it applies
+   * to the whole buffer, so in that case the file is read in full.
+   */
+  async listObjectsWithType(): Promise<ObjectsWithType> {
+    const refs = await this.listRefs();
+    const concurrency = concurrentIOLimit();
+    logger.debug(`Repository.listObjectsWithType, classifying ${refs.length} objects`);
+    const results = await pMapPool(refs, (ref) => this.getObjectTypeGracefully(ref), { concurrency });
+    const objects = compact(results.map((result) => result.object));
+    const unreadable = compact(results.map((result) => result.unreadable));
+    if (unreadable.length) {
+      logger.warn(`Repository.listObjectsWithType, unable to classify ${unreadable.length} objects`);
+    }
+    return { objects, unreadable };
+  }
+
+  /**
+   * the objects that couldn't be classified are reported rather than skipped. a caller that acts on
+   * this inventory (the garbage collector) has to know that it's incomplete, because an object it
+   * can't see is one it can't reason about.
+   */
+  private async getObjectTypeGracefully(ref: Ref): Promise<{ object?: ObjectWithType; unreadable?: Ref }> {
+    const objectPath = this.objectPath(ref);
+    try {
+      const stat = await fs.stat(objectPath);
+      const type = await this.readObjectType(objectPath);
+      return { object: { ref, type, size: stat.size, mtimeMs: stat.mtimeMs } };
+    } catch (err: any) {
+      logger.warn(`Repository.listObjectsWithType, failed reading ${objectPath}. Error: ${err.message}`);
+      return { unreadable: ref };
+    }
+  }
+
+  private async readObjectType(objectPath: string): Promise<string> {
+    const readInFull = async () => BitObject.parseObjectType(this.onRead(await fs.readFile(objectPath)), objectPath);
+    if (this.isContentTransformed()) return readInFull();
+    return (await this.readTypeFromHeader(objectPath)) || readInFull();
+  }
+
+  /**
+   * whether object files have to go through `onPostObjectRead` before their header can be read.
+   * when there's no way to tell, assume they do. it's the slow answer, never the wrong one.
+   */
+  private isContentTransformed(): boolean {
+    return Repository.hasPostObjectReadTransformer
+      ? Repository.hasPostObjectReadTransformer()
+      : Boolean(Repository.onPostObjectRead);
+  }
+
+  /**
+   * the type an object file names in its header, read from the first chunk of the file only.
+   * null when that chunk doesn't hold the whole header, in which case the file has to be read in
+   * full. throws `UnknownObjectType` when the header names a type nothing is registered under.
+   */
+  private async readTypeFromHeader(objectPath: string): Promise<string | null> {
+    // `Uint8Array` rather than `Buffer` - see `parseObjectTypeFromChunk`
+    const chunk = new Uint8Array(OBJECT_HEADER_CHUNK_SIZE);
+    const fileDescriptor = await fs.open(objectPath, 'r');
+    let bytesRead: number;
+    try {
+      ({ bytesRead } = await fs.read(fileDescriptor, chunk, 0, chunk.length, 0));
+    } finally {
+      await fs.close(fileDescriptor);
+    }
+    return BitObject.parseObjectTypeFromChunk(chunk.subarray(0, bytesRead));
+  }
+
   async loadRefDeleteIfInvalid(ref: Ref) {
     try {
       return await this.load(ref, true);
@@ -276,11 +376,23 @@ export default class Repository {
 
   async loadRefOnlyIfType(ref: Ref, types: Types): Promise<BitObject | null> {
     const objectPath = this.objectPath(ref);
-    const fileContentsRaw = await fs.readFile(objectPath);
-    const fileContents = this.onRead(fileContentsRaw);
     const typeNames = types.map((type) => type.name);
-    const parsedObject = await BitObject.parseObjectOnlyIfType(fileContents, typeNames, objectPath);
-    return parsedObject;
+    // most of a scope is file contents, which callers of this never ask for. the header says
+    // whether an object is wanted, so the rest of the file is only read and inflated when it is.
+    // transformed content has no header to read on its own, so it keeps to the single full read.
+    if (!this.isContentTransformed()) {
+      let type: string | null;
+      try {
+        type = await this.readTypeFromHeader(objectPath);
+      } catch (err: any) {
+        // an unregistered type is not one of the types asked for - the same answer the full read gives
+        if (err instanceof UnknownObjectType) return null;
+        throw err;
+      }
+      if (type && !typeNames.includes(type)) return null;
+    }
+    const fileContents = this.onRead(await fs.readFile(objectPath));
+    return BitObject.parseObjectOnlyIfType(fileContents, typeNames, objectPath);
   }
 
   async listRefs(cwd = this.getPath()): Promise<Array<Ref>> {

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@teambit/objects",
-  "version": "0.0.683",
+  "version": "0.0.685",
   "homepage": "https://bit.cloud/teambit/scope/objects",
   "main": "dist/index.js",
   "componentId": {
     "scope": "teambit.scope",
     "name": "objects",
-    "version": "0.0.683"
+    "version": "0.0.685"
   },
   "dependencies": {
     "lodash": "4.17.21",
@@ -19,36 +19,36 @@
     "semver": "7.7.1",
     "uuid": "8.3.2",
     "@teambit/harmony": "0.4.12",
+    "@teambit/cli": "0.0.1396",
     "@teambit/component-id": "1.2.4",
+    "@teambit/component.snap-distance": "0.0.168",
     "@teambit/graph.cleargraph": "0.0.11",
     "@teambit/harmony.modules.concurrency": "0.0.46",
     "@teambit/legacy.logger": "0.0.62",
+    "@teambit/legacy.scope": "0.0.167",
     "@teambit/toolbox.promise.map-pool": "0.0.21",
     "@teambit/legacy.constants": "0.0.45",
+    "@teambit/legacy.utils": "0.0.52",
     "@teambit/toolbox.crypto.sha1": "0.0.22",
     "@teambit/bit-error": "0.0.404",
     "@teambit/component-version": "1.0.4",
     "@teambit/harmony.modules.in-memory-cache": "0.0.49",
     "@teambit/toolbox.fs.remove-empty-dir": "0.0.20",
     "@teambit/lane-id": "0.0.312",
+    "@teambit/harmony.modules.get-basic-log": "0.0.168",
+    "@teambit/config-store": "0.0.277",
     "@teambit/legacy.cli.error": "0.0.62",
+    "@teambit/component.sources": "0.0.219",
     "@teambit/legacy-bit-id": "1.1.3",
     "@teambit/legacy-component-log": "0.0.432",
+    "@teambit/legacy.consumer-component": "0.0.168",
+    "@teambit/legacy.consumer-config": "0.0.167",
+    "@teambit/legacy.extension-data": "0.0.169",
     "@teambit/pkg.modules.semver-helper": "0.0.36",
     "@teambit/toolbox.array.duplications-finder": "0.0.16",
     "@teambit/bit.get-bit-version": "0.0.29",
-    "@teambit/cli": "0.0.1395",
-    "@teambit/component.snap-distance": "0.0.167",
-    "@teambit/graph": "1.0.1176",
-    "@teambit/legacy.scope": "0.0.166",
-    "@teambit/legacy.utils": "0.0.52",
-    "@teambit/harmony.modules.get-basic-log": "0.0.167",
-    "@teambit/config-store": "0.0.276",
-    "@teambit/component.sources": "0.0.218",
-    "@teambit/legacy.consumer-component": "0.0.167",
-    "@teambit/legacy.consumer-config": "0.0.166",
-    "@teambit/legacy.extension-data": "0.0.168",
-    "@teambit/semantics.doc-parser": "0.0.174"
+    "@teambit/semantics.doc-parser": "0.0.175",
+    "@teambit/graph": "1.0.1178"
   },
   "devDependencies": {
     "@types/lodash": "4.14.165",