@teambit/objects 0.0.703 → 0.0.705

package/objects/object-list.ts

@@ -46,6 +46,16 @@ export const FETCH_FORMAT_OBJECT_LIST = 'ObjectList';
  */
 export type ObjectItemsStream = Readable;
 
+/**
+ * compressed buffer of an object (as stored on the filesystem) and the size of its inflated content.
+ */
+export type CompressedObject = { buffer: Buffer; inflatedSize: number };
+
+/**
+ * hash => compressed object
+ */
+export type RawObjectsMap = Map<BitObject, CompressedObject & { ref: Ref }>;
+
 export class ObjectList {
   constructor(public objects: ObjectItem[] = []) {}
 
@@ -124,7 +134,10 @@ export class ObjectList {
         resolve(objects);
       });
 
-      packStream.pipe(extract);
+      // unlike `pipe`, it rejects when the input stream fails or the extractor fails (e.g. a truncated tar).
+      pipeline(packStream, extract, (err) => {
+        if (err) reject(err);
+      });
     });
     return new ObjectList(objectItems);
   }
@@ -248,23 +261,46 @@ export class ObjectList {
   }
 
   addIfNotExist(objectItems: ObjectItem[]) {
+    const existing = new Set(this.objects.map((object) => ObjectList.combineScopeAndHash(object)));
     objectItems.forEach((objectItem) => {
-      const exists = this.objects.find(
-        (object) => object.ref.isEqual(objectItem.ref) && object.scope === objectItem.scope
-      );
-      if (!exists) {
-        this.objects.push(objectItem);
-      }
+      const key = ObjectList.combineScopeAndHash(objectItem);
+      if (existing.has(key)) return;
+      existing.add(key);
+      this.objects.push(objectItem);
     });
   }
 
   async toBitObjects(throwForUnknownTypes = false): Promise<BitObjectList> {
+    const { bitObjectList } = await this.parseObjects(throwForUnknownTypes);
+    return bitObjectList;
+  }
+
+  /**
+   * same as `toBitObjects`, but also returns the original compressed buffers of the Source objects (files/artifacts)
+   * whose content matches their ref. Sources are immutable, so these buffers can be written to the filesystem as is,
+   * saving the costly re-compression. (significant for components with thousands of files).
+   */
+  async toBitObjectsWithRawSources(): Promise<{ bitObjectList: BitObjectList; rawSources: RawObjectsMap }> {
+    return this.parseObjects(false, true);
+  }
+
+  private async parseObjects(
+    throwForUnknownTypes: boolean,
+    collectRawSources = false
+  ): Promise<{ bitObjectList: BitObjectList; rawSources: RawObjectsMap }> {
     const concurrency = concurrentIOLimit();
+    const rawSources: RawObjectsMap = new Map();
     const bitObjects = await pMapPool(
       this.objects,
       async (object) => {
         try {
-          return await BitObject.parseObject(object.buffer);
+          const { object: bitObject, inflatedSize } = await BitObject.parseObjectWithSize(object.buffer);
+          if (collectRawSources && bitObject.getType() === 'Source') {
+            if (bitObject.hash().isEqual(object.ref)) {
+              rawSources.set(bitObject, { buffer: object.buffer, inflatedSize, ref: object.ref });
+            }
+          }
+          return bitObject;
         } catch (err) {
           if (throwForUnknownTypes || !(err instanceof UnknownObjectType)) {
             throw err;
@@ -277,7 +313,7 @@ export class ObjectList {
       },
       { concurrency }
     );
-    return new BitObjectList(compact(bitObjects));
+    return { bitObjectList: new BitObjectList(compact(bitObjects)), rawSources };
   }
 
   static async fromBitObjects(bitObjects: BitObject[]): Promise<ObjectList> {
@@ -302,6 +338,7 @@ export class ObjectList {
 
   splitByScopeName(): { [scopeName: string]: ObjectList } {
     const objectListPerScope: { [scopeName: string]: ObjectList } = {};
+    const added = new Set<string>();
     this.objects.forEach((obj) => {
       if (obj.type === ExportMetadata.name) {
         return; // no scope for this type. it's general for all export data from all scopes
@@ -309,11 +346,11 @@ export class ObjectList {
       if (!obj.scope) {
         throw new Error(`ObjectList: unable to split by scopeName, the scopeName is missing for ${obj.ref.hash}`);
       }
-      if (objectListPerScope[obj.scope]) {
-        objectListPerScope[obj.scope].addIfNotExist([obj]);
-      } else {
-        objectListPerScope[obj.scope] = new ObjectList([obj]);
-      }
+      const key = ObjectList.combineScopeAndHash(obj);
+      if (added.has(key)) return;
+      added.add(key);
+      if (!objectListPerScope[obj.scope]) objectListPerScope[obj.scope] = new ObjectList();
+      objectListPerScope[obj.scope].objects.push(obj);
     });
     return objectListPerScope;
   }

package/objects/repository.spec.ts

@@ -2,9 +2,11 @@ import { expect } from 'chai';
 import fs from 'fs-extra';
 import os from 'os';
 import path from 'path';
+import { Readable } from 'stream';
 import zlib from 'zlib';
 import { ScopeJson } from '@teambit/legacy.scope';
 import { ModelComponent, Source } from '../models';
+import { ObjectList } from './object-list';
 import Ref from './ref';
 import Repository from './repository';
 import { IndexType } from './scope-index';
@@ -90,3 +92,103 @@ describe('Repository.list', () => {
     expect(objects[0]).to.be.instanceOf(ModelComponent);
   });
 });
+
+/**
+ * the export flow moves every object through these methods, so for components with thousands of files, the number of
+ * fs operations and compressions per object is what makes an export fast or slow.
+ */
+describe('Repository writing objects for export', () => {
+  let scopePath: string;
+  let repository: Repository;
+
+  beforeEach(async () => {
+    scopePath = await fs.mkdtemp(path.join(os.tmpdir(), 'bit-repository-spec-'));
+    const scopeJson = new ScopeJson(
+      { name: SCOPE_NAME, version: '1.0.0', groupName: null },
+      path.join(scopePath, 'scope.json')
+    );
+    repository = await Repository.create({ scopePath, scopeJson });
+  });
+
+  afterEach(async () => {
+    await fs.remove(scopePath);
+  });
+
+  const createObjectList = async (count: number) => {
+    const sources = Array.from({ length: count }, (_, i) => Source.from(Buffer.from(`file ${i}`)));
+    return ObjectList.fromBitObjects(sources);
+  };
+
+  it('should write a file per pending object and read them back', async () => {
+    const objectList = await createObjectList(SOURCES_COUNT);
+    const pendingDir = path.join(scopePath, 'pending-objects', 'client-1');
+    await repository.writeObjectsToPendingDir(objectList, pendingDir);
+    for (const objectItem of objectList.objects) {
+      const hash = objectItem.ref.toString();
+      const written = await fs.readFile(path.join(pendingDir, hash.slice(0, 2), hash.slice(2)));
+      expect(written.equals(objectItem.buffer)).to.be.true;
+    }
+    const loaded = await repository.readObjectsFromPendingDir(pendingDir);
+    expect(loaded.objects.map((o) => o.ref.toString()).sort()).to.deep.equal(
+      objectList.objects.map((o) => o.ref.toString()).sort()
+    );
+  });
+
+  it('should write the given raw buffers of sources without re-compressing them, and leave no tmp files', async () => {
+    const objectList = await createObjectList(SOURCES_COUNT);
+    const { bitObjectList, rawSources } = await objectList.toBitObjectsWithRawSources();
+    expect(rawSources.size).to.equal(SOURCES_COUNT);
+    bitObjectList.getAll().forEach((obj) => {
+      obj.compressWithSize = async () => {
+        throw new Error('should not re-compress');
+      };
+    });
+    await repository.writeObjectsToTheFS(bitObjectList.getAll(), rawSources);
+    for (const objectItem of objectList.objects) {
+      const written = await fs.readFile(repository.objectPath(objectItem.ref));
+      expect(written.equals(objectItem.buffer)).to.be.true;
+    }
+    const objectDirs = await fs.readdir(repository.getPath());
+    const allFiles = (
+      await Promise.all(objectDirs.map((dir) => fs.readdir(path.join(repository.getPath(), dir))))
+    ).flat();
+    expect(allFiles).to.have.lengthOf(SOURCES_COUNT);
+    expect(allFiles.filter((fileName) => fileName.startsWith('.'))).to.have.lengthOf(0);
+  });
+
+  it('should not use a raw buffer of a source whose content does not match its ref', async () => {
+    const [first, second] = (await createObjectList(2)).objects;
+    const tampered = new ObjectList([{ ref: first.ref, buffer: second.buffer }]);
+    const { rawSources } = await tampered.toBitObjectsWithRawSources();
+    expect(rawSources.size).to.equal(0);
+  });
+
+  it('should re-create an object dir that was removed after it was written to', async () => {
+    const source = Source.from(Buffer.from('some content'));
+    await repository.writeObjectsToTheFS([source]);
+    await fs.remove(path.dirname(repository.objectPath(source.hash())));
+    await repository.writeObjectsToTheFS([source]);
+    expect(await fs.pathExists(repository.objectPath(source.hash()))).to.be.true;
+  });
+
+  it('should keep the mode of an existing object file it overwrites, as write-file-atomic does', async () => {
+    const source = Source.from(Buffer.from('some content'));
+    await repository.writeObjectsToTheFS([source]);
+    const objectPath = repository.objectPath(source.hash());
+    await fs.chmod(objectPath, 0o640);
+    await repository.writeObjectsToTheFS([source]);
+    expect((await fs.stat(objectPath)).mode & 0o777).to.equal(0o640);
+  });
+
+  it('should reject reading a truncated tar of objects rather than hang', async () => {
+    const objectList = await createObjectList(SOURCES_COUNT);
+    const chunks: Buffer[] = [];
+    for await (const chunk of objectList.toTar() as AsyncIterable<Buffer>) chunks.push(chunk);
+    const truncated = Buffer.concat(chunks).subarray(0, 700);
+    let error: Error | undefined;
+    await ObjectList.fromTar(Readable.from([truncated])).catch((err) => {
+      error = err;
+    });
+    expect(error).to.be.instanceOf(Error);
+  });
+});

package/objects/repository.ts

@@ -5,11 +5,11 @@ import { BitError } from '@teambit/bit-error';
 import type { ComponentID } from '@teambit/component-id';
 import { HASH_SIZE, isSnap } from '@teambit/component-version';
 import * as path from 'path';
+import { userInfo } from 'os';
 import { pMapPool } from '@teambit/toolbox.promise.map-pool';
 import { OBJECTS_DIR } from '@teambit/legacy.constants';
 import { logger } from '@teambit/legacy.logger';
-import type { ChownOptions, PathOsBasedAbsolute } from '@teambit/legacy.utils';
-import { writeFile } from '@teambit/legacy.utils';
+import type { PathOsBasedAbsolute } from '@teambit/legacy.utils';
 import { glob } from 'glob';
 import { removeEmptyDir } from '@teambit/toolbox.fs.remove-empty-dir';
 import { concurrentIOLimit } from '@teambit/harmony.modules.concurrency';
@@ -24,7 +24,7 @@ import {
 import type { IndexType, IndexItem } from './scope-index';
 import { ScopeIndex } from './scope-index';
 import BitObject from './object';
-import type { ObjectItem } from './object-list';
+import type { ObjectItem, RawObjectsMap, CompressedObject } from './object-list';
 import { ObjectList } from './object-list';
 import BitRawObject from './raw-object';
 import Ref from './ref';
@@ -37,6 +37,8 @@ type ContentTransformer = (content: Buffer) => Buffer;
 const OBJECTS_BACKUP_DIR = `${OBJECTS_DIR}.bak`;
 const TRASH_DIR = 'trash';
 const MAX_COMPRESSED_SIZE_TO_CACHE = 100 * 1024; // don't cache big files (mainly artifacts) to prevent out-of-memory
+let objectFileWriteCounter = 0;
+type ObjectChownOptions = { uid: number; gid: number } | null;
 /**
  * how much of an object file to read when all that's needed is its header. the header holds the
  * type and is a few dozen bytes, which 512 compressed bytes always cover - and every byte past it
@@ -790,14 +792,13 @@ export default class Repository {
   /**
    * write all objects to the FS and index the components/lanes/symlink objects
    */
-  async writeObjectsToTheFS(objects: BitObject[]): Promise<void> {
+  async writeObjectsToTheFS(objects: BitObject[], rawObjects?: RawObjectsMap): Promise<void> {
     const count = objects.length;
     if (!count) return;
     logger.trace(`Repository.writeObjectsToTheFS: started writing ${count} objects`);
     const concurrency = concurrentIOLimit();
-    await pMapPool(objects, (obj) => this._writeOne(obj), {
-      concurrency,
-    });
+    const chownOptions = await this.getChownOptions();
+    await pMapPool(objects, (obj) => this._writeOne(obj, rawObjects?.get(obj), chownOptions), { concurrency });
     logger.trace(`Repository.writeObjectsToTheFS: completed writing ${count} objects`);
 
     const added = this.scopeIndex.addMany(objects);
@@ -820,16 +821,21 @@ export default class Repository {
    * this method doesn't write to scopeIndex. so using this method for ModelComponent or
    * Symlink makes the index outdated.
    */
-  async _writeOne(object: BitObject): Promise<void> {
-    const { buffer: contents, inflatedSize } = await object.compressWithSize();
-    const options: ChownOptions = {};
-    if (this.scopeJson.groupName) options.gid = await resolveGroupId(this.scopeJson.groupName);
-    const hash = object.hash();
+  async _writeOne(
+    object: BitObject,
+    raw?: CompressedObject & { ref: Ref },
+    chownOptions?: ObjectChownOptions
+  ): Promise<void> {
+    // when the already-compressed buffer of an unchanged object is available, skip the costly re-compression.
+    const { buffer: contents, inflatedSize } = raw || (await object.compressWithSize());
+    const options = chownOptions === undefined ? await this.getChownOptions() : chownOptions;
+    // hashing a Source is sha1 of its content. the raw buffer's ref was already checked against it.
+    const hash = raw?.ref || object.hash();
     const objectPath = this.objectPath(hash);
     logger.trace(`repository._writeOne: ${objectPath}`);
     // Run hook to transform content pre persisting
     const transformedContent = this.onPersist(contents);
-    await writeFile(objectPath, transformedContent, options);
+    await this.writeObjectFile(objectPath, transformedContent, options);
     // once written, the object is the up-to-date one. this also replaces the size-estimate of objects that
     // were cached by `add()`.
     if (this.cache.has(hash.toString())) this.cache.set(hash.toString(), object, inflatedSize);
@@ -850,27 +856,73 @@ export default class Repository {
       const ref = object.ref.toString();
       if (!isSnap(ref)) throw new BitError(`invalid object ref: ${JSON.stringify(ref)}`);
     }
-    const options: ChownOptions = {};
-    if (this.scopeJson.groupName) options.gid = await resolveGroupId(this.scopeJson.groupName);
-    await Promise.all(
-      objectList.objects.map(async (object) => {
-        const objPath = path.join(pendingDir, this.hashPath(object.ref));
-        await writeFile(objPath, object.buffer, options);
-      })
+    const chownOptions = await this.getChownOptions();
+    await pMapPool(
+      objectList.objects,
+      (object) => this.writeObjectFile(path.join(pendingDir, this.hashPath(object.ref)), object.buffer, chownOptions),
+      { concurrency: concurrentIOLimit() }
     );
   }
 
   async readObjectsFromPendingDir(pendingDir: PathOsBasedAbsolute) {
     const refs = await this.listRefs(pendingDir);
-    const objects = await Promise.all(
-      refs.map(async (ref) => {
+    const objects = await pMapPool(
+      refs,
+      async (ref) => {
         const buffer = await fs.readFile(path.join(pendingDir, this.hashPath(ref)));
         return { ref, buffer };
-      })
+      },
+      { concurrency: concurrentIOLimit() }
     );
     return new ObjectList(objects);
   }
 
+  /**
+   * the owner to set on written object files. `null` when the scope has no group (or it can't be resolved), in which
+   * case the owner of an existing file is kept.
+   */
+  private async getChownOptions(): Promise<ObjectChownOptions> {
+    if (!this.scopeJson.groupName) return null;
+    const gid = await resolveGroupId(this.scopeJson.groupName);
+    return gid ? { uid: userInfo().uid, gid } : null;
+  }
+
+  /**
+   * writes the file atomically (tmp file + rename), the same way `writeFile` of legacy.utils does with write-file-atomic,
+   * including keeping the mode (and owner, when no chown is given) of an existing file and ignoring the same chown/chmod
+   * errors. it's called for every object, so it saves the fs calls write-file-atomic and `ensureDir` do on every write
+   * (realpath, stat of the dir, exit handlers). the parent dir is created only when the first write fails with ENOENT.
+   * the tmp file is a dotfile, which `listRefs()` ignores in case it's left behind.
+   */
+  private async writeObjectFile(filePath: string, contents: Buffer, chownOptions: ObjectChownOptions) {
+    const existing = await fs.stat(filePath).catch(() => undefined);
+    const mode = existing?.mode;
+    // process.getuid is undefined on Windows, where there's no owner to keep.
+    const keepOwner =
+      existing && typeof process.getuid === 'function' ? { uid: existing.uid, gid: existing.gid } : null;
+    const chown = chownOptions || keepOwner;
+    objectFileWriteCounter += 1;
+    const tmpPath = path.join(
+      path.dirname(filePath),
+      `.${path.basename(filePath)}.${process.pid}.${objectFileWriteCounter}`
+    );
+    try {
+      try {
+        await fs.writeFile(tmpPath, contents, { mode });
+      } catch (err: any) {
+        if (err.code !== 'ENOENT') throw err;
+        await fs.mkdir(path.dirname(filePath), { recursive: true });
+        await fs.writeFile(tmpPath, contents, { mode });
+      }
+      if (chown) await fs.chown(tmpPath, chown.uid, chown.gid).catch(throwUnlessChownErrOk);
+      if (mode) await fs.chmod(tmpPath, mode).catch(throwUnlessChownErrOk);
+      await fs.rename(tmpPath, filePath);
+    } catch (err) {
+      await fs.remove(tmpPath).catch(() => {});
+      throw err;
+    }
+  }
+
   private hashPath(ref: Ref) {
     const hash = ref.toString();
     return path.join(hash.slice(0, 2), hash.slice(2));
@@ -893,6 +945,16 @@ async function removeFile(filePath: string, propagateDirs = false): Promise<bool
   return true;
 }
 
+/**
+ * same as write-file-atomic: a non-root user can't always chown/chmod, and it's not an error.
+ */
+function throwUnlessChownErrOk(err: any) {
+  if (err.code === 'ENOSYS') return;
+  const nonRoot = !process.getuid || process.getuid() !== 0;
+  if (nonRoot && (err.code === 'EINVAL' || err.code === 'EPERM')) return;
+  throw err;
+}
+
 async function resolveGroupId(groupName: string): Promise<number | null | undefined> {
   // unix group ids are meaningless on Windows, which has no /etc/group and no process.getgid
   if (!process.getgid) return null;

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@teambit/objects",
-  "version": "0.0.703",
+  "version": "0.0.705",
   "homepage": "https://bit.cloud/teambit/scope/objects",
   "main": "dist/index.js",
   "componentId": {
     "scope": "teambit.scope",
     "name": "objects",
-    "version": "0.0.703"
+    "version": "0.0.705"
   },
   "dependencies": {
     "lodash": "4.17.21",
@@ -21,11 +21,9 @@
     "@teambit/harmony": "0.4.12",
     "@teambit/harmony.modules.runtimes": "0.0.1",
     "@teambit/component-id": "1.2.4",
-    "@teambit/component.snap-distance": "0.0.174",
     "@teambit/graph.cleargraph": "0.0.11",
     "@teambit/harmony.modules.concurrency": "0.0.47",
     "@teambit/legacy.logger": "0.0.63",
-    "@teambit/legacy.scope": "0.0.173",
     "@teambit/toolbox.promise.map-pool": "0.0.21",
     "@teambit/legacy.constants": "0.0.46",
     "@teambit/legacy.utils": "0.0.52",
@@ -35,19 +33,21 @@
     "@teambit/harmony.modules.in-memory-cache": "0.0.51",
     "@teambit/toolbox.fs.remove-empty-dir": "0.0.20",
     "@teambit/lane-id": "0.0.312",
-    "@teambit/harmony.modules.get-basic-log": "0.0.174",
-    "@teambit/config-store": "0.0.285",
     "@teambit/legacy.cli.error": "0.0.63",
-    "@teambit/component.sources": "0.0.225",
     "@teambit/legacy-bit-id": "1.1.3",
     "@teambit/legacy-component-log": "0.0.432",
-    "@teambit/legacy.consumer-component": "0.0.174",
-    "@teambit/legacy.consumer-config": "0.0.173",
-    "@teambit/legacy.extension-data": "0.0.175",
     "@teambit/pkg.modules.semver-helper": "0.0.36",
     "@teambit/toolbox.array.duplications-finder": "0.0.16",
     "@teambit/bit.get-bit-version": "0.0.29",
-    "@teambit/semantics.doc-parser": "0.0.181"
+    "@teambit/component.snap-distance": "0.0.176",
+    "@teambit/legacy.scope": "0.0.175",
+    "@teambit/harmony.modules.get-basic-log": "0.0.176",
+    "@teambit/config-store": "0.0.287",
+    "@teambit/component.sources": "0.0.227",
+    "@teambit/legacy.consumer-component": "0.0.176",
+    "@teambit/legacy.consumer-config": "0.0.175",
+    "@teambit/legacy.extension-data": "0.0.177",
+    "@teambit/semantics.doc-parser": "0.0.183"
   },
   "devDependencies": {
     "@types/lodash": "4.14.165",