@salesforce/source-tracking 1.2.0 → 1.4.0

package/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.4.0](https://github.com/forcedotcom/source-tracking/compare/v1.3.1...v1.4.0) (2022-04-27)
+
+### Features
+
+- comp set for pulls ([4b361a4](https://github.com/forcedotcom/source-tracking/commit/4b361a4f31ac4f810762ee8d6f8447f7eef1be31))
+
+### [1.3.1](https://github.com/forcedotcom/source-tracking/compare/v1.3.0...v1.3.1) (2022-03-25)
+
+### Bug Fixes
+
+- tracking really large repos in chunks, lower limit for windows ([0cb2ce5](https://github.com/forcedotcom/source-tracking/commit/0cb2ce5f3b65b8be9f4e4210aba010e919f692a3))
+
+## [1.3.0](https://github.com/forcedotcom/source-tracking/compare/v1.2.0...v1.3.0) (2022-03-25)
+
+### Features
+
+- gracful-fs for EMFILE: too many open files ([1573828](https://github.com/forcedotcom/source-tracking/commit/1573828f5b3cf5f4f8b2023ff31b5764214d4b06))
+
 ## [1.2.0](https://github.com/forcedotcom/source-tracking/compare/v1.1.7...v1.2.0) (2022-03-23)
 
 ### Features

package/lib/shared/functions.d.ts

@@ -8,3 +8,4 @@ export declare const isBundle: (cmp: SourceComponent) => boolean;
  * ex: '/foo/bar-extra/baz'.startsWith('foo/bar') would be true, but this function understands that they are not in the same folder
  */
 export declare const pathIsInFolder: (filePath: string, folder: string) => boolean;
+export declare const chunkArray: <T>(arr: T[], size: number) => T[][];

package/lib/shared/functions.js

@@ -6,7 +6,7 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.pathIsInFolder = exports.isBundle = exports.getKeyFromObject = exports.getMetadataKey = void 0;
+exports.chunkArray = exports.pathIsInFolder = exports.isBundle = exports.getKeyFromObject = exports.getMetadataKey = void 0;
 const path_1 = require("path");
 const ts_types_1 = require("@salesforce/ts-types");
 const getMetadataKey = (metadataType, metadataName) => {
@@ -37,4 +37,7 @@ exports.pathIsInFolder = pathIsInFolder;
 const nonEmptyStringFilter = (value) => {
     return (0, ts_types_1.isString)(value) && value.length > 0;
 };
+// adapted for TS from https://github.com/30-seconds/30-seconds-of-code/blob/master/snippets/chunk.md
+const chunkArray = (arr, size) => Array.from({ length: Math.ceil(arr.length / size) }, (v, i) => arr.slice(i * size, i * size + size));
+exports.chunkArray = chunkArray;
 //# sourceMappingURL=functions.js.map

package/lib/shared/localShadowRepo.d.ts

@@ -18,6 +18,9 @@ export declare class ShadowRepo {
     private packageDirs;
     private status;
     private logger;
+    private isWindows;
+    /** do not try to add more than this many files at a time through isogit.  You'll hit EMFILE: too many open files even with graceful-fs */
+    private maxFileAdd;
     private constructor();
     static getInstance(options: ShadowRepoOptions): Promise<ShadowRepo>;
     init(): Promise<void>;

package/lib/shared/localShadowRepo.js

@@ -5,19 +5,15 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-/* eslint-disable no-console */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ShadowRepo = void 0;
 const path = require("path");
 const os = require("os");
-const fs = require("fs");
+const fs = require("graceful-fs");
 const core_1 = require("@salesforce/core");
 const git = require("isomorphic-git");
 const functions_1 = require("./functions");
-const gitIgnoreFileName = '.gitignore';
-/**
- * returns the full path to where we store the shadow repo
- */
+/** returns the full path to where we store the shadow repo */
 const getGitDir = (orgId, projectPath) => {
     return path.join(projectPath, '.sfdx', 'orgs', orgId, 'localSourceTracking');
 };
@@ -32,6 +28,8 @@ class ShadowRepo {
         this.gitDir = getGitDir(options.orgId, options.projectPath);
         this.projectPath = options.projectPath;
         this.packageDirs = options.packageDirs;
+        this.isWindows = os.type() === 'Windows_NT';
+        this.maxFileAdd = this.isWindows ? 8000 : 15000;
     }
     // think of singleton behavior but unique to the projectPath
     static async getInstance(options) {
@@ -83,14 +81,12 @@ class ShadowRepo {
      */
     async getStatus(noCache = false) {
         if (!this.status || noCache) {
-            // only ask about OS once but use twice
-            const isWindows = os.type() === 'Windows_NT';
             // iso-git uses relative, posix paths
             // but packageDirs has already resolved / normalized them
             // so we need to make them project-relative again and convert if windows
             const filepaths = this.packageDirs
                 .map((dir) => path.relative(this.projectPath, dir.fullPath))
-                .map((p) => (isWindows ? p.split(path.sep).join(path.posix.sep) : p));
+                .map((p) => (this.isWindows ? p.split(path.sep).join(path.posix.sep) : p));
             // status hasn't been initalized yet
             this.status = await git.statusMatrix({
                 fs,
@@ -104,12 +100,12 @@ class ShadowRepo {
                     // no lwc tests
                     !f.includes('__tests__') &&
                     // no gitignore files
-                    !f.endsWith(gitIgnoreFileName) &&
+                    !f.endsWith('.gitignore') &&
                     // isogit uses `startsWith` for filepaths so it's possible to get a false positive
                     filepaths.some((pkgDir) => (0, functions_1.pathIsInFolder)(f, pkgDir)),
             });
             // isomorphic-git stores things in unix-style tree.  Convert to windows-style if necessary
-            if (isWindows) {
+            if (this.isWindows) {
                 this.status = this.status.map((row) => [path.normalize(row[FILE]), row[HEAD], row[WORKDIR], row[3]]);
             }
         }
@@ -180,13 +176,27 @@ class ShadowRepo {
             deletedFiles = deletedFiles.map((filepath) => path.normalize(filepath).split(path.sep).join(path.posix.sep));
         }
         if (deployedFiles.length) {
-            await git.add({
-                fs,
-                dir: this.projectPath,
-                gitdir: this.gitDir,
-                filepath: [...new Set(deployedFiles)],
-                force: true,
-            });
+            const chunks = (0, functions_1.chunkArray)([...new Set(deployedFiles)], this.maxFileAdd);
+            for (const chunk of chunks) {
+                try {
+                    await git.add({
+                        fs,
+                        dir: this.projectPath,
+                        gitdir: this.gitDir,
+                        filepath: chunk,
+                        force: true,
+                    });
+                }
+                catch (e) {
+                    if (e instanceof git.Errors.MultipleGitError) {
+                        this.logger.error('multiple errors on git.add', e.errors.slice(0, 5));
+                        const error = new core_1.SfdxError(e.message, e.name, [], 1);
+                        error.setData(e.errors);
+                        throw error;
+                    }
+                    throw e;
+                }
+            }
         }
         for (const filepath of [...new Set(deletedFiles)]) {
             await git.remove({ fs, dir: this.projectPath, gitdir: this.gitDir, filepath });

package/lib/sourceTracking.d.ts

@@ -33,6 +33,7 @@ export declare class SourceTracking extends AsyncCreatable {
      * @returns ComponentSet[]
      */
     localChangesAsComponentSet(byPackageDir?: boolean): Promise<ComponentSet[]>;
+    remoteNonDeletesAsComponentSet(): Promise<ComponentSet>;
     /**
      * Does most of the work for the force:source:status command.
      * Outputs need a bit of massage since this aims to provide nice json.

package/lib/sourceTracking.js

@@ -116,6 +116,33 @@ class SourceTracking extends kit_1.AsyncCreatable {
         })
             .filter((componentSet) => componentSet.size > 0);
     }
+    async remoteNonDeletesAsComponentSet() {
+        const [changeResults, sourceBackedComponents] = await Promise.all([
+            // all changes based on remote tracking
+            this.getChanges({
+                origin: 'remote',
+                state: 'nondelete',
+                format: 'ChangeResult',
+            }),
+            // only returns source-backed components (SBC)
+            this.getChanges({
+                origin: 'remote',
+                state: 'nondelete',
+                format: 'SourceComponent',
+            }),
+        ]);
+        const componentSet = new source_deploy_retrieve_1.ComponentSet(sourceBackedComponents);
+        // there may be remote adds not in the SBC.  So we add those manually
+        changeResults.forEach((cr) => {
+            if (cr.type && cr.name && !componentSet.has({ type: cr.type, fullName: cr.name })) {
+                componentSet.add({
+                    type: cr.type,
+                    fullName: cr.name,
+                });
+            }
+        });
+        return componentSet;
+    }
     /**
      * Does most of the work for the force:source:status command.
      * Outputs need a bit of massage since this aims to provide nice json.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@salesforce/source-tracking",
   "description": "API for tracking local and remote Salesforce metadata changes",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "author": "Salesforce",
   "license": "BSD-3-Clause",
   "main": "lib/index.js",
@@ -46,7 +46,8 @@
     "@salesforce/core": "^2.33.1",
     "@salesforce/kit": "^1.5.17",
     "@salesforce/source-deploy-retrieve": "^5.9.4",
-    "isomorphic-git": "1.16.0",
+    "graceful-fs": "^4.2.9",
+    "isomorphic-git": "1.17.0",
     "ts-retry-promise": "^0.6.0"
   },
   "devDependencies": {