@salesforce/source-tracking 1.3.0 → 1.3.1

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.3.1](https://github.com/forcedotcom/source-tracking/compare/v1.3.0...v1.3.1) (2022-03-25)
+
+### Bug Fixes
+
+- tracking really large repos in chunks, lower limit for windows ([0cb2ce5](https://github.com/forcedotcom/source-tracking/commit/0cb2ce5f3b65b8be9f4e4210aba010e919f692a3))
+
 ## [1.3.0](https://github.com/forcedotcom/source-tracking/compare/v1.2.0...v1.3.0) (2022-03-25)
 
 ### Features

package/lib/shared/functions.d.ts

@@ -8,3 +8,4 @@ export declare const isBundle: (cmp: SourceComponent) => boolean;
  * ex: '/foo/bar-extra/baz'.startsWith('foo/bar') would be true, but this function understands that they are not in the same folder
  */
 export declare const pathIsInFolder: (filePath: string, folder: string) => boolean;
+export declare const chunkArray: <T>(arr: T[], size: number) => T[][];

package/lib/shared/functions.js

@@ -6,7 +6,7 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.pathIsInFolder = exports.isBundle = exports.getKeyFromObject = exports.getMetadataKey = void 0;
+exports.chunkArray = exports.pathIsInFolder = exports.isBundle = exports.getKeyFromObject = exports.getMetadataKey = void 0;
 const path_1 = require("path");
 const ts_types_1 = require("@salesforce/ts-types");
 const getMetadataKey = (metadataType, metadataName) => {
@@ -37,4 +37,7 @@ exports.pathIsInFolder = pathIsInFolder;
 const nonEmptyStringFilter = (value) => {
     return (0, ts_types_1.isString)(value) && value.length > 0;
 };
+// adapted for TS from https://github.com/30-seconds/30-seconds-of-code/blob/master/snippets/chunk.md
+const chunkArray = (arr, size) => Array.from({ length: Math.ceil(arr.length / size) }, (v, i) => arr.slice(i * size, i * size + size));
+exports.chunkArray = chunkArray;
 //# sourceMappingURL=functions.js.map

package/lib/shared/localShadowRepo.d.ts

@@ -18,6 +18,9 @@ export declare class ShadowRepo {
     private packageDirs;
     private status;
     private logger;
+    private isWindows;
+    /** do not try to add more than this many files at a time through isogit.  You'll hit EMFILE: too many open files even with graceful-fs */
+    private maxFileAdd;
     private constructor();
     static getInstance(options: ShadowRepoOptions): Promise<ShadowRepo>;
     init(): Promise<void>;

package/lib/shared/localShadowRepo.js

@@ -13,10 +13,7 @@ const fs = require("graceful-fs");
 const core_1 = require("@salesforce/core");
 const git = require("isomorphic-git");
 const functions_1 = require("./functions");
-const gitIgnoreFileName = '.gitignore';
-/**
- * returns the full path to where we store the shadow repo
- */
+/** returns the full path to where we store the shadow repo */
 const getGitDir = (orgId, projectPath) => {
     return path.join(projectPath, '.sfdx', 'orgs', orgId, 'localSourceTracking');
 };
@@ -31,6 +28,8 @@ class ShadowRepo {
         this.gitDir = getGitDir(options.orgId, options.projectPath);
         this.projectPath = options.projectPath;
         this.packageDirs = options.packageDirs;
+        this.isWindows = os.type() === 'Windows_NT';
+        this.maxFileAdd = this.isWindows ? 8000 : 15000;
     }
     // think of singleton behavior but unique to the projectPath
     static async getInstance(options) {
@@ -82,14 +81,12 @@ class ShadowRepo {
      */
     async getStatus(noCache = false) {
         if (!this.status || noCache) {
-            // only ask about OS once but use twice
-            const isWindows = os.type() === 'Windows_NT';
             // iso-git uses relative, posix paths
             // but packageDirs has already resolved / normalized them
             // so we need to make them project-relative again and convert if windows
             const filepaths = this.packageDirs
                 .map((dir) => path.relative(this.projectPath, dir.fullPath))
-                .map((p) => (isWindows ? p.split(path.sep).join(path.posix.sep) : p));
+                .map((p) => (this.isWindows ? p.split(path.sep).join(path.posix.sep) : p));
             // status hasn't been initalized yet
             this.status = await git.statusMatrix({
                 fs,
@@ -103,12 +100,12 @@ class ShadowRepo {
                     // no lwc tests
                     !f.includes('__tests__') &&
                     // no gitignore files
-                    !f.endsWith(gitIgnoreFileName) &&
+                    !f.endsWith('.gitignore') &&
                     // isogit uses `startsWith` for filepaths so it's possible to get a false positive
                     filepaths.some((pkgDir) => (0, functions_1.pathIsInFolder)(f, pkgDir)),
             });
             // isomorphic-git stores things in unix-style tree.  Convert to windows-style if necessary
-            if (isWindows) {
+            if (this.isWindows) {
                 this.status = this.status.map((row) => [path.normalize(row[FILE]), row[HEAD], row[WORKDIR], row[3]]);
             }
         }
@@ -179,13 +176,27 @@ class ShadowRepo {
             deletedFiles = deletedFiles.map((filepath) => path.normalize(filepath).split(path.sep).join(path.posix.sep));
         }
         if (deployedFiles.length) {
-            await git.add({
-                fs,
-                dir: this.projectPath,
-                gitdir: this.gitDir,
-                filepath: [...new Set(deployedFiles)],
-                force: true,
-            });
+            const chunks = (0, functions_1.chunkArray)([...new Set(deployedFiles)], this.maxFileAdd);
+            for (const chunk of chunks) {
+                try {
+                    await git.add({
+                        fs,
+                        dir: this.projectPath,
+                        gitdir: this.gitDir,
+                        filepath: chunk,
+                        force: true,
+                    });
+                }
+                catch (e) {
+                    if (e instanceof git.Errors.MultipleGitError) {
+                        this.logger.error('multiple errors on git.add', e.errors.slice(0, 5));
+                        const error = new core_1.SfdxError(e.message, e.name, [], 1);
+                        error.setData(e.errors);
+                        throw error;
+                    }
+                    throw e;
+                }
+            }
         }
         for (const filepath of [...new Set(deletedFiles)]) {
             await git.remove({ fs, dir: this.projectPath, gitdir: this.gitDir, filepath });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@salesforce/source-tracking",
   "description": "API for tracking local and remote Salesforce metadata changes",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "author": "Salesforce",
   "license": "BSD-3-Clause",
   "main": "lib/index.js",
@@ -47,7 +47,7 @@
     "@salesforce/kit": "^1.5.17",
     "@salesforce/source-deploy-retrieve": "^5.9.4",
     "graceful-fs": "^4.2.9",
-    "isomorphic-git": "1.16.0",
+    "isomorphic-git": "1.17.0",
     "ts-retry-promise": "^0.6.0"
   },
   "devDependencies": {