@salesforce/source-tracking 1.1.7 → 1.3.1

package/CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.3.1](https://github.com/forcedotcom/source-tracking/compare/v1.3.0...v1.3.1) (2022-03-25)
+
+### Bug Fixes
+
+- tracking really large repos in chunks, lower limit for windows ([0cb2ce5](https://github.com/forcedotcom/source-tracking/commit/0cb2ce5f3b65b8be9f4e4210aba010e919f692a3))
+
+## [1.3.0](https://github.com/forcedotcom/source-tracking/compare/v1.2.0...v1.3.0) (2022-03-25)
+
+### Features
+
+- gracful-fs for EMFILE: too many open files ([1573828](https://github.com/forcedotcom/source-tracking/commit/1573828f5b3cf5f4f8b2023ff31b5764214d4b06))
+
+## [1.2.0](https://github.com/forcedotcom/source-tracking/compare/v1.1.7...v1.2.0) (2022-03-23)
+
+### Features
+
+- let isogit deal with ignore files ([#135](https://github.com/forcedotcom/source-tracking/issues/135)) ([1ddb2cd](https://github.com/forcedotcom/source-tracking/commit/1ddb2cdb8f23688f7bb4876a893097a85581f4c1))
+
+### Bug Fixes
+
+- ga tracking commands compatibility ([3a31a0d](https://github.com/forcedotcom/source-tracking/commit/3a31a0de448993c643ea4661a7a37772e46e8f51))
+- support pkgDir with ./foo ([3b46454](https://github.com/forcedotcom/source-tracking/commit/3b46454b3e57f653cbe80c66fbfee1cac121c2a8))
+
 ### [1.1.7](https://github.com/forcedotcom/source-tracking/compare/v1.1.6...v1.1.7) (2022-03-16)
 
 ### Bug Fixes

package/lib/compatibility.js

@@ -57,13 +57,18 @@ const throwIfInvalid = ({ org, projectPath, toValidate, command, }) => {
         throw new core_1.SfdxError(`${messages.getMessage('sourceTrackingFileVersionMismatch', ['new/beta'])}\n\nTry this:\n${messages.getMessage('useOtherVersion', ['new/beta', (0, exports.replaceRenamedCommands)(command.replace(':legacy', ''))])}.\n${messages.getMessage('clearSuggestion', [
             'new/beta',
             (0, exports.replaceRenamedCommands)('sfdx force:source:tracking:clear'),
+            (0, exports.replaceRenamedCommands)('sfdx force:source:tracking:reset', true),
         ])}.`, 'SourceTrackingFileVersionMismatch');
     }
     // We expected it to be the plugin-source version but it is using the old tracking files
     if (toValidate === 'plugin-source') {
         throw new core_1.SfdxError(messages.getMessage('sourceTrackingFileVersionMismatch', ['old/legacy']), 'SourceTrackingFileVersionMismatch', [
             messages.getMessage('useOtherVersion', ['old/legacy', (0, exports.replaceRenamedCommands)(command, true)]),
-            messages.getMessage('clearSuggestion', ['old/legacy', 'sfdx force:source:tracking:clear']),
+            messages.getMessage('clearSuggestion', [
+                'old/legacy',
+                'sfdx force:source:legacy:tracking:clear',
+                'sfdx force:source:tracking:reset',
+            ]),
         ]);
     }
 };
@@ -82,10 +87,10 @@ const replaceRenamedCommands = (input, reverse = false) => {
 };
 exports.replaceRenamedCommands = replaceRenamedCommands;
 const renames = new Map([
-    ['force:source:status', 'force:source:beta:status'],
-    ['force:source:push', 'force:source:beta:push'],
-    ['force:source:pull', 'force:source:beta:pull'],
-    ['force:source:tracking:reset', 'force:source:beta:tracking:reset'],
-    ['force:source:tracking:clear', 'force:source:beta:tracking:clear'],
+    ['force:source:legacy:status', 'force:source:status'],
+    ['force:source:legacy:push', 'force:source:push'],
+    ['force:source:legacy:pull', 'force:source:pull'],
+    ['force:source:legacy:tracking:reset', 'force:source:tracking:reset'],
+    ['force:source:legacy:tracking:clear', 'force:source:tracking:clear'],
 ]);
 //# sourceMappingURL=compatibility.js.map

package/lib/shared/functions.d.ts

@@ -8,3 +8,4 @@ export declare const isBundle: (cmp: SourceComponent) => boolean;
  * ex: '/foo/bar-extra/baz'.startsWith('foo/bar') would be true, but this function understands that they are not in the same folder
  */
 export declare const pathIsInFolder: (filePath: string, folder: string) => boolean;
+export declare const chunkArray: <T>(arr: T[], size: number) => T[][];

package/lib/shared/functions.js

@@ -6,7 +6,7 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.pathIsInFolder = exports.isBundle = exports.getKeyFromObject = exports.getMetadataKey = void 0;
+exports.chunkArray = exports.pathIsInFolder = exports.isBundle = exports.getKeyFromObject = exports.getMetadataKey = void 0;
 const path_1 = require("path");
 const ts_types_1 = require("@salesforce/ts-types");
 const getMetadataKey = (metadataType, metadataName) => {
@@ -37,4 +37,7 @@ exports.pathIsInFolder = pathIsInFolder;
 const nonEmptyStringFilter = (value) => {
     return (0, ts_types_1.isString)(value) && value.length > 0;
 };
+// adapted for TS from https://github.com/30-seconds/30-seconds-of-code/blob/master/snippets/chunk.md
+const chunkArray = (arr, size) => Array.from({ length: Math.ceil(arr.length / size) }, (v, i) => arr.slice(i * size, i * size + size));
+exports.chunkArray = chunkArray;
 //# sourceMappingURL=functions.js.map

package/lib/shared/localShadowRepo.d.ts

@@ -18,7 +18,9 @@ export declare class ShadowRepo {
     private packageDirs;
     private status;
     private logger;
-    private gitIgnoreLocations;
+    private isWindows;
+    /** do not try to add more than this many files at a time through isogit.  You'll hit EMFILE: too many open files even with graceful-fs */
+    private maxFileAdd;
     private constructor();
     static getInstance(options: ShadowRepoOptions): Promise<ShadowRepo>;
     init(): Promise<void>;
@@ -76,8 +78,5 @@ export declare class ShadowRepo {
      * @returns sha (string)
      */
     commitChanges({ deployedFiles, deletedFiles, message, needsUpdatedStatus, }?: CommitRequest): Promise<string>;
-    private locateIgnoreFiles;
-    private stashIgnoreFile;
-    private unStashIgnoreFile;
 }
 export {};

package/lib/shared/localShadowRepo.js

@@ -5,20 +5,15 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-/* eslint-disable no-console */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ShadowRepo = void 0;
 const path = require("path");
 const os = require("os");
-const fs = require("fs");
+const fs = require("graceful-fs");
 const core_1 = require("@salesforce/core");
 const git = require("isomorphic-git");
 const functions_1 = require("./functions");
-const gitIgnoreFileName = '.gitignore';
-const stashedGitIgnoreFileName = '.BAK.gitignore';
-/**
- * returns the full path to where we store the shadow repo
- */
+/** returns the full path to where we store the shadow repo */
 const getGitDir = (orgId, projectPath) => {
     return path.join(projectPath, '.sfdx', 'orgs', orgId, 'localSourceTracking');
 };
@@ -30,10 +25,11 @@ const HEAD = 1;
 const WORKDIR = 2;
 class ShadowRepo {
     constructor(options) {
-        this.gitIgnoreLocations = [];
         this.gitDir = getGitDir(options.orgId, options.projectPath);
         this.projectPath = options.projectPath;
         this.packageDirs = options.packageDirs;
+        this.isWindows = os.type() === 'Windows_NT';
+        this.maxFileAdd = this.isWindows ? 8000 : 15000;
     }
     // think of singleton behavior but unique to the projectPath
     static async getInstance(options) {
@@ -51,7 +47,6 @@ class ShadowRepo {
             this.logger.debug('initializing git repo');
             await this.gitInit();
         }
-        await this.locateIgnoreFiles();
     }
     /**
      * Initialize a new source tracking shadow repo.  Think of git init
@@ -86,37 +81,32 @@ class ShadowRepo {
      */
     async getStatus(noCache = false) {
         if (!this.status || noCache) {
-            try {
-                // only ask about OS once but use twice
-                const isWindows = os.type() === 'Windows_NT';
-                await this.stashIgnoreFile();
-                const filepaths = isWindows
-                    ? // iso-git uses posix paths, but packageDirs has already normalized them so we need to convert if windows
-                        this.packageDirs.map((dir) => dir.path.split(path.sep).join(path.posix.sep))
-                    : this.packageDirs.map((dir) => dir.path);
-                // status hasn't been initalized yet
-                this.status = await git.statusMatrix({
-                    fs,
-                    dir: this.projectPath,
-                    gitdir: this.gitDir,
-                    filepaths,
-                    filter: (f) => 
-                    // no hidden files
-                    !f.includes(`${path.sep}.`) &&
-                        // no lwc tests
-                        !f.includes('__tests__') &&
-                        // no gitignore files
-                        ![gitIgnoreFileName, stashedGitIgnoreFileName].includes(path.basename(f)) &&
-                        // isogit uses `startsWith` for filepaths so it's possible to get a false positive
-                        filepaths.some((pkgDir) => (0, functions_1.pathIsInFolder)(f, pkgDir)),
-                });
-                // isomorphic-git stores things in unix-style tree.  Convert to windows-style if necessary
-                if (isWindows) {
-                    this.status = this.status.map((row) => [path.normalize(row[FILE]), row[HEAD], row[WORKDIR], row[3]]);
-                }
-            }
-            finally {
-                await this.unStashIgnoreFile();
+            // iso-git uses relative, posix paths
+            // but packageDirs has already resolved / normalized them
+            // so we need to make them project-relative again and convert if windows
+            const filepaths = this.packageDirs
+                .map((dir) => path.relative(this.projectPath, dir.fullPath))
+                .map((p) => (this.isWindows ? p.split(path.sep).join(path.posix.sep) : p));
+            // status hasn't been initalized yet
+            this.status = await git.statusMatrix({
+                fs,
+                dir: this.projectPath,
+                gitdir: this.gitDir,
+                filepaths,
+                ignored: true,
+                filter: (f) => 
+                // no hidden files
+                !f.includes(`${path.sep}.`) &&
+                    // no lwc tests
+                    !f.includes('__tests__') &&
+                    // no gitignore files
+                    !f.endsWith('.gitignore') &&
+                    // isogit uses `startsWith` for filepaths so it's possible to get a false positive
+                    filepaths.some((pkgDir) => (0, functions_1.pathIsInFolder)(f, pkgDir)),
+            });
+            // isomorphic-git stores things in unix-style tree.  Convert to windows-style if necessary
+            if (this.isWindows) {
+                this.status = this.status.map((row) => [path.normalize(row[FILE]), row[HEAD], row[WORKDIR], row[3]]);
             }
         }
         return this.status;
@@ -180,56 +170,49 @@ class ShadowRepo {
             // this is valid, might not be an error
             return 'no files to commit';
         }
-        await this.stashIgnoreFile();
         // these are stored in posix/style/path format.  We have to convert inbound stuff from windows
         if (os.type() === 'Windows_NT') {
             deployedFiles = deployedFiles.map((filepath) => path.normalize(filepath).split(path.sep).join(path.posix.sep));
             deletedFiles = deletedFiles.map((filepath) => path.normalize(filepath).split(path.sep).join(path.posix.sep));
         }
-        try {
-            if (deployedFiles.length) {
-                await git.add({ fs, dir: this.projectPath, gitdir: this.gitDir, filepath: [...new Set(deployedFiles)] });
-            }
-            for (const filepath of [...new Set(deletedFiles)]) {
-                await git.remove({ fs, dir: this.projectPath, gitdir: this.gitDir, filepath });
-            }
-            const sha = await git.commit({
-                fs,
-                dir: this.projectPath,
-                gitdir: this.gitDir,
-                message,
-                author: { name: 'sfdx source tracking' },
-            });
-            // status changed as a result of the commit.  This prevents users from having to run getStatus(true) to avoid cache
-            if (needsUpdatedStatus) {
-                await this.getStatus(true);
+        if (deployedFiles.length) {
+            const chunks = (0, functions_1.chunkArray)([...new Set(deployedFiles)], this.maxFileAdd);
+            for (const chunk of chunks) {
+                try {
+                    await git.add({
+                        fs,
+                        dir: this.projectPath,
+                        gitdir: this.gitDir,
+                        filepath: chunk,
+                        force: true,
+                    });
+                }
+                catch (e) {
+                    if (e instanceof git.Errors.MultipleGitError) {
+                        this.logger.error('multiple errors on git.add', e.errors.slice(0, 5));
+                        const error = new core_1.SfdxError(e.message, e.name, [], 1);
+                        error.setData(e.errors);
+                        throw error;
+                    }
+                    throw e;
+                }
             }
-            return sha;
         }
-        finally {
-            await this.unStashIgnoreFile();
+        for (const filepath of [...new Set(deletedFiles)]) {
+            await git.remove({ fs, dir: this.projectPath, gitdir: this.gitDir, filepath });
         }
-    }
-    async locateIgnoreFiles() {
-        // set the gitIgnoreLocations so we only have to do it once
-        this.gitIgnoreLocations = (await git.walk({
+        const sha = await git.commit({
             fs,
             dir: this.projectPath,
             gitdir: this.gitDir,
-            trees: [git.WORKDIR()],
-            // eslint-disable-next-line @typescript-eslint/require-await
-            map: async (filepath) => filepath,
-        }))
-            .filter((filepath) => filepath.includes(gitIgnoreFileName))
-            .map((ignoreFile) => path.join(this.projectPath, ignoreFile));
-    }
-    async stashIgnoreFile() {
-        // allSettled allows them to fail (example, the file wasn't where it was expected).
-        await Promise.allSettled(this.gitIgnoreLocations.map((originalLocation) => fs.promises.rename(originalLocation, originalLocation.replace(gitIgnoreFileName, stashedGitIgnoreFileName))));
-    }
-    async unStashIgnoreFile() {
-        // allSettled allows them to fail (example, the file wasn't where it was expected).
-        await Promise.allSettled(this.gitIgnoreLocations.map((originalLocation) => fs.promises.rename(originalLocation.replace(gitIgnoreFileName, stashedGitIgnoreFileName), originalLocation)));
+            message,
+            author: { name: 'sfdx source tracking' },
+        });
+        // status changed as a result of the commit.  This prevents users from having to run getStatus(true) to avoid cache
+        if (needsUpdatedStatus) {
+            await this.getStatus(true);
+        }
+        return sha;
     }
 }
 exports.ShadowRepo = ShadowRepo;

package/messages/compatibility.json

@@ -1,5 +1,5 @@
 {
   "sourceTrackingFileVersionMismatch": "This project uses the %s version of source tracking files.",
-  "clearSuggestion": "Clear the %s version of the tracking files by running '%s'",
+  "clearSuggestion": "Push/Pull any local or remote changes.  Then, clear the %s version of the tracking files by running '%s' followed by '%s'.",
   "useOtherVersion": "Use the %s version of the command, '%s' with your existing tracking files."
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@salesforce/source-tracking",
   "description": "API for tracking local and remote Salesforce metadata changes",
-  "version": "1.1.7",
+  "version": "1.3.1",
   "author": "Salesforce",
   "license": "BSD-3-Clause",
   "main": "lib/index.js",
@@ -46,7 +46,8 @@
     "@salesforce/core": "^2.33.1",
     "@salesforce/kit": "^1.5.17",
     "@salesforce/source-deploy-retrieve": "^5.9.4",
-    "isomorphic-git": "1.14.0",
+    "graceful-fs": "^4.2.9",
+    "isomorphic-git": "1.17.0",
     "ts-retry-promise": "^0.6.0"
   },
   "devDependencies": {