@gadgetinc/ggt 0.2.0 → 0.2.2

package/lib/commands/sync.js

@@ -5,7 +5,6 @@ import { _ as _define_property } from "@swc/helpers/_/_define_property";
 import { Args, Flags } from "@oclif/core";
 import assert from "assert";
 import chalkTemplate from "chalk-template";
-import { FSWatcher } from "chokidar";
 import { format } from "date-fns";
 import { execa } from "execa";
 import fs from "fs-extra";
@@ -17,15 +16,16 @@ import PQueue from "p-queue";
 import path from "path";
 import pluralize from "pluralize";
 import { dedent } from "ts-dedent";
+import FSWatcher from "watcher";
 import which from "which";
 import { FileSyncEncoding } from "../__generated__/graphql.js";
-import { BaseCommand } from "../utils/base-command.js";
-import { Client } from "../utils/client.js";
-import { context } from "../utils/context.js";
-import { InvalidSyncAppFlagError, InvalidSyncFileError, YarnNotFoundError } from "../utils/errors.js";
-import { app } from "../utils/flags.js";
-import { FSIgnorer, ignoreEnoent, isEmptyDir, walkDir } from "../utils/fs-utils.js";
-import { PromiseSignal } from "../utils/promise.js";
+import { BaseCommand } from "../services/base-command.js";
+import { Client } from "../services/client.js";
+import { context } from "../services/context.js";
+import { InvalidSyncAppFlagError, InvalidSyncFileError, YarnNotFoundError } from "../services/errors.js";
+import { app } from "../services/flags.js";
+import { FSIgnorer, ignoreEnoent, isEmptyDir, walkDir } from "../services/fs-utils.js";
+import { PromiseSignal } from "../services/promise.js";
 class Sync extends BaseCommand {
     /**
    * Turns an absolute filepath into a relative one from {@linkcode dir}.
@@ -67,10 +67,10 @@ class Sync extends BaseCommand {
    * @param changed The normalized paths that have changed.
    * @param deleted The normalized paths that have been deleted.
    * @param options.limit The maximum number of lines to print. Defaults to 10. If debug is enabled, this is ignored.
-   */ logPaths(prefix, changed, deleted, { limit =10  } = {}) {
+   */ logPaths(prefix, changed, deleted, { limit = 10 } = {}) {
         const lines = _.sortBy([
-            ...changed.map((normalizedPath)=>chalkTemplate`{green ${prefix}} ${normalizedPath} {gray (changed)}`),
-            ...deleted.map((normalizedPath)=>chalkTemplate`{red ${prefix}} ${normalizedPath} {gray (deleted)}`)
+            ..._.map(changed, (normalizedPath)=>chalkTemplate`{green ${prefix}} ${normalizedPath} {gray (changed)}`),
+            ..._.map(deleted, (normalizedPath)=>chalkTemplate`{red ${prefix}} ${normalizedPath} {gray (deleted)}`)
         ], (line)=>line.slice(line.indexOf(" ") + 1));
         let logged = 0;
         for (const line of lines){
@@ -92,16 +92,21 @@ class Sync extends BaseCommand {
    * - Prompts the user how to resolve conflicts if the local filesystem has changed since the last sync.
    */ async init() {
         await super.init();
-        assert(_.isString(this.args["directory"]));
-        this.dir = this.config.windows && this.args["directory"].startsWith("~/") ? path.join(this.config.home, this.args["directory"].slice(2)) : path.resolve(this.args["directory"]);
+        this.dir = this.config.windows && _.startsWith(this.args.directory, "~/") ? path.join(this.config.home, this.args.directory.slice(2)) : path.resolve(this.args.directory);
         const getApp = async ()=>{
-            if (this.flags.app) return this.flags.app;
-            if (this.state?.app) return this.state.app;
+            if (this.flags.app) {
+                return this.flags.app;
+            }
+            // this.state can be undefined if the user is running `ggt sync` for the first time
+            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+            if (this.state?.app) {
+                return this.state.app;
+            }
             const selected = await inquirer.prompt({
                 type: "list",
                 name: "app",
                 message: "Please select the app to sync to.",
-                choices: await context.getAvailableApps().then((apps)=>apps.map((app)=>app.slug))
+                choices: await context.getAvailableApps().then((apps)=>_.map(apps, "slug"))
             });
             return selected.app;
         };
@@ -134,26 +139,13 @@ class Sync extends BaseCommand {
             ".gadget",
             ".git"
         ]);
-        this.watcher = new FSWatcher({
-            ignored: (filepath)=>this.ignorer.ignores(filepath),
-            // don't emit an event for every watched file on boot
-            ignoreInitial: true,
-            // make sure stats are always present on add/change events
-            alwaysStat: true,
-            // wait for the entire file to be written before emitting add/change events
-            awaitWriteFinish: {
-                pollInterval: this.flags["file-poll-interval"],
-                stabilityThreshold: this.flags["file-stability-threshold"]
-            }
-        });
-        this.debug("starting");
         if (!which.sync("yarn", {
             nothrow: true
         })) {
             throw new YarnNotFoundError();
         }
         await fs.ensureDir(this.dir);
-        const { remoteFilesVersion  } = await this.client.queryUnwrap({
+        const { remoteFilesVersion } = await this.client.queryUnwrap({
             query: REMOTE_FILES_VERSION_QUERY
         });
         const hasRemoteChanges = BigInt(remoteFilesVersion) > BigInt(this.state.filesVersion);
@@ -171,7 +163,7 @@ class Sync extends BaseCommand {
             files.delete("/");
             return files;
         };
-        const changedFiles = await getChangedFiles();
+        let changedFiles = await getChangedFiles();
         const hasLocalChanges = changedFiles.size > 0;
         if (hasLocalChanges) {
             this.log("Local files have changed since you last synced");
@@ -180,15 +172,20 @@ class Sync extends BaseCommand {
             });
             this.log();
         }
-        this.debug("init %O", {
-            metadata: this.state,
-            remoteFilesVersion,
-            hasRemoteChanges,
-            hasLocalChanges
+        context.addBreadcrumb({
+            category: "sync",
+            message: "Initializing",
+            data: {
+                state: this.state,
+                remoteFilesVersion,
+                hasRemoteChanges,
+                hasLocalChanges,
+                changed: Array.from(changedFiles.keys())
+            }
         });
         let action;
         if (hasLocalChanges) {
-            ({ action  } = await inquirer.prompt({
+            ({ action } = await inquirer.prompt({
                 type: "list",
                 name: "action",
                 choices: [
@@ -199,11 +196,20 @@ class Sync extends BaseCommand {
                 message: hasRemoteChanges ? "Remote files have also changed. How would you like to proceed?" : "How would you like to proceed?"
             }));
         }
+        // get all the changed files again in case more changed
+        changedFiles = await getChangedFiles();
         switch(action){
             case Action.MERGE:
                 {
-                    // get all the changed files again in case more changed
-                    const files = await getChangedFiles();
+                    context.addBreadcrumb({
+                        category: "sync",
+                        message: "Merging local changes",
+                        data: {
+                            state: this.state,
+                            remoteFilesVersion,
+                            changed: Array.from(changedFiles.keys())
+                        }
+                    });
                     // We purposefully don't set the returned remoteFilesVersion here because we haven't received the remote changes
                     // yet. This will cause us to receive the local files that we just published + the remote files that were
                     // changed since the last sync.
@@ -212,7 +218,7 @@ class Sync extends BaseCommand {
                         variables: {
                             input: {
                                 expectedRemoteFilesVersion: remoteFilesVersion,
-                                changed: await pMap(files, async ([normalizedPath, stats])=>{
+                                changed: await pMap(changedFiles, async ([normalizedPath, stats])=>{
                                     if (stats.mtime.getTime() > this.state.mtime) {
                                         this.state.mtime = stats.mtime.getTime();
                                     }
@@ -231,6 +237,15 @@ class Sync extends BaseCommand {
                 }
             case Action.RESET:
                 {
+                    context.addBreadcrumb({
+                        category: "sync",
+                        message: "Resetting local changes",
+                        data: {
+                            state: this.state,
+                            remoteFilesVersion,
+                            changed: Array.from(changedFiles.keys())
+                        }
+                    });
                     // delete all the local files that have changed since the last sync and set the files version to 0 so we receive
                     // all the remote files again, including any files that we just deleted that still exist
                     await pMap(changedFiles.keys(), (normalizedPath)=>this.softDelete(normalizedPath));
@@ -242,7 +257,13 @@ class Sync extends BaseCommand {
                     process.exit(0);
                 }
         }
-        this.debug("started");
+        context.addBreadcrumb({
+            category: "sync",
+            message: "Initialized",
+            data: {
+                state: this.state
+            }
+        });
     }
     /**
    * Runs the sync process until it is stopped or an error occurs.
@@ -251,9 +272,17 @@ class Sync extends BaseCommand {
         const stopped = new PromiseSignal();
         this.stop = async (e)=>{
             if (this.status != SyncStatus.RUNNING) return;
-            error = e;
-            this.debug("stopping");
             this.status = SyncStatus.STOPPING;
+            error = e;
+            context.addBreadcrumb({
+                category: "sync",
+                message: "Stopping",
+                level: error ? "error" : undefined,
+                data: {
+                    state: this.state,
+                    error
+                }
+            });
             try {
                 unsubscribe();
                 this.watcher.removeAllListeners();
@@ -265,9 +294,15 @@ class Sync extends BaseCommand {
                     this.watcher.close(),
                     this.client.dispose()
                 ]);
-                this.debug("stopped");
                 this.status = SyncStatus.STOPPED;
                 stopped.resolve();
+                context.addBreadcrumb({
+                    category: "sync",
+                    message: "Stopped",
+                    data: {
+                        state: this.state
+                    }
+                });
             }
         };
         for (const signal of [
@@ -295,23 +330,50 @@ class Sync extends BaseCommand {
                 })
         }, {
             error: (error)=>void this.stop(error),
-            next: ({ remoteFileSyncEvents  })=>{
+            next: ({ remoteFileSyncEvents })=>{
+                context.addBreadcrumb({
+                    category: "sync",
+                    message: "Received file sync events",
+                    data: {
+                        state: this.state,
+                        remoteFilesVersion: remoteFileSyncEvents.remoteFilesVersion,
+                        changed: _.map(remoteFileSyncEvents.changed, "path"),
+                        deleted: _.map(remoteFileSyncEvents.deleted, "path")
+                    }
+                });
                 const remoteFilesVersion = remoteFileSyncEvents.remoteFilesVersion;
                 // we always ignore .gadget/ files so that we don't publish them (they're managed by gadget), but we still want to receive them
-                const filter = (event)=>event.path.startsWith(".gadget/") || !this.ignorer.ignores(event.path);
-                const changed = remoteFileSyncEvents.changed.filter(filter);
-                const deleted = remoteFileSyncEvents.deleted.filter(filter);
+                const filter = (event)=>_.startsWith(event.path, ".gadget/") || !this.ignorer.ignores(event.path);
+                const changed = _.filter(remoteFileSyncEvents.changed, filter);
+                const deleted = _.filter(remoteFileSyncEvents.deleted, filter);
                 this._enqueue(async ()=>{
+                    context.addBreadcrumb({
+                        category: "sync",
+                        message: "Processing received file sync events",
+                        data: {
+                            state: this.state,
+                            remoteFilesVersion: remoteFileSyncEvents.remoteFilesVersion,
+                            changed: _.map(remoteFileSyncEvents.changed, "path"),
+                            deleted: _.map(remoteFileSyncEvents.deleted, "path")
+                        }
+                    });
                     if (!changed.length && !deleted.length) {
                         if (BigInt(remoteFilesVersion) > this.state.filesVersion) {
                             // we still need to update filesVersion, otherwise our expectedFilesVersion will be behind the next time we publish
-                            this.debug("updated local files version from %s to %s", this.state.filesVersion, remoteFilesVersion);
                             this.state.filesVersion = remoteFilesVersion;
+                            context.addBreadcrumb({
+                                category: "sync",
+                                message: "Received empty file sync events",
+                                data: {
+                                    state: this.state,
+                                    remoteFilesVersion: remoteFileSyncEvents.remoteFilesVersion
+                                }
+                            });
                         }
                         return;
                     }
                     this.log(chalkTemplate`Received {gray ${format(new Date(), "pp")}}`);
-                    this.logPaths("←", changed.map((x)=>x.path), deleted.map((x)=>x.path));
+                    this.logPaths("←", _.map(changed, "path"), _.map(deleted, "path"));
                     // we need to processed deleted files first as we may delete an empty directory after a file has been put
                     // into it. if processed out of order the new file will be deleted as well
                     await pMap(deleted, async (file)=>{
@@ -321,14 +383,16 @@ class Sync extends BaseCommand {
                     await pMap(changed, async (file)=>{
                         this.recentRemoteChanges.add(file.path);
                         const absolutePath = this.absolute(file.path);
-                        if (file.path.endsWith("/")) {
+                        if (_.endsWith(file.path, "/")) {
                             await fs.ensureDir(absolutePath, {
                                 mode: 0o755
                             });
                             return;
                         }
-                        // we need to add the parent directory to recentRemoteChanges so that we don't try to publish it
-                        this.recentRemoteChanges.add(path.dirname(file.path) + "/");
+                        // we need to add all parent directories to recentRemoteChanges so that we don't re-publish them
+                        for (const dir of _.split(path.dirname(file.path), "/")){
+                            this.recentRemoteChanges.add(dir + "/");
+                        }
                         await fs.ensureDir(path.dirname(absolutePath), {
                             mode: 0o755
                         });
@@ -340,16 +404,22 @@ class Sync extends BaseCommand {
                                 "install"
                             ], {
                                 cwd: this.dir
-                            }).catch((err)=>{
-                                this.debug("yarn install failed");
-                                this.debug(err.message);
+                            }).catch((error)=>{
+                                context.addBreadcrumb({
+                                    category: "sync",
+                                    message: "Yarn install failed",
+                                    level: "error",
+                                    data: {
+                                        state: this.state,
+                                        error
+                                    }
+                                });
                             });
                         }
                         if (absolutePath == this.ignorer.filepath) {
                             this.ignorer.reload();
                         }
                     });
-                    this.debug("updated local files version from %s to %s", this.state.filesVersion, remoteFilesVersion);
                     this.state.filesVersion = remoteFilesVersion;
                     // always remove the root directory from recentRemoteChanges
                     this.recentRemoteChanges.delete("./");
@@ -359,6 +429,17 @@ class Sync extends BaseCommand {
                             this.recentRemoteChanges.delete(filepath);
                         }
                     }
+                    context.addBreadcrumb({
+                        category: "sync",
+                        message: "Processed received file sync events",
+                        data: {
+                            state: this.state,
+                            remoteFilesVersion: remoteFileSyncEvents.remoteFilesVersion,
+                            changed: _.map(remoteFileSyncEvents.changed, "path"),
+                            deleted: _.map(remoteFileSyncEvents.deleted, "path"),
+                            recentRemoteChanges: Array.from(this.recentRemoteChanges.keys())
+                        }
+                    });
                 });
             }
         });
@@ -367,6 +448,14 @@ class Sync extends BaseCommand {
             const localFiles = new Map(localFilesBuffer.entries());
             localFilesBuffer.clear();
             this._enqueue(async ()=>{
+                context.addBreadcrumb({
+                    category: "sync",
+                    message: "Publishing file sync events",
+                    data: {
+                        state: this.state,
+                        localFiles: Array.from(localFiles.keys())
+                    }
+                });
                 const changed = [];
                 const deleted = [];
                 await pMap(localFiles, async ([normalizedPath, file])=>{
@@ -379,8 +468,9 @@ class Sync extends BaseCommand {
                     try {
                         changed.push({
                             path: normalizedPath,
+                            oldPath: "oldPath" in file ? file.oldPath : undefined,
                             mode: file.mode,
-                            content: file.isDirectory ? "" : await fs.readFile(this.absolute(normalizedPath), "base64"),
+                            content: file.isDirectory ? "" : await fs.readFile(this.absolute(normalizedPath), FileSyncEncoding.Base64),
                             encoding: FileSyncEncoding.Base64
                         });
                     } catch (error) {
@@ -392,7 +482,7 @@ class Sync extends BaseCommand {
                 if (!changed.length && !deleted.length) {
                     return;
                 }
-                const data = await this.client.queryUnwrap({
+                const { publishFileSyncEvents } = await this.client.queryUnwrap({
                     query: PUBLISH_FILE_SYNC_EVENTS_MUTATION,
                     variables: {
                         input: {
@@ -402,28 +492,52 @@ class Sync extends BaseCommand {
                         }
                     }
                 });
-                this.log(chalkTemplate`Sent {gray ${format(new Date(), "pp")}}`);
-                this.logPaths("→", changed.map((x)=>x.path), deleted.map((x)=>x.path));
-                const { remoteFilesVersion  } = data.publishFileSyncEvents;
-                this.debug("remote files version after publishing %s", remoteFilesVersion);
-                if (BigInt(remoteFilesVersion) > this.state.filesVersion) {
-                    this.debug("updated local files version from %s to %s", this.state.filesVersion, remoteFilesVersion);
-                    this.state.filesVersion = remoteFilesVersion;
+                context.addBreadcrumb({
+                    category: "sync",
+                    message: "Published file sync events",
+                    data: {
+                        state: this.state,
+                        remoteFilesVersion: publishFileSyncEvents.remoteFilesVersion,
+                        changed: _.map(changed, "path"),
+                        deleted: _.map(deleted, "path")
+                    }
+                });
+                if (BigInt(publishFileSyncEvents.remoteFilesVersion) > this.state.filesVersion) {
+                    this.state.filesVersion = publishFileSyncEvents.remoteFilesVersion;
                 }
+                this.log(chalkTemplate`Sent {gray ${format(new Date(), "pp")}}`);
+                this.logPaths("→", _.map(changed, "path"), _.map(deleted, "path"));
             });
         }, this.flags["file-push-delay"]);
-        this.watcher.add(`${this.dir}/**/*`).on("error", (error)=>void this.stop(error)).on("all", (event, absolutePath, stats)=>{
-            const normalizedPath = this.normalize(absolutePath, event == "addDir" || event == "unlinkDir");
-            if (stats?.isSymbolicLink?.()) {
-                this.debug("skipping event caused by symlink %s", normalizedPath);
-                return;
-            }
-            if (absolutePath == this.ignorer.filepath) {
+        this.watcher = new FSWatcher(this.dir, {
+            // paths that we never want to publish
+            ignore: /(\.gadget|\.git|node_modules)/,
+            // don't emit an event for every watched file on boot
+            ignoreInitial: true,
+            renameDetection: true,
+            recursive: true,
+            debounce: this.flags["file-watch-debounce"],
+            pollingInterval: this.flags["file-watch-poll-interval"],
+            pollingTimeout: this.flags["file-watch-poll-timeout"],
+            renameTimeout: this.flags["file-watch-rename-timeout"]
+        });
+        this.watcher.once("error", (error)=>void this.stop(error));
+        this.watcher.on("all", (event, absolutePath, renamedPath)=>{
+            const filepath = event === "rename" || event === "renameDir" ? renamedPath : absolutePath;
+            const isDirectory = event === "renameDir" || event === "addDir" || event === "unlinkDir";
+            const normalizedPath = this.normalize(filepath, isDirectory);
+            if (filepath == this.ignorer.filepath) {
                 this.ignorer.reload();
-            } else if (this.ignorer.ignores(absolutePath)) {
+            } else if (this.ignorer.ignores(filepath)) {
                 this.debug("skipping event caused by ignored file %s", normalizedPath);
                 return;
             }
+            let stats;
+            try {
+                stats = fs.statSync(filepath);
+            } catch (error) {
+                ignoreEnoent(error);
+            }
             // we only update the mtime if the file is not ignored, because if we restart and the mtime is set to an ignored
             // file, then it could be greater than the mtime of all non ignored files and we'll think that local files have
             // changed when only an ignored one has
@@ -434,7 +548,7 @@ class Sync extends BaseCommand {
                 this.debug("skipping event caused by recent write %s", normalizedPath);
                 return;
             }
-            this.debug("file changed %s", normalizedPath, event);
+            this.debug("%s %s", event, normalizedPath);
             switch(event){
                 case "add":
                 case "change":
@@ -458,6 +572,16 @@ class Sync extends BaseCommand {
                         isDirectory: event === "unlinkDir"
                     });
                     break;
+                case "rename":
+                case "renameDir":
+                    assert(stats, "missing stats on rename/renameDir event");
+                    localFilesBuffer.set(normalizedPath, {
+                        oldPath: this.normalize(absolutePath, isDirectory),
+                        newPath: normalizedPath,
+                        isDirectory: event === "renameDir",
+                        mode: stats.mode
+                    });
+                    break;
             }
             this.publish();
         });
@@ -586,20 +710,33 @@ _define_property(Sync, "flags", {
         default: 100,
         hidden: true
     }),
-    "file-stability-threshold": Flags.integer({
-        name: "file-stability-threshold",
-        summary: "Time in milliseconds a file's size must remain the same.",
+    // The following flags are passed to FSWatcher (https://github.com/fabiospampinato/watcher)
+    "file-watch-debounce": Flags.integer({
+        summary: "Amount of milliseconds to debounce file changed events",
         helpGroup: "file",
         helpValue: "ms",
-        default: 500,
+        default: 300,
         hidden: true
     }),
-    "file-poll-interval": Flags.integer({
-        name: "file-poll-interval",
-        description: "Interval in milliseconds between polling a file's size.",
+    "file-watch-poll-interval": Flags.integer({
+        summary: "Polling is used as a last resort measure when watching non-existent paths inside non-existent directories, this controls how often polling is performed, in milliseconds. You can set it to a lower value to make the app detect events much more quickly, but don't set it too low if you are watching many paths that require polling as polling is expensive.",
         helpGroup: "file",
         helpValue: "ms",
-        default: 100,
+        default: 3_000,
+        hidden: true
+    }),
+    "file-watch-poll-timeout": Flags.integer({
+        summary: "Sometimes polling will fail, for example if there are too many file descriptors currently open, usually eventually polling will succeed after a few tries though, this controls the amount of milliseconds the library should keep retrying for.",
+        helpGroup: "file",
+        helpValue: "ms",
+        default: 20_000,
+        hidden: true
+    }),
+    "file-watch-rename-timeout": Flags.integer({
+        summary: "Amount of milliseconds to wait for a potential rename/renameDir event to be detected. The higher this value is the more reliably renames will be detected, but don't set this too high, or the emission of some events could be delayed by that amount. The higher this value is the longer the library will take to emit add/addDir/unlink/unlinkDir events.",
+        helpGroup: "file",
+        helpValue: "ms",
+        default: 1_250,
         hidden: true
     })
 });
@@ -689,6 +826,13 @@ var /**
    * @returns The SyncFile instance.
    */ static load(rootDir) {
         const state = fs.readJsonSync(path.join(rootDir, ".gadget/sync.json"));
+        context.addBreadcrumb({
+            category: "sync",
+            message: "Loaded sync state",
+            data: {
+                state
+            }
+        });
         assert(_.isString(state.app), "missing or invalid app");
         assert(_.isString(state.filesVersion), "missing or invalid filesVersion");
         assert(_.isNumber(state.mtime), "missing or invalid mtime");
@@ -703,6 +847,11 @@ var /**
    */ flush() {
         _class_private_field_get(this, _save).flush();
     }
+    /**
+   * @returns The JSON representation of this instance.
+   */ toJSON() {
+        return this._inner;
+    }
     constructor(_rootDir, inner){
         _define_property(this, "_rootDir", void 0);
         _define_property(this, "_inner", void 0);
@@ -715,6 +864,13 @@ var /**
             fs.outputJSONSync(path.join(this._rootDir, ".gadget/sync.json"), this._inner, {
                 spaces: 2
             });
+            context.addBreadcrumb({
+                category: "sync",
+                message: "Saved sync state",
+                data: {
+                    state: this._inner
+                }
+            });
         }, 100));
         this._inner = inner;
     }