dxfl 0.1.4 → 0.1.6

package/README.md

@@ -4,18 +4,10 @@ A CLI tool to manage your Deuxfleurs account.
 
 ## Install
 
-From NPM:
-
 ```
 npm install -g dxfl
 ```
 
-From git:
-
-```
-npm install -g git+https://git.deuxfleurs.fr/Deuxfleurs/dxfl
-```
-
 ## Usage
 
 *Not ready*
@@ -44,7 +36,27 @@ dxfl deploy example.com _public
 git clone https://git.deuxfleurs.fr/Deuxfleurs/dxfl
 cd dxfl
 npm install
-node bootstrap.js
+# build and install the tool as the `dxfl` command
+npm link
+dxfl
+# alternatively, run the tool from the sources directly
+npx dxfl
+```
+
+## Release
+
+First you need an account on npmjs.com and be a maintainer of the `dxfl` package (ask quentin).
+Do not forget also to run `npm login` to bind your account with the CLI.
+
+Then to publish a release:
+
+```bash
+vim package.json # update the version in this file
+git commit -a -m 'set version 0.1.5' # commit your change
+git push # send update
+git tag -m 'v0.1.5' v0.1.5 # create associated tag
+git push --tags # push tag
+npm publish # build and push the package
 ```
 
 ## License

package/dist/deploy.js

@@ -7,14 +7,23 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 import fs from "fs";
 import path from "path";
+import crypto from "crypto";
 import mime from "mime";
 import { WebsiteApi } from "guichet-sdk-ts";
-import { S3Client, ListObjectsV2Command, DeleteObjectsCommand, } from "@aws-sdk/client-s3";
+import { S3Client, ListObjectsV2Command, DeleteObjectsCommand, HeadObjectCommand, } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { PromisePool } from "@supercharge/promise-pool";
 import { openApiConf } from "./auth.js";
+const MD5METAFIELD = "dfl-md5sum";
 // Walks through the local directory at path `dir`, and for each file it contains, returns :
 // - `localPath`: its path on the local filesystem (includes `dir`). On windows, this path
 //    will typically use `\` as separator.
@@ -37,9 +46,31 @@ function getLocalFiles(dir, s3Prefix) {
         return files.flat();
     });
 }
+function getFileMd5(file) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a, e_1, _b, _c;
+        const hash = crypto.createHash('md5');
+        try {
+            for (var _d = true, _e = __asyncValues(fs.createReadStream(file)), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                _c = _f.value;
+                _d = false;
+                const chunk = _c;
+                hash.update(chunk);
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return hash.digest('hex');
+    });
+}
 function getBucketFiles(client, Bucket) {
     return __awaiter(this, void 0, void 0, function* () {
-        const files = [];
+        const files = new Map();
         let done = false;
         let cmd = new ListObjectsV2Command({ Bucket });
         while (!done) {
@@ -49,8 +80,12 @@ function getBucketFiles(client, Bucket) {
                 console.error(resp);
                 process.exit(1);
             }
-            for (var item of resp.Contents) {
-                files.push(item.Key);
+            if (resp.Contents) {
+                for (const item of resp.Contents) {
+                    if (item.Key) {
+                        files.set(item.Key, { size: item.Size });
+                    }
+                }
             }
             if (resp.NextContinuationToken) {
                 cmd = new ListObjectsV2Command({
@@ -65,7 +100,22 @@ function getBucketFiles(client, Bucket) {
         return files;
     });
 }
-function uploadFile(client, Bucket, Key, Body) {
+function formatBytes(bytes) {
+    if (bytes < 1000) {
+        return `${bytes}B`;
+    }
+    bytes /= 1000;
+    if (bytes < 1000) {
+        return `${bytes.toFixed(2)}KB`;
+    }
+    bytes /= 1000;
+    if (bytes < 1000) {
+        return `${bytes.toFixed(2)}MB`;
+    }
+    bytes /= 1000;
+    return `${bytes.toFixed(2)}GB`;
+}
+function uploadFile(client, Bucket, Key, Body, md5, display) {
     return __awaiter(this, void 0, void 0, function* () {
         var _a;
         // use `path.posix` because `Key` is a path in a bucket that uses `/` as separator.
@@ -74,15 +124,35 @@ function uploadFile(client, Bucket, Key, Body) {
         if (ContentType && ContentType.startsWith("text/")) {
             ContentType = ContentType + "; charset=utf-8";
         }
-        const parallelUpload = new Upload({ client, params: { Bucket, Key, Body, ContentType } });
+        // store the md5 checksum in the object metadata; it will be used to skip
+        // subsequent uploads if the file has not changed.
+        const Metadata = { [MD5METAFIELD]: md5 };
+        const params = { Bucket, Key, Body, ContentType, Metadata };
+        const parallelUpload = new Upload({ client, params });
         parallelUpload.on("httpUploadProgress", progress => {
-            process.stdout.moveCursor(0, -1);
-            process.stdout.clearLine(1);
-            process.stdout.write("Sending " + progress.Key);
-            if (!(progress.loaded == progress.total && progress.part == 1)) {
-                process.stdout.write(" (" + progress.loaded + "/" + progress.total + ")");
+            function writeLine() {
+                process.stdout.write(`Sending ${progress.Key}`);
+                if (progress.loaded && progress.total &&
+                    !(progress.loaded == progress.total && progress.part == 1)) {
+                    process.stdout.write(` (${formatBytes(progress.loaded)}/${formatBytes(progress.total)})`);
+                }
+                process.stdout.write("\n");
+            }
+            ;
+            if (progress.Key) {
+                if (display.lines.has(progress.Key)) {
+                    const offset = display.nblines - display.lines.get(progress.Key);
+                    process.stdout.moveCursor(0, -offset);
+                    process.stdout.clearLine(1);
+                    writeLine();
+                    process.stdout.moveCursor(0, offset - 1);
+                }
+                else {
+                    display.lines.set(progress.Key, display.nblines);
+                    display.nblines++;
+                    writeLine();
+                }
             }
-            process.stdout.write("\n");
         });
         yield parallelUpload.done();
     });
@@ -95,16 +165,44 @@ function deleteFiles(client, Bucket, files) {
         return yield client.send(new DeleteObjectsCommand({
             Bucket,
             Delete: {
-                Objects: files.map(f => { return { Key: f }; }),
+                Objects: files.map(f => { return { Key: f.name }; }),
             },
         }));
     });
 }
-export function deploy(vhost, localFolder) {
+// Checks whether a remote file needs to be updated by its local copy.
+//
+// We first check whether files differ, and if not compare the md5 checksum we
+// previously stored in the object metadata (if it exists) with the local file's
+// md5 checksum.
+function needsUpdate(client, localFile, localMd5, Bucket, Key, remoteSize) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (remoteSize) {
+            const localSize = (yield fs.promises.stat(localFile)).size;
+            if (localSize == 0 /* stat can return 0 in case of error */
+                || localSize != remoteSize) {
+                return true;
+            }
+        }
+        // fetch metadata for the object and see if we previously stored its md5
+        const resp = yield client.send(new HeadObjectCommand({ Bucket, Key }));
+        if (resp.$metadata.httpStatusCode != 200) {
+            // TODO: better error handling?
+            throw resp;
+        }
+        const remoteMd5 = resp.Metadata ? resp.Metadata[MD5METAFIELD] : null;
+        if (!remoteMd5) {
+            return true;
+        }
+        // we have a remote md5, compare it with the local one
+        return (localMd5 != remoteMd5);
+    });
+}
+export function deploy(vhost, localFolder, options) {
     return __awaiter(this, void 0, void 0, function* () {
         const conf = yield openApiConf();
         // Get paths of the local files to deploy
-        const localFiles = yield getLocalFiles(localFolder, "").catch(err => {
+        const localPaths = yield getLocalFiles(localFolder, "").catch(err => {
             if (err.errno = -2) {
                 console.error(`Error: directory '${localFolder}' does not exist`);
             }
@@ -113,6 +211,10 @@ export function deploy(vhost, localFolder) {
             }
             process.exit(1);
         });
+        // Compute local file sizes
+        const localFiles = yield Promise.all(localPaths.map((_a) => __awaiter(this, [_a], void 0, function* ({ localPath, s3Path }) {
+            return { localPath, s3Path, size: (yield fs.promises.stat(localPath)).size };
+        })));
         // Get website info from guichet (bucket name and keys)
         const api = new WebsiteApi(conf);
         let vhostInfo = yield api.getWebsite({ vhost }).catch(err => {
@@ -140,16 +242,69 @@ export function deploy(vhost, localFolder) {
         // Delete files that are present in the bucket but not locally.
         // Do this before sending the new files to avoid hitting the size quota
         // unnecessarily.
-        const resp = yield deleteFiles(s3client, Bucket, remoteFiles.filter(f => !localFiles.find(({ s3Path }) => s3Path == f)));
-        if (resp && resp.$metadata.httpStatusCode != 200) {
-            // TODO: better error handling?
-            console.error(resp);
-            process.exit(1);
+        const filesToDelete = [...remoteFiles]
+            .filter(([name, _]) => !localFiles.find(({ s3Path }) => s3Path == name))
+            .map(([name, { size }]) => ({ name, size }));
+        for (const file of filesToDelete) {
+            process.stdout.write(`Deleting ${file.name}\n`);
         }
-        // Control concurrence while uploading
+        // If not in dry-run mode, send the delete command
+        if (!options.dryRun) {
+            const resp = yield deleteFiles(s3client, Bucket, filesToDelete);
+            if (resp && resp.$metadata.httpStatusCode != 200) {
+                // TODO: better error handling?
+                console.error(resp);
+                process.exit(1);
+            }
+        }
+        // Upload files.
+        let displaystate = {
+            lines: new Map(),
+            nblines: 0
+        };
+        let filesSent = [];
+        function processFile(f) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const remoteFile = remoteFiles.get(f.s3Path);
+                const localMd5 = yield getFileMd5(f.localPath);
+                if (!remoteFile ||
+                    (yield needsUpdate(s3client, f.localPath, localMd5, Bucket, f.s3Path, remoteFile.size))) {
+                    filesSent.push({ s3Path: f.s3Path, size: f.size });
+                    if (options.dryRun) {
+                        process.stdout.write(`Sending ${f.s3Path}\n`);
+                    }
+                    else {
+                        yield uploadFile(s3client, Bucket, f.s3Path, fs.createReadStream(f.localPath), localMd5, displaystate);
+                    }
+                }
+            });
+        }
+        ;
+        // Control concurrency while uploading
         yield PromisePool
             .for(localFiles)
             .withConcurrency(6)
-            .process(({ localPath, s3Path }) => uploadFile(s3client, Bucket, s3Path, fs.createReadStream(localPath)));
+            .process(processFile);
+        // Display a summary
+        function sum(a) { return a.reduce(((x, y) => x + y), 0); }
+        function formatFiles(n) {
+            if (n == 1) {
+                return `${n} file `;
+            }
+            else {
+                return `${n} files`;
+            }
+        }
+        const sizeSent = sum(filesSent.map((f) => f.size));
+        const sizeDeleted = sum(filesToDelete.map((f) => { var _a; return (_a = f.size) !== null && _a !== void 0 ? _a : 0; }));
+        let filesUnchanged = new Map(localFiles.map((f) => [f.s3Path, f.size]));
+        for (const f of filesSent) {
+            filesUnchanged.delete(f.s3Path);
+        }
+        const sizeUnchanged = sum([...filesUnchanged.values()]);
+        process.stdout.write("\nSummary:\n");
+        process.stdout.write(`${formatFiles(filesSent.length)} uploaded (${formatBytes(sizeSent)})\n`);
+        process.stdout.write(`${formatFiles(filesToDelete.length)} deleted (${formatBytes(sizeDeleted)})\n`);
+        process.stdout.write(`${formatFiles(filesUnchanged.size)} unchanged (${formatBytes(sizeUnchanged)})\n`);
     });
 }

package/dist/index.js

@@ -18,5 +18,6 @@ program.command('deploy')
     .description('Deploy your website')
     .argument('<vhost>', 'selected vhost')
     .argument('<local_folder>', 'your local folder')
+    .option('-n, --dry-run', 'do a trial run without making actual changes')
     .action(deploy);
 program.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dxfl",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "",
   "license": "EUPL-1.2",
   "author": "Deuxfleurs Team <coucou@deuxfleurs.fr>",