dxfl 0.1.4 → 0.1.5

package/README.md

@@ -4,18 +4,10 @@ A CLI tool to manage your Deuxfleurs account.
 
 ## Install
 
-From NPM:
-
 ```
 npm install -g dxfl
 ```
 
-From git:
-
-```
-npm install -g git+https://git.deuxfleurs.fr/Deuxfleurs/dxfl
-```
-
 ## Usage
 
 *Not ready*
@@ -44,7 +36,24 @@ dxfl deploy example.com _public
 git clone https://git.deuxfleurs.fr/Deuxfleurs/dxfl
 cd dxfl
 npm install
-node bootstrap.js
+npm link
+dxfl
+```
+
+## Release
+
+First you need an account on npmjs.com and be a maintainer of the `dxfl` package (ask quentin).
+Do not forget also to run `npm login` to bind your account with the CLI.
+
+Then to publish a release:
+
+```bash
+vim package.json # update the version in this file
+git commit -a -m 'set version 0.1.5' # commit your change
+git push # send update
+git tag -m 'v0.1.5' v0.1.5 # create associated tag
+git push --tags # push tag
+npm publish # build and push the package
 ```
 
 ## License

package/dist/deploy.js

@@ -7,14 +7,23 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 import fs from "fs";
 import path from "path";
+import crypto from "crypto";
 import mime from "mime";
 import { WebsiteApi } from "guichet-sdk-ts";
-import { S3Client, ListObjectsV2Command, DeleteObjectsCommand, } from "@aws-sdk/client-s3";
+import { S3Client, ListObjectsV2Command, DeleteObjectsCommand, HeadObjectCommand, } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { PromisePool } from "@supercharge/promise-pool";
 import { openApiConf } from "./auth.js";
+const MD5METAFIELD = "dfl-md5sum";
 // Walks through the local directory at path `dir`, and for each file it contains, returns :
 // - `localPath`: its path on the local filesystem (includes `dir`). On windows, this path
 //    will typically use `\` as separator.
@@ -37,9 +46,31 @@ function getLocalFiles(dir, s3Prefix) {
         return files.flat();
     });
 }
+function getFileMd5(file) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a, e_1, _b, _c;
+        const hash = crypto.createHash('md5');
+        try {
+            for (var _d = true, _e = __asyncValues(fs.createReadStream(file)), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
+                _c = _f.value;
+                _d = false;
+                const chunk = _c;
+                hash.update(chunk);
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return hash.digest('hex');
+    });
+}
 function getBucketFiles(client, Bucket) {
     return __awaiter(this, void 0, void 0, function* () {
-        const files = [];
+        const files = new Map();
         let done = false;
         let cmd = new ListObjectsV2Command({ Bucket });
         while (!done) {
@@ -49,8 +80,12 @@ function getBucketFiles(client, Bucket) {
                 console.error(resp);
                 process.exit(1);
             }
-            for (var item of resp.Contents) {
-                files.push(item.Key);
+            if (resp.Contents) {
+                for (const item of resp.Contents) {
+                    if (item.Key) {
+                        files.set(item.Key, { size: item.Size });
+                    }
+                }
             }
             if (resp.NextContinuationToken) {
                 cmd = new ListObjectsV2Command({
@@ -65,7 +100,7 @@ function getBucketFiles(client, Bucket) {
         return files;
     });
 }
-function uploadFile(client, Bucket, Key, Body) {
+function uploadFile(client, Bucket, Key, Body, md5) {
     return __awaiter(this, void 0, void 0, function* () {
         var _a;
         // use `path.posix` because `Key` is a path in a bucket that uses `/` as separator.
@@ -74,13 +109,17 @@ function uploadFile(client, Bucket, Key, Body) {
         if (ContentType && ContentType.startsWith("text/")) {
             ContentType = ContentType + "; charset=utf-8";
         }
-        const parallelUpload = new Upload({ client, params: { Bucket, Key, Body, ContentType } });
+        // store the md5 checksum in the object metadata; it will be used to skip
+        // subsequent uploads if the file has not changed.
+        const Metadata = { [MD5METAFIELD]: md5 };
+        const params = { Bucket, Key, Body, ContentType, Metadata };
+        const parallelUpload = new Upload({ client, params });
         parallelUpload.on("httpUploadProgress", progress => {
             process.stdout.moveCursor(0, -1);
             process.stdout.clearLine(1);
-            process.stdout.write("Sending " + progress.Key);
+            process.stdout.write(`Sending ${progress.Key}`);
             if (!(progress.loaded == progress.total && progress.part == 1)) {
-                process.stdout.write(" (" + progress.loaded + "/" + progress.total + ")");
+                process.stdout.write(` (${progress.loaded}/${progress.total})`);
             }
             process.stdout.write("\n");
         });
@@ -100,6 +139,34 @@ function deleteFiles(client, Bucket, files) {
         }));
     });
 }
+// Checks whether a remote file needs to be updated by its local copy.
+//
+// We first check whether files differ, and if not compare the md5 checksum we
+// previously stored in the object metadata (if it exists) with the local file's
+// md5 checksum.
+function needsUpdate(client, localFile, localMd5, Bucket, Key, remoteSize) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (remoteSize) {
+            const localSize = (yield fs.promises.stat(localFile)).size;
+            if (localSize == 0 /* stat can return 0 in case of error */
+                || localSize != remoteSize) {
+                return true;
+            }
+        }
+        // fetch metadata for the object and see if we previously stored its md5
+        const resp = yield client.send(new HeadObjectCommand({ Bucket, Key }));
+        if (resp.$metadata.httpStatusCode != 200) {
+            // TODO: better error handling?
+            throw resp;
+        }
+        const remoteMd5 = resp.Metadata ? resp.Metadata[MD5METAFIELD] : null;
+        if (!remoteMd5) {
+            return true;
+        }
+        // we have a remote md5, compare it with the local one
+        return (localMd5 != remoteMd5);
+    });
+}
 export function deploy(vhost, localFolder) {
     return __awaiter(this, void 0, void 0, function* () {
         const conf = yield openApiConf();
@@ -140,16 +207,30 @@ export function deploy(vhost, localFolder) {
         // Delete files that are present in the bucket but not locally.
         // Do this before sending the new files to avoid hitting the size quota
         // unnecessarily.
-        const resp = yield deleteFiles(s3client, Bucket, remoteFiles.filter(f => !localFiles.find(({ s3Path }) => s3Path == f)));
+        const resp = yield deleteFiles(s3client, Bucket, [...remoteFiles]
+            .filter(([name, _]) => !localFiles.find(({ s3Path }) => s3Path == name))
+            .map(([name, _]) => name));
         if (resp && resp.$metadata.httpStatusCode != 200) {
             // TODO: better error handling?
             console.error(resp);
             process.exit(1);
         }
-        // Control concurrence while uploading
+        // Uploads a local file unless the remote copy is the same
+        function processFile(localPath, s3Path) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const localMd5 = yield getFileMd5(localPath);
+                const remoteFile = remoteFiles.get(s3Path);
+                if (!remoteFile ||
+                    (yield needsUpdate(s3client, localPath, localMd5, Bucket, s3Path, remoteFile.size))) {
+                    uploadFile(s3client, Bucket, s3Path, fs.createReadStream(localPath), localMd5);
+                }
+            });
+        }
+        ;
+        // Control concurrency while uploading
         yield PromisePool
             .for(localFiles)
             .withConcurrency(6)
-            .process(({ localPath, s3Path }) => uploadFile(s3client, Bucket, s3Path, fs.createReadStream(localPath)));
+            .process(({ localPath, s3Path }) => processFile(localPath, s3Path));
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dxfl",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "",
   "license": "EUPL-1.2",
   "author": "Deuxfleurs Team <coucou@deuxfleurs.fr>",