dxfl 0.1.7 → 0.1.8

package/.editorconfig

@@ -0,0 +1,22 @@
+# Stop the editor from looking for .editorconfig files in the parent directories
+root = true
+
+[*]
+indent_style = tab
+# Preferred indentation size is a user preference
+# tab_width = 4
+end_of_line = lf
+insert_final_newline = false
+trim_trailing_whitespace = true
+charset = utf-8
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2
+
+[*.json]
+indent_style = space
+indent_size = 2

package/.prettierignore

@@ -0,0 +1,3 @@
+# Ignore artifacts:
+dist
+node_modules

package/.prettierrc.json

@@ -0,0 +1,3 @@
+{
+  "arrowParens": "avoid"
+}

package/CHANGELOG.md

@@ -0,0 +1,48 @@
+- `deploy`: add support for redirects, configured from a new `deuxfleurs.toml` file.
+  - supports S3 "object redirects" (redirects from a path in the website)
+  - supports global "bucket redirects" (redirects from any path in the website
+    that starts with a specified prefix)
+  - allows specifying a default index page
+  - allows specifying a error page for 404 errors
+- new `empty` command: removes all contents of a website (guichet requires emptying
+  a website before deleting it)
+- `deploy`: instead of unconditionally deploying by default, ask for interactive
+  confirmation after displaying a summary of planned changes. Add a `--yes`
+  option to skip confirmation and automatically apply the changes.
+- `deploy`: improve performance by increasing concurrency of S3 operations
+- `deploy`: drop multipart upload support & simplify the strategy for
+  incremental uploads: simply use ETags now.
+
+# v0.1.7
+
+- internal packaging fixes
+
+# v0.1.6
+
+- improve the output of `deploy`
+  - show a full list of uploaded files, keeping multipart uploads on one line
+  - list deleted files
+  - show human-friendly file sizes instead of number of bytes
+- `deploy`: add a `--dry-run` option to see planned changes without doing anything
+
+# v0.1.5
+
+- make `deploy` incremental: only upload files that have changed.
+  The implementation strategy is by the one used in rclone: store the file md5
+  in a custom metadata header.
+- fix a crash in case of empty ListObject reply
+
+# v0.1.3, v0.1.4
+
+- misc internal tooling changes
+
+# v0.1.1, v0.1.2
+
+- rename the tool to `dxfl`
+
+# v0.1.0
+
+- new `login` command: authenticates to guichet & stores credentials locally
+- new `list` command: list websites of user (after logging in)
+- new `deploy` command: deploys a local directory to a user's website.
+  Also detects the MIME type of files and sets ContentType accordingly.

package/README.md

@@ -10,7 +10,7 @@ npm install -g dxfl
 
 ## Usage
 
-*Not ready*
+_Not ready_
 
 Start by login with your username, for example for `john`:
 
@@ -30,6 +30,56 @@ And then to deploy your `_public` folder on `example.com`:
 dxfl deploy example.com _public
 ```
 
+Or potentially preview the upcoming changes with the `--dry-run` option:
+
+```
+dxfl deploy example.com _public --dry-run
+```
+
+If you need to empty a website from its contents:
+
+```
+dxfl empty example.com
+```
+
+## Website configuration
+
+`dxfl deploy` reads a `deuxfleurs.toml` configuration file (if it exists in the current directory).
+This file can be used to specify website configuration metadata, such as redirections.
+
+Your `deuxfleurs.toml` should follow the following structure:
+
+```toml
+# Filename added after URLs that point to directories.
+# Default value: "index.html"
+index_page = "index.html"
+# Path to the file to serve in case of 404 error
+# Default value: none.
+error_page = "404.html"
+
+# A redirect entry. There can be as many as desired.
+[[redirects]]
+# Source of the redirection.
+from = "old/path"
+# Target of the redirection.
+to = "new/path"
+
+# Redirects can also match multiple paths, by specifying
+# a prefix followed by *
+[[redirects]]
+# Source of the redirection: all paths that match the prefix
+# foobar/
+from = "foobar/*"
+# Target: redirect all foobar/XXX to baz/XXX.
+# The target can also be an URL outside of the website, or a
+# single path.
+to = "baz/*"
+# Optional: custom HTTP status code for the redirection.
+status = 302
+# Optional: only redirect if the original request triggered a 404 error
+if_error = 404
+```
+
 ## Development
 
 ```
@@ -43,6 +93,18 @@ dxfl
 npx dxfl
 ```
 
+### Code formatting
+
+[Prettier](https://prettier.io/) is used to assure a certain consistency in style (and [accessibility](https://adamtuttle.codes/blog/2021/tabs-vs-spaces-its-an-accessibility-issue/)) through the codebase. An [EditorConfig](https://editorconfig.org/) file is also here with a similar goal.
+
+You can format your changes with the dedicated npm command lines:
+
+```bash
+npm run prettier-check # check formatting without making change
+npm run prettier # fix formatting
+npm run prettier-watch # watch upcoming changes to fix
+```
+
 ## Release
 
 First you need an account on npmjs.com and be a maintainer of the `dxfl` package (ask quentin).
@@ -54,6 +116,7 @@ Then to publish a release:
 vim package.json # update the version in this file
 vim index.ts # update the version in this file
 npm install # update the version in the package-lock.json
+npm run prettier # fix potential coding style problem
 git commit -a -m 'set version 0.1.5' # commit your change
 git push # send update
 git tag -m 'v0.1.5' v0.1.5 # create associated tag

package/dist/_empty.js

@@ -0,0 +1,64 @@
+// // TODO: Refacto the codebase global before integrate this feature
+export {};
+// export async function empty(
+// 	vhost: string,
+// 	options: { dryRun: boolean | undefined },
+// ) {
+// 	const conf = await openApiConf();
+// 	// Get website info from guichet (bucket name and keys)
+// 	const api = new WebsiteApi(conf);
+// 	let vhostInfo = await api.getWebsite({ vhost }).catch(err => {
+// 		if (err.response.status == 404) {
+// 			console.error(`Error: website '${vhost}' does not exist`);
+// 		} else {
+// 			console.error(err);
+// 		}
+// 		process.exit(1);
+// 	});
+// 	// List the files currently stored in the bucket
+// 	// @FIXME this info could be returned by the guichet API
+// 	const s3client = new S3Client({
+// 		endpoint: "https://garage.deuxfleurs.fr",
+// 		region: "garage",
+// 		forcePathStyle: true,
+// 		credentials: {
+// 			accessKeyId: vhostInfo.accessKeyId!,
+// 			secretAccessKey: vhostInfo.secretAccessKey!,
+// 		},
+// 	});
+// 	const Bucket = vhostInfo.vhost!.name!;
+// 	const filesToDelete = [...(await getBucketFiles(s3client, Bucket))].map(
+// 		([name, { size }]) => ({
+// 			name,
+// 			size,
+// 		}),
+// 	);
+// 	for (const file of filesToDelete) {
+// 		process.stdout.write(`Deleting ${file.name}\n`);
+// 	}
+// 	// If not in dry-run mode, send the delete command
+// 	if (!options.dryRun) {
+// 		const resp = await deleteFiles(s3client, Bucket, filesToDelete);
+// 		if (resp && resp!.$metadata.httpStatusCode != 200) {
+// 			// TODO: better error handling?
+// 			console.error(resp);
+// 			process.exit(1);
+// 		}
+// 	}
+// 	// Display a summary
+// 	function sum(a: number[]) {
+// 		return a.reduce((x, y) => x + y, 0);
+// 	}
+// 	function formatFiles(n: number) {
+// 		if (n == 1) {
+// 			return `${n} file `;
+// 		} else {
+// 			return `${n} files`;
+// 		}
+// 	}
+// 	const sizeDeleted = sum(filesToDelete.map(f => f.size ?? 0));
+// 	process.stdout.write("\nSummary:\n");
+// 	process.stdout.write(
+// 		`${formatFiles(filesToDelete.length)} deleted (${formatBytes(sizeDeleted)})\n`,
+// 	);
+// }

package/dist/auth.js

@@ -8,9 +8,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { Configuration, WebsiteApi } from "guichet-sdk-ts";
-import { read } from 'read';
-import path from 'node:path';
-import fs from 'node:fs/promises';
+import { read } from "read";
+import path from "node:path";
+import fs from "node:fs/promises";
 function configPath() {
     let path = ".dfl/config.json";
     if (process.env.XDG_CONFIG_HOME) {
@@ -27,7 +27,7 @@ export function openApiConf() {
         let dictConf;
         const configFile = configPath();
         try {
-            strConf = yield fs.readFile(configFile, { encoding: 'utf8' });
+            strConf = yield fs.readFile(configFile, { encoding: "utf8" });
         }
         catch (err) {
             console.error(err, `\n\nUnable to read ${configFile}, run 'dfl login' first.`);
@@ -52,7 +52,7 @@ export function login(username) {
         const password = yield read({
             prompt: "password: ",
             silent: true,
-            replace: "*"
+            replace: "*",
         });
         // check config
         const testConf = new Configuration({
@@ -76,6 +76,6 @@ export function login(username) {
         const serializedConfig = JSON.stringify(configData);
         yield fs.writeFile(configFile, serializedConfig, { mode: 0o600 });
         // @FIXME: we would like to avoid storing the password in clear text in the future.
-        console.log('ok');
+        console.log("ok");
     });
 }

package/dist/bucket.js

@@ -0,0 +1,174 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from "fs";
+import mime from "mime";
+import { DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client, } from "@aws-sdk/client-s3";
+import { PromisePool } from "@supercharge/promise-pool";
+import { parseEtag, toChunks, formatBytesHuman } from "./utils.js";
+export function getBucket(api, vhost) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // Get website info from guichet (bucket name and keys)
+        const bucket = yield api.getWebsite({ vhost }).catch(err => {
+            var _a;
+            if (((_a = err === null || err === void 0 ? void 0 : err.response) === null || _a === void 0 ? void 0 : _a.status) == 404) {
+                console.error(`Error: website '${vhost}' does not exist`);
+            }
+            else {
+                console.error(err);
+            }
+            throw new Error(err);
+        });
+        // Initialize a S3 Client
+        const client = new S3Client({
+            endpoint: "https://garage.deuxfleurs.fr",
+            region: "garage",
+            forcePathStyle: true,
+            credentials: {
+                accessKeyId: bucket.accessKeyId,
+                secretAccessKey: bucket.secretAccessKey,
+            },
+        });
+        return {
+            client: client,
+            name: bucket.vhost.name,
+        };
+    });
+}
+export function getBucketFiles(bucket) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const files = new Map();
+        let done = false;
+        let cmd = new ListObjectsV2Command({ Bucket: bucket.name });
+        while (!done) {
+            const resp = yield bucket.client.send(cmd);
+            if (resp.$metadata.httpStatusCode != 200) {
+                // TODO: better error handling?
+                throw resp;
+            }
+            if (resp.Contents) {
+                for (const item of resp.Contents) {
+                    const etag = item.ETag ? parseEtag(item.ETag) : undefined;
+                    if (item.Key) {
+                        files.set(item.Key, { size: item.Size, etag });
+                    }
+                }
+            }
+            if (resp.NextContinuationToken) {
+                cmd = new ListObjectsV2Command({
+                    Bucket: bucket.name,
+                    ContinuationToken: resp.NextContinuationToken,
+                });
+            }
+            else {
+                done = true;
+            }
+        }
+        return files;
+    });
+}
+export function getBucketFilesDetails(bucket, files) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let res = new Map();
+        function doFile(file) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const resp = yield bucket.client.send(new HeadObjectCommand({ Bucket: bucket.name, Key: file }));
+                if (resp.$metadata.httpStatusCode != 200) {
+                    // TODO: better error handling?
+                    throw resp;
+                }
+                res.set(file, {
+                    redirect: resp.WebsiteRedirectLocation,
+                });
+            });
+        }
+        yield PromisePool.for(files)
+            .withConcurrency(50)
+            .handleError(err => {
+            throw err;
+        })
+            .process(doFile);
+        return res;
+    });
+}
+export function deleteBucketFile(bucket, name) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const resp = yield bucket.client.send(new DeleteObjectCommand({
+            Bucket: bucket.name,
+            Key: name,
+        }));
+        if (resp && resp.$metadata.httpStatusCode != 204) {
+            // TODO: better error handling?
+            throw resp;
+        }
+    });
+}
+export function deleteBucketFiles(bucket, files) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const chunks = toChunks(files, 20);
+        yield PromisePool.for(chunks)
+            .withConcurrency(50)
+            .handleError(err => {
+            throw err;
+        })
+            .process((files) => __awaiter(this, void 0, void 0, function* () {
+            for (const file of files) {
+                process.stdout.write(`  Delete ${file.name} (${file.size ? formatBytesHuman(file.size) : "?B"})\n`);
+            }
+            const resp = yield bucket.client.send(new DeleteObjectsCommand({
+                Bucket: bucket.name,
+                Delete: {
+                    Objects: files.map(f => {
+                        return { Key: f.name };
+                    }),
+                },
+            }));
+            if (resp && resp.$metadata.httpStatusCode != 200) {
+                // TODO: better error handling?
+                throw resp;
+            }
+        }));
+    });
+}
+export function uploadFile(bucket, s3Path, localPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        // use `path.posix` because `Key` is a path in a bucket that uses `/` as separator.
+        let ContentType = (_a = mime.getType(localPath)) !== null && _a !== void 0 ? _a : undefined;
+        // add charset=utf-8 by default on text files (TODO: allow the user to override this)
+        if (ContentType && ContentType.startsWith("text/")) {
+            ContentType = ContentType + "; charset=utf-8";
+        }
+        const Body = fs.createReadStream(localPath);
+        const params = { Bucket: bucket.name, Key: s3Path, Body, ContentType };
+        const resp = yield bucket.client.send(new PutObjectCommand(params));
+        if (resp && resp.$metadata.httpStatusCode != 200) {
+            throw resp;
+        }
+    });
+}
+// the 'source' object may or may not exist already in the bucket.
+// if it exists, its contents will be discarded, otherwise it
+// will be created with empty contents.
+export function putEmptyObjectRedirect(bucket, source, target) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // use a non-empty Body to sidestep
+        // https://github.com/aws/aws-sdk-js-v3/issues/5722
+        const params = {
+            Bucket: bucket.name,
+            Key: source,
+            Body: "source of object redirect",
+            WebsiteRedirectLocation: target,
+        };
+        const resp = yield bucket.client.send(new PutObjectCommand(params));
+        if (resp && resp.$metadata.httpStatusCode != 200) {
+            throw resp;
+        }
+    });
+}

package/dist/config.js

@@ -0,0 +1,185 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from "fs";
+import TOML from "smol-toml";
+import URI from "fast-uri";
+import { z as zod } from "zod";
+import { fromError as zodError } from "zod-validation-error";
+const DXFL_CONFIG = "deuxfleurs.toml";
+// Parsing: TOML -> untyped object
+function readConfigFile() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!fs.existsSync(DXFL_CONFIG)) {
+            return {};
+        }
+        let strConf;
+        let dictConf;
+        try {
+            strConf = yield fs.promises.readFile(DXFL_CONFIG, { encoding: "utf8" });
+        }
+        catch (err) {
+            console.error(err, `\n\nUnable to read ${DXFL_CONFIG}`);
+            process.exit(1);
+        }
+        try {
+            dictConf = TOML.parse(strConf);
+        }
+        catch (err) {
+            console.error(err, `\n\nUnable to parse ${DXFL_CONFIG} as TOML, please check it for syntax errors.`);
+            process.exit(1);
+        }
+        return dictConf;
+    });
+}
+// Parsing: untyped object -> RawConfig
+const RawRedirectSchema = zod.object({
+    from: zod.string(),
+    to: zod.string(),
+    if_error: zod.number().int().optional(),
+    code: zod.number().int().positive().optional(),
+});
+const RawConfigSchema = zod.object({
+    index_page: zod.string().optional(),
+    error_page: zod.string().optional(),
+    redirects: zod.array(RawRedirectSchema),
+});
+function interpRawConfig(cfg) {
+    try {
+        return RawConfigSchema.parse(cfg);
+    }
+    catch (err) {
+        const validationError = zodError(err);
+        throw validationError.toString();
+    }
+}
+// Parsing: RawConfig -> DxflConfig
+function unescape(s) {
+    let out = "";
+    const iter = s[Symbol.iterator]();
+    while (true) {
+        const elt = iter.next();
+        if (elt.done == true) {
+            return { s: out, is_prefix: false };
+        }
+        if (elt.value == "*") {
+            const next = iter.next();
+            if (next.done == true) {
+                return { s: out, is_prefix: true };
+            }
+            else if (next.value == "*") {
+                out += "*";
+            }
+            else {
+                throw `a single '*' is only allowed at the end for wildcards; please use '**' instead`;
+            }
+        }
+        else {
+            out += elt.value;
+        }
+    }
+}
+function interpConfig(rawcfg) {
+    function interpPath(name, str) {
+        try {
+            return unescape(str);
+        }
+        catch (err) {
+            throw `${name}: ${err}`;
+        }
+    }
+    function interpRedirect(i, r) {
+        const rfrom = interpPath(`redirect ${i}, from`, r.from);
+        const rto = interpPath(`redirect ${i}, to`, r.to);
+        let redirect;
+        if (rfrom.is_prefix) {
+            // This is a bucket redirect
+            const toURI = URI.parse(rto.s);
+            if (toURI.error) {
+                throw `redirect ${i}, 'to': ${toURI.error}`;
+            }
+            if (toURI.scheme && toURI.scheme != "http" && toURI.scheme != "https") {
+                throw `redirect ${i}, 'to': unsupported URI scheme ${toURI.scheme}; http or https required`;
+            }
+            if (toURI.port) {
+                throw `redirect ${i}, 'to': specifying a port is not supported`;
+            }
+            if (toURI.query) {
+                throw `redirect ${i}, 'to': specifying a query is not supported`;
+            }
+            let bredirect = {
+                prefix: rfrom.s,
+                if_error: undefined,
+                hostname: toURI.host,
+                protocol: toURI.scheme,
+                code: undefined,
+                to: rto.is_prefix
+                    ? { kind: "replace_prefix", prefix: rto.s }
+                    : { kind: "replace", target: rto.s },
+            };
+            if (r.if_error) {
+                if (r.if_error != 404) {
+                    throw `redirect ${i}: the only currently supportetd value for 'if_error' is 404`;
+                }
+                bredirect.if_error = 404;
+            }
+            if (r.code) {
+                if ([301, 302, 303, 307, 308].includes(r.code)) {
+                    bredirect.code = r.code;
+                }
+                else if ([200, 404].includes(r.code)) {
+                    if (toURI.host || toURI.scheme) {
+                        throw `request ${i}: a code of 200 or 404 is not accepted while specifying a protocol and hostname in the destination`;
+                    }
+                    bredirect.code = r.code;
+                }
+                else {
+                    throw `redirect ${i}: unsupported error code ${r.code}`;
+                }
+            }
+            redirect = { kind: "bucket", r: bredirect };
+        }
+        else {
+            // This is an object redirect
+            if (r.if_error) {
+                throw `redirect ${i}: 'if_error' is not supported for single redirections`;
+            }
+            if (r.code) {
+                throw `redirect ${i}: 'code' is not supported for single redirections`;
+            }
+            if (rto.is_prefix) {
+                throw `redirect ${i}: cannot use a prefix target in 'to' when 'from' is a single address`;
+            }
+            // pass the 'to' field as-is
+            const oredirect = { from: rfrom.s, to: rto.s };
+            redirect = { kind: "object", r: oredirect };
+        }
+        return redirect;
+    }
+    let cfg = {
+        index_page: rawcfg.index_page,
+        error_page: rawcfg.error_page,
+        redirects: [],
+    };
+    for (const [i, r] of rawcfg.redirects.entries()) {
+        cfg.redirects.push(interpRedirect(i, r));
+    }
+    return cfg;
+}
+export function readConfig() {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            return interpConfig(interpRawConfig(yield readConfigFile()));
+        }
+        catch (err) {
+            console.error(`Error while reading ${DXFL_CONFIG}: ${err}`);
+            process.exit(1);
+        }
+    });
+}