dxfl 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +22 -0
- package/.prettierignore +3 -0
- package/.prettierrc.json +3 -0
- package/CHANGELOG.md +48 -0
- package/README.md +64 -1
- package/dist/_empty.js +64 -0
- package/dist/auth.js +6 -6
- package/dist/bucket.js +174 -0
- package/dist/config.js +185 -0
- package/dist/deploy.js +300 -241
- package/dist/empty.js +73 -0
- package/dist/index.js +23 -14
- package/dist/utils.js +113 -0
- package/dist/website_config.js +410 -0
- package/package.json +15 -4
- package/tsconfig.json +13 -14
package/dist/deploy.js
CHANGED
|
@@ -7,28 +7,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
-
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
11
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
12
|
-
var m = o[Symbol.asyncIterator], i;
|
|
13
|
-
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
14
|
-
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
15
|
-
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
16
|
-
};
|
|
17
10
|
import fs from "fs";
|
|
18
11
|
import path from "path";
|
|
19
|
-
import crypto from "crypto";
|
|
20
|
-
import mime from "mime";
|
|
21
12
|
import { WebsiteApi } from "guichet-sdk-ts";
|
|
22
|
-
import { S3Client, ListObjectsV2Command, DeleteObjectsCommand, HeadObjectCommand, } from "@aws-sdk/client-s3";
|
|
23
|
-
import { Upload } from "@aws-sdk/lib-storage";
|
|
24
13
|
import { PromisePool } from "@supercharge/promise-pool";
|
|
25
14
|
import { openApiConf } from "./auth.js";
|
|
26
|
-
|
|
15
|
+
import { deleteBucketFile, deleteBucketFiles, getBucket, getBucketFiles, putEmptyObjectRedirect, uploadFile, } from "./bucket.js";
|
|
16
|
+
import { confirmationPrompt, filterMap, formatBytesHuman, formatCount, getFileMd5, sum, } from "./utils.js";
|
|
17
|
+
import { equalBucketRedirect, getBucketConfig, putBucketWebsiteConfig, readConfigFile, } from "./website_config.js";
|
|
27
18
|
// Walks through the local directory at path `dir`, and for each file it contains, returns :
|
|
28
19
|
// - `localPath`: its path on the local filesystem (includes `dir`). On windows, this path
|
|
29
20
|
// will typically use `\` as separator.
|
|
30
21
|
// - `s3Path`: an equivalent path as we would store it in an S3 bucket, using '/' as separator.
|
|
31
|
-
// This path includes `s3Prefix` as a prefix if provided. If `s3Prefix` is
|
|
22
|
+
// This path includes `s3Prefix` as a prefix if provided. If `s3Prefix` is undefined, `s3Path`
|
|
32
23
|
// is relative to the root (of the form "a/b/c", instead of "/a/b/c" if `s3Prefix` is "").
|
|
33
24
|
function getLocalFiles(dir, s3Prefix) {
|
|
34
25
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -46,265 +37,333 @@ function getLocalFiles(dir, s3Prefix) {
|
|
|
46
37
|
return files.flat();
|
|
47
38
|
});
|
|
48
39
|
}
|
|
49
|
-
function
|
|
40
|
+
function getLocalFilesWithInfo(localFolder) {
|
|
50
41
|
return __awaiter(this, void 0, void 0, function* () {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
for (var _d = true, _e = __asyncValues(fs.createReadStream(file)), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
55
|
-
_c = _f.value;
|
|
56
|
-
_d = false;
|
|
57
|
-
const chunk = _c;
|
|
58
|
-
hash.update(chunk);
|
|
42
|
+
const localPaths = yield getLocalFiles(localFolder).catch(err => {
|
|
43
|
+
if (err.errno == -2) {
|
|
44
|
+
throw `directory '${localFolder}' does not exist`;
|
|
59
45
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
finally {
|
|
63
|
-
try {
|
|
64
|
-
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
46
|
+
else {
|
|
47
|
+
throw err;
|
|
65
48
|
}
|
|
66
|
-
|
|
67
|
-
}
|
|
68
|
-
|
|
49
|
+
});
|
|
50
|
+
return yield Promise.all(localPaths.map((_a) => __awaiter(this, [_a], void 0, function* ({ localPath, s3Path }) {
|
|
51
|
+
const [stat, localMd5] = yield Promise.all([
|
|
52
|
+
fs.promises.stat(localPath),
|
|
53
|
+
getFileMd5(localPath),
|
|
54
|
+
]);
|
|
55
|
+
return {
|
|
56
|
+
localPath,
|
|
57
|
+
s3Path,
|
|
58
|
+
size: stat.size,
|
|
59
|
+
md5: localMd5,
|
|
60
|
+
};
|
|
61
|
+
})));
|
|
69
62
|
});
|
|
70
63
|
}
|
|
71
|
-
function
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
64
|
+
function computeDeployPlan(localFiles, remoteFiles, localCfg, remoteCfg) {
|
|
65
|
+
// We raise an error if a file is both present locally and is the source of a
|
|
66
|
+
// redirection.
|
|
67
|
+
// This is to avoid any ambiguous behavior in dxfl: since the contents of the source of
|
|
68
|
+
// a redirection do not matter, we ask the user to clarify by deleting either the file
|
|
69
|
+
// or the redirection.
|
|
70
|
+
const localRedirectSources = filterMap([...localFiles], f => {
|
|
71
|
+
const target = localCfg.object_redirects.get(f.s3Path);
|
|
72
|
+
const ret = target
|
|
73
|
+
? [f, target]
|
|
74
|
+
: undefined;
|
|
75
|
+
return ret;
|
|
76
|
+
});
|
|
77
|
+
if (localRedirectSources.length > 0) {
|
|
78
|
+
console.error("Error: the following local files are also the source of a redirection:");
|
|
79
|
+
for (const [f, target] of localRedirectSources) {
|
|
80
|
+
console.error(` ${f.localPath} (redirect to: ${target})`);
|
|
81
|
+
}
|
|
82
|
+
console.error("\nIt does not make sense for a local file to also be the source of a redirection,\n" +
|
|
83
|
+
"because its contents will never be read.");
|
|
84
|
+
console.error("Please delete these files or the corresponding redirections.");
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
87
|
+
// Compute object redirects that need to be set because the redirection is new,
|
|
88
|
+
// the target was changed, or because the source object changed.
|
|
89
|
+
// (See `redirectsToDelete` below for the "delete" case.)
|
|
90
|
+
const redirectsToApply = filterMap([...localCfg.object_redirects], ([source, target]) => {
|
|
91
|
+
let prev_target = remoteCfg.object_redirects.get(source);
|
|
92
|
+
if (prev_target === undefined) {
|
|
93
|
+
return { action: "add", source, target };
|
|
94
|
+
}
|
|
95
|
+
else if (prev_target != target) {
|
|
96
|
+
return { action: "update", source, target, prev_target };
|
|
99
97
|
}
|
|
100
|
-
return files;
|
|
101
98
|
});
|
|
99
|
+
// Compute object redirects that need to be removed
|
|
100
|
+
const redirectsToDelete = filterMap([...remoteCfg.object_redirects], ([source, target]) => {
|
|
101
|
+
if (localCfg.object_redirects.get(source) == undefined) {
|
|
102
|
+
return { action: "delete", source, prev_target: target };
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
const redirects = redirectsToApply.concat(redirectsToDelete);
|
|
106
|
+
// Compute files to delete: files that are present in the bucket but not locally,
|
|
107
|
+
// and were not the source of a redirection.
|
|
108
|
+
// NOTE: the code to apply redirect changes takes care of deleting redirect sources
|
|
109
|
+
// when needed; `filesToDelete` is only about user-managed files. Thus, we do not
|
|
110
|
+
// delete files that were source of a redirection here.
|
|
111
|
+
// NOTE: we delete (and show it to the user) files that were a user file but become
|
|
112
|
+
// the source of a redirection; this reflects the fact that their contents is deleted.
|
|
113
|
+
const filesToDelete = [...remoteFiles]
|
|
114
|
+
.filter(([name, _]) => !localFiles.find(({ s3Path }) => s3Path == name) &&
|
|
115
|
+
!remoteCfg.object_redirects.has(name))
|
|
116
|
+
.map(([name, { size }]) => ({ name, size }));
|
|
117
|
+
// Compute files to upload, i.e files that either:
|
|
118
|
+
// - are missing on the remote
|
|
119
|
+
// - have a md5 that differs from their remote ETag,
|
|
120
|
+
// - have a local size that differs from its remote size
|
|
121
|
+
let filesToUpload = localFiles.filter(({ s3Path, size, md5 }) => {
|
|
122
|
+
const remoteFile = remoteFiles.get(s3Path);
|
|
123
|
+
return !remoteFile || remoteFile.etag != md5 || remoteFile.size != size;
|
|
124
|
+
});
|
|
125
|
+
return {
|
|
126
|
+
localFiles,
|
|
127
|
+
remoteFiles,
|
|
128
|
+
filesToDelete,
|
|
129
|
+
filesToUpload,
|
|
130
|
+
redirects,
|
|
131
|
+
index_page: { from: remoteCfg.index_page, to: localCfg.index_page },
|
|
132
|
+
error_page: { from: remoteCfg.error_page, to: localCfg.error_page },
|
|
133
|
+
bucket_redirects: {
|
|
134
|
+
from: remoteCfg.bucket_redirects,
|
|
135
|
+
to: localCfg.bucket_redirects,
|
|
136
|
+
},
|
|
137
|
+
};
|
|
102
138
|
}
|
|
103
|
-
function
|
|
104
|
-
|
|
105
|
-
|
|
139
|
+
function diffBucketRedirects(from, to) {
|
|
140
|
+
// best-effort diff of bucket redirects
|
|
141
|
+
const redirects_from = new Map(from.map(r => [r.prefix, r]));
|
|
142
|
+
const redirects_to = new Map(to.map(r => [r.prefix, r]));
|
|
143
|
+
const added = new Map(redirects_to);
|
|
144
|
+
for (const [p, _] of redirects_from) {
|
|
145
|
+
added.delete(p);
|
|
106
146
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
147
|
+
const updated = new Map();
|
|
148
|
+
for (const [p, r] of redirects_from) {
|
|
149
|
+
const r2 = redirects_to.get(p);
|
|
150
|
+
if (r2 && !equalBucketRedirect(r, r2)) {
|
|
151
|
+
updated.set(p, [r, r2]);
|
|
152
|
+
}
|
|
110
153
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
154
|
+
const deleted = new Map(redirects_from);
|
|
155
|
+
for (const [p, _] of redirects_to) {
|
|
156
|
+
deleted.delete(p);
|
|
114
157
|
}
|
|
115
|
-
|
|
116
|
-
return `${bytes.toFixed(2)}GB`;
|
|
158
|
+
return { added, updated, deleted };
|
|
117
159
|
}
|
|
118
|
-
function
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
process.stdout.write(`
|
|
135
|
-
if (progress.loaded && progress.total &&
|
|
136
|
-
!(progress.loaded == progress.total && progress.part == 1)) {
|
|
137
|
-
process.stdout.write(` (${formatBytes(progress.loaded)}/${formatBytes(progress.total)})`);
|
|
138
|
-
}
|
|
139
|
-
process.stdout.write("\n");
|
|
140
|
-
}
|
|
141
|
-
;
|
|
142
|
-
if (progress.Key) {
|
|
143
|
-
if (display.lines.has(progress.Key)) {
|
|
144
|
-
const offset = display.nblines - display.lines.get(progress.Key);
|
|
145
|
-
process.stdout.moveCursor(0, -offset);
|
|
146
|
-
process.stdout.clearLine(1);
|
|
147
|
-
writeLine();
|
|
148
|
-
process.stdout.moveCursor(0, offset - 1);
|
|
149
|
-
}
|
|
150
|
-
else {
|
|
151
|
-
display.lines.set(progress.Key, display.nblines);
|
|
152
|
-
display.nblines++;
|
|
153
|
-
writeLine();
|
|
154
|
-
}
|
|
160
|
+
function printPlan(plan, details) {
|
|
161
|
+
function showBucketRedirectTarget(r) {
|
|
162
|
+
const to = r.to.kind == "replace" ? `${r.to.target}` : `${r.to.prefix}*`;
|
|
163
|
+
const proto = r.protocol ? `${r.protocol}://` : "";
|
|
164
|
+
const hostname = r.hostname ? `${r.hostname}/` : "";
|
|
165
|
+
const status = r.status ? ` status=${r.status}` : "";
|
|
166
|
+
const if_error = r.if_error ? ` if_error=${r.if_error}` : "";
|
|
167
|
+
return `${proto}${hostname}${to}${status}${if_error}`;
|
|
168
|
+
}
|
|
169
|
+
function showBucketRedirect(r) {
|
|
170
|
+
return `${r.prefix}* -> ${showBucketRedirectTarget(r)}`;
|
|
171
|
+
}
|
|
172
|
+
function printSummary(nb, action, bytes) {
|
|
173
|
+
if (nb > 0) {
|
|
174
|
+
process.stdout.write(` ${formatCount(nb, "file")} ${action}`);
|
|
175
|
+
if (bytes) {
|
|
176
|
+
process.stdout.write(` (${formatBytesHuman(bytes)})`);
|
|
155
177
|
}
|
|
156
|
-
|
|
157
|
-
yield parallelUpload.done();
|
|
158
|
-
});
|
|
159
|
-
}
|
|
160
|
-
function deleteFiles(client, Bucket, files) {
|
|
161
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
162
|
-
if (files.length == 0) {
|
|
163
|
-
return null;
|
|
178
|
+
process.stdout.write("\n");
|
|
164
179
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
180
|
+
}
|
|
181
|
+
// collect bucket redirects
|
|
182
|
+
const bredirects = diffBucketRedirects(plan.bucket_redirects.from, plan.bucket_redirects.to);
|
|
183
|
+
// collect object redirects
|
|
184
|
+
const oredirects_added = plan.redirects.filter(r => r.action == "add");
|
|
185
|
+
const oredirects_updated = plan.redirects.filter(r => r.action == "update");
|
|
186
|
+
const oredirects_deleted = plan.redirects.filter(r => r.action == "delete");
|
|
187
|
+
// print
|
|
188
|
+
if (details == "summary") {
|
|
189
|
+
const sizeRemote = sum([...plan.remoteFiles.values()].map(f => { var _a; return (_a = f.size) !== null && _a !== void 0 ? _a : 0; }));
|
|
190
|
+
const sizeLocal = sum(plan.localFiles.map(f => f.size));
|
|
191
|
+
const sizeSent = sum(plan.filesToUpload.map(f => f.size));
|
|
192
|
+
const sizeDeleted = sum(plan.filesToDelete.map(f => { var _a; return (_a = f.size) !== null && _a !== void 0 ? _a : 0; }));
|
|
193
|
+
let filesUnchanged = new Map(plan.localFiles.map(f => [f.s3Path, f.size]));
|
|
194
|
+
for (const f of plan.filesToUpload) {
|
|
195
|
+
filesUnchanged.delete(f.s3Path);
|
|
196
|
+
}
|
|
197
|
+
const sizeUnchanged = sum([...filesUnchanged.values()]);
|
|
198
|
+
printSummary(plan.filesToUpload.length, "uploaded", sizeSent);
|
|
199
|
+
printSummary(plan.filesToDelete.length, "deleted", sizeDeleted);
|
|
200
|
+
printSummary(filesUnchanged.size, "unchanged", sizeUnchanged);
|
|
201
|
+
const items = [
|
|
202
|
+
[bredirects.added.size + oredirects_added.length, "added"],
|
|
203
|
+
[bredirects.updated.size + oredirects_updated.length, "modified"],
|
|
204
|
+
[bredirects.deleted.size + oredirects_deleted.length, "deleted"],
|
|
205
|
+
];
|
|
206
|
+
const sentence = filterMap(items, ([nb, action]) => {
|
|
207
|
+
if (nb > 0) {
|
|
208
|
+
return `${formatCount(nb, "redirect")} ${action}`;
|
|
185
209
|
}
|
|
210
|
+
}).join(", ");
|
|
211
|
+
if (sentence != "") {
|
|
212
|
+
process.stdout.write(` ${sentence}\n`);
|
|
186
213
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
214
|
+
process.stdout.write(` size: ${formatBytesHuman(sizeRemote)} -> ${formatBytesHuman(sizeLocal)}\n`);
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
for (const file of plan.filesToDelete) {
|
|
218
|
+
process.stdout.write(` Delete ${file.name} (${file.size ? formatBytesHuman(file.size) : "?B"})\n`);
|
|
192
219
|
}
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
return true;
|
|
220
|
+
for (const file of plan.filesToUpload) {
|
|
221
|
+
process.stdout.write(` Send ${file.s3Path} (${formatBytesHuman(file.size)})\n`);
|
|
196
222
|
}
|
|
197
|
-
//
|
|
198
|
-
|
|
199
|
-
|
|
223
|
+
// print redirects
|
|
224
|
+
for (const [_, r] of bredirects.added) {
|
|
225
|
+
process.stdout.write(` Add redirect ${showBucketRedirect(r)}\n`);
|
|
226
|
+
}
|
|
227
|
+
for (const { source, target } of oredirects_added) {
|
|
228
|
+
process.stdout.write(` Add redirect ${source} -> ${target}\n`);
|
|
229
|
+
}
|
|
230
|
+
for (const [_, [r_before, r]] of bredirects.updated) {
|
|
231
|
+
process.stdout.write(` Update redirect ${showBucketRedirect(r)} (was: ${showBucketRedirectTarget(r_before)})`);
|
|
232
|
+
}
|
|
233
|
+
for (const { source, target, prev_target } of oredirects_updated) {
|
|
234
|
+
process.stdout.write(` Update redirect ${source} -> ${target} (was: ${prev_target})\n`);
|
|
235
|
+
}
|
|
236
|
+
for (const [_, r] of bredirects.deleted) {
|
|
237
|
+
process.stdout.write(` Delete redirect ${showBucketRedirect(r)}\n`);
|
|
238
|
+
}
|
|
239
|
+
for (const { source, prev_target } of oredirects_deleted) {
|
|
240
|
+
process.stdout.write(` Delete redirect ${source} -> ${prev_target}\n`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
// print other (index, error page)
|
|
244
|
+
function s(s) {
|
|
245
|
+
return s !== null && s !== void 0 ? s : "<undefined>";
|
|
246
|
+
}
|
|
247
|
+
if (plan.index_page.from != plan.index_page.to) {
|
|
248
|
+
process.stdout.write(` Update index_page: ${s(plan.index_page.from)} -> ${s(plan.index_page.to)}\n`);
|
|
249
|
+
}
|
|
250
|
+
if (plan.error_page.from != plan.error_page.to) {
|
|
251
|
+
process.stdout.write(` Update error_page: ${s(plan.error_page.from)} -> ${s(plan.error_page.to)}\n`);
|
|
252
|
+
}
|
|
200
253
|
}
|
|
201
|
-
|
|
254
|
+
function applyDeployPlan(bucket, plan) {
|
|
202
255
|
return __awaiter(this, void 0, void 0, function* () {
|
|
203
|
-
|
|
204
|
-
//
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
console.error(`Error: website '${vhost}' does not exist`);
|
|
256
|
+
// Delete files.
|
|
257
|
+
// Do this before sending new files to avoid hitting the size quota
|
|
258
|
+
// unnecessarily.
|
|
259
|
+
yield deleteBucketFiles(bucket, plan.filesToDelete);
|
|
260
|
+
// Apply object redirects.
|
|
261
|
+
// Since the source of object redirects cannot be user files (cf the
|
|
262
|
+
// check at the beginning of `computeDeployPlan`, we can do this before
|
|
263
|
+
// uploading user files.
|
|
264
|
+
// Furthermore, in the case of a previously uploaded user file that now
|
|
265
|
+
// becomes the source of a redirection (thus truncating it), we free
|
|
266
|
+
// space in the bucket by doing so, so we should do it before uploading.
|
|
267
|
+
yield PromisePool.for(plan.redirects)
|
|
268
|
+
.withConcurrency(50)
|
|
269
|
+
.handleError(err => {
|
|
270
|
+
throw err;
|
|
271
|
+
})
|
|
272
|
+
.process((r) => __awaiter(this, void 0, void 0, function* () {
|
|
273
|
+
if (r.action == "delete") {
|
|
274
|
+
yield deleteBucketFile(bucket, r.source);
|
|
223
275
|
}
|
|
224
276
|
else {
|
|
225
|
-
|
|
277
|
+
yield putEmptyObjectRedirect(bucket, r.source, r.target);
|
|
226
278
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
279
|
+
}));
|
|
280
|
+
// Upload files
|
|
281
|
+
yield PromisePool.for(plan.filesToUpload)
|
|
282
|
+
.withConcurrency(50)
|
|
283
|
+
.handleError(err => {
|
|
284
|
+
throw err;
|
|
285
|
+
})
|
|
286
|
+
.process((f) => __awaiter(this, void 0, void 0, function* () {
|
|
287
|
+
process.stdout.write(` Send ${f.s3Path} (${formatBytesHuman(f.size)})\n`);
|
|
288
|
+
yield uploadFile(bucket, f.s3Path, f.localPath);
|
|
289
|
+
}));
|
|
290
|
+
// Apply bucket redirects & global config
|
|
291
|
+
yield putBucketWebsiteConfig(bucket, plan.index_page.to, plan.error_page.to, plan.bucket_redirects.to);
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
function deployMain(website, localFolder, options) {
|
|
295
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
296
|
+
if (options.dryRun && options.yes) {
|
|
297
|
+
throw "options --yes and --dry-run cannot be passed at the same time";
|
|
298
|
+
}
|
|
299
|
+
// TODO: make this configurable
|
|
300
|
+
const website_config_path = "deuxfleurs.toml";
|
|
301
|
+
// Read and validate the local configuration file before doing anything else
|
|
302
|
+
const localWebsiteConfig = yield readConfigFile(website_config_path).catch(err => {
|
|
303
|
+
throw `while reading ${website_config_path}:\n${err}`;
|
|
239
304
|
});
|
|
240
|
-
|
|
241
|
-
const remoteFiles = yield
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
305
|
+
process.stdout.write("Fetching the website configuration and metadata...\n");
|
|
306
|
+
const [localFiles, [bucket, remoteFiles, remoteWebsiteConfig]] = yield Promise.all([
|
|
307
|
+
// Get paths & size of the local files to deploy
|
|
308
|
+
getLocalFilesWithInfo(localFolder),
|
|
309
|
+
// Get the bucket, list of files stored in the bucket, and bucket website config
|
|
310
|
+
(() => __awaiter(this, void 0, void 0, function* () {
|
|
311
|
+
const guichet = yield openApiConf();
|
|
312
|
+
const api = new WebsiteApi(guichet);
|
|
313
|
+
const bucket = yield getBucket(api, website);
|
|
314
|
+
const remoteFiles = yield getBucketFiles(bucket);
|
|
315
|
+
// This can be slow because it needs to query each object in the bucket.
|
|
316
|
+
const remoteWebsiteConfig = yield getBucketConfig(bucket, [
|
|
317
|
+
...remoteFiles.keys(),
|
|
318
|
+
]);
|
|
319
|
+
return [bucket, remoteFiles, remoteWebsiteConfig];
|
|
320
|
+
}))(),
|
|
321
|
+
]);
|
|
322
|
+
// Compute the deploy plan
|
|
323
|
+
const plan = computeDeployPlan(localFiles, remoteFiles, localWebsiteConfig, remoteWebsiteConfig);
|
|
324
|
+
// If --dry-run: display the plan and return
|
|
325
|
+
if (options.dryRun) {
|
|
326
|
+
printPlan(plan, "full");
|
|
327
|
+
process.stdout.write("\nSummary:\n");
|
|
328
|
+
printPlan(plan, "summary");
|
|
329
|
+
return;
|
|
250
330
|
}
|
|
251
|
-
// If not
|
|
252
|
-
if (!options.
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
process.
|
|
331
|
+
// If not --yes: show the plan summary, ask for confirmation before proceeding
|
|
332
|
+
if (!options.yes) {
|
|
333
|
+
process.stdout.write("The following changes will be made:\n");
|
|
334
|
+
printPlan(plan, "summary");
|
|
335
|
+
process.stdout.write("\n");
|
|
336
|
+
const ok = yield confirmationPrompt(() => {
|
|
337
|
+
process.stdout.write("Details of planned changes:\n");
|
|
338
|
+
printPlan(plan, "full");
|
|
339
|
+
});
|
|
340
|
+
if (!ok) {
|
|
341
|
+
return;
|
|
258
342
|
}
|
|
259
343
|
}
|
|
260
|
-
//
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
function processFile(f) {
|
|
267
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
268
|
-
const remoteFile = remoteFiles.get(f.s3Path);
|
|
269
|
-
const localMd5 = yield getFileMd5(f.localPath);
|
|
270
|
-
if (!remoteFile ||
|
|
271
|
-
(yield needsUpdate(s3client, f.localPath, localMd5, Bucket, f.s3Path, remoteFile.size))) {
|
|
272
|
-
filesSent.push({ s3Path: f.s3Path, size: f.size });
|
|
273
|
-
if (options.dryRun) {
|
|
274
|
-
process.stdout.write(`Sending ${f.s3Path}\n`);
|
|
275
|
-
}
|
|
276
|
-
else {
|
|
277
|
-
yield uploadFile(s3client, Bucket, f.s3Path, fs.createReadStream(f.localPath), localMd5, displaystate);
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
});
|
|
344
|
+
// Apply the deploy plan
|
|
345
|
+
yield applyDeployPlan(bucket, plan);
|
|
346
|
+
// Display a summary if --yes was used
|
|
347
|
+
if (options.yes) {
|
|
348
|
+
process.stdout.write("\nSummary:\n");
|
|
349
|
+
printPlan(plan, "summary");
|
|
281
350
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
return `${n} file `;
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
export function deploy(website, localFolder, options) {
|
|
354
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
355
|
+
try {
|
|
356
|
+
yield deployMain(website, localFolder, options);
|
|
357
|
+
}
|
|
358
|
+
catch (err) {
|
|
359
|
+
if (typeof err == "string" || (err === null || err === void 0 ? void 0 : err.name) === "AbortError") {
|
|
360
|
+
console.error(`Error: ${err}`);
|
|
293
361
|
}
|
|
294
362
|
else {
|
|
295
|
-
|
|
363
|
+
console.error("Error:");
|
|
364
|
+
console.error(err);
|
|
296
365
|
}
|
|
366
|
+
process.exit(1);
|
|
297
367
|
}
|
|
298
|
-
const sizeSent = sum(filesSent.map((f) => f.size));
|
|
299
|
-
const sizeDeleted = sum(filesToDelete.map((f) => { var _a; return (_a = f.size) !== null && _a !== void 0 ? _a : 0; }));
|
|
300
|
-
let filesUnchanged = new Map(localFiles.map((f) => [f.s3Path, f.size]));
|
|
301
|
-
for (const f of filesSent) {
|
|
302
|
-
filesUnchanged.delete(f.s3Path);
|
|
303
|
-
}
|
|
304
|
-
const sizeUnchanged = sum([...filesUnchanged.values()]);
|
|
305
|
-
process.stdout.write("\nSummary:\n");
|
|
306
|
-
process.stdout.write(`${formatFiles(filesSent.length)} uploaded (${formatBytes(sizeSent)})\n`);
|
|
307
|
-
process.stdout.write(`${formatFiles(filesToDelete.length)} deleted (${formatBytes(sizeDeleted)})\n`);
|
|
308
|
-
process.stdout.write(`${formatFiles(filesUnchanged.size)} unchanged (${formatBytes(sizeUnchanged)})\n`);
|
|
309
368
|
});
|
|
310
369
|
}
|