@coderook/cli 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -2
- package/dist/cli/src/cli.js +167 -10
- package/dist/cli/src/mcp.js +349 -0
- package/dist/cli/src/track_commands.js +157 -0
- package/dist/desktop-app/src/main/compress.js +95 -0
- package/dist/desktop-app/src/main/detect.js +137 -0
- package/dist/desktop-app/src/main/download.js +162 -15
- package/dist/desktop-app/src/main/profile.js +73 -0
- package/dist/desktop-app/src/main/retry.js +96 -0
- package/dist/desktop-app/src/main/solid.js +103 -0
- package/dist/desktop-app/src/main/staging.js +140 -0
- package/dist/desktop-app/src/main/tracks.js +132 -0
- package/dist/desktop-app/src/main/upload.js +1039 -55
- package/dist/desktop-app/src/main/worktree.js +214 -25
- package/package.json +1 -1
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.suggestExclusions = suggestExclusions;
|
|
7
|
+
exports.rulesFromSuggestions = rulesFromSuggestions;
|
|
8
|
+
/**
|
|
9
|
+
* Looking at a folder and saying what probably should not be uploaded.
|
|
10
|
+
*
|
|
11
|
+
* Choosing what goes up is the part of adding a project that people get wrong,
|
|
12
|
+
* and they get it wrong in one direction: a folder that has been worked in for
|
|
13
|
+
* a year holds a virtual environment, a build directory, somebody's browser
|
|
14
|
+
* profile and three copies of a release zip, and none of it is the work. The
|
|
15
|
+
* cost is not only storage — it is a save that takes twenty minutes and can be
|
|
16
|
+
* stopped outright by one file another program holds open.
|
|
17
|
+
*
|
|
18
|
+
* So this proposes. It never decides: everything it finds comes back as a
|
|
19
|
+
* suggestion with a reason and a size, the person ticks what they agree with,
|
|
20
|
+
* and a folder it recognises nothing in produces no suggestions rather than a
|
|
21
|
+
* default nobody asked for. Being wrong here has to be cheap, which it is
|
|
22
|
+
* exactly as long as nothing is applied without somebody saying so.
|
|
23
|
+
*/
|
|
24
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
25
|
+
/*
|
|
26
|
+
Everything here is recoverable from something else in the project, or is
|
|
27
|
+
another program's working state. Nothing that could be somebody's only copy
|
|
28
|
+
of anything is marked recommended.
|
|
29
|
+
*/
|
|
30
|
+
const RULES = [
|
|
31
|
+
{ directory: "node_modules", reason: "Installed packages — restored by installing again", recommended: true },
|
|
32
|
+
{ directory: ".venv", reason: "Python virtual environment — rebuilt from requirements", recommended: true },
|
|
33
|
+
{ directory: "venv", reason: "Python virtual environment — rebuilt from requirements", recommended: true },
|
|
34
|
+
{ directory: "__pycache__", reason: "Python bytecode cache", recommended: true },
|
|
35
|
+
{ directory: ".pytest_cache", reason: "Test runner cache", recommended: true },
|
|
36
|
+
{ directory: ".mypy_cache", reason: "Type checker cache", recommended: true },
|
|
37
|
+
{ directory: ".ruff_cache", reason: "Linter cache", recommended: true },
|
|
38
|
+
{ directory: "target", reason: "Rust build output", recommended: true },
|
|
39
|
+
{ directory: ".next", reason: "Next.js build output", recommended: true },
|
|
40
|
+
{ directory: ".gradle", reason: "Gradle build state", recommended: true },
|
|
41
|
+
{ directory: ".terraform", reason: "Downloaded Terraform providers", recommended: true },
|
|
42
|
+
/*
|
|
43
|
+
A Chromium or Electron profile left in the project folder. Worth its own
|
|
44
|
+
reason because the consequence is not only size: these hold lock files
|
|
45
|
+
another program keeps open, and one unreadable file stops the whole save.
|
|
46
|
+
*/
|
|
47
|
+
{ directory: "IndexedDB", reason: "Browser profile data — holds files another program keeps open", recommended: true },
|
|
48
|
+
{ directory: "Local Storage", reason: "Browser profile data — holds files another program keeps open", recommended: true },
|
|
49
|
+
{ directory: "Session Storage", reason: "Browser profile data — holds files another program keeps open", recommended: true },
|
|
50
|
+
{ directory: "Service Worker", reason: "Browser profile data", recommended: true },
|
|
51
|
+
{ directory: "GPUCache", reason: "Browser cache", recommended: true },
|
|
52
|
+
{ directory: "Code Cache", reason: "Browser cache", recommended: true },
|
|
53
|
+
{ directory: "blob_storage", reason: "Browser profile data", recommended: true },
|
|
54
|
+
/*
|
|
55
|
+
Build output and archives are usually regenerable, but "dist" is also a
|
|
56
|
+
perfectly ordinary folder name for hand-written files, so these are
|
|
57
|
+
offered rather than recommended.
|
|
58
|
+
*/
|
|
59
|
+
{ directory: "dist", reason: "Usually build output — check before excluding", recommended: false },
|
|
60
|
+
{ directory: "build", reason: "Usually build output — check before excluding", recommended: false },
|
|
61
|
+
{ directory: "out", reason: "Usually build output — check before excluding", recommended: false },
|
|
62
|
+
{ extension: ".safetensors", reason: "Model weights — large, and usually downloadable", recommended: false },
|
|
63
|
+
{ extension: ".ckpt", reason: "Model weights — large, and usually downloadable", recommended: false },
|
|
64
|
+
{ extension: ".pt", reason: "Model weights — large, and usually downloadable", recommended: false },
|
|
65
|
+
{ extension: ".pth", reason: "Model weights — large, and usually downloadable", recommended: false },
|
|
66
|
+
{ extension: ".cbx", reason: "A CodeBox bundle of this project, inside the project", recommended: true },
|
|
67
|
+
{ extension: ".pyc", reason: "Python bytecode", recommended: true },
|
|
68
|
+
{ extension: ".log", reason: "Log files", recommended: false },
|
|
69
|
+
{ file: "Local State", reason: "Browser profile data", recommended: true },
|
|
70
|
+
{ file: "Preferences", reason: "Browser profile data", recommended: true },
|
|
71
|
+
];
|
|
72
|
+
const norm = (value) => value.split("\\").join("/");
|
|
73
|
+
/**
|
|
74
|
+
* What this folder appears to contain that is not the work.
|
|
75
|
+
*
|
|
76
|
+
* Ordered by size, because the decision somebody is making is about how long
|
|
77
|
+
* their upload will take and the biggest item answers it. Anything matching
|
|
78
|
+
* nothing is simply absent — silence is the right output for a tidy folder.
|
|
79
|
+
*/
|
|
80
|
+
function suggestExclusions(files) {
|
|
81
|
+
const totals = new Map();
|
|
82
|
+
for (const entry of files) {
|
|
83
|
+
const relative = norm(entry.path);
|
|
84
|
+
const segments = relative.split("/");
|
|
85
|
+
const name = segments[segments.length - 1] ?? "";
|
|
86
|
+
const extension = node_path_1.default.extname(name).toLowerCase();
|
|
87
|
+
for (const rule of RULES) {
|
|
88
|
+
let pattern = null;
|
|
89
|
+
if (rule.directory && segments.slice(0, -1).includes(rule.directory)) {
|
|
90
|
+
pattern = `${rule.directory}/`;
|
|
91
|
+
}
|
|
92
|
+
else if (rule.file && name === rule.file) {
|
|
93
|
+
pattern = rule.file;
|
|
94
|
+
}
|
|
95
|
+
else if (rule.extension && extension === rule.extension) {
|
|
96
|
+
pattern = `*${rule.extension}`;
|
|
97
|
+
}
|
|
98
|
+
if (!pattern)
|
|
99
|
+
continue;
|
|
100
|
+
const held = totals.get(pattern) ?? { bytes: 0, files: 0, rule };
|
|
101
|
+
held.bytes += entry.size;
|
|
102
|
+
held.files += 1;
|
|
103
|
+
totals.set(pattern, held);
|
|
104
|
+
/*
|
|
105
|
+
One rule per file. The list is ordered from most specific to least, so
|
|
106
|
+
a .pyc inside __pycache__ is counted once, under the directory that
|
|
107
|
+
explains it, rather than inflating two separate suggestions.
|
|
108
|
+
*/
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return [...totals.entries()]
|
|
113
|
+
.map(([pattern, held]) => ({
|
|
114
|
+
pattern,
|
|
115
|
+
reason: held.rule.reason,
|
|
116
|
+
bytes: held.bytes,
|
|
117
|
+
files: held.files,
|
|
118
|
+
recommended: held.rule.recommended,
|
|
119
|
+
}))
|
|
120
|
+
.sort((left, right) => right.bytes - left.bytes);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The suggestions a person accepted, as lines for the rules file.
|
|
124
|
+
*
|
|
125
|
+
* Written as its own step so that nothing is ever applied as a side effect of
|
|
126
|
+
* looking: the detector proposes, this records a decision, and the two are
|
|
127
|
+
* only connected by somebody choosing.
|
|
128
|
+
*/
|
|
129
|
+
function rulesFromSuggestions(accepted) {
|
|
130
|
+
if (!accepted.length)
|
|
131
|
+
return "";
|
|
132
|
+
return [
|
|
133
|
+
"# Suggested when this folder was added, and accepted.",
|
|
134
|
+
...accepted.map((one) => one.pattern),
|
|
135
|
+
"",
|
|
136
|
+
].join("\n");
|
|
137
|
+
}
|
|
@@ -14,10 +14,14 @@ exports.underway = underway;
|
|
|
14
14
|
* not a file quietly written wrong.
|
|
15
15
|
*/
|
|
16
16
|
const node_crypto_1 = require("node:crypto");
|
|
17
|
+
const node_fs_1 = require("node:fs");
|
|
17
18
|
const promises_1 = require("node:fs/promises");
|
|
19
|
+
const promises_2 = require("node:stream/promises");
|
|
20
|
+
const node_stream_1 = require("node:stream");
|
|
18
21
|
const node_path_1 = __importDefault(require("node:path"));
|
|
19
22
|
const identify_js_1 = require("./identify.js");
|
|
20
23
|
const faults_js_1 = require("./faults.js");
|
|
24
|
+
const retry_js_1 = require("./retry.js");
|
|
21
25
|
async function exists(target) {
|
|
22
26
|
try {
|
|
23
27
|
await (0, promises_1.stat)(target);
|
|
@@ -92,7 +96,38 @@ class Downloader {
|
|
|
92
96
|
if (this.aborted)
|
|
93
97
|
throw new DownloadCancelled();
|
|
94
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* One request, repeated while repeating it might help.
|
|
101
|
+
*
|
|
102
|
+
* A restore makes at least one request per file and, for a file kept in
|
|
103
|
+
* pieces, one per piece — three thousand seven hundred and fifty-six for a
|
|
104
|
+
* single file here. At those numbers a dropped connection is not a risk but
|
|
105
|
+
* a certainty, and without this the whole restore ends on the first one.
|
|
106
|
+
*/
|
|
95
107
|
async request(route) {
|
|
108
|
+
let wait = retry_js_1.RETRY_FIRST_WAIT_MS;
|
|
109
|
+
for (let attempt = 1;; attempt += 1) {
|
|
110
|
+
this.check();
|
|
111
|
+
try {
|
|
112
|
+
return await this.attempt(route);
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
// Its own cancellation first: `worthRetrying` knows nothing of it.
|
|
116
|
+
if (error instanceof DownloadCancelled)
|
|
117
|
+
throw error;
|
|
118
|
+
if (attempt >= retry_js_1.RETRY_ATTEMPTS || !(0, retry_js_1.worthRetrying)(error))
|
|
119
|
+
throw error;
|
|
120
|
+
try {
|
|
121
|
+
await (0, retry_js_1.pauseFor)(this.controller.signal, wait);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
throw new DownloadCancelled();
|
|
125
|
+
}
|
|
126
|
+
wait *= 2;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
async attempt(route) {
|
|
96
131
|
const token = await this.credentials.token();
|
|
97
132
|
if (!token)
|
|
98
133
|
throw new Error("Sign in again before downloading");
|
|
@@ -116,13 +151,26 @@ class Downloader {
|
|
|
116
151
|
`update from ${tooOld.upgradeUrl}.`);
|
|
117
152
|
}
|
|
118
153
|
let message = `${route} failed (${response.status})`;
|
|
154
|
+
let code;
|
|
119
155
|
try {
|
|
120
|
-
|
|
156
|
+
const body = JSON.parse(text);
|
|
157
|
+
message = body?.error?.message ?? message;
|
|
158
|
+
code = body?.error?.code;
|
|
121
159
|
}
|
|
122
160
|
catch {
|
|
123
161
|
/* the body was not JSON; the status will have to do */
|
|
124
162
|
}
|
|
125
|
-
|
|
163
|
+
/*
|
|
164
|
+
The status travels with the error.
|
|
165
|
+
|
|
166
|
+
Callers already reason about it — the retry rule asks whether a
|
|
167
|
+
failure is worth trying again, and the sync asks whether a project is
|
|
168
|
+
gone or merely unreachable — and both were reading a property that
|
|
169
|
+
was never set. A plain Error made every failure look identical, so a
|
|
170
|
+
404 for a deleted project was retried five times and then reported as
|
|
171
|
+
though the network were down.
|
|
172
|
+
*/
|
|
173
|
+
throw Object.assign(new Error(message), { status: response.status, code });
|
|
126
174
|
}
|
|
127
175
|
return response;
|
|
128
176
|
}
|
|
@@ -140,11 +188,106 @@ class Downloader {
|
|
|
140
188
|
}
|
|
141
189
|
async files(repositoryId, versionId) {
|
|
142
190
|
const body = (await (await this.request(`/v1/repositories/${repositoryId}/versions/${versionId}/files`)).json());
|
|
143
|
-
return (body.files ?? []).map((row) =>
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
191
|
+
return (body.files ?? []).map((row) => {
|
|
192
|
+
const pieces = Array.isArray(row.chunks) ? row.chunks : null;
|
|
193
|
+
return {
|
|
194
|
+
path: String(row.path ?? ""),
|
|
195
|
+
sha256: String(row.sha256 ?? ""),
|
|
196
|
+
sourceSize: Number(row.sourceSize ?? 0),
|
|
197
|
+
...(pieces && pieces.length
|
|
198
|
+
? {
|
|
199
|
+
chunks: pieces.map((piece) => {
|
|
200
|
+
const one = piece;
|
|
201
|
+
return {
|
|
202
|
+
objectId: String(one.objectId ?? ""),
|
|
203
|
+
sha256: String(one.sha256 ?? ""),
|
|
204
|
+
sourceSize: Number(one.sourceSize ?? 0),
|
|
205
|
+
};
|
|
206
|
+
}),
|
|
207
|
+
}
|
|
208
|
+
: {}),
|
|
209
|
+
};
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Fetch one file to `target`, verifying it as it lands, and answer with the
|
|
214
|
+
* digest of what was actually written.
|
|
215
|
+
*
|
|
216
|
+
* A file kept in pieces is fetched piece by piece and assembled here rather
|
|
217
|
+
* than by the service. That is the whole point of this method. Asking the
|
|
218
|
+
* service to hand back one reassembled file made it loop over every piece
|
|
219
|
+
* inside a single edge invocation, and for a file of three thousand seven
|
|
220
|
+
* hundred and fifty-six pieces it ran out of whatever it runs out of and
|
|
221
|
+
* stopped — after the two hundred and its headers had already gone out. A
|
|
222
|
+
* 7.34 GB file came back as 5.09 GB under a clean `200`, with nothing in
|
|
223
|
+
* the response saying otherwise. Pulling the pieces from here bounds each
|
|
224
|
+
* request to one piece and puts the loop somewhere that can fail honestly.
|
|
225
|
+
*
|
|
226
|
+
* Nothing is held whole. The previous version read the entire file into one
|
|
227
|
+
* buffer to hash it, which for this file would have been 7.34 GB resident
|
|
228
|
+
* before the check it was there to perform could even run.
|
|
229
|
+
*/
|
|
230
|
+
/**
|
|
231
|
+
* Fetch one file from a version to a chosen path, verified, and answer with
|
|
232
|
+
* the digest of what was written.
|
|
233
|
+
*
|
|
234
|
+
* Restoring a whole version is the usual case. This is the same machinery
|
|
235
|
+
* for somebody who wants one file out of one — and it is how a very large
|
|
236
|
+
* file can be checked on its own, without pulling down the project it
|
|
237
|
+
* belongs to in order to find out whether it survives the trip.
|
|
238
|
+
*/
|
|
239
|
+
async fileTo(repositoryId, versionId, file, target) {
|
|
240
|
+
await (0, promises_1.mkdir)(node_path_1.default.dirname(target), { recursive: true });
|
|
241
|
+
return this.fetchInto(repositoryId, versionId, file, target);
|
|
242
|
+
}
|
|
243
|
+
async fetchInto(repositoryId, versionId, file, target) {
|
|
244
|
+
const whole = (0, node_crypto_1.createHash)("sha256");
|
|
245
|
+
const pieces = file.chunks ?? [];
|
|
246
|
+
// Captured so the generators below do not need `this` rebound.
|
|
247
|
+
const request = (route) => this.request(route);
|
|
248
|
+
const check = () => this.check();
|
|
249
|
+
const source = pieces.length
|
|
250
|
+
? (async function* () {
|
|
251
|
+
for (let at = 0; at < pieces.length; at += 1) {
|
|
252
|
+
check();
|
|
253
|
+
const piece = pieces[at];
|
|
254
|
+
const reply = await request(`/v1/repositories/${repositoryId}/objects/${piece.objectId}`);
|
|
255
|
+
const body = Buffer.from(await reply.arrayBuffer());
|
|
256
|
+
/*
|
|
257
|
+
Checked against what the upload recorded for this piece, before
|
|
258
|
+
a byte of it reaches the file. Without the digest the only
|
|
259
|
+
available check was the whole file's, which says that something
|
|
260
|
+
in seven gigabytes is wrong and nothing about what.
|
|
261
|
+
*/
|
|
262
|
+
const digest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
|
|
263
|
+
if (piece.sha256 && digest !== piece.sha256) {
|
|
264
|
+
throw new Error(`${file.path}: piece ${at + 1} of ${pieces.length} ` +
|
|
265
|
+
`did not arrive intact`);
|
|
266
|
+
}
|
|
267
|
+
whole.update(body);
|
|
268
|
+
yield body;
|
|
269
|
+
}
|
|
270
|
+
})()
|
|
271
|
+
: (async function* () {
|
|
272
|
+
const reply = await request(`/v1/repositories/${repositoryId}/versions/${versionId}/file` +
|
|
273
|
+
`?path=${encodeURIComponent(file.path)}`);
|
|
274
|
+
if (!reply.body)
|
|
275
|
+
return;
|
|
276
|
+
for await (const block of node_stream_1.Readable.fromWeb(reply.body)) {
|
|
277
|
+
check();
|
|
278
|
+
const buffer = Buffer.from(block);
|
|
279
|
+
whole.update(buffer);
|
|
280
|
+
yield buffer;
|
|
281
|
+
}
|
|
282
|
+
})();
|
|
283
|
+
/*
|
|
284
|
+
`pipeline` rather than a write loop: it applies backpressure, and it
|
|
285
|
+
destroys the file handle when the source throws. A partially written
|
|
286
|
+
file left open on a failed verification is how a staging directory ends
|
|
287
|
+
up holding something that looks finished.
|
|
288
|
+
*/
|
|
289
|
+
await (0, promises_2.pipeline)(source, (0, node_fs_1.createWriteStream)(target));
|
|
290
|
+
return whole.digest("hex");
|
|
148
291
|
}
|
|
149
292
|
/**
|
|
150
293
|
* Write a version into `destination`.
|
|
@@ -194,19 +337,23 @@ class Downloader {
|
|
|
194
337
|
if (parts.some((part) => part === ".." || part.includes("\0"))) {
|
|
195
338
|
throw new Error(`That version contains an unsafe path: ${file.path}`);
|
|
196
339
|
}
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
|
|
340
|
+
const full = node_path_1.default.join(staging, ...parts);
|
|
341
|
+
await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
|
|
342
|
+
const digest = await this.fetchInto(repositoryId, versionId, file, full);
|
|
343
|
+
/*
|
|
344
|
+
The whole file's digest, checked after every piece has already been
|
|
345
|
+
checked individually. Both are needed: the pieces catch a corrupt or
|
|
346
|
+
truncated piece and name it, and this catches a set of individually
|
|
347
|
+
valid pieces assembled in the wrong order or with one missing —
|
|
348
|
+
which is the failure that produces bytes that are all present, all
|
|
349
|
+
verified, and wrong.
|
|
350
|
+
*/
|
|
201
351
|
if (file.sha256 && digest !== file.sha256) {
|
|
202
352
|
throw new Error(`${file.path} did not arrive intact`);
|
|
203
353
|
}
|
|
204
|
-
const full = node_path_1.default.join(staging, ...parts);
|
|
205
|
-
await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
|
|
206
|
-
await (0, promises_1.writeFile)(full, body);
|
|
207
354
|
manifest[file.path] = digest;
|
|
208
355
|
written += 1;
|
|
209
|
-
bytes +=
|
|
356
|
+
bytes += file.sourceSize;
|
|
210
357
|
}
|
|
211
358
|
/*
|
|
212
359
|
Everything has arrived and verified, so it can go into place. Each
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.timed = timed;
|
|
4
|
+
exports.counted = counted;
|
|
5
|
+
exports.profileReport = profileReport;
|
|
6
|
+
exports.resetProfile = resetProfile;
|
|
7
|
+
/**
|
|
8
|
+
* Where an upload's time actually goes.
|
|
9
|
+
*
|
|
10
|
+
* Written because guessing was wrong twice. The first batching attempt was
|
|
11
|
+
* built on the assumption that HTTP round trips were the cost; they were about
|
|
12
|
+
* a tenth of it, and the real expense was ten database queries per object. The
|
|
13
|
+
* fix for that then revealed object storage as the next bottleneck. Neither
|
|
14
|
+
* would have been found by reading the code.
|
|
15
|
+
*
|
|
16
|
+
* Inert unless CODEROOK_PROFILE is set, so it costs a boolean check on a hot
|
|
17
|
+
* path and nothing else. It reports totals per named phase rather than a trace,
|
|
18
|
+
* because the question is always "which stage is the upload" and never "what
|
|
19
|
+
* happened at 14:32:07".
|
|
20
|
+
*/
|
|
21
|
+
const enabled = Boolean(process.env.CODEROOK_PROFILE);
|
|
22
|
+
const phases = new Map();
|
|
23
|
+
/** Time one awaited step and attribute it to a phase. */
|
|
24
|
+
async function timed(phase, work) {
|
|
25
|
+
if (!enabled)
|
|
26
|
+
return work();
|
|
27
|
+
const started = performance.now();
|
|
28
|
+
try {
|
|
29
|
+
return await work();
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
const held = phases.get(phase) ?? { total: 0, count: 0 };
|
|
33
|
+
held.total += performance.now() - started;
|
|
34
|
+
held.count += 1;
|
|
35
|
+
phases.set(phase, held);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/** Count something that is not a duration — files batched, files sent alone. */
|
|
39
|
+
function counted(phase, by = 1) {
|
|
40
|
+
if (!enabled)
|
|
41
|
+
return;
|
|
42
|
+
const held = phases.get(phase) ?? { total: 0, count: 0 };
|
|
43
|
+
held.count += by;
|
|
44
|
+
phases.set(phase, held);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* What was measured, slowest first.
|
|
48
|
+
*
|
|
49
|
+
* Returns an empty string when profiling is off, so a caller can print it
|
|
50
|
+
* unconditionally without deciding whether there is anything to print.
|
|
51
|
+
*/
|
|
52
|
+
function profileReport() {
|
|
53
|
+
if (!enabled || !phases.size)
|
|
54
|
+
return "";
|
|
55
|
+
const rows = [...phases.entries()].sort((left, right) => right[1].total - left[1].total);
|
|
56
|
+
const width = Math.max(...rows.map(([name]) => name.length));
|
|
57
|
+
return [
|
|
58
|
+
"",
|
|
59
|
+
"where the time went:",
|
|
60
|
+
...rows.map(([name, phase]) => {
|
|
61
|
+
const seconds = (phase.total / 1000).toFixed(1);
|
|
62
|
+
const each = phase.count && phase.total
|
|
63
|
+
? ` (${(phase.total / phase.count).toFixed(1)}ms each)`
|
|
64
|
+
: "";
|
|
65
|
+
return phase.total
|
|
66
|
+
? ` ${name.padEnd(width)} ${seconds.padStart(7)}s ×${phase.count}${each}`
|
|
67
|
+
: ` ${name.padEnd(width)} ${String(phase.count).padStart(8)}`;
|
|
68
|
+
}),
|
|
69
|
+
].join("\n");
|
|
70
|
+
}
|
|
71
|
+
function resetProfile() {
|
|
72
|
+
phases.clear();
|
|
73
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Trying again, and knowing when not to.
|
|
4
|
+
*
|
|
5
|
+
* Both directions need this and neither had it. The uploader's absence of
|
|
6
|
+
* retries ended a twenty-eight gigabyte run at section fifty-one of
|
|
7
|
+
* eighty-seven, twenty-four minutes in, because one request came back `fetch
|
|
8
|
+
* failed`. The downloader has the same shape of problem and a worse
|
|
9
|
+
* consequence: a restore that gives up part way through has already written
|
|
10
|
+
* nothing usable, and the person is left without the thing they were
|
|
11
|
+
* restoring.
|
|
12
|
+
*
|
|
13
|
+
* It lives apart from both because the rule is the same for both and the
|
|
14
|
+
* cancellation is not. Each side has its own cancelled-error type, so each
|
|
15
|
+
* side checks for its own before asking this whether the failure was worth
|
|
16
|
+
* repeating. That keeps this module ignorant of either, rather than importing
|
|
17
|
+
* one into the other and coupling upload to download for a single `instanceof`.
|
|
18
|
+
*/
|
|
19
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
+
exports.RETRY_FIRST_WAIT_MS = exports.RETRY_ATTEMPTS = void 0;
|
|
21
|
+
exports.worthRetrying = worthRetrying;
|
|
22
|
+
exports.pauseFor = pauseFor;
|
|
23
|
+
/**
|
|
24
|
+
* How many times one request is attempted before giving up.
|
|
25
|
+
*
|
|
26
|
+
* Five attempts with doubling backoff covers an outage of roughly fifteen
|
|
27
|
+
* seconds, which is far longer than the reconnects that actually end long
|
|
28
|
+
* transfers. Longer than that and a genuine outage is being waited on rather
|
|
29
|
+
* than reported, which helps nobody.
|
|
30
|
+
*/
|
|
31
|
+
exports.RETRY_ATTEMPTS = 5;
|
|
32
|
+
/** The first backoff. Doubles each attempt, and is jittered when used. */
|
|
33
|
+
exports.RETRY_FIRST_WAIT_MS = 500;
|
|
34
|
+
/**
|
|
35
|
+
* Whether a failure is worth trying again.
|
|
36
|
+
*
|
|
37
|
+
* The service's refusals are deliberate and none of them change on a second
|
|
38
|
+
* attempt: every 4xx it returns names a settled condition — the version is
|
|
39
|
+
* gone, the body is malformed, the quota is full — that a retry would meet
|
|
40
|
+
* again identically. Repeating those turns a clear answer into a long pause
|
|
41
|
+
* followed by the same answer, with the message that explained it buried under
|
|
42
|
+
* four pointless attempts.
|
|
43
|
+
*
|
|
44
|
+
* What is worth repeating is everything that never reached a verdict: a
|
|
45
|
+
* dropped connection, a gateway mid-restart, a request that timed out. Those
|
|
46
|
+
* carry no decision at all, and they are the whole failure population that
|
|
47
|
+
* ends long transfers.
|
|
48
|
+
*
|
|
49
|
+
* Cancellation is not considered here. It is a decision too — the user's —
|
|
50
|
+
* and each caller recognises its own before asking.
|
|
51
|
+
*/
|
|
52
|
+
function worthRetrying(error) {
|
|
53
|
+
const status = error.status;
|
|
54
|
+
if (typeof status !== "number") {
|
|
55
|
+
/*
|
|
56
|
+
No status means the request never got an answer — `fetch failed`, a
|
|
57
|
+
reset socket, a name that did not resolve. This is the case that ended
|
|
58
|
+
the twenty-eight gigabyte upload.
|
|
59
|
+
*/
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
if (status === 429) {
|
|
63
|
+
/*
|
|
64
|
+
The one 4xx worth repeating, and only sometimes. A 429 from the edge is
|
|
65
|
+
throttling and clears in seconds. A 429 from the upload window is a
|
|
66
|
+
refusal that stands for days, so retrying it would hang the transfer
|
|
67
|
+
instead of showing the person the message that explains it.
|
|
68
|
+
*/
|
|
69
|
+
return error.code !== "upload_rate_exceeded";
|
|
70
|
+
}
|
|
71
|
+
if (status === 408)
|
|
72
|
+
return true;
|
|
73
|
+
return status >= 500;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Wait, unless the signal aborts while waiting.
|
|
77
|
+
*
|
|
78
|
+
* Jittered, because the lanes fail together. Six requests against a gateway
|
|
79
|
+
* that has just gone away all fail within milliseconds of each other, and six
|
|
80
|
+
* retries timed from those failures would arrive together too — which turns
|
|
81
|
+
* one outage into a second one at precisely the wrong moment.
|
|
82
|
+
*/
|
|
83
|
+
function pauseFor(signal, ms) {
|
|
84
|
+
const delay = ms * (0.5 + Math.random());
|
|
85
|
+
return new Promise((resolve, reject) => {
|
|
86
|
+
const stop = () => {
|
|
87
|
+
clearTimeout(timer);
|
|
88
|
+
reject(new Error("aborted"));
|
|
89
|
+
};
|
|
90
|
+
const timer = setTimeout(() => {
|
|
91
|
+
signal.removeEventListener("abort", stop);
|
|
92
|
+
resolve();
|
|
93
|
+
}, delay);
|
|
94
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
95
|
+
});
|
|
96
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildSolidPack = buildSolidPack;
|
|
4
|
+
/**
|
|
5
|
+
* Many files compressed as one stream, the way the .cbx format does it.
|
|
6
|
+
*
|
|
7
|
+
* The upload path used to gzip each file on its own. That is the wrong shape
|
|
8
|
+
* for what a project actually holds: measured on a real folder, fifteen
|
|
9
|
+
* hundred small source files came to 42.9% of their size compressed
|
|
10
|
+
* individually and 30.2% compressed together. A two-kilobyte file gives a
|
|
11
|
+
* compressor nothing to work with — it starts cold, builds a dictionary out of
|
|
12
|
+
* two kilobytes, and throws it away. A thousand files from the same project
|
|
13
|
+
* share an enormous vocabulary, and only a shared stream can spend it.
|
|
14
|
+
*
|
|
15
|
+
* Notably the codec is not where the win is. Per-file zstd measured *worse*
|
|
16
|
+
* than per-file gzip on the same files (44.4% against 42.9%); it is the
|
|
17
|
+
* sharing that pays, not the algorithm.
|
|
18
|
+
*
|
|
19
|
+
* Gzip is used here rather than the format's Zstandard because the service has
|
|
20
|
+
* to be able to open a pack to serve one file out of it, and the Worker
|
|
21
|
+
* runtime can decompress gzip and cannot decompress zstd. That costs some
|
|
22
|
+
* ratio — zstd reached 30.2% where gzip reaches 36.2% — and it is the whole
|
|
23
|
+
* of the difference between this and what `packBundle` writes to disk.
|
|
24
|
+
*/
|
|
25
|
+
const node_crypto_1 = require("node:crypto");
|
|
26
|
+
const node_zlib_1 = require("node:zlib");
|
|
27
|
+
const node_util_1 = require("node:util");
|
|
28
|
+
const deflate = (0, node_util_1.promisify)(node_zlib_1.gzip);
|
|
29
|
+
/**
|
|
30
|
+
* Build one pack from a set of files.
|
|
31
|
+
*
|
|
32
|
+
* Members are laid end to end in the order given and each one's position is
|
|
33
|
+
* recorded, so the service can hand back any single file by decompressing up
|
|
34
|
+
* to its offset. Order is therefore not arbitrary: it is the index.
|
|
35
|
+
*
|
|
36
|
+
* Returns null when packing is not worth it — a pack of one file is just a
|
|
37
|
+
* compressed file with extra bookkeeping, and a pack that did not compress is
|
|
38
|
+
* a decompression step the service would pay for on every read forever.
|
|
39
|
+
*/
|
|
40
|
+
async function buildSolidPack(files) {
|
|
41
|
+
if (files.length < 2)
|
|
42
|
+
return null;
|
|
43
|
+
const members = [];
|
|
44
|
+
let offset = 0;
|
|
45
|
+
for (const file of files) {
|
|
46
|
+
members.push({
|
|
47
|
+
sha256: file.sha256,
|
|
48
|
+
offset,
|
|
49
|
+
length: file.body.byteLength,
|
|
50
|
+
});
|
|
51
|
+
offset += file.body.byteLength;
|
|
52
|
+
}
|
|
53
|
+
const stream = new Uint8Array(offset);
|
|
54
|
+
let at = 0;
|
|
55
|
+
for (const file of files) {
|
|
56
|
+
stream.set(file.body, at);
|
|
57
|
+
at += file.body.byteLength;
|
|
58
|
+
}
|
|
59
|
+
/*
|
|
60
|
+
Ask a sample before compressing the whole thing.
|
|
61
|
+
|
|
62
|
+
Measured across a hundred and eighty real files, trial-compressing sixty
|
|
63
|
+
four kilobytes predicted the whole file's answer in all but one case — and
|
|
64
|
+
that one erred towards trying, which costs milliseconds, rather than
|
|
65
|
+
towards skipping, which would lose a real saving.
|
|
66
|
+
|
|
67
|
+
It matters most exactly where packing is pointless: a pack of already
|
|
68
|
+
compressed images or video would otherwise be gzipped in full, at some
|
|
69
|
+
tens of megabytes per second, to discover what a sixty four kilobyte probe
|
|
70
|
+
says in under two milliseconds.
|
|
71
|
+
*/
|
|
72
|
+
const probe = stream.subarray(0, Math.min(64 * 1024, stream.byteLength));
|
|
73
|
+
try {
|
|
74
|
+
const sampled = await deflate(probe, { level: 6 });
|
|
75
|
+
if (sampled.byteLength >= probe.byteLength * 0.95)
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
let packed;
|
|
82
|
+
try {
|
|
83
|
+
packed = await deflate(stream, { level: 6 });
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
/*
|
|
89
|
+
Measured, not assumed. A pack of already-compressed files — a folder of
|
|
90
|
+
images, a directory of archives — comes out no smaller, and storing it as
|
|
91
|
+
a pack would buy nothing while making every later read of a single file
|
|
92
|
+
decompress the whole thing up to its offset.
|
|
93
|
+
*/
|
|
94
|
+
if (packed.byteLength >= stream.byteLength * 0.95)
|
|
95
|
+
return null;
|
|
96
|
+
return {
|
|
97
|
+
body: new Uint8Array(packed),
|
|
98
|
+
storedSha256: (0, node_crypto_1.createHash)("sha256").update(packed).digest("hex"),
|
|
99
|
+
sha256: (0, node_crypto_1.createHash)("sha256").update(stream).digest("hex"),
|
|
100
|
+
size: stream.byteLength,
|
|
101
|
+
members,
|
|
102
|
+
};
|
|
103
|
+
}
|