@coderook/cli 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,137 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.suggestExclusions = suggestExclusions;
7
+ exports.rulesFromSuggestions = rulesFromSuggestions;
8
+ /**
9
+ * Looking at a folder and saying what probably should not be uploaded.
10
+ *
11
+ * Choosing what goes up is the part of adding a project that people get wrong,
12
+ * and they get it wrong in one direction: a folder that has been worked in for
13
+ * a year holds a virtual environment, a build directory, somebody's browser
14
+ * profile and three copies of a release zip, and none of it is the work. The
15
+ * cost is not only storage — it is a save that takes twenty minutes and can be
16
+ * stopped outright by one file another program holds open.
17
+ *
18
+ * So this proposes. It never decides: everything it finds comes back as a
19
+ * suggestion with a reason and a size, the person ticks what they agree with,
20
+ * and a folder it recognises nothing in produces no suggestions rather than a
21
+ * default nobody asked for. Being wrong here has to be cheap, which it is
22
+ * exactly as long as nothing is applied without somebody saying so.
23
+ */
24
+ const node_path_1 = __importDefault(require("node:path"));
25
+ /*
26
+ Everything here is recoverable from something else in the project, or is
27
+ another program's working state. Nothing that could be somebody's only copy
28
+ of anything is marked recommended.
29
+ */
30
+ const RULES = [
31
+ { directory: "node_modules", reason: "Installed packages — restored by installing again", recommended: true },
32
+ { directory: ".venv", reason: "Python virtual environment — rebuilt from requirements", recommended: true },
33
+ { directory: "venv", reason: "Python virtual environment — rebuilt from requirements", recommended: true },
34
+ { directory: "__pycache__", reason: "Python bytecode cache", recommended: true },
35
+ { directory: ".pytest_cache", reason: "Test runner cache", recommended: true },
36
+ { directory: ".mypy_cache", reason: "Type checker cache", recommended: true },
37
+ { directory: ".ruff_cache", reason: "Linter cache", recommended: true },
38
+ { directory: "target", reason: "Rust build output", recommended: true },
39
+ { directory: ".next", reason: "Next.js build output", recommended: true },
40
+ { directory: ".gradle", reason: "Gradle build state", recommended: true },
41
+ { directory: ".terraform", reason: "Downloaded Terraform providers", recommended: true },
42
+ /*
43
+ A Chromium or Electron profile left in the project folder. Worth its own
44
+ reason because the consequence is not only size: these hold lock files
45
+ another program keeps open, and one unreadable file stops the whole save.
46
+ */
47
+ { directory: "IndexedDB", reason: "Browser profile data — holds files another program keeps open", recommended: true },
48
+ { directory: "Local Storage", reason: "Browser profile data — holds files another program keeps open", recommended: true },
49
+ { directory: "Session Storage", reason: "Browser profile data — holds files another program keeps open", recommended: true },
50
+ { directory: "Service Worker", reason: "Browser profile data", recommended: true },
51
+ { directory: "GPUCache", reason: "Browser cache", recommended: true },
52
+ { directory: "Code Cache", reason: "Browser cache", recommended: true },
53
+ { directory: "blob_storage", reason: "Browser profile data", recommended: true },
54
+ /*
55
+ Build output and archives are usually regenerable, but "dist" is also a
56
+ perfectly ordinary folder name for hand-written files, so these are
57
+ offered rather than recommended.
58
+ */
59
+ { directory: "dist", reason: "Usually build output — check before excluding", recommended: false },
60
+ { directory: "build", reason: "Usually build output — check before excluding", recommended: false },
61
+ { directory: "out", reason: "Usually build output — check before excluding", recommended: false },
62
+ { extension: ".safetensors", reason: "Model weights — large, and usually downloadable", recommended: false },
63
+ { extension: ".ckpt", reason: "Model weights — large, and usually downloadable", recommended: false },
64
+ { extension: ".pt", reason: "Model weights — large, and usually downloadable", recommended: false },
65
+ { extension: ".pth", reason: "Model weights — large, and usually downloadable", recommended: false },
66
+ { extension: ".cbx", reason: "A CodeBox bundle of this project, inside the project", recommended: true },
67
+ { extension: ".pyc", reason: "Python bytecode", recommended: true },
68
+ { extension: ".log", reason: "Log files", recommended: false },
69
+ { file: "Local State", reason: "Browser profile data", recommended: true },
70
+ { file: "Preferences", reason: "Browser profile data", recommended: true },
71
+ ];
72
+ const norm = (value) => value.split("\\").join("/");
73
+ /**
74
+ * What this folder appears to contain that is not the work.
75
+ *
76
+ * Ordered by size, because the decision somebody is making is about how long
77
+ * their upload will take and the biggest item answers it. Anything matching
78
+ * nothing is simply absent — silence is the right output for a tidy folder.
79
+ */
80
+ function suggestExclusions(files) {
81
+ const totals = new Map();
82
+ for (const entry of files) {
83
+ const relative = norm(entry.path);
84
+ const segments = relative.split("/");
85
+ const name = segments[segments.length - 1] ?? "";
86
+ const extension = node_path_1.default.extname(name).toLowerCase();
87
+ for (const rule of RULES) {
88
+ let pattern = null;
89
+ if (rule.directory && segments.slice(0, -1).includes(rule.directory)) {
90
+ pattern = `${rule.directory}/`;
91
+ }
92
+ else if (rule.file && name === rule.file) {
93
+ pattern = rule.file;
94
+ }
95
+ else if (rule.extension && extension === rule.extension) {
96
+ pattern = `*${rule.extension}`;
97
+ }
98
+ if (!pattern)
99
+ continue;
100
+ const held = totals.get(pattern) ?? { bytes: 0, files: 0, rule };
101
+ held.bytes += entry.size;
102
+ held.files += 1;
103
+ totals.set(pattern, held);
104
+ /*
105
+ One rule per file. The list is ordered from most specific to least, so
106
+ a .pyc inside __pycache__ is counted once, under the directory that
107
+ explains it, rather than inflating two separate suggestions.
108
+ */
109
+ break;
110
+ }
111
+ }
112
+ return [...totals.entries()]
113
+ .map(([pattern, held]) => ({
114
+ pattern,
115
+ reason: held.rule.reason,
116
+ bytes: held.bytes,
117
+ files: held.files,
118
+ recommended: held.rule.recommended,
119
+ }))
120
+ .sort((left, right) => right.bytes - left.bytes);
121
+ }
122
+ /**
123
+ * The suggestions a person accepted, as lines for the rules file.
124
+ *
125
+ * Written as its own step so that nothing is ever applied as a side effect of
126
+ * looking: the detector proposes, this records a decision, and the two are
127
+ * only connected by somebody choosing.
128
+ */
129
+ function rulesFromSuggestions(accepted) {
130
+ if (!accepted.length)
131
+ return "";
132
+ return [
133
+ "# Suggested when this folder was added, and accepted.",
134
+ ...accepted.map((one) => one.pattern),
135
+ "",
136
+ ].join("\n");
137
+ }
@@ -14,10 +14,14 @@ exports.underway = underway;
14
14
  * not a file quietly written wrong.
15
15
  */
16
16
  const node_crypto_1 = require("node:crypto");
17
+ const node_fs_1 = require("node:fs");
17
18
  const promises_1 = require("node:fs/promises");
19
+ const promises_2 = require("node:stream/promises");
20
+ const node_stream_1 = require("node:stream");
18
21
  const node_path_1 = __importDefault(require("node:path"));
19
22
  const identify_js_1 = require("./identify.js");
20
23
  const faults_js_1 = require("./faults.js");
24
+ const retry_js_1 = require("./retry.js");
21
25
  async function exists(target) {
22
26
  try {
23
27
  await (0, promises_1.stat)(target);
@@ -92,7 +96,38 @@ class Downloader {
92
96
  if (this.aborted)
93
97
  throw new DownloadCancelled();
94
98
  }
99
+ /**
100
+ * One request, repeated while repeating it might help.
101
+ *
102
+ * A restore makes at least one request per file and, for a file kept in
103
+ * pieces, one per piece — three thousand seven hundred and fifty-six for a
104
+ * single file here. At those numbers a dropped connection is not a risk but
105
+ * a certainty, and without this the whole restore ends on the first one.
106
+ */
95
107
  async request(route) {
108
+ let wait = retry_js_1.RETRY_FIRST_WAIT_MS;
109
+ for (let attempt = 1;; attempt += 1) {
110
+ this.check();
111
+ try {
112
+ return await this.attempt(route);
113
+ }
114
+ catch (error) {
115
+ // Its own cancellation first: `worthRetrying` knows nothing of it.
116
+ if (error instanceof DownloadCancelled)
117
+ throw error;
118
+ if (attempt >= retry_js_1.RETRY_ATTEMPTS || !(0, retry_js_1.worthRetrying)(error))
119
+ throw error;
120
+ try {
121
+ await (0, retry_js_1.pauseFor)(this.controller.signal, wait);
122
+ }
123
+ catch {
124
+ throw new DownloadCancelled();
125
+ }
126
+ wait *= 2;
127
+ }
128
+ }
129
+ }
130
+ async attempt(route) {
96
131
  const token = await this.credentials.token();
97
132
  if (!token)
98
133
  throw new Error("Sign in again before downloading");
@@ -116,13 +151,26 @@ class Downloader {
116
151
  `update from ${tooOld.upgradeUrl}.`);
117
152
  }
118
153
  let message = `${route} failed (${response.status})`;
154
+ let code;
119
155
  try {
120
- message = JSON.parse(text)?.error?.message ?? message;
156
+ const body = JSON.parse(text);
157
+ message = body?.error?.message ?? message;
158
+ code = body?.error?.code;
121
159
  }
122
160
  catch {
123
161
  /* the body was not JSON; the status will have to do */
124
162
  }
125
- throw new Error(message);
163
+ /*
164
+ The status travels with the error.
165
+
166
+ Callers already reason about it — the retry rule asks whether a
167
+ failure is worth trying again, and the sync asks whether a project is
168
+ gone or merely unreachable — and both were reading a property that
169
+ was never set. A plain Error made every failure look identical, so a
170
+ 404 for a deleted project was retried five times and then reported as
171
+ though the network were down.
172
+ */
173
+ throw Object.assign(new Error(message), { status: response.status, code });
126
174
  }
127
175
  return response;
128
176
  }
@@ -140,11 +188,106 @@ class Downloader {
140
188
  }
141
189
  async files(repositoryId, versionId) {
142
190
  const body = (await (await this.request(`/v1/repositories/${repositoryId}/versions/${versionId}/files`)).json());
143
- return (body.files ?? []).map((row) => ({
144
- path: String(row.path ?? ""),
145
- sha256: String(row.sha256 ?? ""),
146
- sourceSize: Number(row.sourceSize ?? 0),
147
- }));
191
+ return (body.files ?? []).map((row) => {
192
+ const pieces = Array.isArray(row.chunks) ? row.chunks : null;
193
+ return {
194
+ path: String(row.path ?? ""),
195
+ sha256: String(row.sha256 ?? ""),
196
+ sourceSize: Number(row.sourceSize ?? 0),
197
+ ...(pieces && pieces.length
198
+ ? {
199
+ chunks: pieces.map((piece) => {
200
+ const one = piece;
201
+ return {
202
+ objectId: String(one.objectId ?? ""),
203
+ sha256: String(one.sha256 ?? ""),
204
+ sourceSize: Number(one.sourceSize ?? 0),
205
+ };
206
+ }),
207
+ }
208
+ : {}),
209
+ };
210
+ });
211
+ }
212
+ /**
213
+ * Fetch one file to `target`, verifying it as it lands, and answer with the
214
+ * digest of what was actually written.
215
+ *
216
+ * A file kept in pieces is fetched piece by piece and assembled here rather
217
+ * than by the service. That is the whole point of this method. Asking the
218
+ * service to hand back one reassembled file made it loop over every piece
219
+ * inside a single edge invocation, and for a file of three thousand seven
220
+ * hundred and fifty-six pieces it ran out of whatever it runs out of and
221
+ * stopped — after the two hundred and its headers had already gone out. A
222
+ * 7.34 GB file came back as 5.09 GB under a clean `200`, with nothing in
223
+ * the response saying otherwise. Pulling the pieces from here bounds each
224
+ * request to one piece and puts the loop somewhere that can fail honestly.
225
+ *
226
+ * Nothing is held whole. The previous version read the entire file into one
227
+ * buffer to hash it, which for this file would have been 7.34 GB resident
228
+ * before the check it was there to perform could even run.
229
+ */
230
+ /**
231
+ * Fetch one file from a version to a chosen path, verified, and answer with
232
+ * the digest of what was written.
233
+ *
234
+ * Restoring a whole version is the usual case. This is the same machinery
235
+ * for somebody who wants one file out of one — and it is how a very large
236
+ * file can be checked on its own, without pulling down the project it
237
+ * belongs to in order to find out whether it survives the trip.
238
+ */
239
+ async fileTo(repositoryId, versionId, file, target) {
240
+ await (0, promises_1.mkdir)(node_path_1.default.dirname(target), { recursive: true });
241
+ return this.fetchInto(repositoryId, versionId, file, target);
242
+ }
243
+ async fetchInto(repositoryId, versionId, file, target) {
244
+ const whole = (0, node_crypto_1.createHash)("sha256");
245
+ const pieces = file.chunks ?? [];
246
+ // Captured so the generators below do not need `this` rebound.
247
+ const request = (route) => this.request(route);
248
+ const check = () => this.check();
249
+ const source = pieces.length
250
+ ? (async function* () {
251
+ for (let at = 0; at < pieces.length; at += 1) {
252
+ check();
253
+ const piece = pieces[at];
254
+ const reply = await request(`/v1/repositories/${repositoryId}/objects/${piece.objectId}`);
255
+ const body = Buffer.from(await reply.arrayBuffer());
256
+ /*
257
+ Checked against what the upload recorded for this piece, before
258
+ a byte of it reaches the file. Without the digest the only
259
+ available check was the whole file's, which says that something
260
+ in seven gigabytes is wrong and nothing about what.
261
+ */
262
+ const digest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
263
+ if (piece.sha256 && digest !== piece.sha256) {
264
+ throw new Error(`${file.path}: piece ${at + 1} of ${pieces.length} ` +
265
+ `did not arrive intact`);
266
+ }
267
+ whole.update(body);
268
+ yield body;
269
+ }
270
+ })()
271
+ : (async function* () {
272
+ const reply = await request(`/v1/repositories/${repositoryId}/versions/${versionId}/file` +
273
+ `?path=${encodeURIComponent(file.path)}`);
274
+ if (!reply.body)
275
+ return;
276
+ for await (const block of node_stream_1.Readable.fromWeb(reply.body)) {
277
+ check();
278
+ const buffer = Buffer.from(block);
279
+ whole.update(buffer);
280
+ yield buffer;
281
+ }
282
+ })();
283
+ /*
284
+ `pipeline` rather than a write loop: it applies backpressure, and it
285
+ destroys the file handle when the source throws. A partially written
286
+ file left open on a failed verification is how a staging directory ends
287
+ up holding something that looks finished.
288
+ */
289
+ await (0, promises_2.pipeline)(source, (0, node_fs_1.createWriteStream)(target));
290
+ return whole.digest("hex");
148
291
  }
149
292
  /**
150
293
  * Write a version into `destination`.
@@ -194,19 +337,23 @@ class Downloader {
194
337
  if (parts.some((part) => part === ".." || part.includes("\0"))) {
195
338
  throw new Error(`That version contains an unsafe path: ${file.path}`);
196
339
  }
197
- const response = await this.request(`/v1/repositories/${repositoryId}/versions/${versionId}/file` +
198
- `?path=${encodeURIComponent(file.path)}`);
199
- const body = Buffer.from(await response.arrayBuffer());
200
- const digest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
340
+ const full = node_path_1.default.join(staging, ...parts);
341
+ await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
342
+ const digest = await this.fetchInto(repositoryId, versionId, file, full);
343
+ /*
344
+ The whole file's digest, checked after every piece has already been
345
+ checked individually. Both are needed: the pieces catch a corrupt or
346
+ truncated piece and name it, and this catches a set of individually
347
+ valid pieces assembled in the wrong order or with one missing —
348
+ which is the failure that produces bytes that are all present, all
349
+ verified, and wrong.
350
+ */
201
351
  if (file.sha256 && digest !== file.sha256) {
202
352
  throw new Error(`${file.path} did not arrive intact`);
203
353
  }
204
- const full = node_path_1.default.join(staging, ...parts);
205
- await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
206
- await (0, promises_1.writeFile)(full, body);
207
354
  manifest[file.path] = digest;
208
355
  written += 1;
209
- bytes += body.length;
356
+ bytes += file.sourceSize;
210
357
  }
211
358
  /*
212
359
  Everything has arrived and verified, so it can go into place. Each
@@ -0,0 +1,73 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.timed = timed;
4
+ exports.counted = counted;
5
+ exports.profileReport = profileReport;
6
+ exports.resetProfile = resetProfile;
7
+ /**
8
+ * Where an upload's time actually goes.
9
+ *
10
+ * Written because guessing was wrong twice. The first batching attempt was
11
+ * built on the assumption that HTTP round trips were the cost; they were about
12
+ * a tenth of it, and the real expense was ten database queries per object. The
13
+ * fix for that then revealed object storage as the next bottleneck. Neither
14
+ * would have been found by reading the code.
15
+ *
16
+ * Inert unless CODEROOK_PROFILE is set, so it costs a boolean check on a hot
17
+ * path and nothing else. It reports totals per named phase rather than a trace,
18
+ * because the question is always "which stage is the upload" and never "what
19
+ * happened at 14:32:07".
20
+ */
21
+ const enabled = Boolean(process.env.CODEROOK_PROFILE);
22
+ const phases = new Map();
23
+ /** Time one awaited step and attribute it to a phase. */
24
+ async function timed(phase, work) {
25
+ if (!enabled)
26
+ return work();
27
+ const started = performance.now();
28
+ try {
29
+ return await work();
30
+ }
31
+ finally {
32
+ const held = phases.get(phase) ?? { total: 0, count: 0 };
33
+ held.total += performance.now() - started;
34
+ held.count += 1;
35
+ phases.set(phase, held);
36
+ }
37
+ }
38
+ /** Count something that is not a duration — files batched, files sent alone. */
39
+ function counted(phase, by = 1) {
40
+ if (!enabled)
41
+ return;
42
+ const held = phases.get(phase) ?? { total: 0, count: 0 };
43
+ held.count += by;
44
+ phases.set(phase, held);
45
+ }
46
+ /**
47
+ * What was measured, slowest first.
48
+ *
49
+ * Returns an empty string when profiling is off, so a caller can print it
50
+ * unconditionally without deciding whether there is anything to print.
51
+ */
52
+ function profileReport() {
53
+ if (!enabled || !phases.size)
54
+ return "";
55
+ const rows = [...phases.entries()].sort((left, right) => right[1].total - left[1].total);
56
+ const width = Math.max(...rows.map(([name]) => name.length));
57
+ return [
58
+ "",
59
+ "where the time went:",
60
+ ...rows.map(([name, phase]) => {
61
+ const seconds = (phase.total / 1000).toFixed(1);
62
+ const each = phase.count && phase.total
63
+ ? ` (${(phase.total / phase.count).toFixed(1)}ms each)`
64
+ : "";
65
+ return phase.total
66
+ ? ` ${name.padEnd(width)} ${seconds.padStart(7)}s ×${phase.count}${each}`
67
+ : ` ${name.padEnd(width)} ${String(phase.count).padStart(8)}`;
68
+ }),
69
+ ].join("\n");
70
+ }
71
+ function resetProfile() {
72
+ phases.clear();
73
+ }
@@ -0,0 +1,96 @@
1
+ "use strict";
2
+ /**
3
+ * Trying again, and knowing when not to.
4
+ *
5
+ * Both directions need this and neither had it. The uploader's absence of
6
+ * retries ended a twenty-eight gigabyte run at section fifty-one of
7
+ * eighty-seven, twenty-four minutes in, because one request came back `fetch
8
+ * failed`. The downloader has the same shape of problem and a worse
9
+ * consequence: a restore that gives up part way through has already written
10
+ * nothing usable, and the person is left without the thing they were
11
+ * restoring.
12
+ *
13
+ * It lives apart from both because the rule is the same for both and the
14
+ * cancellation is not. Each side has its own cancelled-error type, so each
15
+ * side checks for its own before asking this whether the failure was worth
16
+ * repeating. That keeps this module ignorant of either, rather than importing
17
+ * one into the other and coupling upload to download for a single `instanceof`.
18
+ */
19
+ Object.defineProperty(exports, "__esModule", { value: true });
20
+ exports.RETRY_FIRST_WAIT_MS = exports.RETRY_ATTEMPTS = void 0;
21
+ exports.worthRetrying = worthRetrying;
22
+ exports.pauseFor = pauseFor;
23
+ /**
24
+ * How many times one request is attempted before giving up.
25
+ *
26
+ * Five attempts with doubling backoff covers an outage of roughly fifteen
27
+ * seconds, which is far longer than the reconnects that actually end long
28
+ * transfers. Longer than that and a genuine outage is being waited on rather
29
+ * than reported, which helps nobody.
30
+ */
31
+ exports.RETRY_ATTEMPTS = 5;
32
+ /** The first backoff. Doubles each attempt, and is jittered when used. */
33
+ exports.RETRY_FIRST_WAIT_MS = 500;
34
+ /**
35
+ * Whether a failure is worth trying again.
36
+ *
37
+ * The service's refusals are deliberate and none of them change on a second
38
+ * attempt: every 4xx it returns names a settled condition — the version is
39
+ * gone, the body is malformed, the quota is full — that a retry would meet
40
+ * again identically. Repeating those turns a clear answer into a long pause
41
+ * followed by the same answer, with the message that explained it buried under
42
+ * four pointless attempts.
43
+ *
44
+ * What is worth repeating is everything that never reached a verdict: a
45
+ * dropped connection, a gateway mid-restart, a request that timed out. Those
46
+ * carry no decision at all, and they are the whole failure population that
47
+ * ends long transfers.
48
+ *
49
+ * Cancellation is not considered here. It is a decision too — the user's —
50
+ * and each caller recognises its own before asking.
51
+ */
52
+ function worthRetrying(error) {
53
+ const status = error.status;
54
+ if (typeof status !== "number") {
55
+ /*
56
+ No status means the request never got an answer — `fetch failed`, a
57
+ reset socket, a name that did not resolve. This is the case that ended
58
+ the twenty-eight gigabyte upload.
59
+ */
60
+ return true;
61
+ }
62
+ if (status === 429) {
63
+ /*
64
+ The one 4xx worth repeating, and only sometimes. A 429 from the edge is
65
+ throttling and clears in seconds. A 429 from the upload window is a
66
+ refusal that stands for days, so retrying it would hang the transfer
67
+ instead of showing the person the message that explains it.
68
+ */
69
+ return error.code !== "upload_rate_exceeded";
70
+ }
71
+ if (status === 408)
72
+ return true;
73
+ return status >= 500;
74
+ }
75
+ /**
76
+ * Wait, unless the signal aborts while waiting.
77
+ *
78
+ * Jittered, because the lanes fail together. Six requests against a gateway
79
+ * that has just gone away all fail within milliseconds of each other, and six
80
+ * retries timed from those failures would arrive together too — which turns
81
+ * one outage into a second one at precisely the wrong moment.
82
+ */
83
+ function pauseFor(signal, ms) {
84
+ const delay = ms * (0.5 + Math.random());
85
+ return new Promise((resolve, reject) => {
86
+ const stop = () => {
87
+ clearTimeout(timer);
88
+ reject(new Error("aborted"));
89
+ };
90
+ const timer = setTimeout(() => {
91
+ signal.removeEventListener("abort", stop);
92
+ resolve();
93
+ }, delay);
94
+ signal.addEventListener("abort", stop, { once: true });
95
+ });
96
+ }
@@ -0,0 +1,103 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.buildSolidPack = buildSolidPack;
4
+ /**
5
+ * Many files compressed as one stream, the way the .cbx format does it.
6
+ *
7
+ * The upload path used to gzip each file on its own. That is the wrong shape
8
+ * for what a project actually holds: measured on a real folder, fifteen
9
+ * hundred small source files came to 42.9% of their size compressed
10
+ * individually and 30.2% compressed together. A two-kilobyte file gives a
11
+ * compressor nothing to work with — it starts cold, builds a dictionary out of
12
+ * two kilobytes, and throws it away. A thousand files from the same project
13
+ * share an enormous vocabulary, and only a shared stream can spend it.
14
+ *
15
+ * Notably the codec is not where the win is. Per-file zstd measured *worse*
16
+ * than per-file gzip on the same files (44.4% against 42.9%); it is the
17
+ * sharing that pays, not the algorithm.
18
+ *
19
+ * Gzip is used here rather than the format's Zstandard because the service has
20
+ * to be able to open a pack to serve one file out of it, and the Worker
21
+ * runtime can decompress gzip and cannot decompress zstd. That costs some
22
+ * ratio — zstd reached 30.2% where gzip reaches 36.2% — and it is the whole
23
+ * of the difference between this and what `packBundle` writes to disk.
24
+ */
25
+ const node_crypto_1 = require("node:crypto");
26
+ const node_zlib_1 = require("node:zlib");
27
+ const node_util_1 = require("node:util");
28
+ const deflate = (0, node_util_1.promisify)(node_zlib_1.gzip);
29
+ /**
30
+ * Build one pack from a set of files.
31
+ *
32
+ * Members are laid end to end in the order given and each one's position is
33
+ * recorded, so the service can hand back any single file by decompressing up
34
+ * to its offset. Order is therefore not arbitrary: it is the index.
35
+ *
36
+ * Returns null when packing is not worth it — a pack of one file is just a
37
+ * compressed file with extra bookkeeping, and a pack that did not compress is
38
+ * a decompression step the service would pay for on every read forever.
39
+ */
40
+ async function buildSolidPack(files) {
41
+ if (files.length < 2)
42
+ return null;
43
+ const members = [];
44
+ let offset = 0;
45
+ for (const file of files) {
46
+ members.push({
47
+ sha256: file.sha256,
48
+ offset,
49
+ length: file.body.byteLength,
50
+ });
51
+ offset += file.body.byteLength;
52
+ }
53
+ const stream = new Uint8Array(offset);
54
+ let at = 0;
55
+ for (const file of files) {
56
+ stream.set(file.body, at);
57
+ at += file.body.byteLength;
58
+ }
59
+ /*
60
+ Ask a sample before compressing the whole thing.
61
+
62
+ Measured across a hundred and eighty real files, trial-compressing sixty
63
+ four kilobytes predicted the whole file's answer in all but one case — and
64
+ that one erred towards trying, which costs milliseconds, rather than
65
+ towards skipping, which would lose a real saving.
66
+
67
+ It matters most exactly where packing is pointless: a pack of already
68
+ compressed images or video would otherwise be gzipped in full, at some
69
+ tens of megabytes per second, to discover what a sixty four kilobyte probe
70
+ says in under two milliseconds.
71
+ */
72
+ const probe = stream.subarray(0, Math.min(64 * 1024, stream.byteLength));
73
+ try {
74
+ const sampled = await deflate(probe, { level: 6 });
75
+ if (sampled.byteLength >= probe.byteLength * 0.95)
76
+ return null;
77
+ }
78
+ catch {
79
+ return null;
80
+ }
81
+ let packed;
82
+ try {
83
+ packed = await deflate(stream, { level: 6 });
84
+ }
85
+ catch {
86
+ return null;
87
+ }
88
+ /*
89
+ Measured, not assumed. A pack of already-compressed files — a folder of
90
+ images, a directory of archives — comes out no smaller, and storing it as
91
+ a pack would buy nothing while making every later read of a single file
92
+ decompress the whole thing up to its offset.
93
+ */
94
+ if (packed.byteLength >= stream.byteLength * 0.95)
95
+ return null;
96
+ return {
97
+ body: new Uint8Array(packed),
98
+ storedSha256: (0, node_crypto_1.createHash)("sha256").update(packed).digest("hex"),
99
+ sha256: (0, node_crypto_1.createHash)("sha256").update(stream).digest("hex"),
100
+ size: stream.byteLength,
101
+ members,
102
+ };
103
+ }