@coderook/cli 0.30.2 → 0.31.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
  "name": "coderook",
3
3
  "displayName": "CodeRook",
4
4
  "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
5
- "version": "0.30.2",
5
+ "version": "0.31.1",
6
6
  "author": {
7
7
  "name": "ACCA Gaming Productions",
8
8
  "url": "https://coderook.com"
@@ -623,7 +623,21 @@ async function commandSubmit(parsed) {
623
623
  which is the wrong way round — so this is that capability, asked for
624
624
  plainly.
625
625
  */
626
- const files = await (0, worktree_js_1.changedFiles)(folder, rules, baseline, hasFlag(parsed, "sync") ? "synchronize" : "add-and-update");
626
+ /*
627
+ Digests this machine has taken before, read before the scan rather than
628
+ before the upload.
629
+
630
+ The uploader has always accepted these and the desktop has always passed
631
+ them, but on the command line the expensive part happens earlier: the
632
+ scan below reads and hashes the whole folder to decide what changed, and
633
+ that is where a 21,071-file project spent 169 of its 299 seconds. The
634
+ same map serves both — `changedFiles` fills it in place, so what the scan
635
+ learns is what the upload is handed.
636
+ */
637
+ const knownDigests = link?.repositoryId
638
+ ? await (0, config_js_1.readDigests)(link.repositoryId)
639
+ : new Map();
640
+ const files = await (0, worktree_js_1.changedFiles)(folder, rules, baseline, hasFlag(parsed, "sync") ? "synchronize" : "add-and-update", knownDigests);
627
641
  /*
628
642
  An upgraded folder with no materialisation record and something that
629
643
  differs from its recorded version is ambiguous in the one way that
@@ -788,6 +802,7 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
788
802
  */
789
803
  const uploadRequest = (0, publish_js_1.uploadRequestFor)({
790
804
  localPath: folder,
805
+ ...(knownDigests.size ? { knownDigests } : {}),
791
806
  changed: files.filter((file) => !file.deleted).map((file) => file.path),
792
807
  deleted: files.filter((file) => file.deleted).map((file) => file.path),
793
808
  /*
@@ -994,6 +1009,15 @@ The connection failed: ${text}`));
994
1009
  local: result.local,
995
1010
  manifest: result.manifest,
996
1011
  });
1012
+ /*
1013
+ What this save hashed, kept for the next one.
1014
+
1015
+ Merged rather than replaced: `digestsTaken` holds only the files actually
1016
+ read this time, and everything served from the existing record has to
1017
+ survive or the next save reads it all again — which is precisely the cost
1018
+ this exists to avoid.
1019
+ */
1020
+ await (0, config_js_1.writeDigests)(result.repositoryId, new Map([...knownDigests, ...uploader.digestsTaken()]));
997
1021
  if (result.repeated) {
998
1022
  console.log(`Already saved as ${accent(`v${result.sequence}`)} by an earlier attempt;` +
999
1023
  ` nothing was sent again.`);
@@ -3,16 +3,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.credentials = exports.DEFAULT_API = void 0;
6
+ exports.DamagedLinkError = exports.credentials = exports.DEFAULT_API = void 0;
7
7
  exports.apiOrigin = apiOrigin;
8
8
  exports.configDirectory = configDirectory;
9
9
  exports.storeToken = storeToken;
10
10
  exports.loadToken = loadToken;
11
11
  exports.clearToken = clearToken;
12
12
  exports.keyFor = keyFor;
13
+ exports.linkFileFor = linkFileFor;
13
14
  exports.readLink = readLink;
14
15
  exports.forgetLink = forgetLink;
15
16
  exports.writeLink = writeLink;
17
+ exports.readDigests = readDigests;
18
+ exports.writeDigests = writeDigests;
16
19
  /**
17
20
  * Where the command-line tool keeps its credential and its project links.
18
21
  *
@@ -21,6 +24,7 @@ exports.writeLink = writeLink;
21
24
  * location follows each platform's convention rather than scattering dot
22
25
  * directories about.
23
26
  */
27
+ const node_crypto_1 = require("node:crypto");
24
28
  const node_fs_1 = require("node:fs");
25
29
  const promises_1 = require("node:fs/promises");
26
30
  const node_os_1 = require("node:os");
@@ -42,7 +46,21 @@ function configDirectory() {
42
46
  return node_path_1.default.join(process.env.XDG_CONFIG_HOME ?? node_path_1.default.join((0, node_os_1.homedir)(), ".config"), "coderook");
43
47
  }
44
48
  const tokenFile = () => node_path_1.default.join(configDirectory(), "token");
45
- const linksFile = () => node_path_1.default.join(configDirectory(), "links.json");
49
+ /*
50
+ One small file per folder, under `links/`.
51
+
52
+ It was one file for every folder this machine had ever linked, read whole and
53
+ rewritten whole — pretty-printed — by every command. On a real machine that
54
+ file reached 419 MB across 549 folders, 536 of which no longer existed, and
55
+ every `cbx` paid about three seconds to read it before doing anything and
56
+ another two and a half to write it back after a save. It also lost updates:
57
+ two commands in two folders each rewrote the lot, and the later one erased
58
+ the earlier one's change.
59
+
60
+ The old file is read once, split into these, and renamed out of the way.
61
+ */
62
+ const linksDirectory = () => node_path_1.default.join(configDirectory(), "links");
63
+ const legacyLinksFile = () => node_path_1.default.join(configDirectory(), "links.json");
46
64
  /**
47
65
  * Write a file only its owner can read, and atomically.
48
66
  *
@@ -134,17 +152,165 @@ function keyFor(localPath) {
134
152
  }
135
153
  /** How the key used to be worked out, so existing links keep working. */
136
154
  const legacyKeyFor = (localPath) => node_path_1.default.resolve(localPath).toLowerCase();
137
- async function readLinks() {
155
+ /**
156
+ * A link record that exists and cannot be read.
157
+ *
158
+ * Deliberately not the same as "no link". Reading a damaged record as absent
159
+ * makes a linked folder look new, and the next save creates a second project
160
+ * beside the real one — so this stops and names the file instead.
161
+ */
162
+ class DamagedLinkError extends Error {
163
+ file;
164
+ constructor(file, cause) {
165
+ super(`CodeRook's record of which project a folder belongs to is damaged: ${file}\n` +
166
+ "Nothing was changed. Restore that file from a backup, or move it aside " +
167
+ "to treat the folder as not yet linked.");
168
+ this.file = file;
169
+ this.name = "DamagedLinkError";
170
+ this.cause = cause;
171
+ }
172
+ }
173
+ exports.DamagedLinkError = DamagedLinkError;
174
+ /** The file one folder's link lives in. Hashed, because a key is a path. */
175
+ function linkFileFor(localPath) {
176
+ return storedFileFor(keyFor(localPath));
177
+ }
178
+ const storedFileFor = (key) => node_path_1.default.join(linksDirectory(), `${(0, node_crypto_1.createHash)("sha256").update(key).digest("hex").slice(0, 32)}.json`);
179
+ function sameMap(left, right) {
180
+ const keys = Object.keys(left);
181
+ if (keys.length !== Object.keys(right).length)
182
+ return false;
183
+ for (const key of keys)
184
+ if (left[key] !== right[key])
185
+ return false;
186
+ return true;
187
+ }
188
+ function pack(key, link) {
189
+ const { local, ...rest } = link;
190
+ if (local && sameMap(local, link.manifest)) {
191
+ return { key, link: { ...rest, localIsManifest: true } };
192
+ }
193
+ return { key, link: local ? { ...rest, local } : rest };
194
+ }
195
+ function unpack(stored) {
196
+ const { localIsManifest, ...link } = stored.link;
197
+ /* A copy, so a caller changing one cannot quietly change the other. */
198
+ return localIsManifest ? { ...link, local: { ...link.manifest } } : link;
199
+ }
200
+ async function exists(file) {
201
+ return (0, promises_1.access)(file).then(() => true, () => false);
202
+ }
203
+ /** One folder's record: undefined when there is none, and loud when damaged. */
204
+ async function readStored(key) {
205
+ const file = storedFileFor(key);
206
+ let text;
138
207
  try {
139
- return JSON.parse(await (0, promises_1.readFile)(linksFile(), "utf8"));
208
+ text = await (0, promises_1.readFile)(file, "utf8");
140
209
  }
141
- catch {
142
- return {};
210
+ catch (error) {
211
+ if (error.code === "ENOENT")
212
+ return undefined;
213
+ throw error;
214
+ }
215
+ try {
216
+ const stored = JSON.parse(text);
217
+ if (stored?.key !== key)
218
+ throw new Error("the record is for another folder");
219
+ return stored;
220
+ }
221
+ catch (error) {
222
+ throw new DamagedLinkError(file, error);
223
+ }
224
+ }
225
+ /**
226
+ * Write a record only if there is not one already.
227
+ *
228
+ * The migration's writer. A link written by a command running at the same
229
+ * moment is newer than anything in the old file, and must win. A hard link
230
+ * makes "create only if absent" one atomic step, and a reader never sees a
231
+ * half-written file; where the disk cannot do hard links the check and the
232
+ * move are two steps, which is only a race against a command in that same
233
+ * folder in the same instant.
234
+ */
235
+ async function writeStoredIfAbsent(stored) {
236
+ const target = storedFileFor(stored.key);
237
+ if (await exists(target))
238
+ return;
239
+ await (0, promises_1.mkdir)(node_path_1.default.dirname(target), { recursive: true });
240
+ const temporary = `${target}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
241
+ await (0, promises_1.writeFile)(temporary, JSON.stringify(stored), "utf8");
242
+ await (0, promises_1.chmod)(temporary, 0o600).catch(() => undefined);
243
+ try {
244
+ await (0, promises_1.link)(temporary, target);
245
+ }
246
+ catch (error) {
247
+ const code = error.code;
248
+ if (code !== "EEXIST" && !(await exists(target))) {
249
+ await (0, promises_1.rename)(temporary, target);
250
+ return;
251
+ }
252
+ }
253
+ finally {
254
+ await (0, promises_1.rm)(temporary, { force: true }).catch(() => undefined);
143
255
  }
144
256
  }
257
+ /**
258
+ * Split the old single file into one per folder, once.
259
+ *
260
+ * Whoever renames it to `.migrating` first does the work, and anyone arriving
261
+ * while that is going on — or after a run that was interrupted — works from
262
+ * the `.migrating` copy too. Every write here is "only if absent", so running
263
+ * it twice is harmless and a newer link written meanwhile is never overwritten.
264
+ *
265
+ * A damaged old file is put back and reported rather than migrated as empty:
266
+ * that would silently unlink every folder on the machine.
267
+ */
268
+ async function migrateLegacyLinks() {
269
+ const legacy = legacyLinksFile();
270
+ const claimed = `${legacy}.migrating`;
271
+ await (0, promises_1.rename)(legacy, claimed).catch(() => undefined);
272
+ let text;
273
+ try {
274
+ text = await (0, promises_1.readFile)(claimed, "utf8");
275
+ }
276
+ catch (error) {
277
+ if (error.code === "ENOENT")
278
+ return;
279
+ throw error;
280
+ }
281
+ let links;
282
+ try {
283
+ links = JSON.parse(text);
284
+ }
285
+ catch (error) {
286
+ await (0, promises_1.rename)(claimed, legacy).catch(() => undefined);
287
+ throw new DamagedLinkError(legacy, error);
288
+ }
289
+ for (const [key, link] of Object.entries(links)) {
290
+ await writeStoredIfAbsent(pack(key, link));
291
+ }
292
+ /* Kept rather than deleted: it is somebody's data, and moving it is enough. */
293
+ await (0, promises_1.rename)(claimed, `${legacy}.migrated`).catch(() => undefined);
294
+ }
295
+ async function lookUp(localPath) {
296
+ for (const key of new Set([keyFor(localPath), legacyKeyFor(localPath)])) {
297
+ const stored = await readStored(key);
298
+ if (!stored)
299
+ continue;
300
+ if ("forgotten" in stored)
301
+ return null;
302
+ return unpack(stored);
303
+ }
304
+ return null;
305
+ }
145
306
  async function readLink(localPath) {
146
- const links = await readLinks();
147
- const link = links[keyFor(localPath)] ?? links[legacyKeyFor(localPath)] ?? null;
307
+ let link = await lookUp(localPath);
308
+ if (!link &&
309
+ ((await exists(legacyLinksFile())) ||
310
+ (await exists(`${legacyLinksFile()}.migrating`)))) {
311
+ await migrateLegacyLinks();
312
+ link = await lookUp(localPath);
313
+ }
148
314
  if (!link)
149
315
  return null;
150
316
  return {
@@ -160,25 +326,46 @@ async function readLink(localPath) {
160
326
  * afterwards — which is the one thing this is for.
161
327
  */
162
328
  async function forgetLink(localPath) {
163
- const links = await readLinks();
164
- const had = links[keyFor(localPath)] !== undefined ||
165
- links[legacyKeyFor(localPath)] !== undefined;
166
- if (!had)
329
+ if (!(await readLink(localPath)))
167
330
  return false;
168
- delete links[keyFor(localPath)];
169
- delete links[legacyKeyFor(localPath)];
170
- await writePrivate(linksFile(), JSON.stringify(links, null, 2));
331
+ for (const key of new Set([keyFor(localPath), legacyKeyFor(localPath)])) {
332
+ const forgotten = { key, forgotten: true };
333
+ await writePrivate(storedFileFor(key), JSON.stringify(forgotten));
334
+ }
171
335
  return true;
172
336
  }
173
337
  async function writeLink(localPath, link) {
174
- const links = await readLinks();
175
- // A folder recorded under the old key moves to the new one rather than
176
- // being left behind as a second connection to the same place.
177
- delete links[legacyKeyFor(localPath)];
338
+ const key = keyFor(localPath);
178
339
  const baseVersionId = link.baseVersionId ?? link.versionId;
179
- links[keyFor(localPath)] = {
340
+ const stored = pack(key, {
180
341
  ...link,
181
342
  ...(baseVersionId ? { baseVersionId, versionId: baseVersionId } : {}),
182
- };
183
- await writePrivate(linksFile(), JSON.stringify(links, null, 2));
343
+ });
344
+ await writePrivate(storedFileFor(key), JSON.stringify(stored));
345
+ // A folder recorded under the old key moves to the new one rather than
346
+ // being left behind as a second connection to the same place.
347
+ const legacy = legacyKeyFor(localPath);
348
+ if (legacy !== key)
349
+ await (0, promises_1.rm)(storedFileFor(legacy), { force: true });
350
+ }
351
+ const digestsFile = (projectId) => node_path_1.default.join(configDirectory(), "digests", `${projectId}.json`);
352
+ async function readDigests(projectId) {
353
+ try {
354
+ const raw = await (0, promises_1.readFile)(digestsFile(projectId), "utf8");
355
+ return new Map(Object.entries(JSON.parse(raw)));
356
+ }
357
+ catch {
358
+ /* No record yet, or one this build cannot read: hash everything. */
359
+ return new Map();
360
+ }
361
+ }
362
+ async function writeDigests(projectId, digests) {
363
+ if (!digests.size)
364
+ return;
365
+ try {
366
+ await writePrivate(digestsFile(projectId), JSON.stringify(Object.fromEntries(digests)));
367
+ }
368
+ catch {
369
+ /* The next save hashes again, which is where this started. */
370
+ }
184
371
  }
@@ -114,6 +114,7 @@ const node_os_1 = __importDefault(require("node:os"));
114
114
  const node_path_1 = __importDefault(require("node:path"));
115
115
  const node_process_1 = __importDefault(require("node:process"));
116
116
  const upload_js_1 = require("../../desktop-app/src/main/upload.js");
117
+ const stored_zip_js_1 = require("./stored_zip.js");
117
118
  const tracks_js_1 = require("../../desktop-app/src/main/tracks.js");
118
119
  const identify_js_1 = require("../../desktop-app/src/main/identify.js");
119
120
  const config_js_1 = require("./config.js");
@@ -533,6 +534,17 @@ async function recordPushedCommits(marks) {
533
534
  * Contents are deduplicated by SHA-256 across the whole history, so a file
534
535
  * unchanged across a thousand versions is downloaded and sent to git once.
535
536
  */
537
+ /*
538
+ How many blobs a clone fetches at once, and how much it will hold doing it.
539
+
540
+ Eight lanes for the same reason the folder walks use eight: these are
541
+ latency, not work, and the measured curve flattens there. The byte ceiling
542
+ is what stops a window of large files becoming a window of memory — one
543
+ oversized file still goes through on its own, because the count check comes
544
+ first and an empty window always accepts one more.
545
+ */
546
+ const BLOB_LANES = 8;
547
+ const BLOB_WINDOW_BYTES = 64 * 1024 * 1024;
536
548
  async function doImport(refs, url) {
537
549
  const { owner, slug } = (0, git_history_js_1.parseRemoteUrl)(url);
538
550
  const project = await (0, api_js_1.findProject)(owner ? `${owner}/${slug}` : slug);
@@ -639,22 +651,99 @@ async function doImport(refs, url) {
639
651
  let done = 0;
640
652
  for (const version of fresh) {
641
653
  const files = await downloader.files(repositoryId, version.id);
642
- // Blobs first: fast-import requires a mark to exist before a commit
643
- // names it.
644
- for (const file of files) {
645
- if (blobMark.has(file.sha256))
646
- continue;
647
- const target = node_path_1.default.join(scratch, "blob.bin");
648
- await (0, promises_1.rm)(target, { force: true });
649
- await downloader.fileTo(repositoryId, version.id, file, target);
650
- const bytes = await (0, promises_1.readFile)(target);
651
- const mark = nextMark++;
652
- blobMark.set(file.sha256, mark);
653
- send("blob");
654
- send(`mark :${mark}`);
655
- send(`data ${bytes.length}`);
656
- node_process_1.default.stdout.write(bytes);
657
- node_process_1.default.stdout.write("\n");
654
+ /*
655
+ Blobs first: fast-import requires a mark to exist before a commit
656
+ names it. Fetched a window at a time and written out in order.
657
+
658
+ This asked for one file, waited for it, wrote it to a scratch path,
659
+ read it back, and only then asked for the next — so a clone cost one
660
+ network round trip per file, in series. Against a 21,071-file project
661
+ that is 21,071 round trips at about seven tenths of a second each,
662
+ and a clone that never finished: measured over twenty seconds it was
663
+ using 0.17s of processor and moving two kilobytes.
664
+
665
+ The stream fast-import reads has to stay ordered, so the fetching is
666
+ what overlaps, not the writing. The window is bounded by count and by
667
+ bytes together, because sixteen source files and sixteen video files
668
+ are not the same amount to be holding at once.
669
+ */
670
+ /*
671
+ The whole version in one request, where that works.
672
+
673
+ Fetching per file is one round trip each: 21,071 of them on a real
674
+ Unity project, which is four hours in series and about half an hour
675
+ even eight at a time. The archive is the same bytes in one request.
676
+
677
+ Anything unexpected about it — an entry this cannot read, a short
678
+ download, a service that does not offer the route — falls through to
679
+ the per-file path below, which is slow and known to work. A clone
680
+ that is slow is a nuisance; a clone that writes the wrong bytes is
681
+ not recoverable by trying again.
682
+ */
683
+ const fromArchive = new Map();
684
+ try {
685
+ const packed = await downloader.archive(repositoryId, version.id);
686
+ for (const entry of (0, stored_zip_js_1.readStoredZip)(packed)) {
687
+ fromArchive.set(entry.path, entry.bytes);
688
+ }
689
+ }
690
+ catch (error) {
691
+ /*
692
+ Said out loud, not swallowed.
693
+
694
+ This fell back silently, and the silence cost an hour: a deadline
695
+ meant for ordinary requests was aborting the archive, the import
696
+ dropped to fetching every file on its own, and from the outside it
697
+ looked exactly like the slow clone the archive was added to fix.
698
+ A fallback nobody can see is a fallback nobody can diagnose.
699
+ */
700
+ say(` the archive for v${version.sequence} could not be used ` +
701
+ `(${error instanceof Error ? error.message : String(error)}); ` +
702
+ `fetching its files one by one`);
703
+ fromArchive.clear();
704
+ }
705
+ const seenBlob = new Set();
706
+ const wantedBlobs = files.filter((file) => {
707
+ if (blobMark.has(file.sha256) || seenBlob.has(file.sha256))
708
+ return false;
709
+ seenBlob.add(file.sha256);
710
+ return true;
711
+ });
712
+ for (let at = 0; at < wantedBlobs.length;) {
713
+ const window = [];
714
+ let held = 0;
715
+ while (at < wantedBlobs.length && window.length < BLOB_LANES) {
716
+ const size = Number(wantedBlobs[at].sourceSize ?? 0);
717
+ if (window.length > 0 && held + size > BLOB_WINDOW_BYTES)
718
+ break;
719
+ held += size;
720
+ window.push(wantedBlobs[at]);
721
+ at += 1;
722
+ }
723
+ const fetched = await Promise.all(window.map(async (file, lane) => {
724
+ /* Already in hand, if the archive carried it. */
725
+ const packed = fromArchive.get(file.path);
726
+ if (packed)
727
+ return { file, bytes: packed };
728
+ /*
729
+ A scratch path per lane. They shared one name, which is fine
730
+ while only one download is ever in flight and silently wrong
731
+ the moment two are.
732
+ */
733
+ const target = node_path_1.default.join(scratch, `blob-${lane}.bin`);
734
+ await (0, promises_1.rm)(target, { force: true });
735
+ await downloader.fileTo(repositoryId, version.id, file, target);
736
+ return { file, bytes: await (0, promises_1.readFile)(target) };
737
+ }));
738
+ for (const { file, bytes } of fetched) {
739
+ const mark = nextMark++;
740
+ blobMark.set(file.sha256, mark);
741
+ send("blob");
742
+ send(`mark :${mark}`);
743
+ send(`data ${bytes.length}`);
744
+ node_process_1.default.stdout.write(bytes);
745
+ node_process_1.default.stdout.write("\n");
746
+ }
658
747
  }
659
748
  const mark = nextMark++;
660
749
  commitMark.set(version.id, mark);
@@ -23,6 +23,11 @@ function uploadRequestFor(input) {
23
23
  ? { expectedHeadVersionId: input.baseVersionId }
24
24
  : {}),
25
25
  ...(input.known ? { known: input.known } : {}),
26
+ /*
27
+ Digests this machine has taken before, so an unchanged file is not read
28
+ again. The desktop has always passed these; the command line never did.
29
+ */
30
+ ...(input.knownDigests ? { knownDigests: input.knownDigests } : {}),
26
31
  };
27
32
  }
28
33
  function classifyPublishFailure(error) {
@@ -0,0 +1,85 @@
1
+ "use strict";
2
+ /**
3
+ * Reading the archive CodeRook builds, without a compression library.
4
+ *
5
+ * A clone fetched every file individually: one HTTP round trip per blob, in
6
+ * series. Against a 21,071-file project that is 21,071 round trips, which at
7
+ * seven tenths of a second each is four hours — and the clone that exposed
8
+ * this simply never finished. Fetching eight at a time cut it to about half
9
+ * an hour; asking for the version as one archive cuts it to one request.
10
+ *
11
+ * Written by hand for the same reason `backend/src/archive.ts` writes it by
12
+ * hand: the entries are *stored*, never deflated, so the whole job is a
13
+ * header, a body, and skipping the directory at the end. Pulling in a zip
14
+ * library to not decompress anything would be a poor trade.
15
+ *
16
+ * Only the format that writer produces is accepted. Anything else — a
17
+ * deflated entry, a size deferred to a data descriptor, a Zip64 field — is
18
+ * refused rather than guessed at, and the caller falls back to fetching the
19
+ * files one by one. A reader that quietly mangles a project is worse than a
20
+ * slow one.
21
+ */
22
+ Object.defineProperty(exports, "__esModule", { value: true });
23
+ exports.readStoredZip = readStoredZip;
24
+ const LOCAL_HEADER = 0x04034b50;
25
+ const CENTRAL_HEADER = 0x02014b50;
26
+ /* An archive of nothing has no entries and no directory records — it opens
27
+ with this, and a reader that only knew the other two would refuse it. */
28
+ const END_OF_DIRECTORY = 0x06054b50;
29
+ /** Set in the general-purpose flags when sizes come after the body. */
30
+ const SIZES_DEFERRED = 0x0008;
31
+ const STORED = 0;
32
+ /**
33
+ * Every entry in a stored archive, in the order it was written.
34
+ *
35
+ * Deliberately whole-buffer rather than streaming: the caller is holding one
36
+ * version's files to hand to fast-import, which needs them in order anyway,
37
+ * and a version large enough for this to matter is one the archive path
38
+ * should not be taking.
39
+ */
40
+ function readStoredZip(archive) {
41
+ const view = new DataView(archive.buffer, archive.byteOffset, archive.byteLength);
42
+ const names = new TextDecoder("utf-8", { fatal: false });
43
+ const entries = [];
44
+ let at = 0;
45
+ while (at + 4 <= archive.length) {
46
+ const signature = view.getUint32(at, true);
47
+ /* Either record marks the end of the entries. */
48
+ if (signature === CENTRAL_HEADER || signature === END_OF_DIRECTORY)
49
+ break;
50
+ if (signature !== LOCAL_HEADER) {
51
+ throw new Error(`Not a stored archive: unexpected record at byte ${at}`);
52
+ }
53
+ if (at + 30 > archive.length)
54
+ throw new Error("Archive ends inside a header");
55
+ const flags = view.getUint16(at + 6, true);
56
+ const method = view.getUint16(at + 8, true);
57
+ const compressed = view.getUint32(at + 18, true);
58
+ const nameLength = view.getUint16(at + 26, true);
59
+ const extraLength = view.getUint16(at + 28, true);
60
+ if (method !== STORED) {
61
+ throw new Error(`Archive entry is compressed (method ${method})`);
62
+ }
63
+ if (flags & SIZES_DEFERRED) {
64
+ throw new Error("Archive entry defers its size to a data descriptor");
65
+ }
66
+ /*
67
+ 0xffffffff is Zip64's marker for "the real size is in an extra field".
68
+ This writer never emits one, so seeing it means the archive did not come
69
+ from where it is assumed to have come from.
70
+ */
71
+ if (compressed === 0xffffffff)
72
+ throw new Error("Archive entry needs Zip64");
73
+ const nameAt = at + 30;
74
+ const bodyAt = nameAt + nameLength + extraLength;
75
+ const end = bodyAt + compressed;
76
+ if (end > archive.length)
77
+ throw new Error("Archive ends inside an entry");
78
+ entries.push({
79
+ path: names.decode(archive.subarray(nameAt, nameAt + nameLength)),
80
+ bytes: archive.subarray(bodyAt, end),
81
+ });
82
+ at = end;
83
+ }
84
+ return entries;
85
+ }
@@ -55,6 +55,27 @@ async function encodeForUpload(original, relativePath, allowGzip) {
55
55
  if (!(0, compression_policy_1.shouldTryGzip)(relativePath, original.byteLength, allowGzip))
56
56
  return plain;
57
57
  try {
58
+ /*
59
+ Ask a little of it before asking all of it.
60
+
61
+ The extension list is a guess and the cost of a miss used to be the whole
62
+ file. Two sampled windows make it an eighth of that, which is what lets
63
+ the decision stay measured — the principle above — for a format nothing
64
+ here has heard of, instead of measured-at-any-price.
65
+ */
66
+ if ((0, compression_policy_1.worthSampling)(original.byteLength)) {
67
+ let anyWindowArguesFor = false;
68
+ for (const window of (0, compression_policy_1.sampleWindows)(original.byteLength)) {
69
+ const sample = original.subarray(window.at, window.at + window.size);
70
+ const tried = await deflate(sample, { level: 6 });
71
+ if (!(0, compression_policy_1.sampleSaysDoNotBother)(sample.byteLength, tried.byteLength)) {
72
+ anyWindowArguesFor = true;
73
+ break;
74
+ }
75
+ }
76
+ if (!anyWindowArguesFor)
77
+ return plain;
78
+ }
58
79
  const packed = await deflate(original, { level: 6 });
59
80
  if (!(0, compression_policy_1.gzipIsWorthKeeping)(original.byteLength, packed.byteLength))
60
81
  return plain;