@coderook/cli 0.29.1 → 0.29.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
  "name": "coderook",
3
3
  "displayName": "CodeRook",
4
4
  "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
5
- "version": "0.29.1",
5
+ "version": "0.29.3",
6
6
  "author": {
7
7
  "name": "ACCA Gaming Productions",
8
8
  "url": "https://coderook.com"
@@ -40,6 +40,7 @@ const download_js_1 = require("../../desktop-app/src/main/download.js");
40
40
  const faults_js_1 = require("../../desktop-app/src/main/faults.js");
41
41
  const version_commands_js_1 = require("./version_commands.js");
42
42
  const identify_js_1 = require("../../desktop-app/src/main/identify.js");
43
+ const profile_js_1 = require("../../desktop-app/src/main/profile.js");
43
44
  const detect_js_1 = require("../../desktop-app/src/main/detect.js");
44
45
  const cbx_js_1 = require("../../desktop-app/src/main/cbx.js");
45
46
  const api_js_2 = require("./api.js");
@@ -193,6 +194,23 @@ async function reconcile(folder, options = {}) {
193
194
  const project = link
194
195
  ? (await (0, api_js_2.projects)()).find((candidate) => candidate.id === link.repositoryId)
195
196
  : await (0, api_js_2.findProject)(reference);
197
+ /*
198
+ A link that has outlived the project it points at.
199
+
200
+ The list above is everything this account can reach — owned, shared with
201
+ them, and through their organisation — so a linked project missing from it
202
+ is gone or no longer theirs. Left alone, the next call that used the
203
+ remembered id failed with "Repository not found", which names neither the
204
+ folder nor anything to do about it; the soak suite hit exactly this and it
205
+ took a debugger to work out which folder was complaining.
206
+ */
207
+ if (link && !project) {
208
+ throw new Error(`This folder is linked to a project that is no longer on your account.\n` +
209
+ ` ${folder}\n` +
210
+ `It may have been deleted, or your access to it withdrawn.\n` +
211
+ `Detach the folder with ${accent("cbx unlink")} — the files here are ` +
212
+ `untouched, and submitting afterwards starts a new project from them.`);
213
+ }
196
214
  if (!project?.versionCount) {
197
215
  // Nothing saved on the account, so everything here is outstanding.
198
216
  return { link: link ?? null, baseline: null, behind: null };
@@ -302,6 +320,19 @@ async function commandProjects() {
302
320
  }
303
321
  return 0;
304
322
  }
323
+ async function commandUnlink(parsed) {
324
+ const folder = folderFor(parsed);
325
+ const link = await (0, config_js_1.readLink)(folder);
326
+ if (!(await (0, config_js_1.forgetLink)(folder))) {
327
+ console.log("This folder was not linked to a project.");
328
+ return 0;
329
+ }
330
+ console.log(`Detached ${bold(node_path_1.default.basename(folder))}` +
331
+ (link?.slug ? dim(` from ${link.slug}`) : "") +
332
+ ".");
333
+ console.log(dim("Nothing was deleted. Submitting from here now starts a new project."));
334
+ return 0;
335
+ }
305
336
  async function commandStatus(parsed) {
306
337
  const folder = folderFor(parsed);
307
338
  let link;
@@ -889,6 +920,16 @@ The connection failed: ${text}`));
889
920
  command does not do.
890
921
  */
891
922
  ` · the save holds ${bytes(result.sourceBytes)}`);
923
+ /*
924
+ Where the time went, when asked for.
925
+
926
+ The profiler has existed for a while and nothing ever printed it, so the
927
+ one question it answers — which stage is the save — still had to be
928
+ guessed at. Silent unless CODEROOK_PROFILE is set.
929
+ */
930
+ const profile = (0, profile_js_1.profileReport)();
931
+ if (profile)
932
+ console.log(profile);
892
933
  return 0;
893
934
  }
894
935
  /**
@@ -1667,6 +1708,20 @@ const SPECS = [
1667
1708
  ],
1668
1709
  run: commandSubmit,
1669
1710
  },
1711
+ {
1712
+ name: "unlink",
1713
+ group: "Working with a folder",
1714
+ summary: "detach this folder from its project",
1715
+ usage: "unlink [folder]",
1716
+ detail: "Forgets which project this folder belongs to. Nothing is deleted:\n" +
1717
+ "the files here are untouched and the project stays on your account.\n\n" +
1718
+ "Mostly useful when a folder points at a project that is gone — one\n" +
1719
+ "that was deleted, or shared with you and since withdrawn — because\n" +
1720
+ "until the folder forgets it, every command asks about a project that\n" +
1721
+ "is no longer there.",
1722
+ examples: ["cbx unlink", "cbx unlink ./old-copy"],
1723
+ run: commandUnlink,
1724
+ },
1670
1725
  {
1671
1726
  name: "import",
1672
1727
  group: "Getting started",
@@ -11,6 +11,7 @@ exports.loadToken = loadToken;
11
11
  exports.clearToken = clearToken;
12
12
  exports.keyFor = keyFor;
13
13
  exports.readLink = readLink;
14
+ exports.forgetLink = forgetLink;
14
15
  exports.writeLink = writeLink;
15
16
  /**
16
17
  * Where the command-line tool keeps its credential and its project links.
@@ -151,6 +152,24 @@ async function readLink(localPath) {
151
152
  baseVersionId: link.baseVersionId ?? link.versionId,
152
153
  };
153
154
  }
155
+ /**
156
+ * Forget which project a folder belongs to.
157
+ *
158
+ * Both keys, because a folder linked by an older build is recorded under the
159
+ * old one and leaving that behind would mean the folder was still linked
160
+ * afterwards — which is the one thing this is for.
161
+ */
162
+ async function forgetLink(localPath) {
163
+ const links = await readLinks();
164
+ const had = links[keyFor(localPath)] !== undefined ||
165
+ links[legacyKeyFor(localPath)] !== undefined;
166
+ if (!had)
167
+ return false;
168
+ delete links[keyFor(localPath)];
169
+ delete links[legacyKeyFor(localPath)];
170
+ await writePrivate(linksFile(), JSON.stringify(links, null, 2));
171
+ return true;
172
+ }
154
173
  async function writeLink(localPath, link) {
155
174
  const links = await readLinks();
156
175
  // A folder recorded under the old key moves to the new one rather than
@@ -473,75 +473,98 @@ class Downloader {
473
473
  packed.set(id, group);
474
474
  }
475
475
  const restoredPacks = new Set();
476
+ /*
477
+ Fetched several at a time, and each pack fetched once.
478
+
479
+ This was one request after another — a pack or a file, wait for it,
480
+ write it, ask for the next — while the sending side has had six lanes
481
+ for a long time. Measured across thirty project shapes, that made
482
+ restoring roughly six times slower than saving the same bytes: an
483
+ Unreal project of four hundred and ninety megabytes went up in ten
484
+ minutes and came back in sixty-one. A restore is mostly waiting on the
485
+ network, and waiting is the thing that overlaps.
486
+
487
+ The work is planned before any of it runs, which also removes the
488
+ double-fetch guard the serial version needed: a pack appears once in
489
+ the list, so it can only be fetched once.
490
+ */
491
+ const jobs = [];
476
492
  for (const file of files) {
477
- this.check();
493
+ const id = file.sealed ? undefined : file.pack?.objectId;
494
+ if (id) {
495
+ if (restoredPacks.has(id))
496
+ continue;
497
+ restoredPacks.add(id);
498
+ jobs.push({ kind: "pack", id });
499
+ }
500
+ else {
501
+ jobs.push({ kind: "file", file });
502
+ }
503
+ }
504
+ /** Refused outright, wherever it appears — inside a pack or beside it. */
505
+ const safeParts = (where) => {
506
+ const parts = where.split("/").filter(Boolean);
507
+ if (parts.some((part) => part === ".." || part.includes("\0"))) {
508
+ throw new Error(`That version contains an unsafe path: ${where}`);
509
+ }
510
+ return parts;
511
+ };
512
+ const told = { at: 0 };
513
+ const sayProgress = (where) => {
514
+ const now = Date.now();
515
+ if (now - told.at < 80)
516
+ return;
517
+ told.at = now;
478
518
  report({
479
519
  files: written,
480
520
  totalFiles: files.length,
481
521
  bytes,
482
522
  totalBytes,
483
- path: file.path,
523
+ path: where,
484
524
  percent: underway(bytes, totalBytes),
485
525
  });
486
- // A path that climbs out of the destination is refused outright.
487
- const parts = file.path.split("/").filter(Boolean);
488
- if (parts.some((part) => part === ".." || part.includes("\0"))) {
489
- throw new Error(`That version contains an unsafe path: ${file.path}`);
526
+ };
527
+ const restorePack = async (packId) => {
528
+ /*
529
+ A solid pack is one compressed object containing many small files.
530
+
531
+ Asking the single-file route for every member made a large restore
532
+ fetch and decompress the same pack thousands of times. The files
533
+ listing already gives us the pack id and each member's exact window,
534
+ so fetch the decoded pack once, verify it, and materialise every
535
+ member before releasing the buffer.
536
+ */
537
+ const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
538
+ const body = Buffer.from(await reply.arrayBuffer());
539
+ const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
540
+ const expectedPackDigest = reply.headers.get("x-coderook-sha256");
541
+ if (expectedPackDigest && packDigest !== expectedPackDigest) {
542
+ throw new Error("A solid pack did not arrive intact");
490
543
  }
491
- const packId = file.sealed ? undefined : file.pack?.objectId;
492
- if (packId && restoredPacks.has(packId))
493
- continue;
494
- if (packId) {
495
- /*
496
- A solid pack is one compressed object containing many small files.
497
-
498
- Asking the single-file route for every member made a large restore
499
- fetch and decompress the same pack thousands of times. The files
500
- listing already gives us the pack id and each member's exact
501
- window, so fetch the decoded pack once, verify it, and materialise
502
- every member before releasing the bounded section-sized buffer.
503
- */
504
- const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
505
- const body = Buffer.from(await reply.arrayBuffer());
506
- const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
507
- const expectedPackDigest = reply.headers.get("x-coderook-sha256");
508
- if (expectedPackDigest && packDigest !== expectedPackDigest) {
509
- throw new Error("A solid pack did not arrive intact");
544
+ for (const member of packed.get(packId) ?? []) {
545
+ this.check();
546
+ const memberParts = safeParts(member.path);
547
+ const offset = member.pack.offset;
548
+ const length = member.pack.length;
549
+ if (offset < 0 || length < 0 || offset + length > body.byteLength) {
550
+ throw new Error(`${member.path} points outside its solid pack`);
510
551
  }
511
- for (const member of packed.get(packId) ?? []) {
512
- this.check();
513
- const memberParts = member.path.split("/").filter(Boolean);
514
- if (memberParts.some((part) => part === ".." || part.includes("\0"))) {
515
- throw new Error(`That version contains an unsafe path: ${member.path}`);
516
- }
517
- const offset = member.pack.offset;
518
- const length = member.pack.length;
519
- if (offset < 0 || length < 0 || offset + length > body.byteLength) {
520
- throw new Error(`${member.path} points outside its solid pack`);
521
- }
522
- const contents = body.subarray(offset, offset + length);
523
- const digest = (0, node_crypto_1.createHash)("sha256").update(contents).digest("hex");
524
- if (member.sha256 && digest !== member.sha256) {
525
- throw new Error(`${member.path} did not arrive intact`);
526
- }
527
- const full = node_path_1.default.join(staging, ...memberParts);
528
- await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
529
- await (0, promises_1.writeFile)(full, contents);
530
- manifest[member.path] = digest;
531
- written += 1;
532
- bytes += member.sourceSize;
533
- report({
534
- files: written,
535
- totalFiles: files.length,
536
- bytes,
537
- totalBytes,
538
- path: member.path,
539
- percent: underway(bytes, totalBytes),
540
- });
552
+ const contents = body.subarray(offset, offset + length);
553
+ const digest = (0, node_crypto_1.createHash)("sha256").update(contents).digest("hex");
554
+ if (member.sha256 && digest !== member.sha256) {
555
+ throw new Error(`${member.path} did not arrive intact`);
541
556
  }
542
- restoredPacks.add(packId);
543
- continue;
557
+ const full = node_path_1.default.join(staging, ...memberParts);
558
+ await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
559
+ await (0, promises_1.writeFile)(full, contents);
560
+ manifest[member.path] = digest;
561
+ written += 1;
562
+ bytes += member.sourceSize;
563
+ sayProgress(member.path);
544
564
  }
565
+ };
566
+ const restoreFile = async (file) => {
567
+ const parts = safeParts(file.path);
545
568
  const full = node_path_1.default.join(staging, ...parts);
546
569
  await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
547
570
  const digest = await this.fetchInto(repositoryId, versionId, file, full);
@@ -559,7 +582,78 @@ class Downloader {
559
582
  manifest[file.path] = digest;
560
583
  written += 1;
561
584
  bytes += file.sourceSize;
562
- }
585
+ sayProgress(file.path);
586
+ };
587
+ /*
588
+ Bounded by bytes in flight, not by how many files are open.
589
+
590
+ Six lanes turned an hour-long restore of many small files into a
591
+ minute, and made a restore of three very large ones eight times
592
+ slower — 93 seconds became 802. Splitting the queue by file size
593
+ fixed that shape and cost another: a project of forty-megabyte video
594
+ files had its six big ones running two at a time while the small ones
595
+ finished and left the line idle, and its fetch went from 54 seconds
596
+ to 151.
597
+
598
+ Neither is really about file count. A file is fetched a piece at a
599
+ time, so each job holds exactly one request open whatever its size —
600
+ what six large files at once actually costs is six streams' worth of
601
+ bytes being buffered, hashed and written against one line. So that is
602
+ what is bounded: lanes take work while the bytes already in flight
603
+ leave room for it. Forty-megabyte files then run five or six at a
604
+ time and hundred-megabyte ones run two, without a threshold having to
605
+ guess which project this is.
606
+ */
607
+ const LANES = 6;
608
+ const BYTES_IN_FLIGHT = 200 * 1024 * 1024;
609
+ const sizeOf = (job) => job.kind === "pack" ? 0 : job.file.sourceSize;
610
+ let next = 0;
611
+ let flying = 0;
612
+ /*
613
+ Every waiting lane, not the most recent one. A single slot would let a
614
+ second waiter overwrite the first, and the first would then never be
615
+ woken — a restore that stops with lanes idle and work left.
616
+ */
617
+ const waiting = [];
618
+ const wakeAll = () => {
619
+ while (waiting.length)
620
+ waiting.pop()();
621
+ };
622
+ const roomFor = async (size) => {
623
+ /*
624
+ One job always proceeds, however large. A file bigger than the whole
625
+ budget would otherwise wait for room that can never appear.
626
+ */
627
+ while (flying > 0 && flying + size > BYTES_IN_FLIGHT) {
628
+ await new Promise((resolve) => {
629
+ waiting.push(resolve);
630
+ });
631
+ }
632
+ };
633
+ const lane = async () => {
634
+ for (;;) {
635
+ const at = next;
636
+ next += 1;
637
+ if (at >= jobs.length)
638
+ return;
639
+ this.check();
640
+ const job = jobs[at];
641
+ const size = sizeOf(job);
642
+ await roomFor(size);
643
+ flying += size;
644
+ try {
645
+ if (job.kind === "pack")
646
+ await restorePack(job.id);
647
+ else
648
+ await restoreFile(job.file);
649
+ }
650
+ finally {
651
+ flying -= size;
652
+ wakeAll();
653
+ }
654
+ }
655
+ };
656
+ await Promise.all(Array.from({ length: Math.min(LANES, jobs.length) }, () => lane()));
563
657
  /*
564
658
  Everything has arrived and verified, so it can go into place. Each
565
659
  file is renamed over its destination individually: a rename within
@@ -65,6 +65,42 @@ function globToSource(pattern) {
65
65
  index += 1;
66
66
  continue;
67
67
  }
68
+ if (character === "[") {
69
+ /*
70
+ A character class, which every engine's published ignore file is
71
+ written in: `/[Ll]ibrary/`, `/[Bb]uild/`, `/[Oo]bj/`. Without this
72
+ they were escaped into literals and matched a directory genuinely
73
+ called "[Ll]ibrary" — so pasting the official Unity template, or
74
+ applying the one this application ships, excluded nothing at all and
75
+ said nothing about it.
76
+
77
+ A `]` straight after the bracket, or after a leading negation, is a
78
+ literal `]` rather than the end of the class — which is how git and
79
+ the shell both read it.
80
+ */
81
+ let end = index + 1;
82
+ if (pattern[end] === "!" || pattern[end] === "^")
83
+ end += 1;
84
+ if (pattern[end] === "]")
85
+ end += 1;
86
+ while (end < pattern.length && pattern[end] !== "]")
87
+ end += 1;
88
+ if (end < pattern.length) {
89
+ const inside = pattern.slice(index + 1, end);
90
+ const negated = inside.startsWith("!") || inside.startsWith("^");
91
+ const members = (negated ? inside.slice(1) : inside).replace(/[\\\^\]]/g, "\\$&");
92
+ /*
93
+ A class never spans directories, so a negated one excludes the
94
+ separator too — otherwise `[!a]` would match a slash and a pattern
95
+ for one segment would quietly reach into the next.
96
+ */
97
+ source += negated ? `[^/${members}]` : `[${members}]`;
98
+ index = end;
99
+ continue;
100
+ }
101
+ source += "\\[";
102
+ continue;
103
+ }
68
104
  if (character === "*") {
69
105
  if (pattern[index + 1] === "*") {
70
106
  source += ".*";
@@ -119,6 +119,22 @@ class Uploader {
119
119
  planning = false;
120
120
  plannedObjects = new Map();
121
121
  plannedQuote = null;
122
+ /*
123
+ Digests worked out while planning, so sending does not work them out again.
124
+
125
+ A save runs the whole pipeline twice: the plan walks, reads, hashes and
126
+ prices every file to produce a quote, then the send does all of it over.
127
+ Measured on a twenty-one thousand file project, that was 42,142 hashes for
128
+ 21,071 files — the count in the profile is exactly double, which is how
129
+ the duplication was finally noticed.
130
+
131
+ Keyed by path and guarded by size and modification time, so a file edited
132
+ while somebody reads the plan is hashed again rather than published under
133
+ the digest it used to have.
134
+ */
135
+ /** Whether this save has already said that packing was refused. */
136
+ packRefusalSaid = false;
137
+ plannedDigests = new Map();
122
138
  planningCreatedRepositoryId = null;
123
139
  /** Attached to every object write after the server admits the exact plan. */
124
140
  activeQuoteId = null;
@@ -258,6 +274,7 @@ class Uploader {
258
274
  */
259
275
  async plan(request, report) {
260
276
  this.planning = true;
277
+ this.packRefusalSaid = false;
261
278
  this.activeQuoteId = null;
262
279
  this.plannedQuote = null;
263
280
  this.planningCreatedRepositoryId = null;
@@ -352,7 +369,27 @@ class Uploader {
352
369
  truncated version of it would propose deleting everything the walk did
353
370
  not reach.
354
371
  */
355
- const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules));
372
+ const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules, (found) =>
373
+ /*
374
+ The walk is the first minute of a large save and said nothing at
375
+ all, so the panel could only show 0%. A running count of what has
376
+ been seen is not a fraction of anything — it is the evidence that
377
+ the thing is working.
378
+ */
379
+ report({
380
+ stage: "plan",
381
+ files: found,
382
+ totalFiles: 0,
383
+ bytes: 0,
384
+ totalBytes: 0,
385
+ path: "Looking through the folder",
386
+ /*
387
+ Between the key check and the reading, and it holds its own place
388
+ on the bar rather than dropping back to nought between them.
389
+ */
390
+ percent: 6,
391
+ bytesPerSecond: 0,
392
+ })));
356
393
  (0, profile_js_1.counted)("files surveyed", everything.length);
357
394
  const unticked = new Set(request.excluded ?? []);
358
395
  const ticked = request.excluded
@@ -454,8 +491,11 @@ class Uploader {
454
491
  this.check();
455
492
  const full = node_path_1.default.join(request.localPath, file.path);
456
493
  let size;
494
+ let mtimeMs;
457
495
  try {
458
- size = (await (0, promises_1.stat)(full)).size;
496
+ const info = await (0, promises_1.stat)(full);
497
+ size = info.size;
498
+ mtimeMs = info.mtimeMs;
459
499
  }
460
500
  catch (error) {
461
501
  // Unreadable now, though it was listed a moment ago. Skipping it
@@ -476,6 +516,28 @@ class Uploader {
476
516
  locked" naming a path they never chose to think about, instead of
477
517
  the sentence this code already writes for exactly this situation.
478
518
  */
519
+ /*
520
+ Read only when the file has actually moved.
521
+
522
+ Size and modification time are the same test the changes list uses,
523
+ and a match there means the digest already taken still describes the
524
+ bytes. Reading is what costs; deciding not to read is free.
525
+ */
526
+ const remembered = this.plannedDigests.get(file.path) ??
527
+ request.knownDigests?.get(file.path);
528
+ if (remembered &&
529
+ remembered.size === size &&
530
+ remembered.mtimeMs === mtimeMs) {
531
+ measured[index] = {
532
+ path: file.path,
533
+ full,
534
+ size,
535
+ sha256: remembered.sha256,
536
+ mediaType: mediaTypeOf(file.path),
537
+ };
538
+ hashed += 1;
539
+ continue;
540
+ }
479
541
  let digest;
480
542
  try {
481
543
  digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
@@ -486,6 +548,7 @@ class Uploader {
486
548
  hashed += 1;
487
549
  continue;
488
550
  }
551
+ this.plannedDigests.set(file.path, { size, mtimeMs, sha256: digest });
489
552
  measured[index] = {
490
553
  path: file.path,
491
554
  full,
@@ -504,7 +567,12 @@ class Uploader {
504
567
  bytes: 0,
505
568
  totalBytes: 0,
506
569
  path: file.path,
507
- percent: Math.round((hashed / sending.length) * 20),
570
+ /*
571
+ Reading is the stretch from just past the walk to a fifth of
572
+ the way through. Starting it at nought put the bar behind where
573
+ it had already been and it went backwards between stages.
574
+ */
575
+ percent: 8 + Math.round((hashed / sending.length) * 12),
508
576
  bytesPerSecond: 0,
509
577
  });
510
578
  }
@@ -702,11 +770,78 @@ class Uploader {
702
770
  const heavy = sections.filter((one) => smallShare(one) < 0.5);
703
771
  (0, profile_js_1.counted)("sections batched", batchy.length);
704
772
  (0, profile_js_1.counted)("sections heavy", heavy.length);
773
+ /*
774
+ How many files the whole save has finished, across every section.
775
+
776
+ The sections are an implementation detail of not holding thirty
777
+ gigabytes in memory at once, and they were leaking into what a person
778
+ reads: the count ran up to twenty-one thousand while reading, then reset
779
+ to "0 of 4,000" for each section and counted up again, over and over. It
780
+ looked like the save was starting over, repeatedly.
781
+
782
+ One denominator for the whole save. Sections still say which one is in
783
+ flight, in words, where that belongs.
784
+ */
785
+ const sectionProgress = new Map();
786
+ /*
787
+ What has already been shown, so the number never goes backwards.
788
+
789
+ Two pipelines run sections at once and each reports its own share, which
790
+ I have now twice failed to combine into a figure that only rises — the
791
+ count kept dropping to zero at section boundaries and reading as though
792
+ the save had restarted. Whatever the accounting does, the panel must not
793
+ be told a smaller number than it was told a moment ago: going backwards
794
+ is the specific thing a person notices, and it is never information.
795
+ */
796
+ let shown = 0;
797
+ const forward = (count) => {
798
+ if (count > shown)
799
+ shown = count;
800
+ return shown;
801
+ };
705
802
  const runSection = async (section) => {
706
803
  const declarations = section.files.map((file) => declarationByPath.get(file.path));
804
+ /*
805
+ This section's share of the one number the panel shows.
806
+
807
+ Two pipelines run at once, so no section can report on its own behalf
808
+ without the count lurching between them. Each records how far it has
809
+ got and the sum is what is shown, which is the only figure that is
810
+ true while both are moving.
811
+ */
812
+ const advance = (done) => {
813
+ /*
814
+ Only ever upward.
815
+
816
+ The workers share one iterator, so the indices arrive out of order —
817
+ a worker holding file 5 reports after one holding file 200. Writing
818
+ each straight in let a later, lower number undo a higher one, and
819
+ the count sat still or fell back to zero at every section boundary,
820
+ which is the exact stutter this was meant to remove.
821
+ */
822
+ const held = sectionProgress.get(section.index) ?? 0;
823
+ if (done > held)
824
+ sectionProgress.set(section.index, done);
825
+ else if (!sectionProgress.has(section.index)) {
826
+ sectionProgress.set(section.index, 0);
827
+ }
828
+ let sum = 0;
829
+ for (const each of sectionProgress.values())
830
+ sum += each;
831
+ return forward(sum);
832
+ };
707
833
  report({
708
834
  stage: "upload",
709
- files: 0,
835
+ /*
836
+ The count so far, not zero.
837
+
838
+ This line announces a new section, and it was reporting nought files
839
+ done — so every section boundary threw the number back to the start
840
+ and the bar with it. Four of the five reports in this stage were
841
+ changed to carry the running total and this one, the only one a
842
+ person is guaranteed to see, was missed.
843
+ */
844
+ files: advance(0),
710
845
  totalFiles: allDeclarations.length,
711
846
  bytes: progressBytes(),
712
847
  totalBytes,
@@ -836,12 +971,23 @@ class Uploader {
836
971
  batched.set(digest, one);
837
972
  }
838
973
  }
839
- catch {
974
+ catch (reason) {
840
975
  /*
841
976
  A refusal here is never fatal: everything in the pack is simply
842
977
  left for the ordinary path below, which sends it one at a time.
843
978
  An older deployment without this route lands here every time.
979
+
980
+ Said once per save, not once per pack. It used to be swallowed
981
+ outright, and when the service began refusing packs for a reason
982
+ that also broke the fallback, the only thing anybody saw was the
983
+ fallback's complaint about an arbitrary file — which described
984
+ neither what went wrong nor where. One line naming the first
985
+ refusal is what makes the next one an hour shorter to find.
844
986
  */
987
+ if (!this.packRefusalSaid) {
988
+ this.packRefusalSaid = true;
989
+ console.warn(`Packing was refused; sending these files individually. ${String(reason).slice(0, 300)}`);
990
+ }
845
991
  }
846
992
  /*
847
993
  Only what actually landed. Counting the whole pack meant a batch the
@@ -861,8 +1007,8 @@ class Uploader {
861
1007
  transferred += exactWireBytes;
862
1008
  report({
863
1009
  stage: "upload",
864
- files: 0,
865
- totalFiles: declarations.length,
1010
+ files: advance(0),
1011
+ totalFiles: allDeclarations.length,
866
1012
  bytes: progressBytes(),
867
1013
  totalBytes,
868
1014
  path: taking[taking.length - 1].declaration.path,
@@ -914,8 +1060,8 @@ class Uploader {
914
1060
  this.check();
915
1061
  report({
916
1062
  stage: "upload",
917
- files: index,
918
- totalFiles: declarations.length,
1063
+ files: advance(index),
1064
+ totalFiles: allDeclarations.length,
919
1065
  bytes: sentBytes,
920
1066
  totalBytes,
921
1067
  path: declaration.path,
@@ -949,8 +1095,8 @@ class Uploader {
949
1095
  inFlight.set(index, offset);
950
1096
  report({
951
1097
  stage: "upload",
952
- files: index,
953
- totalFiles: declarations.length,
1098
+ files: advance(index),
1099
+ totalFiles: allDeclarations.length,
954
1100
  bytes: progressBytes(),
955
1101
  totalBytes,
956
1102
  path: declaration.path,
@@ -1023,8 +1169,8 @@ class Uploader {
1023
1169
  : await this.putMultipart(repositoryId, declaration, (offset) => {
1024
1170
  report({
1025
1171
  stage: "upload",
1026
- files: index,
1027
- totalFiles: declarations.length,
1172
+ files: advance(index),
1173
+ totalFiles: allDeclarations.length,
1028
1174
  bytes: progressBytes(),
1029
1175
  totalBytes,
1030
1176
  path: declaration.path,
@@ -1047,6 +1193,8 @@ class Uploader {
1047
1193
  }
1048
1194
  };
1049
1195
  await Promise.all(Array.from({ length: Math.min(UPLOAD_LANES, declarations.length) }, () => runWorker()));
1196
+ /* Finished, so its whole share counts — the per-file reports stop one short. */
1197
+ advance(declarations.length);
1050
1198
  };
1051
1199
  const pipeline = async (queue) => {
1052
1200
  for (const section of queue) {
@@ -1066,7 +1214,7 @@ class Uploader {
1066
1214
  percent: 96,
1067
1215
  bytesPerSecond: 0,
1068
1216
  });
1069
- this.plannedQuote = await this.call(`/v1/repositories/${repositoryId}/uploads/preflight`, {
1217
+ this.plannedQuote = await (0, profile_js_1.timed)("ask the service to price it", () => this.call(`/v1/repositories/${repositoryId}/uploads/preflight`, {
1070
1218
  method: "POST",
1071
1219
  contentType: "application/json",
1072
1220
  body: JSON.stringify({
@@ -1074,7 +1222,7 @@ class Uploader {
1074
1222
  excludedBytes: 0,
1075
1223
  objects: [...this.plannedObjects.values()],
1076
1224
  }),
1077
- });
1225
+ }));
1078
1226
  return {
1079
1227
  repositoryId,
1080
1228
  versionId: "",
@@ -1201,7 +1349,7 @@ class Uploader {
1201
1349
  objects are immutable and content-addressed, so a retry reuses them.
1202
1350
  */
1203
1351
  (0, faults_js_1.maybeFail)("submit:after-objects");
1204
- const completed = await this.call(`/v1/repositories/${repositoryId}/versions`, {
1352
+ const completed = await (0, profile_js_1.timed)("publish the version", () => this.call(`/v1/repositories/${repositoryId}/versions`, {
1205
1353
  method: "POST",
1206
1354
  contentType: "application/json",
1207
1355
  body: JSON.stringify({
@@ -1266,7 +1414,7 @@ class Uploader {
1266
1414
  executable: false,
1267
1415
  })),
1268
1416
  }),
1269
- });
1417
+ }));
1270
1418
  /*
1271
1419
  A clean stale-base save may contain a server-side merge. The candidate
1272
1420
  manifest built above is then not the Version that landed: it lacks the
@@ -66,6 +66,28 @@ __pycache__/
66
66
  venv/
67
67
  *.log
68
68
 
69
+ # What a game engine rebuilds for itself. These are the largest thing in a
70
+ # folder by a wide margin — a Unity project's Library is routinely thousands
71
+ # of times the size of its Assets — and every byte comes back the next time
72
+ # the editor opens. Left out of this list for a long time, which meant a
73
+ # folder with no rules of its own sent its whole cache and nobody was told.
74
+ /[Ll]ibrary/
75
+ /[Tt]emp/
76
+ /[Ll]ogs/
77
+ /[Uu]ser[Ss]ettings/
78
+ /[Mm]emoryCaptures/
79
+ Intermediate/
80
+ DerivedDataCache/
81
+ Binaries/
82
+ .godot/
83
+ .import/
84
+
85
+ # Compiler and editor output, which the next build writes again.
86
+ obj/
87
+ .vs/
88
+ .gradle/
89
+ DerivedData/
90
+
69
91
  # Large model weights
70
92
  models/**
71
93
  *.safetensors
@@ -497,6 +519,7 @@ outcome) {
497
519
  }
498
520
  if (outcome) {
499
521
  outcome.listed = files.length;
522
+ outcome.truncated = files.length >= limit;
500
523
  /*
501
524
  Deletions are rows the walk never saw, so they are added to both sides
502
525
  rather than left out of the total — otherwise a synchronize scan could
@@ -520,7 +543,17 @@ outcome) {
520
543
  * content at all. That is what makes it affordable to run over everything
521
544
  * before deciding anything.
522
545
  */
523
- async function surveyFiles(root, rules) {
546
+ async function surveyFiles(root, rules,
547
+ /*
548
+ Told how many have been found, every so often.
549
+
550
+ This walk is the first thing a save does and it can take a minute on a
551
+ large folder — during which the only honest thing the progress panel
552
+ could say was 0%, because nothing had told it otherwise. A count of what
553
+ has been seen is not a fraction of anything, but it moves, and it is the
554
+ difference between waiting and wondering whether it has hung.
555
+ */
556
+ onFound) {
524
557
  const layers = await collectLayers(root, rules);
525
558
  const found = [];
526
559
  const pending = [root];
@@ -554,6 +587,8 @@ async function surveyFiles(root, rules) {
554
587
  continue;
555
588
  try {
556
589
  found.push({ path: relative, size: (await (0, promises_1.stat)(full)).size });
590
+ if (onFound && found.length % 500 === 0)
591
+ onFound(found.length);
557
592
  }
558
593
  catch {
559
594
  /* Gone or unreadable since the walk began; the upload reports it. */
@@ -1004,46 +1039,80 @@ const MOST_FINDINGS = 40;
1004
1039
  * already leave behind is not being published, and raising it would train
1005
1040
  * people to dismiss the question that matters.
1006
1041
  */
1007
- async function detectPastedCredentials(root, include) {
1042
+ async function detectPastedCredentials(root, include,
1043
+ /*
1044
+ Told how many have been looked at, every so often.
1045
+
1046
+ This reads up to twenty thousand files before a save begins, and it did so
1047
+ one after another with nothing reported — on Windows, where every open is
1048
+ also an antivirus scan, that is a quarter of an hour in which the panel
1049
+ could only show the words "Working out what to send" and no number at all.
1050
+ The same silence the read stage used to have, in a different module.
1051
+ */
1052
+ onScanned) {
1008
1053
  const found = [];
1009
1054
  let read = 0;
1010
1055
  let bytes = 0;
1011
- for (const relative of include) {
1012
- if (found.length >= MOST_FINDINGS)
1013
- break;
1014
- if (read >= MOST_SCANNED_FILES || bytes >= MOST_SCANNED_BYTES)
1015
- break;
1016
- const name = relative.slice(relative.lastIndexOf("/") + 1);
1017
- if (!(0, secret_patterns_js_1.worthReading)(name))
1018
- continue;
1019
- const full = node_path_1.default.join(root, relative.split("/").join(node_path_1.default.sep));
1020
- let contents;
1021
- try {
1022
- const info = await (0, promises_1.stat)(full);
1023
- if (!info.isFile() || info.size > MOST_FILE_BYTES)
1056
+ /*
1057
+ Read several at a time. This is waiting on the disk, not computing, so
1058
+ overlapping the waits collapses them into one wait — the same change the
1059
+ hashing stage needed, for the same reason.
1060
+ */
1061
+ const LANES = 8;
1062
+ const wanted = include.filter((relative) => (0, secret_patterns_js_1.worthReading)(relative.slice(relative.lastIndexOf("/") + 1)));
1063
+ let cursor = 0;
1064
+ let stop = false;
1065
+ const lane = async () => {
1066
+ for (;;) {
1067
+ if (stop)
1068
+ return;
1069
+ const at = cursor;
1070
+ cursor += 1;
1071
+ if (at >= wanted.length)
1072
+ return;
1073
+ if (found.length >= MOST_FINDINGS) {
1074
+ stop = true;
1075
+ return;
1076
+ }
1077
+ if (read >= MOST_SCANNED_FILES || bytes >= MOST_SCANNED_BYTES) {
1078
+ stop = true;
1079
+ return;
1080
+ }
1081
+ const relative = wanted[at];
1082
+ const full = node_path_1.default.join(root, relative.split("/").join(node_path_1.default.sep));
1083
+ let contents;
1084
+ try {
1085
+ const info = await (0, promises_1.stat)(full);
1086
+ if (!info.isFile() || info.size > MOST_FILE_BYTES)
1087
+ continue;
1088
+ contents = await (0, promises_1.readFile)(full);
1089
+ }
1090
+ catch {
1091
+ /* Unreadable is the scan's problem, not something to report as clean. */
1024
1092
  continue;
1025
- contents = await (0, promises_1.readFile)(full);
1026
- }
1027
- catch {
1028
- /* Unreadable is the scan's problem, not something to report as clean. */
1029
- continue;
1093
+ }
1094
+ read += 1;
1095
+ bytes += contents.byteLength;
1096
+ if (onScanned && read % 50 === 0)
1097
+ onScanned(read, wanted.length);
1098
+ if (!(0, secret_patterns_js_1.looksLikeText)(contents))
1099
+ continue;
1100
+ const credentials = (0, secret_patterns_js_1.findCredentials)(contents.toString("utf8"));
1101
+ if (credentials.length)
1102
+ found.push({ path: relative, found: credentials });
1030
1103
  }
1031
- read += 1;
1032
- bytes += contents.byteLength;
1033
- if (!(0, secret_patterns_js_1.looksLikeText)(contents))
1034
- continue;
1035
- const credentials = (0, secret_patterns_js_1.findCredentials)(contents.toString("utf8"));
1036
- if (credentials.length)
1037
- found.push({ path: relative, found: credentials });
1038
- }
1104
+ };
1105
+ await Promise.all(Array.from({ length: Math.min(LANES, wanted.length) }, () => lane()));
1039
1106
  return found;
1040
1107
  }
1041
- async function uploadConcerns(root, include) {
1108
+ async function uploadConcerns(root, include,
1109
+ /** Passed through so the one slow part of this can say it is working. */
1110
+ onScanned) {
1042
1111
  const chosen = new Set(include);
1043
1112
  const [named, folders, pasted] = await Promise.all([
1044
1113
  detectSecrets(root),
1045
1114
  detectPrivateDirectories(root),
1046
- detectPastedCredentials(root, include),
1115
+ detectPastedCredentials(root, include, onScanned),
1047
1116
  ]);
1048
1117
  const secrets = named.filter((file) => chosen.has(file));
1049
1118
  return {
package/package.json CHANGED
@@ -1,54 +1,54 @@
1
- {
2
- "name": "@coderook/cli",
3
- "version": "0.29.1",
4
- "description": "CodeRook from the command line, on any operating system",
5
- "license": "SEE LICENSE IN LICENSE.txt",
6
- "homepage": "https://coderook.com",
7
- "bugs": {
8
- "url": "https://coderook.com/contact"
9
- },
10
- "keywords": [
11
- "coderook",
12
- "versioning",
13
- "backup",
14
- "cbx"
15
- ],
16
- "engines": {
17
- "node": ">=20.11.0"
18
- },
19
- "bin": {
20
- "cbx": "dist/cli/src/cli.js",
21
- "coderook": "dist/cli/src/cli.js",
22
- "git-remote-coderook": "dist/cli/src/git_remote_bin.js"
23
- },
24
- "files": [
25
- ".claude-plugin",
26
- "dist",
27
- "skills"
28
- ],
29
- "scripts": {
30
- "build": "tsc -p tsconfig.json",
31
- "check": "tsc -p tsconfig.json --noEmit",
32
- "test": "tsc -p tsconfig.json && node --test --experimental-strip-types test/*.test.ts",
33
- "start": "node dist/cli/src/cli.js",
34
- "prepublishOnly": "npm run sync:plugin && npm run build",
35
- "test:e2e": "node test/e2e.mjs",
36
- "test:matrix": "node test/state-matrix.mjs",
37
- "test:attempt": "node test/attempt-identity.mjs",
38
- "test:get": "node test/get-safety.mjs",
39
- "test:live": "node test/e2e.mjs --allow-production && node test/state-matrix.mjs --allow-production && node test/get-safety.mjs --allow-production && node test/get-track.mjs --allow-production && node test/get-protects-edits.mjs --allow-production && node test/upgrade-migration.mjs --allow-production && node test/fault-get.mjs --allow-production && node test/version-floor.mjs --allow-production && node test/fault-submit.mjs --allow-production && node test/race-attempts.mjs --allow-production && node test/attempt-identity.mjs --allow-production",
40
- "test:getedits": "node test/get-protects-edits.mjs",
41
- "test:upgrade": "node test/upgrade-migration.mjs",
42
- "test:faultget": "node test/fault-get.mjs",
43
- "test:floor": "node test/version-floor.mjs",
44
- "test:faultsubmit": "node test/fault-submit.mjs",
45
- "test:race": "node test/race-attempts.mjs",
46
- "test:runner": "node test/runner-live.mjs",
47
- "sync:plugin": "node scripts/sync-plugin-version.mjs",
48
- "test:track": "node test/get-track.mjs"
49
- },
50
- "devDependencies": {
51
- "@types/node": "24.10.1",
52
- "typescript": "5.9.3"
53
- }
54
- }
1
+ {
2
+ "name": "@coderook/cli",
3
+ "version": "0.29.3",
4
+ "description": "CodeRook from the command line, on any operating system",
5
+ "license": "SEE LICENSE IN LICENSE.txt",
6
+ "homepage": "https://coderook.com",
7
+ "bugs": {
8
+ "url": "https://coderook.com/contact"
9
+ },
10
+ "keywords": [
11
+ "coderook",
12
+ "versioning",
13
+ "backup",
14
+ "cbx"
15
+ ],
16
+ "engines": {
17
+ "node": ">=20.11.0"
18
+ },
19
+ "bin": {
20
+ "cbx": "dist/cli/src/cli.js",
21
+ "coderook": "dist/cli/src/cli.js",
22
+ "git-remote-coderook": "dist/cli/src/git_remote_bin.js"
23
+ },
24
+ "files": [
25
+ ".claude-plugin",
26
+ "dist",
27
+ "skills"
28
+ ],
29
+ "scripts": {
30
+ "build": "tsc -p tsconfig.json",
31
+ "check": "tsc -p tsconfig.json --noEmit",
32
+ "test": "tsc -p tsconfig.json && node --test --experimental-strip-types test/*.test.ts",
33
+ "start": "node dist/cli/src/cli.js",
34
+ "prepublishOnly": "npm run sync:plugin && npm run build",
35
+ "test:e2e": "node test/e2e.mjs",
36
+ "test:matrix": "node test/state-matrix.mjs",
37
+ "test:attempt": "node test/attempt-identity.mjs",
38
+ "test:get": "node test/get-safety.mjs",
39
+ "test:live": "node test/e2e.mjs --allow-production && node test/state-matrix.mjs --allow-production && node test/get-safety.mjs --allow-production && node test/get-track.mjs --allow-production && node test/get-protects-edits.mjs --allow-production && node test/upgrade-migration.mjs --allow-production && node test/fault-get.mjs --allow-production && node test/version-floor.mjs --allow-production && node test/fault-submit.mjs --allow-production && node test/race-attempts.mjs --allow-production && node test/attempt-identity.mjs --allow-production",
40
+ "test:getedits": "node test/get-protects-edits.mjs",
41
+ "test:upgrade": "node test/upgrade-migration.mjs",
42
+ "test:faultget": "node test/fault-get.mjs",
43
+ "test:floor": "node test/version-floor.mjs",
44
+ "test:faultsubmit": "node test/fault-submit.mjs",
45
+ "test:race": "node test/race-attempts.mjs",
46
+ "test:runner": "node test/runner-live.mjs",
47
+ "sync:plugin": "node scripts/sync-plugin-version.mjs",
48
+ "test:track": "node test/get-track.mjs"
49
+ },
50
+ "devDependencies": {
51
+ "@types/node": "24.10.1",
52
+ "typescript": "5.9.3"
53
+ }
54
+ }