@coderook/cli 0.29.1 → 0.29.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
  "name": "coderook",
3
3
  "displayName": "CodeRook",
4
4
  "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
5
- "version": "0.29.1",
5
+ "version": "0.29.2",
6
6
  "author": {
7
7
  "name": "ACCA Gaming Productions",
8
8
  "url": "https://coderook.com"
@@ -40,6 +40,7 @@ const download_js_1 = require("../../desktop-app/src/main/download.js");
40
40
  const faults_js_1 = require("../../desktop-app/src/main/faults.js");
41
41
  const version_commands_js_1 = require("./version_commands.js");
42
42
  const identify_js_1 = require("../../desktop-app/src/main/identify.js");
43
+ const profile_js_1 = require("../../desktop-app/src/main/profile.js");
43
44
  const detect_js_1 = require("../../desktop-app/src/main/detect.js");
44
45
  const cbx_js_1 = require("../../desktop-app/src/main/cbx.js");
45
46
  const api_js_2 = require("./api.js");
@@ -889,6 +890,16 @@ The connection failed: ${text}`));
889
890
  command does not do.
890
891
  */
891
892
  ` · the save holds ${bytes(result.sourceBytes)}`);
893
+ /*
894
+ Where the time went, when asked for.
895
+
896
+ The profiler has existed for a while and nothing ever printed it, so the
897
+ one question it answers — which stage is the save — still had to be
898
+ guessed at. Silent unless CODEROOK_PROFILE is set.
899
+ */
900
+ const profile = (0, profile_js_1.profileReport)();
901
+ if (profile)
902
+ console.log(profile);
892
903
  return 0;
893
904
  }
894
905
  /**
@@ -473,75 +473,98 @@ class Downloader {
473
473
  packed.set(id, group);
474
474
  }
475
475
  const restoredPacks = new Set();
476
+ /*
477
+ Fetched several at a time, and each pack fetched once.
478
+
479
+ This was one request after another — a pack or a file, wait for it,
480
+ write it, ask for the next — while the sending side has had six lanes
481
+ for a long time. Measured across thirty project shapes, that made
482
+ restoring roughly six times slower than saving the same bytes: an
483
+ Unreal project of four hundred and ninety megabytes went up in ten
484
+ minutes and came back in sixty-one. A restore is mostly waiting on the
485
+ network, and waiting is the thing that overlaps.
486
+
487
+ The work is planned before any of it runs, which also removes the
488
+ double-fetch guard the serial version needed: a pack appears once in
489
+ the list, so it can only be fetched once.
490
+ */
491
+ const jobs = [];
476
492
  for (const file of files) {
477
- this.check();
493
+ const id = file.sealed ? undefined : file.pack?.objectId;
494
+ if (id) {
495
+ if (restoredPacks.has(id))
496
+ continue;
497
+ restoredPacks.add(id);
498
+ jobs.push({ kind: "pack", id });
499
+ }
500
+ else {
501
+ jobs.push({ kind: "file", file });
502
+ }
503
+ }
504
+ /** Refused outright, wherever it appears — inside a pack or beside it. */
505
+ const safeParts = (where) => {
506
+ const parts = where.split("/").filter(Boolean);
507
+ if (parts.some((part) => part === ".." || part.includes("\0"))) {
508
+ throw new Error(`That version contains an unsafe path: ${where}`);
509
+ }
510
+ return parts;
511
+ };
512
+ const told = { at: 0 };
513
+ const sayProgress = (where) => {
514
+ const now = Date.now();
515
+ if (now - told.at < 80)
516
+ return;
517
+ told.at = now;
478
518
  report({
479
519
  files: written,
480
520
  totalFiles: files.length,
481
521
  bytes,
482
522
  totalBytes,
483
- path: file.path,
523
+ path: where,
484
524
  percent: underway(bytes, totalBytes),
485
525
  });
486
- // A path that climbs out of the destination is refused outright.
487
- const parts = file.path.split("/").filter(Boolean);
488
- if (parts.some((part) => part === ".." || part.includes("\0"))) {
489
- throw new Error(`That version contains an unsafe path: ${file.path}`);
526
+ };
527
+ const restorePack = async (packId) => {
528
+ /*
529
+ A solid pack is one compressed object containing many small files.
530
+
531
+ Asking the single-file route for every member made a large restore
532
+ fetch and decompress the same pack thousands of times. The files
533
+ listing already gives us the pack id and each member's exact window,
534
+ so fetch the decoded pack once, verify it, and materialise every
535
+ member before releasing the buffer.
536
+ */
537
+ const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
538
+ const body = Buffer.from(await reply.arrayBuffer());
539
+ const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
540
+ const expectedPackDigest = reply.headers.get("x-coderook-sha256");
541
+ if (expectedPackDigest && packDigest !== expectedPackDigest) {
542
+ throw new Error("A solid pack did not arrive intact");
490
543
  }
491
- const packId = file.sealed ? undefined : file.pack?.objectId;
492
- if (packId && restoredPacks.has(packId))
493
- continue;
494
- if (packId) {
495
- /*
496
- A solid pack is one compressed object containing many small files.
497
-
498
- Asking the single-file route for every member made a large restore
499
- fetch and decompress the same pack thousands of times. The files
500
- listing already gives us the pack id and each member's exact
501
- window, so fetch the decoded pack once, verify it, and materialise
502
- every member before releasing the bounded section-sized buffer.
503
- */
504
- const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
505
- const body = Buffer.from(await reply.arrayBuffer());
506
- const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
507
- const expectedPackDigest = reply.headers.get("x-coderook-sha256");
508
- if (expectedPackDigest && packDigest !== expectedPackDigest) {
509
- throw new Error("A solid pack did not arrive intact");
544
+ for (const member of packed.get(packId) ?? []) {
545
+ this.check();
546
+ const memberParts = safeParts(member.path);
547
+ const offset = member.pack.offset;
548
+ const length = member.pack.length;
549
+ if (offset < 0 || length < 0 || offset + length > body.byteLength) {
550
+ throw new Error(`${member.path} points outside its solid pack`);
510
551
  }
511
- for (const member of packed.get(packId) ?? []) {
512
- this.check();
513
- const memberParts = member.path.split("/").filter(Boolean);
514
- if (memberParts.some((part) => part === ".." || part.includes("\0"))) {
515
- throw new Error(`That version contains an unsafe path: ${member.path}`);
516
- }
517
- const offset = member.pack.offset;
518
- const length = member.pack.length;
519
- if (offset < 0 || length < 0 || offset + length > body.byteLength) {
520
- throw new Error(`${member.path} points outside its solid pack`);
521
- }
522
- const contents = body.subarray(offset, offset + length);
523
- const digest = (0, node_crypto_1.createHash)("sha256").update(contents).digest("hex");
524
- if (member.sha256 && digest !== member.sha256) {
525
- throw new Error(`${member.path} did not arrive intact`);
526
- }
527
- const full = node_path_1.default.join(staging, ...memberParts);
528
- await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
529
- await (0, promises_1.writeFile)(full, contents);
530
- manifest[member.path] = digest;
531
- written += 1;
532
- bytes += member.sourceSize;
533
- report({
534
- files: written,
535
- totalFiles: files.length,
536
- bytes,
537
- totalBytes,
538
- path: member.path,
539
- percent: underway(bytes, totalBytes),
540
- });
552
+ const contents = body.subarray(offset, offset + length);
553
+ const digest = (0, node_crypto_1.createHash)("sha256").update(contents).digest("hex");
554
+ if (member.sha256 && digest !== member.sha256) {
555
+ throw new Error(`${member.path} did not arrive intact`);
541
556
  }
542
- restoredPacks.add(packId);
543
- continue;
557
+ const full = node_path_1.default.join(staging, ...memberParts);
558
+ await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
559
+ await (0, promises_1.writeFile)(full, contents);
560
+ manifest[member.path] = digest;
561
+ written += 1;
562
+ bytes += member.sourceSize;
563
+ sayProgress(member.path);
544
564
  }
565
+ };
566
+ const restoreFile = async (file) => {
567
+ const parts = safeParts(file.path);
545
568
  const full = node_path_1.default.join(staging, ...parts);
546
569
  await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
547
570
  const digest = await this.fetchInto(repositoryId, versionId, file, full);
@@ -559,7 +582,46 @@ class Downloader {
559
582
  manifest[file.path] = digest;
560
583
  written += 1;
561
584
  bytes += file.sourceSize;
562
- }
585
+ sayProgress(file.path);
586
+ };
587
+ /*
588
+ Wide for the small, narrow for the large.
589
+
590
+ Six lanes turned an hour-long restore of many small files into a
591
+ minute, and made a restore of a few very large ones eight times
592
+ slower: each large file is already fetched in pieces, so six at once
593
+ is six times the outstanding requests against the same line, and they
594
+ all finish later than they would have one after another.
595
+
596
+ Split by what the job is: a pack or a small file is one request and
597
+ belongs in the wide lane; a large file is a stream of its own and
598
+ belongs in the narrow one. Both run at once, so a project holding
599
+ both kinds keeps the line busy without either starving the other.
600
+ The sending side splits its work the same way — see the note beside
601
+ BATCH_LANES in upload.ts.
602
+ */
603
+ const HEAVY_BYTES = 8 * 1024 * 1024;
604
+ const light = jobs.filter((job) => job.kind === "pack" || job.file.sourceSize < HEAVY_BYTES);
605
+ const heavy = jobs.filter((job) => job.kind === "file" && job.file.sourceSize >= HEAVY_BYTES);
606
+ const drain = async (queue, lanes) => {
607
+ let next = 0;
608
+ const lane = async () => {
609
+ for (;;) {
610
+ const at = next;
611
+ next += 1;
612
+ if (at >= queue.length)
613
+ return;
614
+ this.check();
615
+ const job = queue[at];
616
+ if (job.kind === "pack")
617
+ await restorePack(job.id);
618
+ else
619
+ await restoreFile(job.file);
620
+ }
621
+ };
622
+ await Promise.all(Array.from({ length: Math.min(lanes, queue.length) }, () => lane()));
623
+ };
624
+ await Promise.all([drain(light, 6), drain(heavy, 2)]);
563
625
  /*
564
626
  Everything has arrived and verified, so it can go into place. Each
565
627
  file is renamed over its destination individually: a rename within
@@ -65,6 +65,42 @@ function globToSource(pattern) {
65
65
  index += 1;
66
66
  continue;
67
67
  }
68
+ if (character === "[") {
69
+ /*
70
+ A character class, which every engine's published ignore file is
71
+ written in: `/[Ll]ibrary/`, `/[Bb]uild/`, `/[Oo]bj/`. Without this
72
+ they were escaped into literals and matched a directory genuinely
73
+ called "[Ll]ibrary" — so pasting the official Unity template, or
74
+ applying the one this application ships, excluded nothing at all and
75
+ said nothing about it.
76
+
77
+ A `]` straight after the bracket, or after a leading negation, is a
78
+ literal `]` rather than the end of the class — which is how git and
79
+ the shell both read it.
80
+ */
81
+ let end = index + 1;
82
+ if (pattern[end] === "!" || pattern[end] === "^")
83
+ end += 1;
84
+ if (pattern[end] === "]")
85
+ end += 1;
86
+ while (end < pattern.length && pattern[end] !== "]")
87
+ end += 1;
88
+ if (end < pattern.length) {
89
+ const inside = pattern.slice(index + 1, end);
90
+ const negated = inside.startsWith("!") || inside.startsWith("^");
91
+ const members = (negated ? inside.slice(1) : inside).replace(/[\\\^\]]/g, "\\$&");
92
+ /*
93
+ A class never spans directories, so a negated one excludes the
94
+ separator too — otherwise `[!a]` would match a slash and a pattern
95
+ for one segment would quietly reach into the next.
96
+ */
97
+ source += negated ? `[^/${members}]` : `[${members}]`;
98
+ index = end;
99
+ continue;
100
+ }
101
+ source += "\\[";
102
+ continue;
103
+ }
68
104
  if (character === "*") {
69
105
  if (pattern[index + 1] === "*") {
70
106
  source += ".*";
@@ -119,6 +119,20 @@ class Uploader {
119
119
  planning = false;
120
120
  plannedObjects = new Map();
121
121
  plannedQuote = null;
122
+ /*
123
+ Digests worked out while planning, so sending does not work them out again.
124
+
125
+ A save runs the whole pipeline twice: the plan walks, reads, hashes and
126
+ prices every file to produce a quote, then the send does all of it over.
127
+ Measured on a twenty-one thousand file project, that was 42,142 hashes for
128
+ 21,071 files — the count in the profile is exactly double, which is how
129
+ the duplication was finally noticed.
130
+
131
+ Keyed by path and guarded by size and modification time, so a file edited
132
+ while somebody reads the plan is hashed again rather than published under
133
+ the digest it used to have.
134
+ */
135
+ plannedDigests = new Map();
122
136
  planningCreatedRepositoryId = null;
123
137
  /** Attached to every object write after the server admits the exact plan. */
124
138
  activeQuoteId = null;
@@ -352,7 +366,27 @@ class Uploader {
352
366
  truncated version of it would propose deleting everything the walk did
353
367
  not reach.
354
368
  */
355
- const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules));
369
+ const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules, (found) =>
370
+ /*
371
+ The walk is the first minute of a large save and said nothing at
372
+ all, so the panel could only show 0%. A running count of what has
373
+ been seen is not a fraction of anything — it is the evidence that
374
+ the thing is working.
375
+ */
376
+ report({
377
+ stage: "plan",
378
+ files: found,
379
+ totalFiles: 0,
380
+ bytes: 0,
381
+ totalBytes: 0,
382
+ path: "Looking through the folder",
383
+ /*
384
+ Between the key check and the reading, and it holds its own place
385
+ on the bar rather than dropping back to nought between them.
386
+ */
387
+ percent: 6,
388
+ bytesPerSecond: 0,
389
+ })));
356
390
  (0, profile_js_1.counted)("files surveyed", everything.length);
357
391
  const unticked = new Set(request.excluded ?? []);
358
392
  const ticked = request.excluded
@@ -454,8 +488,11 @@ class Uploader {
454
488
  this.check();
455
489
  const full = node_path_1.default.join(request.localPath, file.path);
456
490
  let size;
491
+ let mtimeMs;
457
492
  try {
458
- size = (await (0, promises_1.stat)(full)).size;
493
+ const info = await (0, promises_1.stat)(full);
494
+ size = info.size;
495
+ mtimeMs = info.mtimeMs;
459
496
  }
460
497
  catch (error) {
461
498
  // Unreadable now, though it was listed a moment ago. Skipping it
@@ -476,6 +513,28 @@ class Uploader {
476
513
  locked" naming a path they never chose to think about, instead of
477
514
  the sentence this code already writes for exactly this situation.
478
515
  */
516
+ /*
517
+ Read only when the file has actually moved.
518
+
519
+ Size and modification time are the same test the changes list uses,
520
+ and a match there means the digest already taken still describes the
521
+ bytes. Reading is what costs; deciding not to read is free.
522
+ */
523
+ const remembered = this.plannedDigests.get(file.path) ??
524
+ request.knownDigests?.get(file.path);
525
+ if (remembered &&
526
+ remembered.size === size &&
527
+ remembered.mtimeMs === mtimeMs) {
528
+ measured[index] = {
529
+ path: file.path,
530
+ full,
531
+ size,
532
+ sha256: remembered.sha256,
533
+ mediaType: mediaTypeOf(file.path),
534
+ };
535
+ hashed += 1;
536
+ continue;
537
+ }
479
538
  let digest;
480
539
  try {
481
540
  digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
@@ -486,6 +545,7 @@ class Uploader {
486
545
  hashed += 1;
487
546
  continue;
488
547
  }
548
+ this.plannedDigests.set(file.path, { size, mtimeMs, sha256: digest });
489
549
  measured[index] = {
490
550
  path: file.path,
491
551
  full,
@@ -504,7 +564,12 @@ class Uploader {
504
564
  bytes: 0,
505
565
  totalBytes: 0,
506
566
  path: file.path,
507
- percent: Math.round((hashed / sending.length) * 20),
567
+ /*
568
+ Reading is the stretch from just past the walk to a fifth of
569
+ the way through. Starting it at nought put the bar behind where
570
+ it had already been and it went backwards between stages.
571
+ */
572
+ percent: 8 + Math.round((hashed / sending.length) * 12),
508
573
  bytesPerSecond: 0,
509
574
  });
510
575
  }
@@ -702,11 +767,78 @@ class Uploader {
702
767
  const heavy = sections.filter((one) => smallShare(one) < 0.5);
703
768
  (0, profile_js_1.counted)("sections batched", batchy.length);
704
769
  (0, profile_js_1.counted)("sections heavy", heavy.length);
770
+ /*
771
+ How many files the whole save has finished, across every section.
772
+
773
+ The sections are an implementation detail of not holding thirty
774
+ gigabytes in memory at once, and they were leaking into what a person
775
+ reads: the count ran up to twenty-one thousand while reading, then reset
776
+ to "0 of 4,000" for each section and counted up again, over and over. It
777
+ looked like the save was starting over, repeatedly.
778
+
779
+ One denominator for the whole save. Sections still say which one is in
780
+ flight, in words, where that belongs.
781
+ */
782
+ const sectionProgress = new Map();
783
+ /*
784
+ What has already been shown, so the number never goes backwards.
785
+
786
+ Two pipelines run sections at once and each reports its own share, which
787
+ I have now twice failed to combine into a figure that only rises — the
788
+ count kept dropping to zero at section boundaries and reading as though
789
+ the save had restarted. Whatever the accounting does, the panel must not
790
+ be told a smaller number than it was told a moment ago: going backwards
791
+ is the specific thing a person notices, and it is never information.
792
+ */
793
+ let shown = 0;
794
+ const forward = (count) => {
795
+ if (count > shown)
796
+ shown = count;
797
+ return shown;
798
+ };
705
799
  const runSection = async (section) => {
706
800
  const declarations = section.files.map((file) => declarationByPath.get(file.path));
801
+ /*
802
+ This section's share of the one number the panel shows.
803
+
804
+ Two pipelines run at once, so no section can report on its own behalf
805
+ without the count lurching between them. Each records how far it has
806
+ got and the sum is what is shown, which is the only figure that is
807
+ true while both are moving.
808
+ */
809
+ const advance = (done) => {
810
+ /*
811
+ Only ever upward.
812
+
813
+ The workers share one iterator, so the indices arrive out of order —
814
+ a worker holding file 5 reports after one holding file 200. Writing
815
+ each straight in let a later, lower number undo a higher one, and
816
+ the count sat still or fell back to zero at every section boundary,
817
+ which is the exact stutter this was meant to remove.
818
+ */
819
+ const held = sectionProgress.get(section.index) ?? 0;
820
+ if (done > held)
821
+ sectionProgress.set(section.index, done);
822
+ else if (!sectionProgress.has(section.index)) {
823
+ sectionProgress.set(section.index, 0);
824
+ }
825
+ let sum = 0;
826
+ for (const each of sectionProgress.values())
827
+ sum += each;
828
+ return forward(sum);
829
+ };
707
830
  report({
708
831
  stage: "upload",
709
- files: 0,
832
+ /*
833
+ The count so far, not zero.
834
+
835
+ This line announces a new section, and it was reporting nought files
836
+ done — so every section boundary threw the number back to the start
837
+ and the bar with it. Four of the five reports in this stage were
838
+ changed to carry the running total and this one, the only one a
839
+ person is guaranteed to see, was missed.
840
+ */
841
+ files: advance(0),
710
842
  totalFiles: allDeclarations.length,
711
843
  bytes: progressBytes(),
712
844
  totalBytes,
@@ -861,8 +993,8 @@ class Uploader {
861
993
  transferred += exactWireBytes;
862
994
  report({
863
995
  stage: "upload",
864
- files: 0,
865
- totalFiles: declarations.length,
996
+ files: advance(0),
997
+ totalFiles: allDeclarations.length,
866
998
  bytes: progressBytes(),
867
999
  totalBytes,
868
1000
  path: taking[taking.length - 1].declaration.path,
@@ -914,8 +1046,8 @@ class Uploader {
914
1046
  this.check();
915
1047
  report({
916
1048
  stage: "upload",
917
- files: index,
918
- totalFiles: declarations.length,
1049
+ files: advance(index),
1050
+ totalFiles: allDeclarations.length,
919
1051
  bytes: sentBytes,
920
1052
  totalBytes,
921
1053
  path: declaration.path,
@@ -949,8 +1081,8 @@ class Uploader {
949
1081
  inFlight.set(index, offset);
950
1082
  report({
951
1083
  stage: "upload",
952
- files: index,
953
- totalFiles: declarations.length,
1084
+ files: advance(index),
1085
+ totalFiles: allDeclarations.length,
954
1086
  bytes: progressBytes(),
955
1087
  totalBytes,
956
1088
  path: declaration.path,
@@ -1023,8 +1155,8 @@ class Uploader {
1023
1155
  : await this.putMultipart(repositoryId, declaration, (offset) => {
1024
1156
  report({
1025
1157
  stage: "upload",
1026
- files: index,
1027
- totalFiles: declarations.length,
1158
+ files: advance(index),
1159
+ totalFiles: allDeclarations.length,
1028
1160
  bytes: progressBytes(),
1029
1161
  totalBytes,
1030
1162
  path: declaration.path,
@@ -1047,6 +1179,8 @@ class Uploader {
1047
1179
  }
1048
1180
  };
1049
1181
  await Promise.all(Array.from({ length: Math.min(UPLOAD_LANES, declarations.length) }, () => runWorker()));
1182
+ /* Finished, so its whole share counts — the per-file reports stop one short. */
1183
+ advance(declarations.length);
1050
1184
  };
1051
1185
  const pipeline = async (queue) => {
1052
1186
  for (const section of queue) {
@@ -1066,7 +1200,7 @@ class Uploader {
1066
1200
  percent: 96,
1067
1201
  bytesPerSecond: 0,
1068
1202
  });
1069
- this.plannedQuote = await this.call(`/v1/repositories/${repositoryId}/uploads/preflight`, {
1203
+ this.plannedQuote = await (0, profile_js_1.timed)("ask the service to price it", () => this.call(`/v1/repositories/${repositoryId}/uploads/preflight`, {
1070
1204
  method: "POST",
1071
1205
  contentType: "application/json",
1072
1206
  body: JSON.stringify({
@@ -1074,7 +1208,7 @@ class Uploader {
1074
1208
  excludedBytes: 0,
1075
1209
  objects: [...this.plannedObjects.values()],
1076
1210
  }),
1077
- });
1211
+ }));
1078
1212
  return {
1079
1213
  repositoryId,
1080
1214
  versionId: "",
@@ -1201,7 +1335,7 @@ class Uploader {
1201
1335
  objects are immutable and content-addressed, so a retry reuses them.
1202
1336
  */
1203
1337
  (0, faults_js_1.maybeFail)("submit:after-objects");
1204
- const completed = await this.call(`/v1/repositories/${repositoryId}/versions`, {
1338
+ const completed = await (0, profile_js_1.timed)("publish the version", () => this.call(`/v1/repositories/${repositoryId}/versions`, {
1205
1339
  method: "POST",
1206
1340
  contentType: "application/json",
1207
1341
  body: JSON.stringify({
@@ -1266,7 +1400,7 @@ class Uploader {
1266
1400
  executable: false,
1267
1401
  })),
1268
1402
  }),
1269
- });
1403
+ }));
1270
1404
  /*
1271
1405
  A clean stale-base save may contain a server-side merge. The candidate
1272
1406
  manifest built above is then not the Version that landed: it lacks the
@@ -66,6 +66,28 @@ __pycache__/
66
66
  venv/
67
67
  *.log
68
68
 
69
+ # What a game engine rebuilds for itself. These are the largest thing in a
70
+ # folder by a wide margin — a Unity project's Library is routinely thousands
71
+ # of times the size of its Assets — and every byte comes back the next time
72
+ # the editor opens. Left out of this list for a long time, which meant a
73
+ # folder with no rules of its own sent its whole cache and nobody was told.
74
+ /[Ll]ibrary/
75
+ /[Tt]emp/
76
+ /[Ll]ogs/
77
+ /[Uu]ser[Ss]ettings/
78
+ /[Mm]emoryCaptures/
79
+ Intermediate/
80
+ DerivedDataCache/
81
+ Binaries/
82
+ .godot/
83
+ .import/
84
+
85
+ # Compiler and editor output, which the next build writes again.
86
+ obj/
87
+ .vs/
88
+ .gradle/
89
+ DerivedData/
90
+
69
91
  # Large model weights
70
92
  models/**
71
93
  *.safetensors
@@ -497,6 +519,7 @@ outcome) {
497
519
  }
498
520
  if (outcome) {
499
521
  outcome.listed = files.length;
522
+ outcome.truncated = files.length >= limit;
500
523
  /*
501
524
  Deletions are rows the walk never saw, so they are added to both sides
502
525
  rather than left out of the total — otherwise a synchronize scan could
@@ -520,7 +543,17 @@ outcome) {
520
543
  * content at all. That is what makes it affordable to run over everything
521
544
  * before deciding anything.
522
545
  */
523
- async function surveyFiles(root, rules) {
546
+ async function surveyFiles(root, rules,
547
+ /*
548
+ Told how many have been found, every so often.
549
+
550
+ This walk is the first thing a save does and it can take a minute on a
551
+ large folder — during which the only honest thing the progress panel
552
+ could say was 0%, because nothing had told it otherwise. A count of what
553
+ has been seen is not a fraction of anything, but it moves, and it is the
554
+ difference between waiting and wondering whether it has hung.
555
+ */
556
+ onFound) {
524
557
  const layers = await collectLayers(root, rules);
525
558
  const found = [];
526
559
  const pending = [root];
@@ -554,6 +587,8 @@ async function surveyFiles(root, rules) {
554
587
  continue;
555
588
  try {
556
589
  found.push({ path: relative, size: (await (0, promises_1.stat)(full)).size });
590
+ if (onFound && found.length % 500 === 0)
591
+ onFound(found.length);
557
592
  }
558
593
  catch {
559
594
  /* Gone or unreadable since the walk began; the upload reports it. */
@@ -1004,46 +1039,80 @@ const MOST_FINDINGS = 40;
1004
1039
  * already leave behind is not being published, and raising it would train
1005
1040
  * people to dismiss the question that matters.
1006
1041
  */
1007
- async function detectPastedCredentials(root, include) {
1042
+ async function detectPastedCredentials(root, include,
1043
+ /*
1044
+ Told how many have been looked at, every so often.
1045
+
1046
+ This reads up to twenty thousand files before a save begins, and it did so
1047
+ one after another with nothing reported — on Windows, where every open is
1048
+ also an antivirus scan, that is a quarter of an hour in which the panel
1049
+ could only show the words "Working out what to send" and no number at all.
1050
+ The same silence the read stage used to have, in a different module.
1051
+ */
1052
+ onScanned) {
1008
1053
  const found = [];
1009
1054
  let read = 0;
1010
1055
  let bytes = 0;
1011
- for (const relative of include) {
1012
- if (found.length >= MOST_FINDINGS)
1013
- break;
1014
- if (read >= MOST_SCANNED_FILES || bytes >= MOST_SCANNED_BYTES)
1015
- break;
1016
- const name = relative.slice(relative.lastIndexOf("/") + 1);
1017
- if (!(0, secret_patterns_js_1.worthReading)(name))
1018
- continue;
1019
- const full = node_path_1.default.join(root, relative.split("/").join(node_path_1.default.sep));
1020
- let contents;
1021
- try {
1022
- const info = await (0, promises_1.stat)(full);
1023
- if (!info.isFile() || info.size > MOST_FILE_BYTES)
1056
+ /*
1057
+ Read several at a time. This is waiting on the disk, not computing, so
1058
+ overlapping the waits collapses them into one wait — the same change the
1059
+ hashing stage needed, for the same reason.
1060
+ */
1061
+ const LANES = 8;
1062
+ const wanted = include.filter((relative) => (0, secret_patterns_js_1.worthReading)(relative.slice(relative.lastIndexOf("/") + 1)));
1063
+ let cursor = 0;
1064
+ let stop = false;
1065
+ const lane = async () => {
1066
+ for (;;) {
1067
+ if (stop)
1068
+ return;
1069
+ const at = cursor;
1070
+ cursor += 1;
1071
+ if (at >= wanted.length)
1072
+ return;
1073
+ if (found.length >= MOST_FINDINGS) {
1074
+ stop = true;
1075
+ return;
1076
+ }
1077
+ if (read >= MOST_SCANNED_FILES || bytes >= MOST_SCANNED_BYTES) {
1078
+ stop = true;
1079
+ return;
1080
+ }
1081
+ const relative = wanted[at];
1082
+ const full = node_path_1.default.join(root, relative.split("/").join(node_path_1.default.sep));
1083
+ let contents;
1084
+ try {
1085
+ const info = await (0, promises_1.stat)(full);
1086
+ if (!info.isFile() || info.size > MOST_FILE_BYTES)
1087
+ continue;
1088
+ contents = await (0, promises_1.readFile)(full);
1089
+ }
1090
+ catch {
1091
+ /* Unreadable is the scan's problem, not something to report as clean. */
1024
1092
  continue;
1025
- contents = await (0, promises_1.readFile)(full);
1026
- }
1027
- catch {
1028
- /* Unreadable is the scan's problem, not something to report as clean. */
1029
- continue;
1093
+ }
1094
+ read += 1;
1095
+ bytes += contents.byteLength;
1096
+ if (onScanned && read % 50 === 0)
1097
+ onScanned(read, wanted.length);
1098
+ if (!(0, secret_patterns_js_1.looksLikeText)(contents))
1099
+ continue;
1100
+ const credentials = (0, secret_patterns_js_1.findCredentials)(contents.toString("utf8"));
1101
+ if (credentials.length)
1102
+ found.push({ path: relative, found: credentials });
1030
1103
  }
1031
- read += 1;
1032
- bytes += contents.byteLength;
1033
- if (!(0, secret_patterns_js_1.looksLikeText)(contents))
1034
- continue;
1035
- const credentials = (0, secret_patterns_js_1.findCredentials)(contents.toString("utf8"));
1036
- if (credentials.length)
1037
- found.push({ path: relative, found: credentials });
1038
- }
1104
+ };
1105
+ await Promise.all(Array.from({ length: Math.min(LANES, wanted.length) }, () => lane()));
1039
1106
  return found;
1040
1107
  }
1041
- async function uploadConcerns(root, include) {
1108
+ async function uploadConcerns(root, include,
1109
+ /** Passed through so the one slow part of this can say it is working. */
1110
+ onScanned) {
1042
1111
  const chosen = new Set(include);
1043
1112
  const [named, folders, pasted] = await Promise.all([
1044
1113
  detectSecrets(root),
1045
1114
  detectPrivateDirectories(root),
1046
- detectPastedCredentials(root, include),
1115
+ detectPastedCredentials(root, include, onScanned),
1047
1116
  ]);
1048
1117
  const secrets = named.filter((file) => chosen.has(file));
1049
1118
  return {
package/package.json CHANGED
@@ -1,54 +1,54 @@
1
- {
2
- "name": "@coderook/cli",
3
- "version": "0.29.1",
4
- "description": "CodeRook from the command line, on any operating system",
5
- "license": "SEE LICENSE IN LICENSE.txt",
6
- "homepage": "https://coderook.com",
7
- "bugs": {
8
- "url": "https://coderook.com/contact"
9
- },
10
- "keywords": [
11
- "coderook",
12
- "versioning",
13
- "backup",
14
- "cbx"
15
- ],
16
- "engines": {
17
- "node": ">=20.11.0"
18
- },
19
- "bin": {
20
- "cbx": "dist/cli/src/cli.js",
21
- "coderook": "dist/cli/src/cli.js",
22
- "git-remote-coderook": "dist/cli/src/git_remote_bin.js"
23
- },
24
- "files": [
25
- ".claude-plugin",
26
- "dist",
27
- "skills"
28
- ],
29
- "scripts": {
30
- "build": "tsc -p tsconfig.json",
31
- "check": "tsc -p tsconfig.json --noEmit",
32
- "test": "tsc -p tsconfig.json && node --test --experimental-strip-types test/*.test.ts",
33
- "start": "node dist/cli/src/cli.js",
34
- "prepublishOnly": "npm run sync:plugin && npm run build",
35
- "test:e2e": "node test/e2e.mjs",
36
- "test:matrix": "node test/state-matrix.mjs",
37
- "test:attempt": "node test/attempt-identity.mjs",
38
- "test:get": "node test/get-safety.mjs",
39
- "test:live": "node test/e2e.mjs --allow-production && node test/state-matrix.mjs --allow-production && node test/get-safety.mjs --allow-production && node test/get-track.mjs --allow-production && node test/get-protects-edits.mjs --allow-production && node test/upgrade-migration.mjs --allow-production && node test/fault-get.mjs --allow-production && node test/version-floor.mjs --allow-production && node test/fault-submit.mjs --allow-production && node test/race-attempts.mjs --allow-production && node test/attempt-identity.mjs --allow-production",
40
- "test:getedits": "node test/get-protects-edits.mjs",
41
- "test:upgrade": "node test/upgrade-migration.mjs",
42
- "test:faultget": "node test/fault-get.mjs",
43
- "test:floor": "node test/version-floor.mjs",
44
- "test:faultsubmit": "node test/fault-submit.mjs",
45
- "test:race": "node test/race-attempts.mjs",
46
- "test:runner": "node test/runner-live.mjs",
47
- "sync:plugin": "node scripts/sync-plugin-version.mjs",
48
- "test:track": "node test/get-track.mjs"
49
- },
50
- "devDependencies": {
51
- "@types/node": "24.10.1",
52
- "typescript": "5.9.3"
53
- }
54
- }
1
+ {
2
+ "name": "@coderook/cli",
3
+ "version": "0.29.2",
4
+ "description": "CodeRook from the command line, on any operating system",
5
+ "license": "SEE LICENSE IN LICENSE.txt",
6
+ "homepage": "https://coderook.com",
7
+ "bugs": {
8
+ "url": "https://coderook.com/contact"
9
+ },
10
+ "keywords": [
11
+ "coderook",
12
+ "versioning",
13
+ "backup",
14
+ "cbx"
15
+ ],
16
+ "engines": {
17
+ "node": ">=20.11.0"
18
+ },
19
+ "bin": {
20
+ "cbx": "dist/cli/src/cli.js",
21
+ "coderook": "dist/cli/src/cli.js",
22
+ "git-remote-coderook": "dist/cli/src/git_remote_bin.js"
23
+ },
24
+ "files": [
25
+ ".claude-plugin",
26
+ "dist",
27
+ "skills"
28
+ ],
29
+ "scripts": {
30
+ "build": "tsc -p tsconfig.json",
31
+ "check": "tsc -p tsconfig.json --noEmit",
32
+ "test": "tsc -p tsconfig.json && node --test --experimental-strip-types test/*.test.ts",
33
+ "start": "node dist/cli/src/cli.js",
34
+ "prepublishOnly": "npm run sync:plugin && npm run build",
35
+ "test:e2e": "node test/e2e.mjs",
36
+ "test:matrix": "node test/state-matrix.mjs",
37
+ "test:attempt": "node test/attempt-identity.mjs",
38
+ "test:get": "node test/get-safety.mjs",
39
+ "test:live": "node test/e2e.mjs --allow-production && node test/state-matrix.mjs --allow-production && node test/get-safety.mjs --allow-production && node test/get-track.mjs --allow-production && node test/get-protects-edits.mjs --allow-production && node test/upgrade-migration.mjs --allow-production && node test/fault-get.mjs --allow-production && node test/version-floor.mjs --allow-production && node test/fault-submit.mjs --allow-production && node test/race-attempts.mjs --allow-production && node test/attempt-identity.mjs --allow-production",
40
+ "test:getedits": "node test/get-protects-edits.mjs",
41
+ "test:upgrade": "node test/upgrade-migration.mjs",
42
+ "test:faultget": "node test/fault-get.mjs",
43
+ "test:floor": "node test/version-floor.mjs",
44
+ "test:faultsubmit": "node test/fault-submit.mjs",
45
+ "test:race": "node test/race-attempts.mjs",
46
+ "test:runner": "node test/runner-live.mjs",
47
+ "sync:plugin": "node scripts/sync-plugin-version.mjs",
48
+ "test:track": "node test/get-track.mjs"
49
+ },
50
+ "devDependencies": {
51
+ "@types/node": "24.10.1",
52
+ "typescript": "5.9.3"
53
+ }
54
+ }