@coderook/cli 0.28.0 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
  "name": "coderook",
3
3
  "displayName": "CodeRook",
4
4
  "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
5
- "version": "0.28.0",
5
+ "version": "0.29.1",
6
6
  "author": {
7
7
  "name": "ACCA Gaming Productions",
8
8
  "url": "https://coderook.com"
@@ -661,6 +661,16 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
661
661
  localPath: folder,
662
662
  changed: files.filter((file) => !file.deleted).map((file) => file.path),
663
663
  deleted: files.filter((file) => file.deleted).map((file) => file.path),
664
+ /*
665
+ Nothing is left out, and `changed` above is not the whole of it.
666
+
667
+ `submit` sends whatever the rules allow — there is no per-file choosing
668
+ here — but it was passing the listed rows as the selection, and that list
669
+ stops at twenty thousand. A folder of twenty-one thousand files saved
670
+ twenty thousand of them and said "Saved v1" without mentioning the rest.
671
+ An empty exclusion list is how "all of it" is said.
672
+ */
673
+ excluded: [],
664
674
  message,
665
675
  /*
666
676
  A folder's name is the right default and the wrong answer for import,
@@ -708,11 +718,16 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
708
718
  try {
709
719
  const ahead = await (0, api_js_1.collisionCheck)(link.repositoryId, link.baseVersionId, files.filter((file) => !file.deleted).map((file) => file.path));
710
720
  if (ahead.collidingPaths.length) {
721
+ /*
722
+ "1 of your file is" — the plural was on the wrong noun. It is one
723
+ of *your files*, however many of them collided, so only the verb
724
+ changes with the count.
725
+ */
726
+ const many = ahead.collidingPaths.length !== 1;
711
727
  console.log(accent("Heads up") +
712
728
  ` somebody has saved since you last caught up, and ` +
713
- `${ahead.collidingPaths.length} of your file` +
714
- `${ahead.collidingPaths.length === 1 ? "" : "s"} ` +
715
- `${ahead.collidingPaths.length === 1 ? "is" : "are"} among what they changed.`);
729
+ `${ahead.collidingPaths.length} of your files ` +
730
+ `${many ? "are" : "is"} among what they changed.`);
716
731
  for (const path of ahead.collidingPaths.slice(0, 5)) {
717
732
  console.log(dim(` ${path}`));
718
733
  }
@@ -867,7 +882,13 @@ The connection failed: ${text}`));
867
882
  ? `, ${result.alreadyStoredFiles} already on the account`
868
883
  : "") +
869
884
  (result.reusedFiles ? `, ${result.reusedFiles} unchanged` : "") +
870
- ` · version holds ${bytes(result.sourceBytes)}`);
885
+ /*
886
+ "version holds" was the old fused vocabulary. `cbx submit` makes a
887
+ save; a save becomes a version when somebody names it, which is what
888
+ `cbx release` does. Saying "version" here promised something this
889
+ command does not do.
890
+ */
891
+ ` · the save holds ${bytes(result.sourceBytes)}`);
871
892
  return 0;
872
893
  }
873
894
  /**
@@ -1179,8 +1200,17 @@ trackName) {
1179
1200
  */
1180
1201
  async function suggestRules(folder, apply) {
1181
1202
  const rules = await (0, worktree_js_1.readRules)(folder);
1182
- const files = await (0, worktree_js_1.changedFiles)(folder, rules, null);
1183
- const suggestions = (0, detect_js_1.suggestExclusions)(await (0, worktree_js_1.fileSizes)(folder, files));
1203
+ /*
1204
+ Sizes for the whole folder, not for the listed rows.
1205
+
1206
+ This used to measure `changedFiles`, which stops at twenty thousand — so
1207
+ on the projects with the most to leave out, the sizes here were a fraction
1208
+ of the truth and the biggest item could be missing from the list outright.
1209
+ The scan carries a size for every file it walks past now, capped or not.
1210
+ */
1211
+ const listing = { listed: 0, total: 0, sizes: [] };
1212
+ await (0, worktree_js_1.changedFiles)(folder, rules, null, "add-and-update", undefined, listing);
1213
+ const suggestions = (0, detect_js_1.suggestExclusions)(listing.sizes);
1184
1214
  if (!suggestions.length) {
1185
1215
  console.log(dim("Nothing here looks like it should be left out."));
1186
1216
  return 0;
@@ -1599,11 +1629,14 @@ const SPECS = [
1599
1629
  name: "submit",
1600
1630
  aliases: ["publish", "push"],
1601
1631
  group: "Working with a folder",
1602
- summary: "send the changes as a new version",
1632
+ summary: "send the changes as a new save",
1603
1633
  usage: 'submit [folder] -m "…"',
1604
- detail: "Sends everything that changed since the last version. If the folder is\n" +
1634
+ detail: "Sends everything that changed since the last save. If the folder is\n" +
1605
1635
  "not linked to a project yet, one is created on your account, named\n" +
1606
- "after the folder and private to begin with.",
1636
+ "after the folder and private to begin with.\n\n" +
1637
+ "A save is not a version. Nobody outside the project can see one until\n" +
1638
+ "it is named, which is what `cbx release` does — so submitting is as\n" +
1639
+ "cheap and as private as you want it to be.",
1607
1640
  options: [
1608
1641
  { flags: "-m, --message <text>", description: "what changed, in a sentence" },
1609
1642
  { flags: "-n, --dry-run", description: "show what would be sent, send nothing" },
@@ -1045,12 +1045,18 @@ async function doPush(requests, url) {
1045
1045
  continue;
1046
1046
  }
1047
1047
  /*
1048
- Said before the work starts, not after. Each commit is a published
1049
- version and versions are not free to make, so somebody pushing years
1048
+ Said before the work starts, not after. Each commit becomes a save on
1049
+ the project and saves are not free to make, so somebody pushing years
1050
1050
  of history deserves the chance to stop and import a snapshot instead.
1051
+
1052
+ "to publish as versions" is what this used to say, and it stopped
1053
+ being true when commits and versions separated: a pushed commit is a
1054
+ save that nobody outside the project can see. Publishing one is what
1055
+ pushing a *tag* does, because a tag is a name and a named save is a
1056
+ version.
1051
1057
  */
1052
1058
  const estimate = Math.round((commits.length * 11) / 60);
1053
- say(` ${commits.length} commit${commits.length === 1 ? "" : "s"} to publish as version${commits.length === 1 ? "" : "s"}` +
1059
+ say(` ${commits.length} commit${commits.length === 1 ? "" : "s"} to send as save${commits.length === 1 ? "" : "s"}` +
1054
1060
  (commits.length > 20 ? ` — roughly ${estimate} minute${estimate === 1 ? "" : "s"}` : ""));
1055
1061
  /*
1056
1062
  A push that continues a branch has to start from where the branch
@@ -1114,8 +1120,8 @@ async function doPush(requests, url) {
1114
1120
  .slice(1);
1115
1121
  if (parents.length > 1 && !warnedAboutMerges) {
1116
1122
  warnedAboutMerges = true;
1117
- say(` note: merge commits are published as one version holding the ` +
1118
- `merged tree.\n The contents are exact; the branch shape is not kept.`);
1123
+ say(` note: a merge commit arrives as one save holding the merged ` +
1124
+ `tree.\n The contents are exact; the branch shape is not kept.`);
1119
1125
  }
1120
1126
  const changes = await applyCommit(catFile, scratch, previous, sha);
1121
1127
  previous = sha;
@@ -8,6 +8,7 @@ function uploadRequestFor(input) {
8
8
  localPath: input.localPath,
9
9
  // Both lists. See the note at the top of this file.
10
10
  include: [...input.changed, ...input.deleted],
11
+ ...(input.excluded === undefined ? {} : { excluded: input.excluded }),
11
12
  deletions: input.deleted,
12
13
  message: input.message,
13
14
  projectName: input.projectName,
@@ -39,6 +39,40 @@ const RULES = [
39
39
  { directory: ".next", reason: "Next.js build output", recommended: true },
40
40
  { directory: ".gradle", reason: "Gradle build state", recommended: true },
41
41
  { directory: ".terraform", reason: "Downloaded Terraform providers", recommended: true },
42
+ /*
43
+ Game engines, which is where the size actually is.
44
+
45
+ A Unity project is mostly not the project: `Library/` alone is routinely
46
+ thousands of times the size of `Assets/`, and every byte of it is rebuilt
47
+ from `Assets/` and `Packages/` the next time the editor opens. Until these
48
+ existed a Unity folder produced no suggestions at all — the one shape of
49
+ project where the question "what should I leave out?" has the largest
50
+ possible answer was the one the detector had nothing to say about.
51
+ */
52
+ { directory: "Library", reason: "Unity's import cache — rebuilt when the project next opens", recommended: true },
53
+ { directory: "Temp", reason: "Unity's scratch folder for the running editor", recommended: true },
54
+ { directory: "MemoryCaptures", reason: "Unity memory snapshots — large, and not the project", recommended: true },
55
+ { directory: "Recordings", reason: "Unity recorder output", recommended: true },
56
+ { directory: "UserSettings", reason: "Your own editor layout — not part of the project", recommended: true },
57
+ { directory: "DerivedDataCache", reason: "Unreal's derived data cache — rebuilt on demand", recommended: true },
58
+ { directory: "Intermediate", reason: "Unreal build intermediates — rebuilt when you build", recommended: true },
59
+ { directory: "Binaries", reason: "Unreal compiled output — rebuilt when you build", recommended: true },
60
+ /*
61
+ Named like the work, and it is not: Unreal keeps logs, crash reports and
62
+ autosaves here. Offered rather than ticked, because a folder called
63
+ "Saved" is the one nobody should have excluded on our say-so.
64
+ */
65
+ { directory: "Saved", reason: "Unreal logs and autosaves — check before excluding", recommended: false },
66
+ { directory: ".godot", reason: "Godot's import cache — rebuilt when the project next opens", recommended: true },
67
+ { directory: ".import", reason: "Godot's import cache — rebuilt when the project next opens", recommended: true },
68
+ { directory: "Logs", reason: "Editor logs", recommended: true },
69
+ { directory: "obj", reason: "Compiler intermediates — rebuilt when you build", recommended: true },
70
+ { directory: ".vs", reason: "Visual Studio's local cache", recommended: true },
71
+ /*
72
+ Offered, not ticked. JetBrains keeps run configurations in here that some
73
+ projects deliberately share, so this is a judgement rather than a fact.
74
+ */
75
+ { directory: ".idea", reason: "JetBrains editor state — check before excluding", recommended: false },
42
76
  /*
43
77
  A Chromium or Electron profile left in the project folder. Worth its own
44
78
  reason because the consequence is not only size: these hold lock files
@@ -59,6 +93,7 @@ const RULES = [
59
93
  { directory: "dist", reason: "Usually build output — check before excluding", recommended: false },
60
94
  { directory: "build", reason: "Usually build output — check before excluding", recommended: false },
61
95
  { directory: "out", reason: "Usually build output — check before excluding", recommended: false },
96
+ { directory: "bin", reason: "Usually build output — check before excluding", recommended: false },
62
97
  { extension: ".safetensors", reason: "Model weights — large, and usually downloadable", recommended: false },
63
98
  { extension: ".ckpt", reason: "Model weights — large, and usually downloadable", recommended: false },
64
99
  { extension: ".pt", reason: "Model weights — large, and usually downloadable", recommended: false },
@@ -86,13 +121,29 @@ function suggestExclusions(files) {
86
121
  const extension = node_path_1.default.extname(name).toLowerCase();
87
122
  for (const rule of RULES) {
88
123
  let pattern = null;
89
- if (rule.directory && segments.slice(0, -1).includes(rule.directory)) {
90
- pattern = `${rule.directory}/`;
124
+ if (rule.directory) {
125
+ /*
126
+ Matched without regard to case, and written back with the case the
127
+ folder actually has.
128
+
129
+ Every engine disagrees about capitals — Unity ships `Library` and
130
+ `obj` in the same project, Unreal `Binaries`, Godot `.godot` — and
131
+ an exact-name match meant the detector recognised whichever spelling
132
+ happened to be written here and silently missed the rest. Emitting
133
+ the observed name rather than the rule's own keeps the line that
134
+ gets written matching the folder that is there.
135
+ */
136
+ const wanted = rule.directory.toLowerCase();
137
+ const found = segments
138
+ .slice(0, -1)
139
+ .find((segment) => segment.toLowerCase() === wanted);
140
+ if (found)
141
+ pattern = `${found}/`;
91
142
  }
92
- else if (rule.file && name === rule.file) {
143
+ if (!pattern && rule.file && name === rule.file) {
93
144
  pattern = rule.file;
94
145
  }
95
- else if (rule.extension && extension === rule.extension) {
146
+ if (!pattern && rule.extension && extension === rule.extension) {
96
147
  pattern = `*${rule.extension}`;
97
148
  }
98
149
  if (!pattern)
@@ -354,7 +354,15 @@ class Uploader {
354
354
  */
355
355
  const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules));
356
356
  (0, profile_js_1.counted)("files surveyed", everything.length);
357
- const ticked = new Set(request.include);
357
+ const unticked = new Set(request.excluded ?? []);
358
+ const ticked = request.excluded
359
+ ? new Set([
360
+ ...everything
361
+ .map((file) => file.path)
362
+ .filter((path) => !unticked.has(path)),
363
+ ...(request.deletions ?? []),
364
+ ])
365
+ : new Set(request.include);
358
366
  if (!ticked.size)
359
367
  throw new Error("Nothing is selected to upload");
360
368
  const prior = request.repositoryId
@@ -416,60 +424,98 @@ class Uploader {
416
424
  // 1. Measure and hash. This is what makes an unchanged file free.
417
425
  const declarations = [];
418
426
  let totalBytes = 0;
419
- for (const [index, file] of sending.entries()) {
420
- this.check();
421
- const full = node_path_1.default.join(request.localPath, file.path);
422
- let size;
423
- try {
424
- size = (await (0, promises_1.stat)(full)).size;
425
- }
426
- catch (error) {
427
- // Unreadable now, though it was listed a moment ago. Skipping it
428
- // would publish a version quietly missing a file the person chose.
429
- vanished.push(file.path);
430
- readFailures.set(file.path, error instanceof Error ? error.message : String(error));
431
- continue;
432
- }
433
- report({
434
- stage: "hash",
435
- files: index,
436
- totalFiles: sending.length,
437
- bytes: totalBytes,
438
- totalBytes: 0,
439
- path: file.path,
440
- percent: Math.round((index / sending.length) * 20),
441
- bytesPerSecond: 0,
442
- });
443
- /*
444
- Hashing is the first thing that opens the file, and opening is what
445
- fails on a file another program holds — a database in use, a browser
446
- profile's LOCK, an antivirus mid-scan. On Windows those still answer
447
- stat() perfectly well, so the check three lines above lets them
448
- through and the read below was the one that actually broke.
449
-
450
- It was unguarded, so the person got a raw "EBUSY: resource busy or
451
- locked" naming a path they never chose to think about, instead of the
452
- sentence this code already writes for exactly this situation. The
453
- outcome was always going to be a refusal; what was missing was it
454
- being legible.
455
- */
456
- let digest;
457
- try {
458
- digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
427
+ /*
428
+ Hashed several at a time, and reported by how many are done.
429
+
430
+ Each file here is an open, a read and a close, and on Windows every open
431
+ is also an antivirus scan — so this stage is waiting, not computing.
432
+ Taken strictly one after another, a folder of twenty thousand small
433
+ files spent a quarter of an hour in a step that moves no data at all,
434
+ and the bar sat on 0% for the first thousand of them because it only
435
+ ever spanned a fifth of itself across the whole stage.
436
+
437
+ Overlapping the waits collapses them into one wait. The results are
438
+ written into their own slots rather than pushed, because which disk read
439
+ finishes first must not decide the order of a version's file list.
440
+ */
441
+ const HASH_LANES = 8;
442
+ const measured = new Array(sending.length).fill(null);
443
+ let hashed = 0;
444
+ let cursor = 0;
445
+ /** Throttled: twenty thousand sends is its own cost, and nobody reads them. */
446
+ let told = 0;
447
+ const lane = async () => {
448
+ for (;;) {
449
+ const index = cursor;
450
+ cursor += 1;
451
+ if (index >= sending.length)
452
+ return;
453
+ const file = sending[index];
454
+ this.check();
455
+ const full = node_path_1.default.join(request.localPath, file.path);
456
+ let size;
457
+ try {
458
+ size = (await (0, promises_1.stat)(full)).size;
459
+ }
460
+ catch (error) {
461
+ // Unreadable now, though it was listed a moment ago. Skipping it
462
+ // would publish a version quietly missing a file the person chose.
463
+ vanished.push(file.path);
464
+ readFailures.set(file.path, error instanceof Error ? error.message : String(error));
465
+ hashed += 1;
466
+ continue;
467
+ }
468
+ /*
469
+ Hashing is the first thing that opens the file, and opening is what
470
+ fails on a file another program holds — a database in use, a browser
471
+ profile's LOCK, an antivirus mid-scan. On Windows those still answer
472
+ stat() perfectly well, so the check above lets them through and the
473
+ read below is the one that actually breaks.
474
+
475
+ It was unguarded, so the person got a raw "EBUSY: resource busy or
476
+ locked" naming a path they never chose to think about, instead of
477
+ the sentence this code already writes for exactly this situation.
478
+ */
479
+ let digest;
480
+ try {
481
+ digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
482
+ }
483
+ catch (error) {
484
+ vanished.push(file.path);
485
+ readFailures.set(file.path, error instanceof Error ? error.message : String(error));
486
+ hashed += 1;
487
+ continue;
488
+ }
489
+ measured[index] = {
490
+ path: file.path,
491
+ full,
492
+ size,
493
+ sha256: digest,
494
+ mediaType: mediaTypeOf(file.path),
495
+ };
496
+ hashed += 1;
497
+ const now = Date.now();
498
+ if (now - told > 80 || hashed === sending.length) {
499
+ told = now;
500
+ report({
501
+ stage: "hash",
502
+ files: hashed,
503
+ totalFiles: sending.length,
504
+ bytes: 0,
505
+ totalBytes: 0,
506
+ path: file.path,
507
+ percent: Math.round((hashed / sending.length) * 20),
508
+ bytesPerSecond: 0,
509
+ });
510
+ }
459
511
  }
460
- catch (error) {
461
- vanished.push(file.path);
462
- readFailures.set(file.path, error instanceof Error ? error.message : String(error));
512
+ };
513
+ await Promise.all(Array.from({ length: Math.min(HASH_LANES, sending.length) }, () => lane()));
514
+ for (const one of measured) {
515
+ if (!one)
463
516
  continue;
464
- }
465
- declarations.push({
466
- path: file.path,
467
- full,
468
- size,
469
- sha256: digest,
470
- mediaType: mediaTypeOf(file.path),
471
- });
472
- totalBytes += size;
517
+ declarations.push(one);
518
+ totalBytes += one.size;
473
519
  }
474
520
  /*
475
521
  A deletion-only save has no file to hash or upload, but it is still a
@@ -1052,7 +1098,16 @@ class Uploader {
1052
1098
  totalFiles: declarations.length,
1053
1099
  bytes: sentBytes,
1054
1100
  totalBytes,
1055
- path: "Recording the version",
1101
+ /*
1102
+ A save, not a version.
1103
+
1104
+ Sending produces a commit; it becomes a version when somebody names
1105
+ it, which happens later and on purpose. This line said "Recording the
1106
+ version" while doing neither — it was the last wording left over from
1107
+ before the two were separate, and it sat on the one screen a person
1108
+ watches to the end.
1109
+ */
1110
+ path: "Writing down what was sent",
1056
1111
  percent: 96,
1057
1112
  bytesPerSecond: rate(),
1058
1113
  });
@@ -14,7 +14,6 @@ exports.branchName = branchName;
14
14
  exports.changedFiles = changedFiles;
15
15
  exports.surveyFiles = surveyFiles;
16
16
  exports.totalSize = totalSize;
17
- exports.fileSizes = fileSizes;
18
17
  exports.fileDiff = fileDiff;
19
18
  exports.projectTree = projectTree;
20
19
  exports.evaluateRules = evaluateRules;
@@ -55,49 +54,49 @@ async function git(root, ...args) {
55
54
  * file would produce an incomplete project that looked backed up, so likely
56
55
  * secrets are warned about instead (docs/UPLOAD_POLICY.md).
57
56
  */
58
- exports.STARTER_IGNORE = `# Dependencies and generated output
59
- node_modules/
60
- dist/
61
- build/
62
- out/
63
- .next/
64
- target/
65
- __pycache__/
66
- .venv/
67
- venv/
68
- *.log
69
-
70
- # Large model weights
71
- models/**
72
- *.safetensors
73
- *.ckpt
74
- *.pt
75
- *.pth
76
-
77
- # Caches and local state
78
- .venv/
79
- .pytest_cache/
80
- .mypy_cache/
81
- .ruff_cache/
82
- .cache/
83
- *.pyc
84
-
85
- # A browser or Electron profile that has been left in the project folder.
86
- # These hold files another program keeps open — a LOCK that cannot be read
87
- # while it runs will stop a save outright — and nothing in them is the work.
88
- IndexedDB/
89
- Local Storage/
90
- Session Storage/
91
- Service Worker/
92
- Network/
93
- GPUCache/
94
- Code Cache/
95
- blob_storage/
96
- Local State
97
- Preferences
98
-
99
- # Archives of the project, inside the project
100
- *.cbx
57
+ exports.STARTER_IGNORE = `# Dependencies and generated output
58
+ node_modules/
59
+ dist/
60
+ build/
61
+ out/
62
+ .next/
63
+ target/
64
+ __pycache__/
65
+ .venv/
66
+ venv/
67
+ *.log
68
+
69
+ # Large model weights
70
+ models/**
71
+ *.safetensors
72
+ *.ckpt
73
+ *.pt
74
+ *.pth
75
+
76
+ # Caches and local state
77
+ .venv/
78
+ .pytest_cache/
79
+ .mypy_cache/
80
+ .ruff_cache/
81
+ .cache/
82
+ *.pyc
83
+
84
+ # A browser or Electron profile that has been left in the project folder.
85
+ # These hold files another program keeps open — a LOCK that cannot be read
86
+ # while it runs will stop a save outright — and nothing in them is the work.
87
+ IndexedDB/
88
+ Local Storage/
89
+ Session Storage/
90
+ Service Worker/
91
+ Network/
92
+ GPUCache/
93
+ Code Cache/
94
+ blob_storage/
95
+ Local State
96
+ Preferences
97
+
98
+ # Archives of the project, inside the project
99
+ *.cbx
101
100
  `;
102
101
  /** The shared rules file, committed with the project. */
103
102
  exports.IGNORE_FILE = ".gitignore";
@@ -318,19 +317,69 @@ async function measure(full, size, known) {
318
317
  * upload when no version has ever been saved. The filter rules decide what
319
318
  * is a candidate; the baseline decides what is new.
320
319
  */
320
+ /**
321
+ * The same files, ordered so that a cap spends itself across the folder.
322
+ *
323
+ * Taken in walk order, a cap is handed to whichever directory the walk
324
+ * reached first — and the directory a person cares about is rarely that one.
325
+ * Round-robin by directory means the first row of every directory is chosen
326
+ * before the second row of any, so a folder of forty-five thousand cache
327
+ * files cannot bury the eight hundred beside it that are the work.
328
+ */
329
+ function spreadAcrossDirectories(found) {
330
+ const byDirectory = new Map();
331
+ for (const one of found) {
332
+ const at = one.relative.slice(0, one.relative.lastIndexOf("/") + 1);
333
+ const held = byDirectory.get(at);
334
+ if (held)
335
+ held.push(one);
336
+ else
337
+ byDirectory.set(at, [one]);
338
+ }
339
+ const queues = [...byDirectory.values()];
340
+ const deepest = queues.reduce((most, queue) => Math.max(most, queue.length), 0);
341
+ const spread = [];
342
+ for (let round = 0; round < deepest; round++) {
343
+ for (const queue of queues) {
344
+ const one = queue[round];
345
+ if (one)
346
+ spread.push(one);
347
+ }
348
+ }
349
+ return spread;
350
+ }
321
351
  async function changedFiles(root, rules, baseline = null, mode = "add-and-update",
322
352
  /*
323
353
  Digests already taken, and what the files looked like when they were.
324
354
  Read and written in place, so the caller keeps whatever this learns and
325
355
  the next scan does not read the same unchanged gigabyte again.
326
356
  */
327
- stats) {
357
+ stats,
358
+ /** Filled with how much of the folder the cap allowed through. */
359
+ outcome) {
328
360
  const layers = await collectLayers(root, rules);
329
361
  const files = [];
330
362
  const present = new Set();
331
363
  const pending = [root];
332
- while (pending.length && files.length < LISTED_FILE_LIMIT) {
333
- const directory = pending.pop();
364
+ /** Everything the rules would send, including what the cap keeps out. */
365
+ const found = [];
366
+ const limit = outcome?.limit ?? LISTED_FILE_LIMIT;
367
+ /*
368
+ Walk the whole folder first, and decide what to read afterwards.
369
+
370
+ These used to be one pass: walk until twenty thousand rows had been
371
+ collected, then stop where it stood. That made the cap decide the shape of
372
+ the answer — the walk was depth first, so the first directory it descended
373
+ into took the entire allowance. In a Unity project that is `Library/`,
374
+ forty-five thousand files of import cache, and `Assets/` — the actual game
375
+ — appeared in the list nowhere at all.
376
+
377
+ Walking is cheap: a readdir and a stat, no file is opened. Reading is what
378
+ costs, so reading is what the cap should govern, and it can only be spent
379
+ well once the whole folder is known.
380
+ */
381
+ for (let next = 0; next < pending.length; next++) {
382
+ const directory = pending[next];
334
383
  let entries;
335
384
  try {
336
385
  entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
@@ -338,6 +387,7 @@ stats) {
338
387
  catch {
339
388
  continue;
340
389
  }
390
+ const wanted = [];
341
391
  for (const entry of entries) {
342
392
  const full = node_path_1.default.join(directory, entry.name);
343
393
  if (entry.isSymbolicLink())
@@ -359,43 +409,75 @@ stats) {
359
409
  continue;
360
410
  if ((0, rules_js_1.excludes)(relative, false, layers))
361
411
  continue;
362
- let size;
363
- let mtimeMs;
412
+ wanted.push({ full, relative });
413
+ }
414
+ /*
415
+ Sized a directory at a time rather than a file at a time. One `stat` is
416
+ a syscall's round trip, and forty-seven thousand of them taken strictly
417
+ one after another is most of the scan; asked for together they overlap.
418
+ */
419
+ const sized = await Promise.all(wanted.map(async (one) => {
364
420
  try {
365
- const info = await (0, promises_1.stat)(full);
366
- size = info.size;
367
- mtimeMs = info.mtimeMs;
421
+ const info = await (0, promises_1.stat)(one.full);
422
+ return { ...one, size: info.size, mtimeMs: info.mtimeMs };
368
423
  }
369
424
  catch {
370
- continue;
425
+ /* Gone or unreadable since the walk began; the upload reports it. */
426
+ return null;
371
427
  }
372
- present.add(relative);
373
- const measured = await measure(full, size, {
374
- mtimeMs,
375
- cached: stats?.get(relative),
376
- recorded: baseline?.has(relative) ?? false,
377
- });
428
+ }));
429
+ for (const one of sized) {
430
+ if (!one)
431
+ continue;
378
432
  /*
379
- Remembered whether or not it changed: the next scan wants to skip
380
- reading this file again, and that is just as true of one that was
381
- edited a moment ago as of one that never moves.
433
+ Recorded as present for every file the walk saw, capped or not.
434
+
435
+ Synchronize mode calls anything in the last version that is missing
436
+ from this set a deletion. Filling it only for the rows that fitted
437
+ meant a folder past the cap proposed deleting every file the cap had
438
+ kept out — files sitting right there on disk.
382
439
  */
383
- if (measured.hash)
384
- stats?.set(relative, { size, mtimeMs, sha256: measured.hash });
385
- const saved = baseline?.get(relative);
386
- if (saved && measured.hash && saved === measured.hash)
387
- continue;
388
- files.push({
389
- path: relative,
390
- // Against a saved version the true line delta needs the old copy;
391
- // until a version exists to fetch, a changed file counts as rewritten.
392
- added: measured.lines,
393
- removed: 0,
394
- included: true,
395
- binary: measured.binary,
396
- });
440
+ present.add(one.relative);
441
+ outcome?.sizes.push({ path: one.relative, size: one.size });
442
+ found.push(one);
397
443
  }
398
444
  }
445
+ /*
446
+ Read in an order that gives every directory a share, and stop at the cap.
447
+
448
+ An unchanged file costs a digest and produces no row, so this runs until
449
+ the list is full rather than over a fixed slice — a folder where nothing
450
+ has changed still has to be walked through to find that out.
451
+ */
452
+ for (const one of spreadAcrossDirectories(found)) {
453
+ if (files.length >= limit)
454
+ break;
455
+ const { full, relative, size, mtimeMs } = one;
456
+ const measured = await measure(full, size, {
457
+ mtimeMs,
458
+ cached: stats?.get(relative),
459
+ recorded: baseline?.has(relative) ?? false,
460
+ });
461
+ /*
462
+ Remembered whether or not it changed: the next scan wants to skip
463
+ reading this file again, and that is just as true of one that was
464
+ edited a moment ago as of one that never moves.
465
+ */
466
+ if (measured.hash)
467
+ stats?.set(relative, { size, mtimeMs, sha256: measured.hash });
468
+ const saved = baseline?.get(relative);
469
+ if (saved && measured.hash && saved === measured.hash)
470
+ continue;
471
+ files.push({
472
+ path: relative,
473
+ // Against a saved version the true line delta needs the old copy;
474
+ // until a version exists to fetch, a changed file counts as rewritten.
475
+ added: measured.lines,
476
+ removed: 0,
477
+ included: true,
478
+ binary: measured.binary,
479
+ });
480
+ }
399
481
  // A file that was in the last version and is gone now is only a change in
400
482
  // synchronize mode. The safe default adds and updates, and leaves the
401
483
  // saved copy of a missing file alone (docs/UPLOAD_POLICY.md).
@@ -413,6 +495,15 @@ stats) {
413
495
  });
414
496
  }
415
497
  }
498
+ if (outcome) {
499
+ outcome.listed = files.length;
500
+ /*
501
+ Deletions are rows the walk never saw, so they are added to both sides
502
+ rather than left out of the total — otherwise a synchronize scan could
503
+ report listing more files than the folder was found to hold.
504
+ */
505
+ outcome.total = Math.max(found.length, files.length);
506
+ }
416
507
  return files.sort((left, right) => left.path.localeCompare(right.path));
417
508
  }
418
509
  /** The size on disk of everything the rules would upload. */
@@ -483,29 +574,6 @@ async function totalSize(root, files) {
483
574
  }
484
575
  return total;
485
576
  }
486
- /**
487
- * The same walk, keeping each file's size rather than only the sum.
488
- *
489
- * Used where something has to reason about which files are large — the
490
- * exclusion suggestions, whose whole value is saying which folder is costing
491
- * the upload its time. Separate from totalSize so that the common path still
492
- * carries nothing it does not need.
493
- */
494
- async function fileSizes(root, files) {
495
- const sized = [];
496
- for (const file of files) {
497
- try {
498
- sized.push({
499
- path: file.path,
500
- size: (await (0, promises_1.stat)(node_path_1.default.join(root, file.path))).size,
501
- });
502
- }
503
- catch {
504
- /* deleted since the scan; it cannot be measured and does not count */
505
- }
506
- }
507
- return sized;
508
- }
509
577
  /** The unified diff for one file, including files git does not track yet. */
510
578
  async function fileDiff(root, file, ignoreWhitespace = false) {
511
579
  const output = await git(root, "diff", "--no-ext-diff", "--unified=3", ...(ignoreWhitespace ? ["--ignore-all-space"] : []), "--", file);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@coderook/cli",
3
- "version": "0.28.0",
3
+ "version": "0.29.1",
4
4
  "description": "CodeRook from the command line, on any operating system",
5
5
  "license": "SEE LICENSE IN LICENSE.txt",
6
6
  "homepage": "https://coderook.com",