@coderook/cli 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/dist/cli/src/cli.js +42 -9
- package/dist/cli/src/git_remote.js +11 -5
- package/dist/cli/src/publish.js +1 -0
- package/dist/desktop-app/src/main/detect.js +55 -4
- package/dist/desktop-app/src/main/upload.js +109 -54
- package/dist/desktop-app/src/main/worktree.js +167 -99
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "coderook",
|
|
3
3
|
"displayName": "CodeRook",
|
|
4
4
|
"description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.29.1",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "ACCA Gaming Productions",
|
|
8
8
|
"url": "https://coderook.com"
|
package/dist/cli/src/cli.js
CHANGED
|
@@ -661,6 +661,16 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
|
|
|
661
661
|
localPath: folder,
|
|
662
662
|
changed: files.filter((file) => !file.deleted).map((file) => file.path),
|
|
663
663
|
deleted: files.filter((file) => file.deleted).map((file) => file.path),
|
|
664
|
+
/*
|
|
665
|
+
Nothing is left out, and `changed` above is not the whole of it.
|
|
666
|
+
|
|
667
|
+
`submit` sends whatever the rules allow — there is no per-file choosing
|
|
668
|
+
here — but it was passing the listed rows as the selection, and that list
|
|
669
|
+
stops at twenty thousand. A folder of twenty-one thousand files saved
|
|
670
|
+
twenty thousand of them and said "Saved v1" without mentioning the rest.
|
|
671
|
+
An empty exclusion list is how "all of it" is said.
|
|
672
|
+
*/
|
|
673
|
+
excluded: [],
|
|
664
674
|
message,
|
|
665
675
|
/*
|
|
666
676
|
A folder's name is the right default and the wrong answer for import,
|
|
@@ -708,11 +718,16 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
|
|
|
708
718
|
try {
|
|
709
719
|
const ahead = await (0, api_js_1.collisionCheck)(link.repositoryId, link.baseVersionId, files.filter((file) => !file.deleted).map((file) => file.path));
|
|
710
720
|
if (ahead.collidingPaths.length) {
|
|
721
|
+
/*
|
|
722
|
+
"1 of your file is" — the plural was on the wrong noun. It is one
|
|
723
|
+
of *your files*, however many of them collided, so only the verb
|
|
724
|
+
changes with the count.
|
|
725
|
+
*/
|
|
726
|
+
const many = ahead.collidingPaths.length !== 1;
|
|
711
727
|
console.log(accent("Heads up") +
|
|
712
728
|
` somebody has saved since you last caught up, and ` +
|
|
713
|
-
`${ahead.collidingPaths.length} of your
|
|
714
|
-
`${
|
|
715
|
-
`${ahead.collidingPaths.length === 1 ? "is" : "are"} among what they changed.`);
|
|
729
|
+
`${ahead.collidingPaths.length} of your files ` +
|
|
730
|
+
`${many ? "are" : "is"} among what they changed.`);
|
|
716
731
|
for (const path of ahead.collidingPaths.slice(0, 5)) {
|
|
717
732
|
console.log(dim(` ${path}`));
|
|
718
733
|
}
|
|
@@ -867,7 +882,13 @@ The connection failed: ${text}`));
|
|
|
867
882
|
? `, ${result.alreadyStoredFiles} already on the account`
|
|
868
883
|
: "") +
|
|
869
884
|
(result.reusedFiles ? `, ${result.reusedFiles} unchanged` : "") +
|
|
870
|
-
|
|
885
|
+
/*
|
|
886
|
+
"version holds" was the old fused vocabulary. `cbx submit` makes a
|
|
887
|
+
save; a save becomes a version when somebody names it, which is what
|
|
888
|
+
`cbx release` does. Saying "version" here promised something this
|
|
889
|
+
command does not do.
|
|
890
|
+
*/
|
|
891
|
+
` · the save holds ${bytes(result.sourceBytes)}`);
|
|
871
892
|
return 0;
|
|
872
893
|
}
|
|
873
894
|
/**
|
|
@@ -1179,8 +1200,17 @@ trackName) {
|
|
|
1179
1200
|
*/
|
|
1180
1201
|
async function suggestRules(folder, apply) {
|
|
1181
1202
|
const rules = await (0, worktree_js_1.readRules)(folder);
|
|
1182
|
-
|
|
1183
|
-
|
|
1203
|
+
/*
|
|
1204
|
+
Sizes for the whole folder, not for the listed rows.
|
|
1205
|
+
|
|
1206
|
+
This used to measure `changedFiles`, which stops at twenty thousand — so
|
|
1207
|
+
on the projects with the most to leave out, the sizes here were a fraction
|
|
1208
|
+
of the truth and the biggest item could be missing from the list outright.
|
|
1209
|
+
The scan carries a size for every file it walks past now, capped or not.
|
|
1210
|
+
*/
|
|
1211
|
+
const listing = { listed: 0, total: 0, sizes: [] };
|
|
1212
|
+
await (0, worktree_js_1.changedFiles)(folder, rules, null, "add-and-update", undefined, listing);
|
|
1213
|
+
const suggestions = (0, detect_js_1.suggestExclusions)(listing.sizes);
|
|
1184
1214
|
if (!suggestions.length) {
|
|
1185
1215
|
console.log(dim("Nothing here looks like it should be left out."));
|
|
1186
1216
|
return 0;
|
|
@@ -1599,11 +1629,14 @@ const SPECS = [
|
|
|
1599
1629
|
name: "submit",
|
|
1600
1630
|
aliases: ["publish", "push"],
|
|
1601
1631
|
group: "Working with a folder",
|
|
1602
|
-
summary: "send the changes as a new
|
|
1632
|
+
summary: "send the changes as a new save",
|
|
1603
1633
|
usage: 'submit [folder] -m "…"',
|
|
1604
|
-
detail: "Sends everything that changed since the last
|
|
1634
|
+
detail: "Sends everything that changed since the last save. If the folder is\n" +
|
|
1605
1635
|
"not linked to a project yet, one is created on your account, named\n" +
|
|
1606
|
-
"after the folder and private to begin with
|
|
1636
|
+
"after the folder and private to begin with.\n\n" +
|
|
1637
|
+
"A save is not a version. Nobody outside the project can see one until\n" +
|
|
1638
|
+
"it is named, which is what `cbx release` does — so submitting is as\n" +
|
|
1639
|
+
"cheap and as private as you want it to be.",
|
|
1607
1640
|
options: [
|
|
1608
1641
|
{ flags: "-m, --message <text>", description: "what changed, in a sentence" },
|
|
1609
1642
|
{ flags: "-n, --dry-run", description: "show what would be sent, send nothing" },
|
|
@@ -1045,12 +1045,18 @@ async function doPush(requests, url) {
|
|
|
1045
1045
|
continue;
|
|
1046
1046
|
}
|
|
1047
1047
|
/*
|
|
1048
|
-
Said before the work starts, not after. Each commit
|
|
1049
|
-
|
|
1048
|
+
Said before the work starts, not after. Each commit becomes a save on
|
|
1049
|
+
the project and saves are not free to make, so somebody pushing years
|
|
1050
1050
|
of history deserves the chance to stop and import a snapshot instead.
|
|
1051
|
+
|
|
1052
|
+
"to publish as versions" is what this used to say, and it stopped
|
|
1053
|
+
being true when commits and versions separated: a pushed commit is a
|
|
1054
|
+
save that nobody outside the project can see. Publishing one is what
|
|
1055
|
+
pushing a *tag* does, because a tag is a name and a named save is a
|
|
1056
|
+
version.
|
|
1051
1057
|
*/
|
|
1052
1058
|
const estimate = Math.round((commits.length * 11) / 60);
|
|
1053
|
-
say(` ${commits.length} commit${commits.length === 1 ? "" : "s"} to
|
|
1059
|
+
say(` ${commits.length} commit${commits.length === 1 ? "" : "s"} to send as save${commits.length === 1 ? "" : "s"}` +
|
|
1054
1060
|
(commits.length > 20 ? ` — roughly ${estimate} minute${estimate === 1 ? "" : "s"}` : ""));
|
|
1055
1061
|
/*
|
|
1056
1062
|
A push that continues a branch has to start from where the branch
|
|
@@ -1114,8 +1120,8 @@ async function doPush(requests, url) {
|
|
|
1114
1120
|
.slice(1);
|
|
1115
1121
|
if (parents.length > 1 && !warnedAboutMerges) {
|
|
1116
1122
|
warnedAboutMerges = true;
|
|
1117
|
-
say(` note: merge
|
|
1118
|
-
`
|
|
1123
|
+
say(` note: a merge commit arrives as one save holding the merged ` +
|
|
1124
|
+
`tree.\n The contents are exact; the branch shape is not kept.`);
|
|
1119
1125
|
}
|
|
1120
1126
|
const changes = await applyCommit(catFile, scratch, previous, sha);
|
|
1121
1127
|
previous = sha;
|
package/dist/cli/src/publish.js
CHANGED
|
@@ -8,6 +8,7 @@ function uploadRequestFor(input) {
|
|
|
8
8
|
localPath: input.localPath,
|
|
9
9
|
// Both lists. See the note at the top of this file.
|
|
10
10
|
include: [...input.changed, ...input.deleted],
|
|
11
|
+
...(input.excluded === undefined ? {} : { excluded: input.excluded }),
|
|
11
12
|
deletions: input.deleted,
|
|
12
13
|
message: input.message,
|
|
13
14
|
projectName: input.projectName,
|
|
@@ -39,6 +39,40 @@ const RULES = [
|
|
|
39
39
|
{ directory: ".next", reason: "Next.js build output", recommended: true },
|
|
40
40
|
{ directory: ".gradle", reason: "Gradle build state", recommended: true },
|
|
41
41
|
{ directory: ".terraform", reason: "Downloaded Terraform providers", recommended: true },
|
|
42
|
+
/*
|
|
43
|
+
Game engines, which is where the size actually is.
|
|
44
|
+
|
|
45
|
+
A Unity project is mostly not the project: `Library/` alone is routinely
|
|
46
|
+
thousands of times the size of `Assets/`, and every byte of it is rebuilt
|
|
47
|
+
from `Assets/` and `Packages/` the next time the editor opens. Until these
|
|
48
|
+
existed a Unity folder produced no suggestions at all — the one shape of
|
|
49
|
+
project where the question "what should I leave out?" has the largest
|
|
50
|
+
possible answer was the one the detector had nothing to say about.
|
|
51
|
+
*/
|
|
52
|
+
{ directory: "Library", reason: "Unity's import cache — rebuilt when the project next opens", recommended: true },
|
|
53
|
+
{ directory: "Temp", reason: "Unity's scratch folder for the running editor", recommended: true },
|
|
54
|
+
{ directory: "MemoryCaptures", reason: "Unity memory snapshots — large, and not the project", recommended: true },
|
|
55
|
+
{ directory: "Recordings", reason: "Unity recorder output", recommended: true },
|
|
56
|
+
{ directory: "UserSettings", reason: "Your own editor layout — not part of the project", recommended: true },
|
|
57
|
+
{ directory: "DerivedDataCache", reason: "Unreal's derived data cache — rebuilt on demand", recommended: true },
|
|
58
|
+
{ directory: "Intermediate", reason: "Unreal build intermediates — rebuilt when you build", recommended: true },
|
|
59
|
+
{ directory: "Binaries", reason: "Unreal compiled output — rebuilt when you build", recommended: true },
|
|
60
|
+
/*
|
|
61
|
+
Named like the work, and it is not: Unreal keeps logs, crash reports and
|
|
62
|
+
autosaves here. Offered rather than ticked, because a folder called
|
|
63
|
+
"Saved" is the one nobody should have excluded on our say-so.
|
|
64
|
+
*/
|
|
65
|
+
{ directory: "Saved", reason: "Unreal logs and autosaves — check before excluding", recommended: false },
|
|
66
|
+
{ directory: ".godot", reason: "Godot's import cache — rebuilt when the project next opens", recommended: true },
|
|
67
|
+
{ directory: ".import", reason: "Godot's import cache — rebuilt when the project next opens", recommended: true },
|
|
68
|
+
{ directory: "Logs", reason: "Editor logs", recommended: true },
|
|
69
|
+
{ directory: "obj", reason: "Compiler intermediates — rebuilt when you build", recommended: true },
|
|
70
|
+
{ directory: ".vs", reason: "Visual Studio's local cache", recommended: true },
|
|
71
|
+
/*
|
|
72
|
+
Offered, not ticked. JetBrains keeps run configurations in here that some
|
|
73
|
+
projects deliberately share, so this is a judgement rather than a fact.
|
|
74
|
+
*/
|
|
75
|
+
{ directory: ".idea", reason: "JetBrains editor state — check before excluding", recommended: false },
|
|
42
76
|
/*
|
|
43
77
|
A Chromium or Electron profile left in the project folder. Worth its own
|
|
44
78
|
reason because the consequence is not only size: these hold lock files
|
|
@@ -59,6 +93,7 @@ const RULES = [
|
|
|
59
93
|
{ directory: "dist", reason: "Usually build output — check before excluding", recommended: false },
|
|
60
94
|
{ directory: "build", reason: "Usually build output — check before excluding", recommended: false },
|
|
61
95
|
{ directory: "out", reason: "Usually build output — check before excluding", recommended: false },
|
|
96
|
+
{ directory: "bin", reason: "Usually build output — check before excluding", recommended: false },
|
|
62
97
|
{ extension: ".safetensors", reason: "Model weights — large, and usually downloadable", recommended: false },
|
|
63
98
|
{ extension: ".ckpt", reason: "Model weights — large, and usually downloadable", recommended: false },
|
|
64
99
|
{ extension: ".pt", reason: "Model weights — large, and usually downloadable", recommended: false },
|
|
@@ -86,13 +121,29 @@ function suggestExclusions(files) {
|
|
|
86
121
|
const extension = node_path_1.default.extname(name).toLowerCase();
|
|
87
122
|
for (const rule of RULES) {
|
|
88
123
|
let pattern = null;
|
|
89
|
-
if (rule.directory
|
|
90
|
-
|
|
124
|
+
if (rule.directory) {
|
|
125
|
+
/*
|
|
126
|
+
Matched without regard to case, and written back with the case the
|
|
127
|
+
folder actually has.
|
|
128
|
+
|
|
129
|
+
Every engine disagrees about capitals — Unity ships `Library` and
|
|
130
|
+
`obj` in the same project, Unreal `Binaries`, Godot `.godot` — and
|
|
131
|
+
an exact-name match meant the detector recognised whichever spelling
|
|
132
|
+
happened to be written here and silently missed the rest. Emitting
|
|
133
|
+
the observed name rather than the rule's own keeps the line that
|
|
134
|
+
gets written matching the folder that is there.
|
|
135
|
+
*/
|
|
136
|
+
const wanted = rule.directory.toLowerCase();
|
|
137
|
+
const found = segments
|
|
138
|
+
.slice(0, -1)
|
|
139
|
+
.find((segment) => segment.toLowerCase() === wanted);
|
|
140
|
+
if (found)
|
|
141
|
+
pattern = `${found}/`;
|
|
91
142
|
}
|
|
92
|
-
|
|
143
|
+
if (!pattern && rule.file && name === rule.file) {
|
|
93
144
|
pattern = rule.file;
|
|
94
145
|
}
|
|
95
|
-
|
|
146
|
+
if (!pattern && rule.extension && extension === rule.extension) {
|
|
96
147
|
pattern = `*${rule.extension}`;
|
|
97
148
|
}
|
|
98
149
|
if (!pattern)
|
|
@@ -354,7 +354,15 @@ class Uploader {
|
|
|
354
354
|
*/
|
|
355
355
|
const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules));
|
|
356
356
|
(0, profile_js_1.counted)("files surveyed", everything.length);
|
|
357
|
-
const
|
|
357
|
+
const unticked = new Set(request.excluded ?? []);
|
|
358
|
+
const ticked = request.excluded
|
|
359
|
+
? new Set([
|
|
360
|
+
...everything
|
|
361
|
+
.map((file) => file.path)
|
|
362
|
+
.filter((path) => !unticked.has(path)),
|
|
363
|
+
...(request.deletions ?? []),
|
|
364
|
+
])
|
|
365
|
+
: new Set(request.include);
|
|
358
366
|
if (!ticked.size)
|
|
359
367
|
throw new Error("Nothing is selected to upload");
|
|
360
368
|
const prior = request.repositoryId
|
|
@@ -416,60 +424,98 @@ class Uploader {
|
|
|
416
424
|
// 1. Measure and hash. This is what makes an unchanged file free.
|
|
417
425
|
const declarations = [];
|
|
418
426
|
let totalBytes = 0;
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
427
|
+
/*
|
|
428
|
+
Hashed several at a time, and reported by how many are done.
|
|
429
|
+
|
|
430
|
+
Each file here is an open, a read and a close, and on Windows every open
|
|
431
|
+
is also an antivirus scan — so this stage is waiting, not computing.
|
|
432
|
+
Taken strictly one after another, a folder of twenty thousand small
|
|
433
|
+
files spent a quarter of an hour in a step that moves no data at all,
|
|
434
|
+
and the bar sat on 0% for the first thousand of them because it only
|
|
435
|
+
ever spanned a fifth of itself across the whole stage.
|
|
436
|
+
|
|
437
|
+
Overlapping the waits collapses them into one wait. The results are
|
|
438
|
+
written into their own slots rather than pushed, because which disk read
|
|
439
|
+
finishes first must not decide the order of a version's file list.
|
|
440
|
+
*/
|
|
441
|
+
const HASH_LANES = 8;
|
|
442
|
+
const measured = new Array(sending.length).fill(null);
|
|
443
|
+
let hashed = 0;
|
|
444
|
+
let cursor = 0;
|
|
445
|
+
/** Throttled: twenty thousand sends is its own cost, and nobody reads them. */
|
|
446
|
+
let told = 0;
|
|
447
|
+
const lane = async () => {
|
|
448
|
+
for (;;) {
|
|
449
|
+
const index = cursor;
|
|
450
|
+
cursor += 1;
|
|
451
|
+
if (index >= sending.length)
|
|
452
|
+
return;
|
|
453
|
+
const file = sending[index];
|
|
454
|
+
this.check();
|
|
455
|
+
const full = node_path_1.default.join(request.localPath, file.path);
|
|
456
|
+
let size;
|
|
457
|
+
try {
|
|
458
|
+
size = (await (0, promises_1.stat)(full)).size;
|
|
459
|
+
}
|
|
460
|
+
catch (error) {
|
|
461
|
+
// Unreadable now, though it was listed a moment ago. Skipping it
|
|
462
|
+
// would publish a version quietly missing a file the person chose.
|
|
463
|
+
vanished.push(file.path);
|
|
464
|
+
readFailures.set(file.path, error instanceof Error ? error.message : String(error));
|
|
465
|
+
hashed += 1;
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
/*
|
|
469
|
+
Hashing is the first thing that opens the file, and opening is what
|
|
470
|
+
fails on a file another program holds — a database in use, a browser
|
|
471
|
+
profile's LOCK, an antivirus mid-scan. On Windows those still answer
|
|
472
|
+
stat() perfectly well, so the check above lets them through and the
|
|
473
|
+
read below is the one that actually breaks.
|
|
474
|
+
|
|
475
|
+
It was unguarded, so the person got a raw "EBUSY: resource busy or
|
|
476
|
+
locked" naming a path they never chose to think about, instead of
|
|
477
|
+
the sentence this code already writes for exactly this situation.
|
|
478
|
+
*/
|
|
479
|
+
let digest;
|
|
480
|
+
try {
|
|
481
|
+
digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
|
|
482
|
+
}
|
|
483
|
+
catch (error) {
|
|
484
|
+
vanished.push(file.path);
|
|
485
|
+
readFailures.set(file.path, error instanceof Error ? error.message : String(error));
|
|
486
|
+
hashed += 1;
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
measured[index] = {
|
|
490
|
+
path: file.path,
|
|
491
|
+
full,
|
|
492
|
+
size,
|
|
493
|
+
sha256: digest,
|
|
494
|
+
mediaType: mediaTypeOf(file.path),
|
|
495
|
+
};
|
|
496
|
+
hashed += 1;
|
|
497
|
+
const now = Date.now();
|
|
498
|
+
if (now - told > 80 || hashed === sending.length) {
|
|
499
|
+
told = now;
|
|
500
|
+
report({
|
|
501
|
+
stage: "hash",
|
|
502
|
+
files: hashed,
|
|
503
|
+
totalFiles: sending.length,
|
|
504
|
+
bytes: 0,
|
|
505
|
+
totalBytes: 0,
|
|
506
|
+
path: file.path,
|
|
507
|
+
percent: Math.round((hashed / sending.length) * 20),
|
|
508
|
+
bytesPerSecond: 0,
|
|
509
|
+
});
|
|
510
|
+
}
|
|
459
511
|
}
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
512
|
+
};
|
|
513
|
+
await Promise.all(Array.from({ length: Math.min(HASH_LANES, sending.length) }, () => lane()));
|
|
514
|
+
for (const one of measured) {
|
|
515
|
+
if (!one)
|
|
463
516
|
continue;
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
path: file.path,
|
|
467
|
-
full,
|
|
468
|
-
size,
|
|
469
|
-
sha256: digest,
|
|
470
|
-
mediaType: mediaTypeOf(file.path),
|
|
471
|
-
});
|
|
472
|
-
totalBytes += size;
|
|
517
|
+
declarations.push(one);
|
|
518
|
+
totalBytes += one.size;
|
|
473
519
|
}
|
|
474
520
|
/*
|
|
475
521
|
A deletion-only save has no file to hash or upload, but it is still a
|
|
@@ -1052,7 +1098,16 @@ class Uploader {
|
|
|
1052
1098
|
totalFiles: declarations.length,
|
|
1053
1099
|
bytes: sentBytes,
|
|
1054
1100
|
totalBytes,
|
|
1055
|
-
|
|
1101
|
+
/*
|
|
1102
|
+
A save, not a version.
|
|
1103
|
+
|
|
1104
|
+
Sending produces a commit; it becomes a version when somebody names
|
|
1105
|
+
it, which happens later and on purpose. This line said "Recording the
|
|
1106
|
+
version" while doing neither — it was the last wording left over from
|
|
1107
|
+
before the two were separate, and it sat on the one screen a person
|
|
1108
|
+
watches to the end.
|
|
1109
|
+
*/
|
|
1110
|
+
path: "Writing down what was sent",
|
|
1056
1111
|
percent: 96,
|
|
1057
1112
|
bytesPerSecond: rate(),
|
|
1058
1113
|
});
|
|
@@ -14,7 +14,6 @@ exports.branchName = branchName;
|
|
|
14
14
|
exports.changedFiles = changedFiles;
|
|
15
15
|
exports.surveyFiles = surveyFiles;
|
|
16
16
|
exports.totalSize = totalSize;
|
|
17
|
-
exports.fileSizes = fileSizes;
|
|
18
17
|
exports.fileDiff = fileDiff;
|
|
19
18
|
exports.projectTree = projectTree;
|
|
20
19
|
exports.evaluateRules = evaluateRules;
|
|
@@ -55,49 +54,49 @@ async function git(root, ...args) {
|
|
|
55
54
|
* file would produce an incomplete project that looked backed up, so likely
|
|
56
55
|
* secrets are warned about instead (docs/UPLOAD_POLICY.md).
|
|
57
56
|
*/
|
|
58
|
-
exports.STARTER_IGNORE = `# Dependencies and generated output
|
|
59
|
-
node_modules/
|
|
60
|
-
dist/
|
|
61
|
-
build/
|
|
62
|
-
out/
|
|
63
|
-
.next/
|
|
64
|
-
target/
|
|
65
|
-
__pycache__/
|
|
66
|
-
.venv/
|
|
67
|
-
venv/
|
|
68
|
-
*.log
|
|
69
|
-
|
|
70
|
-
# Large model weights
|
|
71
|
-
models/**
|
|
72
|
-
*.safetensors
|
|
73
|
-
*.ckpt
|
|
74
|
-
*.pt
|
|
75
|
-
*.pth
|
|
76
|
-
|
|
77
|
-
# Caches and local state
|
|
78
|
-
.venv/
|
|
79
|
-
.pytest_cache/
|
|
80
|
-
.mypy_cache/
|
|
81
|
-
.ruff_cache/
|
|
82
|
-
.cache/
|
|
83
|
-
*.pyc
|
|
84
|
-
|
|
85
|
-
# A browser or Electron profile that has been left in the project folder.
|
|
86
|
-
# These hold files another program keeps open — a LOCK that cannot be read
|
|
87
|
-
# while it runs will stop a save outright — and nothing in them is the work.
|
|
88
|
-
IndexedDB/
|
|
89
|
-
Local Storage/
|
|
90
|
-
Session Storage/
|
|
91
|
-
Service Worker/
|
|
92
|
-
Network/
|
|
93
|
-
GPUCache/
|
|
94
|
-
Code Cache/
|
|
95
|
-
blob_storage/
|
|
96
|
-
Local State
|
|
97
|
-
Preferences
|
|
98
|
-
|
|
99
|
-
# Archives of the project, inside the project
|
|
100
|
-
*.cbx
|
|
57
|
+
exports.STARTER_IGNORE = `# Dependencies and generated output
|
|
58
|
+
node_modules/
|
|
59
|
+
dist/
|
|
60
|
+
build/
|
|
61
|
+
out/
|
|
62
|
+
.next/
|
|
63
|
+
target/
|
|
64
|
+
__pycache__/
|
|
65
|
+
.venv/
|
|
66
|
+
venv/
|
|
67
|
+
*.log
|
|
68
|
+
|
|
69
|
+
# Large model weights
|
|
70
|
+
models/**
|
|
71
|
+
*.safetensors
|
|
72
|
+
*.ckpt
|
|
73
|
+
*.pt
|
|
74
|
+
*.pth
|
|
75
|
+
|
|
76
|
+
# Caches and local state
|
|
77
|
+
.venv/
|
|
78
|
+
.pytest_cache/
|
|
79
|
+
.mypy_cache/
|
|
80
|
+
.ruff_cache/
|
|
81
|
+
.cache/
|
|
82
|
+
*.pyc
|
|
83
|
+
|
|
84
|
+
# A browser or Electron profile that has been left in the project folder.
|
|
85
|
+
# These hold files another program keeps open — a LOCK that cannot be read
|
|
86
|
+
# while it runs will stop a save outright — and nothing in them is the work.
|
|
87
|
+
IndexedDB/
|
|
88
|
+
Local Storage/
|
|
89
|
+
Session Storage/
|
|
90
|
+
Service Worker/
|
|
91
|
+
Network/
|
|
92
|
+
GPUCache/
|
|
93
|
+
Code Cache/
|
|
94
|
+
blob_storage/
|
|
95
|
+
Local State
|
|
96
|
+
Preferences
|
|
97
|
+
|
|
98
|
+
# Archives of the project, inside the project
|
|
99
|
+
*.cbx
|
|
101
100
|
`;
|
|
102
101
|
/** The shared rules file, committed with the project. */
|
|
103
102
|
exports.IGNORE_FILE = ".gitignore";
|
|
@@ -318,19 +317,69 @@ async function measure(full, size, known) {
|
|
|
318
317
|
* upload when no version has ever been saved. The filter rules decide what
|
|
319
318
|
* is a candidate; the baseline decides what is new.
|
|
320
319
|
*/
|
|
320
|
+
/**
|
|
321
|
+
* The same files, ordered so that a cap spends itself across the folder.
|
|
322
|
+
*
|
|
323
|
+
* Taken in walk order, a cap is handed to whichever directory the walk
|
|
324
|
+
* reached first — and the directory a person cares about is rarely that one.
|
|
325
|
+
* Round-robin by directory means the first row of every directory is chosen
|
|
326
|
+
* before the second row of any, so a folder of forty-five thousand cache
|
|
327
|
+
* files cannot bury the eight hundred beside it that are the work.
|
|
328
|
+
*/
|
|
329
|
+
function spreadAcrossDirectories(found) {
|
|
330
|
+
const byDirectory = new Map();
|
|
331
|
+
for (const one of found) {
|
|
332
|
+
const at = one.relative.slice(0, one.relative.lastIndexOf("/") + 1);
|
|
333
|
+
const held = byDirectory.get(at);
|
|
334
|
+
if (held)
|
|
335
|
+
held.push(one);
|
|
336
|
+
else
|
|
337
|
+
byDirectory.set(at, [one]);
|
|
338
|
+
}
|
|
339
|
+
const queues = [...byDirectory.values()];
|
|
340
|
+
const deepest = queues.reduce((most, queue) => Math.max(most, queue.length), 0);
|
|
341
|
+
const spread = [];
|
|
342
|
+
for (let round = 0; round < deepest; round++) {
|
|
343
|
+
for (const queue of queues) {
|
|
344
|
+
const one = queue[round];
|
|
345
|
+
if (one)
|
|
346
|
+
spread.push(one);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
return spread;
|
|
350
|
+
}
|
|
321
351
|
async function changedFiles(root, rules, baseline = null, mode = "add-and-update",
|
|
322
352
|
/*
|
|
323
353
|
Digests already taken, and what the files looked like when they were.
|
|
324
354
|
Read and written in place, so the caller keeps whatever this learns and
|
|
325
355
|
the next scan does not read the same unchanged gigabyte again.
|
|
326
356
|
*/
|
|
327
|
-
stats
|
|
357
|
+
stats,
|
|
358
|
+
/** Filled with how much of the folder the cap allowed through. */
|
|
359
|
+
outcome) {
|
|
328
360
|
const layers = await collectLayers(root, rules);
|
|
329
361
|
const files = [];
|
|
330
362
|
const present = new Set();
|
|
331
363
|
const pending = [root];
|
|
332
|
-
|
|
333
|
-
|
|
364
|
+
/** Everything the rules would send, including what the cap keeps out. */
|
|
365
|
+
const found = [];
|
|
366
|
+
const limit = outcome?.limit ?? LISTED_FILE_LIMIT;
|
|
367
|
+
/*
|
|
368
|
+
Walk the whole folder first, and decide what to read afterwards.
|
|
369
|
+
|
|
370
|
+
These used to be one pass: walk until twenty thousand rows had been
|
|
371
|
+
collected, then stop where it stood. That made the cap decide the shape of
|
|
372
|
+
the answer — the walk was depth first, so the first directory it descended
|
|
373
|
+
into took the entire allowance. In a Unity project that is `Library/`,
|
|
374
|
+
forty-five thousand files of import cache, and `Assets/` — the actual game
|
|
375
|
+
— appeared in the list nowhere at all.
|
|
376
|
+
|
|
377
|
+
Walking is cheap: a readdir and a stat, no file is opened. Reading is what
|
|
378
|
+
costs, so reading is what the cap should govern, and it can only be spent
|
|
379
|
+
well once the whole folder is known.
|
|
380
|
+
*/
|
|
381
|
+
for (let next = 0; next < pending.length; next++) {
|
|
382
|
+
const directory = pending[next];
|
|
334
383
|
let entries;
|
|
335
384
|
try {
|
|
336
385
|
entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
|
|
@@ -338,6 +387,7 @@ stats) {
|
|
|
338
387
|
catch {
|
|
339
388
|
continue;
|
|
340
389
|
}
|
|
390
|
+
const wanted = [];
|
|
341
391
|
for (const entry of entries) {
|
|
342
392
|
const full = node_path_1.default.join(directory, entry.name);
|
|
343
393
|
if (entry.isSymbolicLink())
|
|
@@ -359,43 +409,75 @@ stats) {
|
|
|
359
409
|
continue;
|
|
360
410
|
if ((0, rules_js_1.excludes)(relative, false, layers))
|
|
361
411
|
continue;
|
|
362
|
-
|
|
363
|
-
|
|
412
|
+
wanted.push({ full, relative });
|
|
413
|
+
}
|
|
414
|
+
/*
|
|
415
|
+
Sized a directory at a time rather than a file at a time. One `stat` is
|
|
416
|
+
a syscall's round trip, and forty-seven thousand of them taken strictly
|
|
417
|
+
one after another is most of the scan; asked for together they overlap.
|
|
418
|
+
*/
|
|
419
|
+
const sized = await Promise.all(wanted.map(async (one) => {
|
|
364
420
|
try {
|
|
365
|
-
const info = await (0, promises_1.stat)(full);
|
|
366
|
-
size
|
|
367
|
-
mtimeMs = info.mtimeMs;
|
|
421
|
+
const info = await (0, promises_1.stat)(one.full);
|
|
422
|
+
return { ...one, size: info.size, mtimeMs: info.mtimeMs };
|
|
368
423
|
}
|
|
369
424
|
catch {
|
|
370
|
-
|
|
425
|
+
/* Gone or unreadable since the walk began; the upload reports it. */
|
|
426
|
+
return null;
|
|
371
427
|
}
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
recorded: baseline?.has(relative) ?? false,
|
|
377
|
-
});
|
|
428
|
+
}));
|
|
429
|
+
for (const one of sized) {
|
|
430
|
+
if (!one)
|
|
431
|
+
continue;
|
|
378
432
|
/*
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
433
|
+
Recorded as present for every file the walk saw, capped or not.
|
|
434
|
+
|
|
435
|
+
Synchronize mode calls anything in the last version that is missing
|
|
436
|
+
from this set a deletion. Filling it only for the rows that fitted
|
|
437
|
+
meant a folder past the cap proposed deleting every file the cap had
|
|
438
|
+
kept out — files sitting right there on disk.
|
|
382
439
|
*/
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
if (saved && measured.hash && saved === measured.hash)
|
|
387
|
-
continue;
|
|
388
|
-
files.push({
|
|
389
|
-
path: relative,
|
|
390
|
-
// Against a saved version the true line delta needs the old copy;
|
|
391
|
-
// until a version exists to fetch, a changed file counts as rewritten.
|
|
392
|
-
added: measured.lines,
|
|
393
|
-
removed: 0,
|
|
394
|
-
included: true,
|
|
395
|
-
binary: measured.binary,
|
|
396
|
-
});
|
|
440
|
+
present.add(one.relative);
|
|
441
|
+
outcome?.sizes.push({ path: one.relative, size: one.size });
|
|
442
|
+
found.push(one);
|
|
397
443
|
}
|
|
398
444
|
}
|
|
445
|
+
/*
|
|
446
|
+
Read in an order that gives every directory a share, and stop at the cap.
|
|
447
|
+
|
|
448
|
+
An unchanged file costs a digest and produces no row, so this runs until
|
|
449
|
+
the list is full rather than over a fixed slice — a folder where nothing
|
|
450
|
+
has changed still has to be walked through to find that out.
|
|
451
|
+
*/
|
|
452
|
+
for (const one of spreadAcrossDirectories(found)) {
|
|
453
|
+
if (files.length >= limit)
|
|
454
|
+
break;
|
|
455
|
+
const { full, relative, size, mtimeMs } = one;
|
|
456
|
+
const measured = await measure(full, size, {
|
|
457
|
+
mtimeMs,
|
|
458
|
+
cached: stats?.get(relative),
|
|
459
|
+
recorded: baseline?.has(relative) ?? false,
|
|
460
|
+
});
|
|
461
|
+
/*
|
|
462
|
+
Remembered whether or not it changed: the next scan wants to skip
|
|
463
|
+
reading this file again, and that is just as true of one that was
|
|
464
|
+
edited a moment ago as of one that never moves.
|
|
465
|
+
*/
|
|
466
|
+
if (measured.hash)
|
|
467
|
+
stats?.set(relative, { size, mtimeMs, sha256: measured.hash });
|
|
468
|
+
const saved = baseline?.get(relative);
|
|
469
|
+
if (saved && measured.hash && saved === measured.hash)
|
|
470
|
+
continue;
|
|
471
|
+
files.push({
|
|
472
|
+
path: relative,
|
|
473
|
+
// Against a saved version the true line delta needs the old copy;
|
|
474
|
+
// until a version exists to fetch, a changed file counts as rewritten.
|
|
475
|
+
added: measured.lines,
|
|
476
|
+
removed: 0,
|
|
477
|
+
included: true,
|
|
478
|
+
binary: measured.binary,
|
|
479
|
+
});
|
|
480
|
+
}
|
|
399
481
|
// A file that was in the last version and is gone now is only a change in
|
|
400
482
|
// synchronize mode. The safe default adds and updates, and leaves the
|
|
401
483
|
// saved copy of a missing file alone (docs/UPLOAD_POLICY.md).
|
|
@@ -413,6 +495,15 @@ stats) {
|
|
|
413
495
|
});
|
|
414
496
|
}
|
|
415
497
|
}
|
|
498
|
+
if (outcome) {
|
|
499
|
+
outcome.listed = files.length;
|
|
500
|
+
/*
|
|
501
|
+
Deletions are rows the walk never saw, so they are added to both sides
|
|
502
|
+
rather than left out of the total — otherwise a synchronize scan could
|
|
503
|
+
report listing more files than the folder was found to hold.
|
|
504
|
+
*/
|
|
505
|
+
outcome.total = Math.max(found.length, files.length);
|
|
506
|
+
}
|
|
416
507
|
return files.sort((left, right) => left.path.localeCompare(right.path));
|
|
417
508
|
}
|
|
418
509
|
/** The size on disk of everything the rules would upload. */
|
|
@@ -483,29 +574,6 @@ async function totalSize(root, files) {
|
|
|
483
574
|
}
|
|
484
575
|
return total;
|
|
485
576
|
}
|
|
486
|
-
/**
|
|
487
|
-
* The same walk, keeping each file's size rather than only the sum.
|
|
488
|
-
*
|
|
489
|
-
* Used where something has to reason about which files are large — the
|
|
490
|
-
* exclusion suggestions, whose whole value is saying which folder is costing
|
|
491
|
-
* the upload its time. Separate from totalSize so that the common path still
|
|
492
|
-
* carries nothing it does not need.
|
|
493
|
-
*/
|
|
494
|
-
async function fileSizes(root, files) {
|
|
495
|
-
const sized = [];
|
|
496
|
-
for (const file of files) {
|
|
497
|
-
try {
|
|
498
|
-
sized.push({
|
|
499
|
-
path: file.path,
|
|
500
|
-
size: (await (0, promises_1.stat)(node_path_1.default.join(root, file.path))).size,
|
|
501
|
-
});
|
|
502
|
-
}
|
|
503
|
-
catch {
|
|
504
|
-
/* deleted since the scan; it cannot be measured and does not count */
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
return sized;
|
|
508
|
-
}
|
|
509
577
|
/** The unified diff for one file, including files git does not track yet. */
|
|
510
578
|
async function fileDiff(root, file, ignoreWhitespace = false) {
|
|
511
579
|
const output = await git(root, "diff", "--no-ext-diff", "--unified=3", ...(ignoreWhitespace ? ["--ignore-all-space"] : []), "--", file);
|