@coderook/cli 0.29.1 → 0.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/dist/cli/src/cli.js +11 -0
- package/dist/desktop-app/src/main/download.js +120 -58
- package/dist/desktop-app/src/main/rules.js +36 -0
- package/dist/desktop-app/src/main/upload.js +150 -16
- package/dist/desktop-app/src/main/worktree.js +99 -30
- package/package.json +54 -54
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "coderook",
|
|
3
3
|
"displayName": "CodeRook",
|
|
4
4
|
"description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
|
|
5
|
-
"version": "0.29.
|
|
5
|
+
"version": "0.29.2",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "ACCA Gaming Productions",
|
|
8
8
|
"url": "https://coderook.com"
|
package/dist/cli/src/cli.js
CHANGED
|
@@ -40,6 +40,7 @@ const download_js_1 = require("../../desktop-app/src/main/download.js");
|
|
|
40
40
|
const faults_js_1 = require("../../desktop-app/src/main/faults.js");
|
|
41
41
|
const version_commands_js_1 = require("./version_commands.js");
|
|
42
42
|
const identify_js_1 = require("../../desktop-app/src/main/identify.js");
|
|
43
|
+
const profile_js_1 = require("../../desktop-app/src/main/profile.js");
|
|
43
44
|
const detect_js_1 = require("../../desktop-app/src/main/detect.js");
|
|
44
45
|
const cbx_js_1 = require("../../desktop-app/src/main/cbx.js");
|
|
45
46
|
const api_js_2 = require("./api.js");
|
|
@@ -889,6 +890,16 @@ The connection failed: ${text}`));
|
|
|
889
890
|
command does not do.
|
|
890
891
|
*/
|
|
891
892
|
` · the save holds ${bytes(result.sourceBytes)}`);
|
|
893
|
+
/*
|
|
894
|
+
Where the time went, when asked for.
|
|
895
|
+
|
|
896
|
+
The profiler has existed for a while and nothing ever printed it, so the
|
|
897
|
+
one question it answers — which stage is the save — still had to be
|
|
898
|
+
guessed at. Silent unless CODEROOK_PROFILE is set.
|
|
899
|
+
*/
|
|
900
|
+
const profile = (0, profile_js_1.profileReport)();
|
|
901
|
+
if (profile)
|
|
902
|
+
console.log(profile);
|
|
892
903
|
return 0;
|
|
893
904
|
}
|
|
894
905
|
/**
|
|
@@ -473,75 +473,98 @@ class Downloader {
|
|
|
473
473
|
packed.set(id, group);
|
|
474
474
|
}
|
|
475
475
|
const restoredPacks = new Set();
|
|
476
|
+
/*
|
|
477
|
+
Fetched several at a time, and each pack fetched once.
|
|
478
|
+
|
|
479
|
+
This was one request after another — a pack or a file, wait for it,
|
|
480
|
+
write it, ask for the next — while the sending side has had six lanes
|
|
481
|
+
for a long time. Measured across thirty project shapes, that made
|
|
482
|
+
restoring roughly six times slower than saving the same bytes: an
|
|
483
|
+
Unreal project of four hundred and ninety megabytes went up in ten
|
|
484
|
+
minutes and came back in sixty-one. A restore is mostly waiting on the
|
|
485
|
+
network, and waiting is the thing that overlaps.
|
|
486
|
+
|
|
487
|
+
The work is planned before any of it runs, which also removes the
|
|
488
|
+
double-fetch guard the serial version needed: a pack appears once in
|
|
489
|
+
the list, so it can only be fetched once.
|
|
490
|
+
*/
|
|
491
|
+
const jobs = [];
|
|
476
492
|
for (const file of files) {
|
|
477
|
-
|
|
493
|
+
const id = file.sealed ? undefined : file.pack?.objectId;
|
|
494
|
+
if (id) {
|
|
495
|
+
if (restoredPacks.has(id))
|
|
496
|
+
continue;
|
|
497
|
+
restoredPacks.add(id);
|
|
498
|
+
jobs.push({ kind: "pack", id });
|
|
499
|
+
}
|
|
500
|
+
else {
|
|
501
|
+
jobs.push({ kind: "file", file });
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
/** Refused outright, wherever it appears — inside a pack or beside it. */
|
|
505
|
+
const safeParts = (where) => {
|
|
506
|
+
const parts = where.split("/").filter(Boolean);
|
|
507
|
+
if (parts.some((part) => part === ".." || part.includes("\0"))) {
|
|
508
|
+
throw new Error(`That version contains an unsafe path: ${where}`);
|
|
509
|
+
}
|
|
510
|
+
return parts;
|
|
511
|
+
};
|
|
512
|
+
const told = { at: 0 };
|
|
513
|
+
const sayProgress = (where) => {
|
|
514
|
+
const now = Date.now();
|
|
515
|
+
if (now - told.at < 80)
|
|
516
|
+
return;
|
|
517
|
+
told.at = now;
|
|
478
518
|
report({
|
|
479
519
|
files: written,
|
|
480
520
|
totalFiles: files.length,
|
|
481
521
|
bytes,
|
|
482
522
|
totalBytes,
|
|
483
|
-
path:
|
|
523
|
+
path: where,
|
|
484
524
|
percent: underway(bytes, totalBytes),
|
|
485
525
|
});
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
526
|
+
};
|
|
527
|
+
const restorePack = async (packId) => {
|
|
528
|
+
/*
|
|
529
|
+
A solid pack is one compressed object containing many small files.
|
|
530
|
+
|
|
531
|
+
Asking the single-file route for every member made a large restore
|
|
532
|
+
fetch and decompress the same pack thousands of times. The files
|
|
533
|
+
listing already gives us the pack id and each member's exact window,
|
|
534
|
+
so fetch the decoded pack once, verify it, and materialise every
|
|
535
|
+
member before releasing the buffer.
|
|
536
|
+
*/
|
|
537
|
+
const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
|
|
538
|
+
const body = Buffer.from(await reply.arrayBuffer());
|
|
539
|
+
const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
|
|
540
|
+
const expectedPackDigest = reply.headers.get("x-coderook-sha256");
|
|
541
|
+
if (expectedPackDigest && packDigest !== expectedPackDigest) {
|
|
542
|
+
throw new Error("A solid pack did not arrive intact");
|
|
490
543
|
}
|
|
491
|
-
const
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
Asking the single-file route for every member made a large restore
|
|
499
|
-
fetch and decompress the same pack thousands of times. The files
|
|
500
|
-
listing already gives us the pack id and each member's exact
|
|
501
|
-
window, so fetch the decoded pack once, verify it, and materialise
|
|
502
|
-
every member before releasing the bounded section-sized buffer.
|
|
503
|
-
*/
|
|
504
|
-
const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
|
|
505
|
-
const body = Buffer.from(await reply.arrayBuffer());
|
|
506
|
-
const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
|
|
507
|
-
const expectedPackDigest = reply.headers.get("x-coderook-sha256");
|
|
508
|
-
if (expectedPackDigest && packDigest !== expectedPackDigest) {
|
|
509
|
-
throw new Error("A solid pack did not arrive intact");
|
|
544
|
+
for (const member of packed.get(packId) ?? []) {
|
|
545
|
+
this.check();
|
|
546
|
+
const memberParts = safeParts(member.path);
|
|
547
|
+
const offset = member.pack.offset;
|
|
548
|
+
const length = member.pack.length;
|
|
549
|
+
if (offset < 0 || length < 0 || offset + length > body.byteLength) {
|
|
550
|
+
throw new Error(`${member.path} points outside its solid pack`);
|
|
510
551
|
}
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
throw new Error(`That version contains an unsafe path: ${member.path}`);
|
|
516
|
-
}
|
|
517
|
-
const offset = member.pack.offset;
|
|
518
|
-
const length = member.pack.length;
|
|
519
|
-
if (offset < 0 || length < 0 || offset + length > body.byteLength) {
|
|
520
|
-
throw new Error(`${member.path} points outside its solid pack`);
|
|
521
|
-
}
|
|
522
|
-
const contents = body.subarray(offset, offset + length);
|
|
523
|
-
const digest = (0, node_crypto_1.createHash)("sha256").update(contents).digest("hex");
|
|
524
|
-
if (member.sha256 && digest !== member.sha256) {
|
|
525
|
-
throw new Error(`${member.path} did not arrive intact`);
|
|
526
|
-
}
|
|
527
|
-
const full = node_path_1.default.join(staging, ...memberParts);
|
|
528
|
-
await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
|
|
529
|
-
await (0, promises_1.writeFile)(full, contents);
|
|
530
|
-
manifest[member.path] = digest;
|
|
531
|
-
written += 1;
|
|
532
|
-
bytes += member.sourceSize;
|
|
533
|
-
report({
|
|
534
|
-
files: written,
|
|
535
|
-
totalFiles: files.length,
|
|
536
|
-
bytes,
|
|
537
|
-
totalBytes,
|
|
538
|
-
path: member.path,
|
|
539
|
-
percent: underway(bytes, totalBytes),
|
|
540
|
-
});
|
|
552
|
+
const contents = body.subarray(offset, offset + length);
|
|
553
|
+
const digest = (0, node_crypto_1.createHash)("sha256").update(contents).digest("hex");
|
|
554
|
+
if (member.sha256 && digest !== member.sha256) {
|
|
555
|
+
throw new Error(`${member.path} did not arrive intact`);
|
|
541
556
|
}
|
|
542
|
-
|
|
543
|
-
|
|
557
|
+
const full = node_path_1.default.join(staging, ...memberParts);
|
|
558
|
+
await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
|
|
559
|
+
await (0, promises_1.writeFile)(full, contents);
|
|
560
|
+
manifest[member.path] = digest;
|
|
561
|
+
written += 1;
|
|
562
|
+
bytes += member.sourceSize;
|
|
563
|
+
sayProgress(member.path);
|
|
544
564
|
}
|
|
565
|
+
};
|
|
566
|
+
const restoreFile = async (file) => {
|
|
567
|
+
const parts = safeParts(file.path);
|
|
545
568
|
const full = node_path_1.default.join(staging, ...parts);
|
|
546
569
|
await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
|
|
547
570
|
const digest = await this.fetchInto(repositoryId, versionId, file, full);
|
|
@@ -559,7 +582,46 @@ class Downloader {
|
|
|
559
582
|
manifest[file.path] = digest;
|
|
560
583
|
written += 1;
|
|
561
584
|
bytes += file.sourceSize;
|
|
562
|
-
|
|
585
|
+
sayProgress(file.path);
|
|
586
|
+
};
|
|
587
|
+
/*
|
|
588
|
+
Wide for the small, narrow for the large.
|
|
589
|
+
|
|
590
|
+
Six lanes turned an hour-long restore of many small files into a
|
|
591
|
+
minute, and made a restore of a few very large ones eight times
|
|
592
|
+
slower: each large file is already fetched in pieces, so six at once
|
|
593
|
+
is six times the outstanding requests against the same line, and they
|
|
594
|
+
all finish later than they would have one after another.
|
|
595
|
+
|
|
596
|
+
Split by what the job is: a pack or a small file is one request and
|
|
597
|
+
belongs in the wide lane; a large file is a stream of its own and
|
|
598
|
+
belongs in the narrow one. Both run at once, so a project holding
|
|
599
|
+
both kinds keeps the line busy without either starving the other.
|
|
600
|
+
The sending side splits its work the same way — see the note beside
|
|
601
|
+
BATCH_LANES in upload.ts.
|
|
602
|
+
*/
|
|
603
|
+
const HEAVY_BYTES = 8 * 1024 * 1024;
|
|
604
|
+
const light = jobs.filter((job) => job.kind === "pack" || job.file.sourceSize < HEAVY_BYTES);
|
|
605
|
+
const heavy = jobs.filter((job) => job.kind === "file" && job.file.sourceSize >= HEAVY_BYTES);
|
|
606
|
+
const drain = async (queue, lanes) => {
|
|
607
|
+
let next = 0;
|
|
608
|
+
const lane = async () => {
|
|
609
|
+
for (;;) {
|
|
610
|
+
const at = next;
|
|
611
|
+
next += 1;
|
|
612
|
+
if (at >= queue.length)
|
|
613
|
+
return;
|
|
614
|
+
this.check();
|
|
615
|
+
const job = queue[at];
|
|
616
|
+
if (job.kind === "pack")
|
|
617
|
+
await restorePack(job.id);
|
|
618
|
+
else
|
|
619
|
+
await restoreFile(job.file);
|
|
620
|
+
}
|
|
621
|
+
};
|
|
622
|
+
await Promise.all(Array.from({ length: Math.min(lanes, queue.length) }, () => lane()));
|
|
623
|
+
};
|
|
624
|
+
await Promise.all([drain(light, 6), drain(heavy, 2)]);
|
|
563
625
|
/*
|
|
564
626
|
Everything has arrived and verified, so it can go into place. Each
|
|
565
627
|
file is renamed over its destination individually: a rename within
|
|
@@ -65,6 +65,42 @@ function globToSource(pattern) {
|
|
|
65
65
|
index += 1;
|
|
66
66
|
continue;
|
|
67
67
|
}
|
|
68
|
+
if (character === "[") {
|
|
69
|
+
/*
|
|
70
|
+
A character class, which every engine's published ignore file is
|
|
71
|
+
written in: `/[Ll]ibrary/`, `/[Bb]uild/`, `/[Oo]bj/`. Without this
|
|
72
|
+
they were escaped into literals and matched a directory genuinely
|
|
73
|
+
called "[Ll]ibrary" — so pasting the official Unity template, or
|
|
74
|
+
applying the one this application ships, excluded nothing at all and
|
|
75
|
+
said nothing about it.
|
|
76
|
+
|
|
77
|
+
A `]` straight after the bracket, or after a leading negation, is a
|
|
78
|
+
literal `]` rather than the end of the class — which is how git and
|
|
79
|
+
the shell both read it.
|
|
80
|
+
*/
|
|
81
|
+
let end = index + 1;
|
|
82
|
+
if (pattern[end] === "!" || pattern[end] === "^")
|
|
83
|
+
end += 1;
|
|
84
|
+
if (pattern[end] === "]")
|
|
85
|
+
end += 1;
|
|
86
|
+
while (end < pattern.length && pattern[end] !== "]")
|
|
87
|
+
end += 1;
|
|
88
|
+
if (end < pattern.length) {
|
|
89
|
+
const inside = pattern.slice(index + 1, end);
|
|
90
|
+
const negated = inside.startsWith("!") || inside.startsWith("^");
|
|
91
|
+
const members = (negated ? inside.slice(1) : inside).replace(/[\\\^\]]/g, "\\$&");
|
|
92
|
+
/*
|
|
93
|
+
A class never spans directories, so a negated one excludes the
|
|
94
|
+
separator too — otherwise `[!a]` would match a slash and a pattern
|
|
95
|
+
for one segment would quietly reach into the next.
|
|
96
|
+
*/
|
|
97
|
+
source += negated ? `[^/${members}]` : `[${members}]`;
|
|
98
|
+
index = end;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
source += "\\[";
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
68
104
|
if (character === "*") {
|
|
69
105
|
if (pattern[index + 1] === "*") {
|
|
70
106
|
source += ".*";
|
|
@@ -119,6 +119,20 @@ class Uploader {
|
|
|
119
119
|
planning = false;
|
|
120
120
|
plannedObjects = new Map();
|
|
121
121
|
plannedQuote = null;
|
|
122
|
+
/*
|
|
123
|
+
Digests worked out while planning, so sending does not work them out again.
|
|
124
|
+
|
|
125
|
+
A save runs the whole pipeline twice: the plan walks, reads, hashes and
|
|
126
|
+
prices every file to produce a quote, then the send does all of it over.
|
|
127
|
+
Measured on a twenty-one thousand file project, that was 42,142 hashes for
|
|
128
|
+
21,071 files — the count in the profile is exactly double, which is how
|
|
129
|
+
the duplication was finally noticed.
|
|
130
|
+
|
|
131
|
+
Keyed by path and guarded by size and modification time, so a file edited
|
|
132
|
+
while somebody reads the plan is hashed again rather than published under
|
|
133
|
+
the digest it used to have.
|
|
134
|
+
*/
|
|
135
|
+
plannedDigests = new Map();
|
|
122
136
|
planningCreatedRepositoryId = null;
|
|
123
137
|
/** Attached to every object write after the server admits the exact plan. */
|
|
124
138
|
activeQuoteId = null;
|
|
@@ -352,7 +366,27 @@ class Uploader {
|
|
|
352
366
|
truncated version of it would propose deleting everything the walk did
|
|
353
367
|
not reach.
|
|
354
368
|
*/
|
|
355
|
-
const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules)
|
|
369
|
+
const everything = await (0, profile_js_1.timed)("survey the project", () => (0, worktree_js_1.surveyFiles)(request.localPath, rules, (found) =>
|
|
370
|
+
/*
|
|
371
|
+
The walk is the first minute of a large save and said nothing at
|
|
372
|
+
all, so the panel could only show 0%. A running count of what has
|
|
373
|
+
been seen is not a fraction of anything — it is the evidence that
|
|
374
|
+
the thing is working.
|
|
375
|
+
*/
|
|
376
|
+
report({
|
|
377
|
+
stage: "plan",
|
|
378
|
+
files: found,
|
|
379
|
+
totalFiles: 0,
|
|
380
|
+
bytes: 0,
|
|
381
|
+
totalBytes: 0,
|
|
382
|
+
path: "Looking through the folder",
|
|
383
|
+
/*
|
|
384
|
+
Between the key check and the reading, and it holds its own place
|
|
385
|
+
on the bar rather than dropping back to nought between them.
|
|
386
|
+
*/
|
|
387
|
+
percent: 6,
|
|
388
|
+
bytesPerSecond: 0,
|
|
389
|
+
})));
|
|
356
390
|
(0, profile_js_1.counted)("files surveyed", everything.length);
|
|
357
391
|
const unticked = new Set(request.excluded ?? []);
|
|
358
392
|
const ticked = request.excluded
|
|
@@ -454,8 +488,11 @@ class Uploader {
|
|
|
454
488
|
this.check();
|
|
455
489
|
const full = node_path_1.default.join(request.localPath, file.path);
|
|
456
490
|
let size;
|
|
491
|
+
let mtimeMs;
|
|
457
492
|
try {
|
|
458
|
-
|
|
493
|
+
const info = await (0, promises_1.stat)(full);
|
|
494
|
+
size = info.size;
|
|
495
|
+
mtimeMs = info.mtimeMs;
|
|
459
496
|
}
|
|
460
497
|
catch (error) {
|
|
461
498
|
// Unreadable now, though it was listed a moment ago. Skipping it
|
|
@@ -476,6 +513,28 @@ class Uploader {
|
|
|
476
513
|
locked" naming a path they never chose to think about, instead of
|
|
477
514
|
the sentence this code already writes for exactly this situation.
|
|
478
515
|
*/
|
|
516
|
+
/*
|
|
517
|
+
Read only when the file has actually moved.
|
|
518
|
+
|
|
519
|
+
Size and modification time are the same test the changes list uses,
|
|
520
|
+
and a match there means the digest already taken still describes the
|
|
521
|
+
bytes. Reading is what costs; deciding not to read is free.
|
|
522
|
+
*/
|
|
523
|
+
const remembered = this.plannedDigests.get(file.path) ??
|
|
524
|
+
request.knownDigests?.get(file.path);
|
|
525
|
+
if (remembered &&
|
|
526
|
+
remembered.size === size &&
|
|
527
|
+
remembered.mtimeMs === mtimeMs) {
|
|
528
|
+
measured[index] = {
|
|
529
|
+
path: file.path,
|
|
530
|
+
full,
|
|
531
|
+
size,
|
|
532
|
+
sha256: remembered.sha256,
|
|
533
|
+
mediaType: mediaTypeOf(file.path),
|
|
534
|
+
};
|
|
535
|
+
hashed += 1;
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
479
538
|
let digest;
|
|
480
539
|
try {
|
|
481
540
|
digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
|
|
@@ -486,6 +545,7 @@ class Uploader {
|
|
|
486
545
|
hashed += 1;
|
|
487
546
|
continue;
|
|
488
547
|
}
|
|
548
|
+
this.plannedDigests.set(file.path, { size, mtimeMs, sha256: digest });
|
|
489
549
|
measured[index] = {
|
|
490
550
|
path: file.path,
|
|
491
551
|
full,
|
|
@@ -504,7 +564,12 @@ class Uploader {
|
|
|
504
564
|
bytes: 0,
|
|
505
565
|
totalBytes: 0,
|
|
506
566
|
path: file.path,
|
|
507
|
-
|
|
567
|
+
/*
|
|
568
|
+
Reading is the stretch from just past the walk to a fifth of
|
|
569
|
+
the way through. Starting it at nought put the bar behind where
|
|
570
|
+
it had already been and it went backwards between stages.
|
|
571
|
+
*/
|
|
572
|
+
percent: 8 + Math.round((hashed / sending.length) * 12),
|
|
508
573
|
bytesPerSecond: 0,
|
|
509
574
|
});
|
|
510
575
|
}
|
|
@@ -702,11 +767,78 @@ class Uploader {
|
|
|
702
767
|
const heavy = sections.filter((one) => smallShare(one) < 0.5);
|
|
703
768
|
(0, profile_js_1.counted)("sections batched", batchy.length);
|
|
704
769
|
(0, profile_js_1.counted)("sections heavy", heavy.length);
|
|
770
|
+
/*
|
|
771
|
+
How many files the whole save has finished, across every section.
|
|
772
|
+
|
|
773
|
+
The sections are an implementation detail of not holding thirty
|
|
774
|
+
gigabytes in memory at once, and they were leaking into what a person
|
|
775
|
+
reads: the count ran up to twenty-one thousand while reading, then reset
|
|
776
|
+
to "0 of 4,000" for each section and counted up again, over and over. It
|
|
777
|
+
looked like the save was starting over, repeatedly.
|
|
778
|
+
|
|
779
|
+
One denominator for the whole save. Sections still say which one is in
|
|
780
|
+
flight, in words, where that belongs.
|
|
781
|
+
*/
|
|
782
|
+
const sectionProgress = new Map();
|
|
783
|
+
/*
|
|
784
|
+
What has already been shown, so the number never goes backwards.
|
|
785
|
+
|
|
786
|
+
Two pipelines run sections at once and each reports its own share, which
|
|
787
|
+
I have now twice failed to combine into a figure that only rises — the
|
|
788
|
+
count kept dropping to zero at section boundaries and reading as though
|
|
789
|
+
the save had restarted. Whatever the accounting does, the panel must not
|
|
790
|
+
be told a smaller number than it was told a moment ago: going backwards
|
|
791
|
+
is the specific thing a person notices, and it is never information.
|
|
792
|
+
*/
|
|
793
|
+
let shown = 0;
|
|
794
|
+
const forward = (count) => {
|
|
795
|
+
if (count > shown)
|
|
796
|
+
shown = count;
|
|
797
|
+
return shown;
|
|
798
|
+
};
|
|
705
799
|
const runSection = async (section) => {
|
|
706
800
|
const declarations = section.files.map((file) => declarationByPath.get(file.path));
|
|
801
|
+
/*
|
|
802
|
+
This section's share of the one number the panel shows.
|
|
803
|
+
|
|
804
|
+
Two pipelines run at once, so no section can report on its own behalf
|
|
805
|
+
without the count lurching between them. Each records how far it has
|
|
806
|
+
got and the sum is what is shown, which is the only figure that is
|
|
807
|
+
true while both are moving.
|
|
808
|
+
*/
|
|
809
|
+
const advance = (done) => {
|
|
810
|
+
/*
|
|
811
|
+
Only ever upward.
|
|
812
|
+
|
|
813
|
+
The workers share one iterator, so the indices arrive out of order —
|
|
814
|
+
a worker holding file 5 reports after one holding file 200. Writing
|
|
815
|
+
each straight in let a later, lower number undo a higher one, and
|
|
816
|
+
the count sat still or fell back to zero at every section boundary,
|
|
817
|
+
which is the exact stutter this was meant to remove.
|
|
818
|
+
*/
|
|
819
|
+
const held = sectionProgress.get(section.index) ?? 0;
|
|
820
|
+
if (done > held)
|
|
821
|
+
sectionProgress.set(section.index, done);
|
|
822
|
+
else if (!sectionProgress.has(section.index)) {
|
|
823
|
+
sectionProgress.set(section.index, 0);
|
|
824
|
+
}
|
|
825
|
+
let sum = 0;
|
|
826
|
+
for (const each of sectionProgress.values())
|
|
827
|
+
sum += each;
|
|
828
|
+
return forward(sum);
|
|
829
|
+
};
|
|
707
830
|
report({
|
|
708
831
|
stage: "upload",
|
|
709
|
-
|
|
832
|
+
/*
|
|
833
|
+
The count so far, not zero.
|
|
834
|
+
|
|
835
|
+
This line announces a new section, and it was reporting nought files
|
|
836
|
+
done — so every section boundary threw the number back to the start
|
|
837
|
+
and the bar with it. Four of the five reports in this stage were
|
|
838
|
+
changed to carry the running total and this one, the only one a
|
|
839
|
+
person is guaranteed to see, was missed.
|
|
840
|
+
*/
|
|
841
|
+
files: advance(0),
|
|
710
842
|
totalFiles: allDeclarations.length,
|
|
711
843
|
bytes: progressBytes(),
|
|
712
844
|
totalBytes,
|
|
@@ -861,8 +993,8 @@ class Uploader {
|
|
|
861
993
|
transferred += exactWireBytes;
|
|
862
994
|
report({
|
|
863
995
|
stage: "upload",
|
|
864
|
-
files: 0,
|
|
865
|
-
totalFiles:
|
|
996
|
+
files: advance(0),
|
|
997
|
+
totalFiles: allDeclarations.length,
|
|
866
998
|
bytes: progressBytes(),
|
|
867
999
|
totalBytes,
|
|
868
1000
|
path: taking[taking.length - 1].declaration.path,
|
|
@@ -914,8 +1046,8 @@ class Uploader {
|
|
|
914
1046
|
this.check();
|
|
915
1047
|
report({
|
|
916
1048
|
stage: "upload",
|
|
917
|
-
files: index,
|
|
918
|
-
totalFiles:
|
|
1049
|
+
files: advance(index),
|
|
1050
|
+
totalFiles: allDeclarations.length,
|
|
919
1051
|
bytes: sentBytes,
|
|
920
1052
|
totalBytes,
|
|
921
1053
|
path: declaration.path,
|
|
@@ -949,8 +1081,8 @@ class Uploader {
|
|
|
949
1081
|
inFlight.set(index, offset);
|
|
950
1082
|
report({
|
|
951
1083
|
stage: "upload",
|
|
952
|
-
files: index,
|
|
953
|
-
totalFiles:
|
|
1084
|
+
files: advance(index),
|
|
1085
|
+
totalFiles: allDeclarations.length,
|
|
954
1086
|
bytes: progressBytes(),
|
|
955
1087
|
totalBytes,
|
|
956
1088
|
path: declaration.path,
|
|
@@ -1023,8 +1155,8 @@ class Uploader {
|
|
|
1023
1155
|
: await this.putMultipart(repositoryId, declaration, (offset) => {
|
|
1024
1156
|
report({
|
|
1025
1157
|
stage: "upload",
|
|
1026
|
-
files: index,
|
|
1027
|
-
totalFiles:
|
|
1158
|
+
files: advance(index),
|
|
1159
|
+
totalFiles: allDeclarations.length,
|
|
1028
1160
|
bytes: progressBytes(),
|
|
1029
1161
|
totalBytes,
|
|
1030
1162
|
path: declaration.path,
|
|
@@ -1047,6 +1179,8 @@ class Uploader {
|
|
|
1047
1179
|
}
|
|
1048
1180
|
};
|
|
1049
1181
|
await Promise.all(Array.from({ length: Math.min(UPLOAD_LANES, declarations.length) }, () => runWorker()));
|
|
1182
|
+
/* Finished, so its whole share counts — the per-file reports stop one short. */
|
|
1183
|
+
advance(declarations.length);
|
|
1050
1184
|
};
|
|
1051
1185
|
const pipeline = async (queue) => {
|
|
1052
1186
|
for (const section of queue) {
|
|
@@ -1066,7 +1200,7 @@ class Uploader {
|
|
|
1066
1200
|
percent: 96,
|
|
1067
1201
|
bytesPerSecond: 0,
|
|
1068
1202
|
});
|
|
1069
|
-
this.plannedQuote = await this.call(`/v1/repositories/${repositoryId}/uploads/preflight`, {
|
|
1203
|
+
this.plannedQuote = await (0, profile_js_1.timed)("ask the service to price it", () => this.call(`/v1/repositories/${repositoryId}/uploads/preflight`, {
|
|
1070
1204
|
method: "POST",
|
|
1071
1205
|
contentType: "application/json",
|
|
1072
1206
|
body: JSON.stringify({
|
|
@@ -1074,7 +1208,7 @@ class Uploader {
|
|
|
1074
1208
|
excludedBytes: 0,
|
|
1075
1209
|
objects: [...this.plannedObjects.values()],
|
|
1076
1210
|
}),
|
|
1077
|
-
});
|
|
1211
|
+
}));
|
|
1078
1212
|
return {
|
|
1079
1213
|
repositoryId,
|
|
1080
1214
|
versionId: "",
|
|
@@ -1201,7 +1335,7 @@ class Uploader {
|
|
|
1201
1335
|
objects are immutable and content-addressed, so a retry reuses them.
|
|
1202
1336
|
*/
|
|
1203
1337
|
(0, faults_js_1.maybeFail)("submit:after-objects");
|
|
1204
|
-
const completed = await this.call(`/v1/repositories/${repositoryId}/versions`, {
|
|
1338
|
+
const completed = await (0, profile_js_1.timed)("publish the version", () => this.call(`/v1/repositories/${repositoryId}/versions`, {
|
|
1205
1339
|
method: "POST",
|
|
1206
1340
|
contentType: "application/json",
|
|
1207
1341
|
body: JSON.stringify({
|
|
@@ -1266,7 +1400,7 @@ class Uploader {
|
|
|
1266
1400
|
executable: false,
|
|
1267
1401
|
})),
|
|
1268
1402
|
}),
|
|
1269
|
-
});
|
|
1403
|
+
}));
|
|
1270
1404
|
/*
|
|
1271
1405
|
A clean stale-base save may contain a server-side merge. The candidate
|
|
1272
1406
|
manifest built above is then not the Version that landed: it lacks the
|
|
@@ -66,6 +66,28 @@ __pycache__/
|
|
|
66
66
|
venv/
|
|
67
67
|
*.log
|
|
68
68
|
|
|
69
|
+
# What a game engine rebuilds for itself. These are the largest thing in a
|
|
70
|
+
# folder by a wide margin — a Unity project's Library is routinely thousands
|
|
71
|
+
# of times the size of its Assets — and every byte comes back the next time
|
|
72
|
+
# the editor opens. Left out of this list for a long time, which meant a
|
|
73
|
+
# folder with no rules of its own sent its whole cache and nobody was told.
|
|
74
|
+
/[Ll]ibrary/
|
|
75
|
+
/[Tt]emp/
|
|
76
|
+
/[Ll]ogs/
|
|
77
|
+
/[Uu]ser[Ss]ettings/
|
|
78
|
+
/[Mm]emoryCaptures/
|
|
79
|
+
Intermediate/
|
|
80
|
+
DerivedDataCache/
|
|
81
|
+
Binaries/
|
|
82
|
+
.godot/
|
|
83
|
+
.import/
|
|
84
|
+
|
|
85
|
+
# Compiler and editor output, which the next build writes again.
|
|
86
|
+
obj/
|
|
87
|
+
.vs/
|
|
88
|
+
.gradle/
|
|
89
|
+
DerivedData/
|
|
90
|
+
|
|
69
91
|
# Large model weights
|
|
70
92
|
models/**
|
|
71
93
|
*.safetensors
|
|
@@ -497,6 +519,7 @@ outcome) {
|
|
|
497
519
|
}
|
|
498
520
|
if (outcome) {
|
|
499
521
|
outcome.listed = files.length;
|
|
522
|
+
outcome.truncated = files.length >= limit;
|
|
500
523
|
/*
|
|
501
524
|
Deletions are rows the walk never saw, so they are added to both sides
|
|
502
525
|
rather than left out of the total — otherwise a synchronize scan could
|
|
@@ -520,7 +543,17 @@ outcome) {
|
|
|
520
543
|
* content at all. That is what makes it affordable to run over everything
|
|
521
544
|
* before deciding anything.
|
|
522
545
|
*/
|
|
523
|
-
async function surveyFiles(root, rules
|
|
546
|
+
async function surveyFiles(root, rules,
|
|
547
|
+
/*
|
|
548
|
+
Told how many have been found, every so often.
|
|
549
|
+
|
|
550
|
+
This walk is the first thing a save does and it can take a minute on a
|
|
551
|
+
large folder — during which the only honest thing the progress panel
|
|
552
|
+
could say was 0%, because nothing had told it otherwise. A count of what
|
|
553
|
+
has been seen is not a fraction of anything, but it moves, and it is the
|
|
554
|
+
difference between waiting and wondering whether it has hung.
|
|
555
|
+
*/
|
|
556
|
+
onFound) {
|
|
524
557
|
const layers = await collectLayers(root, rules);
|
|
525
558
|
const found = [];
|
|
526
559
|
const pending = [root];
|
|
@@ -554,6 +587,8 @@ async function surveyFiles(root, rules) {
|
|
|
554
587
|
continue;
|
|
555
588
|
try {
|
|
556
589
|
found.push({ path: relative, size: (await (0, promises_1.stat)(full)).size });
|
|
590
|
+
if (onFound && found.length % 500 === 0)
|
|
591
|
+
onFound(found.length);
|
|
557
592
|
}
|
|
558
593
|
catch {
|
|
559
594
|
/* Gone or unreadable since the walk began; the upload reports it. */
|
|
@@ -1004,46 +1039,80 @@ const MOST_FINDINGS = 40;
|
|
|
1004
1039
|
* already leave behind is not being published, and raising it would train
|
|
1005
1040
|
* people to dismiss the question that matters.
|
|
1006
1041
|
*/
|
|
1007
|
-
async function detectPastedCredentials(root, include
|
|
1042
|
+
async function detectPastedCredentials(root, include,
|
|
1043
|
+
/*
|
|
1044
|
+
Told how many have been looked at, every so often.
|
|
1045
|
+
|
|
1046
|
+
This reads up to twenty thousand files before a save begins, and it did so
|
|
1047
|
+
one after another with nothing reported — on Windows, where every open is
|
|
1048
|
+
also an antivirus scan, that is a quarter of an hour in which the panel
|
|
1049
|
+
could only show the words "Working out what to send" and no number at all.
|
|
1050
|
+
The same silence the read stage used to have, in a different module.
|
|
1051
|
+
*/
|
|
1052
|
+
onScanned) {
|
|
1008
1053
|
const found = [];
|
|
1009
1054
|
let read = 0;
|
|
1010
1055
|
let bytes = 0;
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1056
|
+
/*
|
|
1057
|
+
Read several at a time. This is waiting on the disk, not computing, so
|
|
1058
|
+
overlapping the waits collapses them into one wait — the same change the
|
|
1059
|
+
hashing stage needed, for the same reason.
|
|
1060
|
+
*/
|
|
1061
|
+
const LANES = 8;
|
|
1062
|
+
const wanted = include.filter((relative) => (0, secret_patterns_js_1.worthReading)(relative.slice(relative.lastIndexOf("/") + 1)));
|
|
1063
|
+
let cursor = 0;
|
|
1064
|
+
let stop = false;
|
|
1065
|
+
const lane = async () => {
|
|
1066
|
+
for (;;) {
|
|
1067
|
+
if (stop)
|
|
1068
|
+
return;
|
|
1069
|
+
const at = cursor;
|
|
1070
|
+
cursor += 1;
|
|
1071
|
+
if (at >= wanted.length)
|
|
1072
|
+
return;
|
|
1073
|
+
if (found.length >= MOST_FINDINGS) {
|
|
1074
|
+
stop = true;
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
if (read >= MOST_SCANNED_FILES || bytes >= MOST_SCANNED_BYTES) {
|
|
1078
|
+
stop = true;
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
const relative = wanted[at];
|
|
1082
|
+
const full = node_path_1.default.join(root, relative.split("/").join(node_path_1.default.sep));
|
|
1083
|
+
let contents;
|
|
1084
|
+
try {
|
|
1085
|
+
const info = await (0, promises_1.stat)(full);
|
|
1086
|
+
if (!info.isFile() || info.size > MOST_FILE_BYTES)
|
|
1087
|
+
continue;
|
|
1088
|
+
contents = await (0, promises_1.readFile)(full);
|
|
1089
|
+
}
|
|
1090
|
+
catch {
|
|
1091
|
+
/* Unreadable is the scan's problem, not something to report as clean. */
|
|
1024
1092
|
continue;
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1093
|
+
}
|
|
1094
|
+
read += 1;
|
|
1095
|
+
bytes += contents.byteLength;
|
|
1096
|
+
if (onScanned && read % 50 === 0)
|
|
1097
|
+
onScanned(read, wanted.length);
|
|
1098
|
+
if (!(0, secret_patterns_js_1.looksLikeText)(contents))
|
|
1099
|
+
continue;
|
|
1100
|
+
const credentials = (0, secret_patterns_js_1.findCredentials)(contents.toString("utf8"));
|
|
1101
|
+
if (credentials.length)
|
|
1102
|
+
found.push({ path: relative, found: credentials });
|
|
1030
1103
|
}
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
if (!(0, secret_patterns_js_1.looksLikeText)(contents))
|
|
1034
|
-
continue;
|
|
1035
|
-
const credentials = (0, secret_patterns_js_1.findCredentials)(contents.toString("utf8"));
|
|
1036
|
-
if (credentials.length)
|
|
1037
|
-
found.push({ path: relative, found: credentials });
|
|
1038
|
-
}
|
|
1104
|
+
};
|
|
1105
|
+
await Promise.all(Array.from({ length: Math.min(LANES, wanted.length) }, () => lane()));
|
|
1039
1106
|
return found;
|
|
1040
1107
|
}
|
|
1041
|
-
async function uploadConcerns(root, include
|
|
1108
|
+
async function uploadConcerns(root, include,
|
|
1109
|
+
/** Passed through so the one slow part of this can say it is working. */
|
|
1110
|
+
onScanned) {
|
|
1042
1111
|
const chosen = new Set(include);
|
|
1043
1112
|
const [named, folders, pasted] = await Promise.all([
|
|
1044
1113
|
detectSecrets(root),
|
|
1045
1114
|
detectPrivateDirectories(root),
|
|
1046
|
-
detectPastedCredentials(root, include),
|
|
1115
|
+
detectPastedCredentials(root, include, onScanned),
|
|
1047
1116
|
]);
|
|
1048
1117
|
const secrets = named.filter((file) => chosen.has(file));
|
|
1049
1118
|
return {
|
package/package.json
CHANGED
|
@@ -1,54 +1,54 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@coderook/cli",
|
|
3
|
-
"version": "0.29.
|
|
4
|
-
"description": "CodeRook from the command line, on any operating system",
|
|
5
|
-
"license": "SEE LICENSE IN LICENSE.txt",
|
|
6
|
-
"homepage": "https://coderook.com",
|
|
7
|
-
"bugs": {
|
|
8
|
-
"url": "https://coderook.com/contact"
|
|
9
|
-
},
|
|
10
|
-
"keywords": [
|
|
11
|
-
"coderook",
|
|
12
|
-
"versioning",
|
|
13
|
-
"backup",
|
|
14
|
-
"cbx"
|
|
15
|
-
],
|
|
16
|
-
"engines": {
|
|
17
|
-
"node": ">=20.11.0"
|
|
18
|
-
},
|
|
19
|
-
"bin": {
|
|
20
|
-
"cbx": "dist/cli/src/cli.js",
|
|
21
|
-
"coderook": "dist/cli/src/cli.js",
|
|
22
|
-
"git-remote-coderook": "dist/cli/src/git_remote_bin.js"
|
|
23
|
-
},
|
|
24
|
-
"files": [
|
|
25
|
-
".claude-plugin",
|
|
26
|
-
"dist",
|
|
27
|
-
"skills"
|
|
28
|
-
],
|
|
29
|
-
"scripts": {
|
|
30
|
-
"build": "tsc -p tsconfig.json",
|
|
31
|
-
"check": "tsc -p tsconfig.json --noEmit",
|
|
32
|
-
"test": "tsc -p tsconfig.json && node --test --experimental-strip-types test/*.test.ts",
|
|
33
|
-
"start": "node dist/cli/src/cli.js",
|
|
34
|
-
"prepublishOnly": "npm run sync:plugin && npm run build",
|
|
35
|
-
"test:e2e": "node test/e2e.mjs",
|
|
36
|
-
"test:matrix": "node test/state-matrix.mjs",
|
|
37
|
-
"test:attempt": "node test/attempt-identity.mjs",
|
|
38
|
-
"test:get": "node test/get-safety.mjs",
|
|
39
|
-
"test:live": "node test/e2e.mjs --allow-production && node test/state-matrix.mjs --allow-production && node test/get-safety.mjs --allow-production && node test/get-track.mjs --allow-production && node test/get-protects-edits.mjs --allow-production && node test/upgrade-migration.mjs --allow-production && node test/fault-get.mjs --allow-production && node test/version-floor.mjs --allow-production && node test/fault-submit.mjs --allow-production && node test/race-attempts.mjs --allow-production && node test/attempt-identity.mjs --allow-production",
|
|
40
|
-
"test:getedits": "node test/get-protects-edits.mjs",
|
|
41
|
-
"test:upgrade": "node test/upgrade-migration.mjs",
|
|
42
|
-
"test:faultget": "node test/fault-get.mjs",
|
|
43
|
-
"test:floor": "node test/version-floor.mjs",
|
|
44
|
-
"test:faultsubmit": "node test/fault-submit.mjs",
|
|
45
|
-
"test:race": "node test/race-attempts.mjs",
|
|
46
|
-
"test:runner": "node test/runner-live.mjs",
|
|
47
|
-
"sync:plugin": "node scripts/sync-plugin-version.mjs",
|
|
48
|
-
"test:track": "node test/get-track.mjs"
|
|
49
|
-
},
|
|
50
|
-
"devDependencies": {
|
|
51
|
-
"@types/node": "24.10.1",
|
|
52
|
-
"typescript": "5.9.3"
|
|
53
|
-
}
|
|
54
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "@coderook/cli",
|
|
3
|
+
"version": "0.29.2",
|
|
4
|
+
"description": "CodeRook from the command line, on any operating system",
|
|
5
|
+
"license": "SEE LICENSE IN LICENSE.txt",
|
|
6
|
+
"homepage": "https://coderook.com",
|
|
7
|
+
"bugs": {
|
|
8
|
+
"url": "https://coderook.com/contact"
|
|
9
|
+
},
|
|
10
|
+
"keywords": [
|
|
11
|
+
"coderook",
|
|
12
|
+
"versioning",
|
|
13
|
+
"backup",
|
|
14
|
+
"cbx"
|
|
15
|
+
],
|
|
16
|
+
"engines": {
|
|
17
|
+
"node": ">=20.11.0"
|
|
18
|
+
},
|
|
19
|
+
"bin": {
|
|
20
|
+
"cbx": "dist/cli/src/cli.js",
|
|
21
|
+
"coderook": "dist/cli/src/cli.js",
|
|
22
|
+
"git-remote-coderook": "dist/cli/src/git_remote_bin.js"
|
|
23
|
+
},
|
|
24
|
+
"files": [
|
|
25
|
+
".claude-plugin",
|
|
26
|
+
"dist",
|
|
27
|
+
"skills"
|
|
28
|
+
],
|
|
29
|
+
"scripts": {
|
|
30
|
+
"build": "tsc -p tsconfig.json",
|
|
31
|
+
"check": "tsc -p tsconfig.json --noEmit",
|
|
32
|
+
"test": "tsc -p tsconfig.json && node --test --experimental-strip-types test/*.test.ts",
|
|
33
|
+
"start": "node dist/cli/src/cli.js",
|
|
34
|
+
"prepublishOnly": "npm run sync:plugin && npm run build",
|
|
35
|
+
"test:e2e": "node test/e2e.mjs",
|
|
36
|
+
"test:matrix": "node test/state-matrix.mjs",
|
|
37
|
+
"test:attempt": "node test/attempt-identity.mjs",
|
|
38
|
+
"test:get": "node test/get-safety.mjs",
|
|
39
|
+
"test:live": "node test/e2e.mjs --allow-production && node test/state-matrix.mjs --allow-production && node test/get-safety.mjs --allow-production && node test/get-track.mjs --allow-production && node test/get-protects-edits.mjs --allow-production && node test/upgrade-migration.mjs --allow-production && node test/fault-get.mjs --allow-production && node test/version-floor.mjs --allow-production && node test/fault-submit.mjs --allow-production && node test/race-attempts.mjs --allow-production && node test/attempt-identity.mjs --allow-production",
|
|
40
|
+
"test:getedits": "node test/get-protects-edits.mjs",
|
|
41
|
+
"test:upgrade": "node test/upgrade-migration.mjs",
|
|
42
|
+
"test:faultget": "node test/fault-get.mjs",
|
|
43
|
+
"test:floor": "node test/version-floor.mjs",
|
|
44
|
+
"test:faultsubmit": "node test/fault-submit.mjs",
|
|
45
|
+
"test:race": "node test/race-attempts.mjs",
|
|
46
|
+
"test:runner": "node test/runner-live.mjs",
|
|
47
|
+
"sync:plugin": "node scripts/sync-plugin-version.mjs",
|
|
48
|
+
"test:track": "node test/get-track.mjs"
|
|
49
|
+
},
|
|
50
|
+
"devDependencies": {
|
|
51
|
+
"@types/node": "24.10.1",
|
|
52
|
+
"typescript": "5.9.3"
|
|
53
|
+
}
|
|
54
|
+
}
|