@coderook/cli 0.34.3 → 0.34.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/dist/cbx/src/core/upload.js +330 -149
- package/dist/cli/src/network.js +25 -5
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "coderook",
|
|
3
3
|
"displayName": "CodeRook",
|
|
4
4
|
"description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, with cbx or plain git, from Claude Code.",
|
|
5
|
-
"version": "0.34.
|
|
5
|
+
"version": "0.34.5",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "ACCA Gaming Productions",
|
|
8
8
|
"url": "https://coderook.com"
|
|
@@ -88,10 +88,14 @@ function mediaTypeOf(file) {
|
|
|
88
88
|
return MEDIA_TYPES[node_path_1.default.extname(file).toLowerCase()] ?? "application/octet-stream";
|
|
89
89
|
}
|
|
90
90
|
/** Hash without holding the file in memory; some of these are large. */
|
|
91
|
-
async function digestOf(full) {
|
|
91
|
+
async function digestOf(full, onBytes) {
|
|
92
92
|
const hash = (0, node_crypto_1.createHash)("sha256");
|
|
93
|
-
|
|
93
|
+
let read = 0;
|
|
94
|
+
for await (const chunk of (0, node_fs_1.createReadStream)(full)) {
|
|
94
95
|
hash.update(chunk);
|
|
96
|
+
read += chunk.byteLength;
|
|
97
|
+
onBytes?.(read);
|
|
98
|
+
}
|
|
95
99
|
return hash.digest("hex");
|
|
96
100
|
}
|
|
97
101
|
/**
|
|
@@ -243,6 +247,38 @@ class Uploader {
|
|
|
243
247
|
/** A single attempt. Everything about repeating it lives in `call`. */
|
|
244
248
|
async attempt(route, init) {
|
|
245
249
|
const started = performance.now();
|
|
250
|
+
/*
|
|
251
|
+
A body that says how much of itself has gone.
|
|
252
|
+
|
|
253
|
+
A batch of small files is one request of a few megabytes, and on an
|
|
254
|
+
ordinary home connection three of them in flight take ten seconds each.
|
|
255
|
+
Counted only when the answer came back, the bar stood still for those
|
|
256
|
+
ten seconds and then jumped — and the last few batches of a save looked
|
|
257
|
+
exactly like a save that had stopped. Handed over in slices, each slice
|
|
258
|
+
taken is counted as it is taken, which is what the network has accepted.
|
|
259
|
+
A fresh stream per attempt, so a retry counts from nothing again.
|
|
260
|
+
*/
|
|
261
|
+
const streamed = init.onSent && init.body instanceof Uint8Array && init.body.byteLength > 0
|
|
262
|
+
? (() => {
|
|
263
|
+
const whole = init.body;
|
|
264
|
+
const report = init.onSent;
|
|
265
|
+
const SLICE = 256 * 1024;
|
|
266
|
+
let at = 0;
|
|
267
|
+
report(0);
|
|
268
|
+
return new ReadableStream({
|
|
269
|
+
pull(controller) {
|
|
270
|
+
if (at >= whole.byteLength) {
|
|
271
|
+
controller.close();
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
const next = whole.subarray(at, Math.min(at + SLICE, whole.byteLength));
|
|
275
|
+
at += next.byteLength;
|
|
276
|
+
controller.enqueue(next);
|
|
277
|
+
report(at / whole.byteLength);
|
|
278
|
+
},
|
|
279
|
+
});
|
|
280
|
+
})()
|
|
281
|
+
: null;
|
|
246
282
|
const token = await this.credentials.token();
|
|
247
283
|
if (!token)
|
|
248
284
|
throw new Error("Sign in again before uploading");
|
|
@@ -258,7 +294,8 @@ class Uploader {
|
|
|
258
294
|
: {}),
|
|
259
295
|
...(init.contentType ? { "content-type": init.contentType } : {}),
|
|
260
296
|
},
|
|
261
|
-
body: init.body,
|
|
297
|
+
body: streamed ?? init.body,
|
|
298
|
+
...(streamed ? { duplex: "half" } : {}),
|
|
262
299
|
/*
|
|
263
300
|
The upload's own cancellation, and a deadline.
|
|
264
301
|
|
|
@@ -575,6 +612,42 @@ class Uploader {
|
|
|
575
612
|
let cursor = 0;
|
|
576
613
|
/** Throttled: twenty thousand sends is its own cost, and nobody reads them. */
|
|
577
614
|
let told = 0;
|
|
615
|
+
/*
|
|
616
|
+
Bytes, not just files.
|
|
617
|
+
|
|
618
|
+
This said how many files were done and nothing else, so a project whose
|
|
619
|
+
weight is four 1.6 GB builds sat on "1 of 11" for three minutes on a
|
|
620
|
+
hard disk and then leapt: every report came after a whole file. The
|
|
621
|
+
sizes are known from the survey, so how much has been read is a share of
|
|
622
|
+
a known total, and the lanes say how far into a large file they are.
|
|
623
|
+
*/
|
|
624
|
+
const readTotal = sending.reduce((sum, file) => sum + file.size, 0);
|
|
625
|
+
let readDone = 0;
|
|
626
|
+
const readingNow = new Map();
|
|
627
|
+
const tell = (path, force = false) => {
|
|
628
|
+
const now = Date.now();
|
|
629
|
+
if (!force && now - told <= 250)
|
|
630
|
+
return;
|
|
631
|
+
told = now;
|
|
632
|
+
let bytes = readDone;
|
|
633
|
+
for (const partial of readingNow.values())
|
|
634
|
+
bytes += partial;
|
|
635
|
+
report({
|
|
636
|
+
stage: "hash",
|
|
637
|
+
files: hashed,
|
|
638
|
+
totalFiles: sending.length,
|
|
639
|
+
bytes: Math.min(bytes, readTotal),
|
|
640
|
+
totalBytes: readTotal,
|
|
641
|
+
path,
|
|
642
|
+
/*
|
|
643
|
+
Reading is the stretch from just past the walk to a fifth of
|
|
644
|
+
the way through. Starting it at nought put the bar behind where
|
|
645
|
+
it had already been and it went backwards between stages.
|
|
646
|
+
*/
|
|
647
|
+
percent: 8 + Math.round((hashed / sending.length) * 12),
|
|
648
|
+
bytesPerSecond: 0,
|
|
649
|
+
});
|
|
650
|
+
};
|
|
578
651
|
const lane = async () => {
|
|
579
652
|
for (;;) {
|
|
580
653
|
const index = cursor;
|
|
@@ -597,6 +670,7 @@ class Uploader {
|
|
|
597
670
|
vanished.push(file.path);
|
|
598
671
|
readFailures.set(file.path, error instanceof Error ? error.message : String(error));
|
|
599
672
|
hashed += 1;
|
|
673
|
+
readDone += file.size;
|
|
600
674
|
continue;
|
|
601
675
|
}
|
|
602
676
|
/*
|
|
@@ -630,13 +704,19 @@ class Uploader {
|
|
|
630
704
|
mediaType: mediaTypeOf(file.path),
|
|
631
705
|
};
|
|
632
706
|
hashed += 1;
|
|
707
|
+
readDone += file.size;
|
|
633
708
|
continue;
|
|
634
709
|
}
|
|
635
710
|
let digest;
|
|
636
711
|
try {
|
|
637
|
-
digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full)
|
|
712
|
+
digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full, (read) => {
|
|
713
|
+
readingNow.set(index, read);
|
|
714
|
+
tell(file.path);
|
|
715
|
+
}));
|
|
638
716
|
}
|
|
639
717
|
catch (error) {
|
|
718
|
+
readingNow.delete(index);
|
|
719
|
+
readDone += file.size;
|
|
640
720
|
vanished.push(file.path);
|
|
641
721
|
readFailures.set(file.path, error instanceof Error ? error.message : String(error));
|
|
642
722
|
hashed += 1;
|
|
@@ -651,25 +731,9 @@ class Uploader {
|
|
|
651
731
|
mediaType: mediaTypeOf(file.path),
|
|
652
732
|
};
|
|
653
733
|
hashed += 1;
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
report({
|
|
658
|
-
stage: "hash",
|
|
659
|
-
files: hashed,
|
|
660
|
-
totalFiles: sending.length,
|
|
661
|
-
bytes: 0,
|
|
662
|
-
totalBytes: 0,
|
|
663
|
-
path: file.path,
|
|
664
|
-
/*
|
|
665
|
-
Reading is the stretch from just past the walk to a fifth of
|
|
666
|
-
the way through. Starting it at nought put the bar behind where
|
|
667
|
-
it had already been and it went backwards between stages.
|
|
668
|
-
*/
|
|
669
|
-
percent: 8 + Math.round((hashed / sending.length) * 12),
|
|
670
|
-
bytesPerSecond: 0,
|
|
671
|
-
});
|
|
672
|
-
}
|
|
734
|
+
readingNow.delete(index);
|
|
735
|
+
readDone += file.size;
|
|
736
|
+
tell(file.path, hashed === sending.length);
|
|
673
737
|
}
|
|
674
738
|
};
|
|
675
739
|
await Promise.all(Array.from({ length: Math.min(HASH_LANES, sending.length) }, () => lane()));
|
|
@@ -1081,6 +1145,16 @@ class Uploader {
|
|
|
1081
1145
|
shown = count;
|
|
1082
1146
|
return shown;
|
|
1083
1147
|
};
|
|
1148
|
+
/*
|
|
1149
|
+
Identical files are one upload.
|
|
1150
|
+
|
|
1151
|
+
A folder holding two copies of the same file declares it twice, and both
|
|
1152
|
+
copies used to go up in full: side by side in one batch, or in two lanes
|
|
1153
|
+
at the same moment. The second copy now waits for the first to finish,
|
|
1154
|
+
after which the service already holds the bytes and the copy costs a
|
|
1155
|
+
question rather than a transfer.
|
|
1156
|
+
*/
|
|
1157
|
+
const inFlightByDigest = new Map();
|
|
1084
1158
|
const runSection = async (section) => {
|
|
1085
1159
|
const declarations = section.files.map((file) => declarationByPath.get(file.path));
|
|
1086
1160
|
/*
|
|
@@ -1151,6 +1225,17 @@ class Uploader {
|
|
|
1151
1225
|
*/
|
|
1152
1226
|
const missing = declarations.filter((one) => !alreadyHeld.has(one.sha256));
|
|
1153
1227
|
const smallOnes = missing.filter((one) => one.size <= BATCH_FILE_LIMIT);
|
|
1228
|
+
/* The first file with each digest is sent; the rest are counted when it lands. */
|
|
1229
|
+
const copiesOf = new Map();
|
|
1230
|
+
const smallUnique = smallOnes.filter((one) => {
|
|
1231
|
+
const copies = copiesOf.get(one.sha256);
|
|
1232
|
+
if (copies) {
|
|
1233
|
+
copies.push(one);
|
|
1234
|
+
return false;
|
|
1235
|
+
}
|
|
1236
|
+
copiesOf.set(one.sha256, []);
|
|
1237
|
+
return true;
|
|
1238
|
+
});
|
|
1154
1239
|
// 3. Send the objects.
|
|
1155
1240
|
/*
|
|
1156
1241
|
What was sent, in whichever shape it was sent. A small file is one
|
|
@@ -1177,10 +1262,23 @@ class Uploader {
|
|
|
1177
1262
|
(0, profile_js_1.counted)("files sent as deltas", 1);
|
|
1178
1263
|
}
|
|
1179
1264
|
}
|
|
1180
|
-
const smallForPacking =
|
|
1265
|
+
const smallForPacking = smallUnique.filter((one) => !batched.has(one.sha256));
|
|
1181
1266
|
if (smallForPacking.length > 1) {
|
|
1182
1267
|
let pack = [];
|
|
1183
1268
|
let packBytes = 0;
|
|
1269
|
+
/* How many batches have had a lane of their own on the bar. */
|
|
1270
|
+
let batchLanes = 0;
|
|
1271
|
+
/*
|
|
1272
|
+
How long a source byte takes to land in a batch on this connection,
|
|
1273
|
+
learned from the batches that have landed.
|
|
1274
|
+
|
|
1275
|
+
Counting the body as it is handed over is not enough on its own: the
|
|
1276
|
+
operating system takes a few megabytes into its send buffer at once,
|
|
1277
|
+
so a whole batch reads as sent within a moment and then waits seconds
|
|
1278
|
+
for the wire. A batch in flight is shown no further along than its
|
|
1279
|
+
time so far at the pace earlier batches actually managed.
|
|
1280
|
+
*/
|
|
1281
|
+
let secondsPerByte = 0;
|
|
1184
1282
|
/*
|
|
1185
1283
|
Batches overlap rather than queue.
|
|
1186
1284
|
|
|
@@ -1195,6 +1293,15 @@ class Uploader {
|
|
|
1195
1293
|
that is reasonable to have outstanding.
|
|
1196
1294
|
*/
|
|
1197
1295
|
const BATCH_LANES = 3;
|
|
1296
|
+
/*
|
|
1297
|
+
Time against the expected time, as a share of a batch: in step until
|
|
1298
|
+
nine tenths, then closing on the end without reaching it, so a batch
|
|
1299
|
+
that runs late keeps moving and never reads as landed before its
|
|
1300
|
+
answer says so.
|
|
1301
|
+
*/
|
|
1302
|
+
const creep = (expectedShare) => expectedShare <= 0.9
|
|
1303
|
+
? Math.max(0, expectedShare)
|
|
1304
|
+
: 0.9 + 0.09 * (1 - Math.exp(-(expectedShare - 0.9) / 0.3));
|
|
1198
1305
|
const flying = new Set();
|
|
1199
1306
|
const flush = async () => {
|
|
1200
1307
|
if (!pack.length)
|
|
@@ -1202,6 +1309,52 @@ class Uploader {
|
|
|
1202
1309
|
const taking = pack;
|
|
1203
1310
|
pack = [];
|
|
1204
1311
|
packBytes = 0;
|
|
1312
|
+
/*
|
|
1313
|
+
This batch's own lane on the bar, keyed below zero so it can never
|
|
1314
|
+
collide with a file's lane. Its source bytes are shared out as its
|
|
1315
|
+
body goes, and taken back off when the batch lands and is counted
|
|
1316
|
+
for real.
|
|
1317
|
+
*/
|
|
1318
|
+
batchLanes += 1;
|
|
1319
|
+
const lane = -batchLanes;
|
|
1320
|
+
const batchSource = taking.reduce((sum, item) => sum + item.declaration.size, 0);
|
|
1321
|
+
let toldAt = 0;
|
|
1322
|
+
let handed = 0;
|
|
1323
|
+
const sentAt = Date.now();
|
|
1324
|
+
const show = () => {
|
|
1325
|
+
/*
|
|
1326
|
+
Until a batch has landed there is no pace of this connection to go
|
|
1327
|
+
by, and the first batches read as sent within a moment and then
|
|
1328
|
+
stood still for seconds. A cautious guess stands in until the first
|
|
1329
|
+
answer: a megabyte and a half a second, shared by the batches in
|
|
1330
|
+
flight. Too slow a guess costs only a jump forward when the batch
|
|
1331
|
+
lands; too fast a one would be the stall again.
|
|
1332
|
+
*/
|
|
1333
|
+
const pace = secondsPerByte > 0 ? secondsPerByte : BATCH_LANES / 1_500_000;
|
|
1334
|
+
const elapsed = (Date.now() - sentAt) / 1000;
|
|
1335
|
+
const share = batchSource > 0 ? Math.min(handed, creep(elapsed / (pace * batchSource))) : handed;
|
|
1336
|
+
progress.lane(lane, Math.round(batchSource * share));
|
|
1337
|
+
const now = Date.now();
|
|
1338
|
+
if (now - toldAt < 250)
|
|
1339
|
+
return;
|
|
1340
|
+
toldAt = now;
|
|
1341
|
+
report({
|
|
1342
|
+
stage: "upload",
|
|
1343
|
+
files: advance(0),
|
|
1344
|
+
totalFiles: everyDeclaration.length,
|
|
1345
|
+
bytes: progress.bytes(),
|
|
1346
|
+
totalBytes,
|
|
1347
|
+
path: taking[taking.length - 1].declaration.path,
|
|
1348
|
+
percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
|
|
1349
|
+
bytesPerSecond: progress.rate(),
|
|
1350
|
+
});
|
|
1351
|
+
};
|
|
1352
|
+
const onSent = (fraction) => {
|
|
1353
|
+
handed = fraction;
|
|
1354
|
+
show();
|
|
1355
|
+
};
|
|
1356
|
+
/* Keeps the estimate moving while the batch waits on the wire. */
|
|
1357
|
+
const pacing = setInterval(show, 300);
|
|
1205
1358
|
const task = (async () => {
|
|
1206
1359
|
let exactWireBytes = null;
|
|
1207
1360
|
try {
|
|
@@ -1233,7 +1386,7 @@ class Uploader {
|
|
|
1233
1386
|
batched.set(digest, one);
|
|
1234
1387
|
}
|
|
1235
1388
|
const placed = !delta && built
|
|
1236
|
-
? await (0, profile_js_1.timed)("send packs", () => this.putPack(repositoryId, packItems, built))
|
|
1389
|
+
? await (0, profile_js_1.timed)("send packs", () => this.putPack(repositoryId, packItems, built, onSent))
|
|
1237
1390
|
: null;
|
|
1238
1391
|
if (placed) {
|
|
1239
1392
|
(0, profile_js_1.counted)("files sent in packs", taking.length);
|
|
@@ -1246,7 +1399,7 @@ class Uploader {
|
|
|
1246
1399
|
Not worth packing — already-compressed content, or too few files.
|
|
1247
1400
|
They still travel together, just as separate objects.
|
|
1248
1401
|
*/
|
|
1249
|
-
const answer = await (0, profile_js_1.timed)("send batches", () => this.putBatch(repositoryId, taking));
|
|
1402
|
+
const answer = await (0, profile_js_1.timed)("send batches", () => this.putBatch(repositoryId, taking, onSent));
|
|
1250
1403
|
(0, profile_js_1.counted)("files sent in batches", taking.length);
|
|
1251
1404
|
for (const [digest, one] of answer)
|
|
1252
1405
|
batched.set(digest, one);
|
|
@@ -1276,11 +1429,20 @@ class Uploader {
|
|
|
1276
1429
|
below and counted a second time — which is how the progress bar came
|
|
1277
1430
|
to report a hundred and twenty percent of a forty megabyte upload.
|
|
1278
1431
|
*/
|
|
1432
|
+
/* Landed or refused, its estimate comes off: what landed is counted below. */
|
|
1433
|
+
clearInterval(pacing);
|
|
1434
|
+
progress.laneDone(lane);
|
|
1435
|
+
if (batchSource > 0 && taking.some((item) => batched.has(item.declaration.sha256) || packedInto.has(item.declaration.sha256))) {
|
|
1436
|
+
const took = (Date.now() - sentAt) / 1000 / batchSource;
|
|
1437
|
+
secondsPerByte = secondsPerByte > 0 ? secondsPerByte * 0.6 + took * 0.4 : took;
|
|
1438
|
+
}
|
|
1279
1439
|
for (const item of taking) {
|
|
1280
1440
|
const digest = item.declaration.sha256;
|
|
1281
1441
|
if (!batched.has(digest) && !packedInto.has(digest))
|
|
1282
1442
|
continue;
|
|
1283
1443
|
progress.finished(item.declaration.size, exactWireBytes === null ? item.encoded.body.byteLength : 0);
|
|
1444
|
+
for (const copy of copiesOf.get(digest) ?? [])
|
|
1445
|
+
progress.finished(copy.size, 0);
|
|
1284
1446
|
}
|
|
1285
1447
|
if (exactWireBytes !== null)
|
|
1286
1448
|
progress.finished(0, exactWireBytes);
|
|
@@ -1294,7 +1456,10 @@ class Uploader {
|
|
|
1294
1456
|
percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
|
|
1295
1457
|
bytesPerSecond: progress.rate(),
|
|
1296
1458
|
});
|
|
1297
|
-
})().finally(() =>
|
|
1459
|
+
})().finally(() => {
|
|
1460
|
+
clearInterval(pacing);
|
|
1461
|
+
flying.delete(task);
|
|
1462
|
+
});
|
|
1298
1463
|
flying.add(task);
|
|
1299
1464
|
if (flying.size >= BATCH_LANES)
|
|
1300
1465
|
await Promise.race(flying);
|
|
@@ -1336,121 +1501,51 @@ class Uploader {
|
|
|
1336
1501
|
const queue = declarations.entries();
|
|
1337
1502
|
const runWorker = async () => {
|
|
1338
1503
|
for (const [index, declaration] of queue) {
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
|
|
1348
|
-
bytesPerSecond: progress.rate(),
|
|
1349
|
-
});
|
|
1350
|
-
/*
|
|
1351
|
-
Content the service already holds does not need sending. Asking costs
|
|
1352
|
-
one round trip and saves the whole file — which after an interrupted
|
|
1353
|
-
save is the difference between re-uploading a project and re-uploading
|
|
1354
|
-
nothing.
|
|
1355
|
-
*/
|
|
1356
|
-
/*
|
|
1357
|
-
A large file goes up in pieces, cut where its own content says.
|
|
1358
|
-
|
|
1359
|
-
This is what CBX is for and what it was never doing on the wire. The
|
|
1360
|
-
boundaries come from the same chunker the .cbx bundle uses, so an
|
|
1361
|
-
edit in the middle of a large file leaves the chunks either side of it
|
|
1362
|
-
untouched — and an untouched chunk is one the service already holds,
|
|
1363
|
-
which costs a question rather than a transfer. A small file is left
|
|
1364
|
-
whole: cutting it up would trade one request for several and save
|
|
1365
|
-
nothing.
|
|
1366
|
-
*/
|
|
1367
|
-
const chunkProfile = (await this.microchunkAllowed())
|
|
1368
|
-
? (0, chunking_js_1.microchunkProfileForFileSize)(declaration.size)
|
|
1369
|
-
: (0, chunking_js_1.profileForFileSize)(declaration.size);
|
|
1370
|
-
if (chunkProfile &&
|
|
1371
|
-
declaration.size >= CHUNK_THRESHOLD &&
|
|
1372
|
-
(await this.chunkingAllowed())) {
|
|
1373
|
-
const pieces = await this.putChunked(repositoryId, declaration, chunkProfile, (offset) => {
|
|
1374
|
-
progress.lane(index, offset);
|
|
1375
|
-
report({
|
|
1376
|
-
stage: "upload",
|
|
1377
|
-
files: advance(index),
|
|
1378
|
-
totalFiles: everyDeclaration.length,
|
|
1379
|
-
bytes: progress.bytes(),
|
|
1380
|
-
totalBytes,
|
|
1381
|
-
path: declaration.path,
|
|
1382
|
-
percent: 20 +
|
|
1383
|
-
Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
|
|
1384
|
-
bytesPerSecond: progress.rate(),
|
|
1385
|
-
});
|
|
1386
|
-
});
|
|
1387
|
-
progress.laneDone(index);
|
|
1388
|
-
/*
|
|
1389
|
-
A chunk is not a file. Counting every reused piece here made a
|
|
1390
|
-
one-file save report a negative number of sent files. The selected
|
|
1391
|
-
file counts as already stored only when every one of its pieces was
|
|
1392
|
-
reused and no payload crossed the wire.
|
|
1393
|
-
*/
|
|
1394
|
-
if (pieces.sentBytes === 0)
|
|
1395
|
-
progress.alreadyHeld();
|
|
1396
|
-
uploaded.push({
|
|
1397
|
-
path: declaration.path,
|
|
1398
|
-
sha256: declaration.sha256,
|
|
1399
|
-
chunks: pieces.chunks,
|
|
1400
|
-
sourceSize: declaration.size,
|
|
1401
|
-
storedSize: pieces.chunks.reduce((sum, one) => sum + one.storedSize, 0),
|
|
1402
|
-
mediaType: declaration.mediaType,
|
|
1403
|
-
});
|
|
1404
|
-
progress.finished(declaration.size, pieces.sentBytes);
|
|
1405
|
-
continue;
|
|
1406
|
-
}
|
|
1407
|
-
const fromPack = packedInto.get(declaration.sha256);
|
|
1408
|
-
if (fromPack) {
|
|
1409
|
-
uploaded.push({
|
|
1410
|
-
path: declaration.path,
|
|
1411
|
-
sha256: declaration.sha256,
|
|
1412
|
-
pack: fromPack,
|
|
1413
|
-
sourceSize: declaration.size,
|
|
1414
|
-
/* Its share of the pack, so the version's totals stay honest. */
|
|
1415
|
-
storedSize: fromPack.storedSize,
|
|
1416
|
-
mediaType: declaration.mediaType,
|
|
1417
|
-
});
|
|
1418
|
-
continue;
|
|
1504
|
+
const earlier = inFlightByDigest.get(declaration.sha256);
|
|
1505
|
+
if (earlier)
|
|
1506
|
+
await earlier;
|
|
1507
|
+
let release = () => { };
|
|
1508
|
+
if (!earlier) {
|
|
1509
|
+
inFlightByDigest.set(declaration.sha256, new Promise((done) => {
|
|
1510
|
+
release = done;
|
|
1511
|
+
}));
|
|
1419
1512
|
}
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1513
|
+
try {
|
|
1514
|
+
this.check();
|
|
1515
|
+
report({
|
|
1516
|
+
stage: "upload",
|
|
1517
|
+
files: advance(index),
|
|
1518
|
+
totalFiles: everyDeclaration.length,
|
|
1519
|
+
bytes: progress.bytes(),
|
|
1520
|
+
totalBytes,
|
|
1423
1521
|
path: declaration.path,
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
storedSize: fromBatch.size,
|
|
1427
|
-
mediaType: declaration.mediaType,
|
|
1522
|
+
percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
|
|
1523
|
+
bytesPerSecond: progress.rate(),
|
|
1428
1524
|
});
|
|
1429
|
-
continue;
|
|
1430
|
-
}
|
|
1431
|
-
const alreadyStored = alreadyHeld.get(declaration.sha256) ??
|
|
1432
1525
|
/*
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1526
|
+
Content the service already holds does not need sending. Asking costs
|
|
1527
|
+
one round trip and saves the whole file — which after an interrupted
|
|
1528
|
+
save is the difference between re-uploading a project and re-uploading
|
|
1529
|
+
nothing.
|
|
1436
1530
|
*/
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1531
|
+
/*
|
|
1532
|
+
A large file goes up in pieces, cut where its own content says.
|
|
1533
|
+
|
|
1534
|
+
This is what CBX is for and what it was never doing on the wire. The
|
|
1535
|
+
boundaries come from the same chunker the .cbx bundle uses, so an
|
|
1536
|
+
edit in the middle of a large file leaves the chunks either side of it
|
|
1537
|
+
untouched — and an untouched chunk is one the service already holds,
|
|
1538
|
+
which costs a question rather than a transfer. A small file is left
|
|
1539
|
+
whole: cutting it up would trade one request for several and save
|
|
1540
|
+
nothing.
|
|
1541
|
+
*/
|
|
1542
|
+
const chunkProfile = (await this.microchunkAllowed())
|
|
1543
|
+
? (0, chunking_js_1.microchunkProfileForFileSize)(declaration.size)
|
|
1544
|
+
: (0, chunking_js_1.profileForFileSize)(declaration.size);
|
|
1545
|
+
if (chunkProfile &&
|
|
1546
|
+
declaration.size >= CHUNK_THRESHOLD &&
|
|
1547
|
+
(await this.chunkingAllowed())) {
|
|
1548
|
+
const pieces = await this.putChunked(repositoryId, declaration, chunkProfile, (offset) => {
|
|
1454
1549
|
progress.lane(index, offset);
|
|
1455
1550
|
report({
|
|
1456
1551
|
stage: "upload",
|
|
@@ -1463,16 +1558,100 @@ class Uploader {
|
|
|
1463
1558
|
Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
|
|
1464
1559
|
bytesPerSecond: progress.rate(),
|
|
1465
1560
|
});
|
|
1466
|
-
})
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1561
|
+
});
|
|
1562
|
+
progress.laneDone(index);
|
|
1563
|
+
/*
|
|
1564
|
+
A chunk is not a file. Counting every reused piece here made a
|
|
1565
|
+
one-file save report a negative number of sent files. The selected
|
|
1566
|
+
file counts as already stored only when every one of its pieces was
|
|
1567
|
+
reused and no payload crossed the wire.
|
|
1568
|
+
*/
|
|
1569
|
+
if (pieces.sentBytes === 0)
|
|
1570
|
+
progress.alreadyHeld();
|
|
1571
|
+
uploaded.push({
|
|
1572
|
+
path: declaration.path,
|
|
1573
|
+
sha256: declaration.sha256,
|
|
1574
|
+
chunks: pieces.chunks,
|
|
1575
|
+
sourceSize: declaration.size,
|
|
1576
|
+
storedSize: pieces.chunks.reduce((sum, one) => sum + one.storedSize, 0),
|
|
1577
|
+
mediaType: declaration.mediaType,
|
|
1578
|
+
});
|
|
1579
|
+
progress.finished(declaration.size, pieces.sentBytes);
|
|
1580
|
+
continue;
|
|
1581
|
+
}
|
|
1582
|
+
const fromPack = packedInto.get(declaration.sha256);
|
|
1583
|
+
if (fromPack) {
|
|
1584
|
+
uploaded.push({
|
|
1585
|
+
path: declaration.path,
|
|
1586
|
+
sha256: declaration.sha256,
|
|
1587
|
+
pack: fromPack,
|
|
1588
|
+
sourceSize: declaration.size,
|
|
1589
|
+
/* Its share of the pack, so the version's totals stay honest. */
|
|
1590
|
+
storedSize: fromPack.storedSize,
|
|
1591
|
+
mediaType: declaration.mediaType,
|
|
1592
|
+
});
|
|
1593
|
+
continue;
|
|
1594
|
+
}
|
|
1595
|
+
const fromBatch = batched.get(declaration.sha256);
|
|
1596
|
+
if (fromBatch) {
|
|
1597
|
+
uploaded.push({
|
|
1598
|
+
path: declaration.path,
|
|
1599
|
+
objectId: fromBatch.objectId,
|
|
1600
|
+
sourceSize: declaration.size,
|
|
1601
|
+
storedSize: fromBatch.size,
|
|
1602
|
+
mediaType: declaration.mediaType,
|
|
1603
|
+
});
|
|
1604
|
+
continue;
|
|
1605
|
+
}
|
|
1606
|
+
const alreadyStored = alreadyHeld.get(declaration.sha256) ??
|
|
1607
|
+
/*
|
|
1608
|
+
Only asked individually when the batch did not cover it — a digest
|
|
1609
|
+
that changed after the bulk question, because the file was rewritten
|
|
1610
|
+
while this upload was running.
|
|
1611
|
+
*/
|
|
1612
|
+
(await this.findStored(repositoryId, declaration));
|
|
1613
|
+
if (alreadyStored)
|
|
1614
|
+
progress.alreadyHeld();
|
|
1615
|
+
const result = alreadyStored ??
|
|
1616
|
+
(declaration.size <= DIRECT_LIMIT
|
|
1617
|
+
? await (0, profile_js_1.timed)("send files one at a time", () => this.putDirect(repositoryId, declaration)).catch((error) => {
|
|
1618
|
+
// Naming the file turns "it failed" into something actionable.
|
|
1619
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1620
|
+
throw new Error(`${declaration.path}: ${reason}`);
|
|
1621
|
+
})
|
|
1622
|
+
: await this.putMultipart(repositoryId, declaration, (offset) => {
|
|
1623
|
+
/*
|
|
1624
|
+
Recorded before the report reads it. Without this the bar does
|
|
1625
|
+
not move at all during a multipart file: `bytes()` sums the
|
|
1626
|
+
settled total and the open lanes, and this lane's offset is
|
|
1627
|
+
only in the second of those once it has been set.
|
|
1628
|
+
*/
|
|
1629
|
+
progress.lane(index, offset);
|
|
1630
|
+
report({
|
|
1631
|
+
stage: "upload",
|
|
1632
|
+
files: advance(index),
|
|
1633
|
+
totalFiles: everyDeclaration.length,
|
|
1634
|
+
bytes: progress.bytes(),
|
|
1635
|
+
totalBytes,
|
|
1636
|
+
path: declaration.path,
|
|
1637
|
+
percent: 20 +
|
|
1638
|
+
Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
|
|
1639
|
+
bytesPerSecond: progress.rate(),
|
|
1640
|
+
});
|
|
1641
|
+
}));
|
|
1642
|
+
progress.laneDone(index);
|
|
1643
|
+
uploaded.push({
|
|
1644
|
+
path: declaration.path,
|
|
1645
|
+
objectId: result.objectId,
|
|
1646
|
+
sourceSize: declaration.size,
|
|
1647
|
+
storedSize: result.size,
|
|
1648
|
+
mediaType: declaration.mediaType,
|
|
1649
|
+
});
|
|
1650
|
+
progress.finished(declaration.size, alreadyStored ? 0 : declaration.size);
|
|
1651
|
+
}
|
|
1652
|
+
finally {
|
|
1653
|
+
release();
|
|
1654
|
+
}
|
|
1476
1655
|
}
|
|
1477
1656
|
};
|
|
1478
1657
|
await Promise.all(Array.from({ length: Math.min(UPLOAD_LANES, declarations.length) }, () => runWorker()));
|
|
@@ -1794,7 +1973,7 @@ class Uploader {
|
|
|
1794
1973
|
* service would not take it — in which case the caller falls back to sending
|
|
1795
1974
|
* them separately, which is slower and larger but always works.
|
|
1796
1975
|
*/
|
|
1797
|
-
async putPack(repositoryId, items, alreadyBuilt) {
|
|
1976
|
+
async putPack(repositoryId, items, alreadyBuilt, onSent) {
|
|
1798
1977
|
const built = alreadyBuilt ?? await (0, solid_js_1.buildSolidPack)(items.map((item) => ({
|
|
1799
1978
|
sha256: item.declaration.sha256,
|
|
1800
1979
|
body: item.body,
|
|
@@ -1830,6 +2009,7 @@ class Uploader {
|
|
|
1830
2009
|
method: "POST",
|
|
1831
2010
|
contentType: "application/octet-stream",
|
|
1832
2011
|
body: packed,
|
|
2012
|
+
...(onSent ? { onSent } : {}),
|
|
1833
2013
|
});
|
|
1834
2014
|
const placed = new Map();
|
|
1835
2015
|
/*
|
|
@@ -2091,7 +2271,7 @@ class Uploader {
|
|
|
2091
2271
|
* content-addressed, so a retry is free and a partial batch costs nothing
|
|
2092
2272
|
* but the objects it missed.
|
|
2093
2273
|
*/
|
|
2094
|
-
async putBatch(repositoryId, items) {
|
|
2274
|
+
async putBatch(repositoryId, items, onSent) {
|
|
2095
2275
|
const landed = new Map();
|
|
2096
2276
|
if (!items.length)
|
|
2097
2277
|
return landed;
|
|
@@ -2133,6 +2313,7 @@ class Uploader {
|
|
|
2133
2313
|
method: "POST",
|
|
2134
2314
|
contentType: "application/octet-stream",
|
|
2135
2315
|
body: packed,
|
|
2316
|
+
...(onSent ? { onSent } : {}),
|
|
2136
2317
|
});
|
|
2137
2318
|
for (const one of answer.objects ?? []) {
|
|
2138
2319
|
landed.set(one.sha256, {
|
package/dist/cli/src/network.js
CHANGED
|
@@ -60,17 +60,37 @@ function describeError(error) {
|
|
|
60
60
|
return "CodeRook did not answer in time. Check the connection, then try again.";
|
|
61
61
|
}
|
|
62
62
|
const cause = error.cause;
|
|
63
|
-
if (error.message === "fetch failed"
|
|
64
|
-
|
|
63
|
+
if (error.message === "fetch failed") {
|
|
64
|
+
/*
|
|
65
|
+
Said in words first, with the code after for whoever needs it. The most
|
|
66
|
+
common one, ENOTFOUND, is simply being offline: the name could not be
|
|
67
|
+
looked up. And a failure that carries no code at all used to fall
|
|
68
|
+
through to the bare "fetch failed" it exists to replace.
|
|
69
|
+
*/
|
|
70
|
+
const attempts = cause?.errors?.length ? cause.errors : cause ? [cause] : [];
|
|
71
|
+
const codes = [...new Set(attempts.map((one) => one.code).filter(Boolean))];
|
|
72
|
+
const said = codes.map((code) => PLAIN[code]).find(Boolean);
|
|
65
73
|
const named = attempts
|
|
66
74
|
.map((one) => [one.code, one.address].filter(Boolean).join(" "))
|
|
67
75
|
.filter(Boolean);
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
76
|
+
const detail = named.length ? [...new Set(named)].join(", ") : cause?.message ?? "";
|
|
77
|
+
return (`Could not reach CodeRook${said ? `: ${said}` : ""}` +
|
|
78
|
+
(detail ? ` (${detail})` : "") +
|
|
79
|
+
". Check the connection, then try again.");
|
|
71
80
|
}
|
|
72
81
|
return error.message;
|
|
73
82
|
}
|
|
83
|
+
/** What each network failure means, for somebody who has not met the code. */
|
|
84
|
+
const PLAIN = {
|
|
85
|
+
ENOTFOUND: "its address could not be looked up, which usually means this computer is offline",
|
|
86
|
+
EAI_AGAIN: "its address could not be looked up, which usually means this computer is offline",
|
|
87
|
+
ECONNREFUSED: "the connection was refused",
|
|
88
|
+
ECONNRESET: "the connection was dropped",
|
|
89
|
+
ETIMEDOUT: "the connection timed out",
|
|
90
|
+
ENETUNREACH: "there is no route to the internet from here",
|
|
91
|
+
EHOSTUNREACH: "there is no route to the internet from here",
|
|
92
|
+
UND_ERR_CONNECT_TIMEOUT: "the connection timed out",
|
|
93
|
+
};
|
|
74
94
|
/** Wait, for a retry. */
|
|
75
95
|
function pause(milliseconds) {
|
|
76
96
|
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@coderook/cli",
|
|
3
|
-
"version": "0.34.
|
|
3
|
+
"version": "0.34.5",
|
|
4
4
|
"description": "The command line for CodeRook (coderook.com): version control and project hosting for projects of any size. Every save is a compressed snapshot, private or public, never used to train AI.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://coderook.com",
|