@coderook/cli 0.21.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "coderook",
|
|
3
3
|
"displayName": "CodeRook",
|
|
4
4
|
"description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.22.1",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "ACCA Gaming Productions",
|
|
8
8
|
"url": "https://coderook.com"
|
|
@@ -27,6 +27,7 @@ const profile_js_1 = require("./profile.js");
|
|
|
27
27
|
const solid_js_1 = require("./solid.js");
|
|
28
28
|
const retry_js_1 = require("./retry.js");
|
|
29
29
|
const staging_js_1 = require("./staging.js");
|
|
30
|
+
const delta_js_1 = require("../shared/delta.js");
|
|
30
31
|
/** Above this the API insists on a multipart session. */
|
|
31
32
|
const DIRECT_LIMIT = 95 * 1024 * 1024;
|
|
32
33
|
/*
|
|
@@ -105,6 +106,10 @@ class Uploader {
|
|
|
105
106
|
chunkingSupported = null;
|
|
106
107
|
/** Null until the service has been asked; see packingAllowed. */
|
|
107
108
|
packingSupported = null;
|
|
109
|
+
/** Null until the service has advertised its best changed-file transport. */
|
|
110
|
+
deltaProtocol = null;
|
|
111
|
+
/** Batch delta is separate so V2 single-file support remains compatible. */
|
|
112
|
+
deltaBatchSupported = null;
|
|
108
113
|
constructor(credentials) {
|
|
109
114
|
this.credentials = credentials;
|
|
110
115
|
}
|
|
@@ -345,8 +350,15 @@ class Uploader {
|
|
|
345
350
|
});
|
|
346
351
|
totalBytes += size;
|
|
347
352
|
}
|
|
348
|
-
|
|
353
|
+
/*
|
|
354
|
+
A deletion-only save has no file to hash or upload, but it is still a
|
|
355
|
+
real new Version: the retained base minus the explicitly selected path.
|
|
356
|
+
Refusing it here made the object phase look healthy while preventing the
|
|
357
|
+
manifest-only operation that deletion is supposed to be.
|
|
358
|
+
*/
|
|
359
|
+
if (!declarations.length && !deletions.size) {
|
|
349
360
|
throw new Error("Nothing is selected to upload");
|
|
361
|
+
}
|
|
350
362
|
// 2. The project needs somewhere on the account to live.
|
|
351
363
|
this.check();
|
|
352
364
|
let repositoryId = request.repositoryId;
|
|
@@ -394,6 +406,24 @@ class Uploader {
|
|
|
394
406
|
*/
|
|
395
407
|
const allDeclarations = declarations;
|
|
396
408
|
const declarationByPath = new Map(allDeclarations.map((one) => [one.path, one]));
|
|
409
|
+
/*
|
|
410
|
+
The narrow first use of the delta transport.
|
|
411
|
+
|
|
412
|
+
One changed small file used to miss solid packing (which correctly
|
|
413
|
+
requires a group) and upload its complete compressed bytes. When more
|
|
414
|
+
than one small file changes, the existing pack remains the better
|
|
415
|
+
answer and is deliberately untouched. New files have no receiver base
|
|
416
|
+
and also stay on the existing path.
|
|
417
|
+
*/
|
|
418
|
+
const logicallyChanged = allDeclarations.filter((one) => prior.get(one.path)?.sha256 !== one.sha256);
|
|
419
|
+
const deltaDeclaration = logicallyChanged.length === 1 &&
|
|
420
|
+
request.baseVersionId &&
|
|
421
|
+
prior.has(logicallyChanged[0].path) &&
|
|
422
|
+
logicallyChanged[0].size <= BATCH_FILE_LIMIT &&
|
|
423
|
+
logicallyChanged[0].size <= delta_js_1.DELTA_MAX_FILE_BYTES &&
|
|
424
|
+
prior.get(logicallyChanged[0].path).sourceSize <= delta_js_1.DELTA_MAX_FILE_BYTES
|
|
425
|
+
? logicallyChanged[0]
|
|
426
|
+
: null;
|
|
397
427
|
const sections = (0, staging_js_1.planSections)(allDeclarations.map((one) => ({ path: one.path, size: one.size })));
|
|
398
428
|
(0, profile_js_1.counted)("sections planned", sections.length);
|
|
399
429
|
const uploaded = [];
|
|
@@ -544,7 +574,22 @@ class Uploader {
|
|
|
544
574
|
an object of their own.
|
|
545
575
|
*/
|
|
546
576
|
const packedInto = new Map();
|
|
547
|
-
if (
|
|
577
|
+
if (deltaDeclaration &&
|
|
578
|
+
smallOnes.some((one) => one.path === deltaDeclaration.path)) {
|
|
579
|
+
const held = prior.get(deltaDeclaration.path);
|
|
580
|
+
const placed = await (0, profile_js_1.timed)("send changed-file delta", () => this.putDelta(repositoryId, request.baseVersionId, held, deltaDeclaration));
|
|
581
|
+
if (placed) {
|
|
582
|
+
batched.set(deltaDeclaration.sha256, {
|
|
583
|
+
objectId: placed.objectId,
|
|
584
|
+
size: placed.storedSize,
|
|
585
|
+
});
|
|
586
|
+
sentBytes += deltaDeclaration.size;
|
|
587
|
+
transferred += placed.sentBytes;
|
|
588
|
+
(0, profile_js_1.counted)("files sent as deltas", 1);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
const smallForPacking = smallOnes.filter((one) => !batched.has(one.sha256));
|
|
592
|
+
if (smallForPacking.length > 1) {
|
|
548
593
|
let pack = [];
|
|
549
594
|
let packBytes = 0;
|
|
550
595
|
/*
|
|
@@ -569,6 +614,7 @@ class Uploader {
|
|
|
569
614
|
pack = [];
|
|
570
615
|
packBytes = 0;
|
|
571
616
|
const task = (async () => {
|
|
617
|
+
let exactWireBytes = null;
|
|
572
618
|
try {
|
|
573
619
|
/*
|
|
574
620
|
Compressed together first. This is what the .cbx format is for and
|
|
@@ -577,18 +623,36 @@ class Uploader {
|
|
|
577
623
|
and 30.2% compressed together, because a two-kilobyte file gives a
|
|
578
624
|
compressor no dictionary worth having.
|
|
579
625
|
*/
|
|
580
|
-
const
|
|
581
|
-
|
|
582
|
-
|
|
626
|
+
const packItems = taking.map((item) => ({
|
|
627
|
+
declaration: item.declaration,
|
|
628
|
+
body: item.body,
|
|
629
|
+
}));
|
|
630
|
+
const built = (await this.packingAllowed())
|
|
631
|
+
? await (0, solid_js_1.buildSolidPack)(packItems.map((item) => ({
|
|
632
|
+
sha256: item.declaration.sha256,
|
|
583
633
|
body: item.body,
|
|
584
|
-
})))
|
|
634
|
+
})))
|
|
635
|
+
: null;
|
|
636
|
+
const packedBytes = built ? this.frameSolidPack(built).byteLength : 0;
|
|
637
|
+
const delta = built
|
|
638
|
+
? await (0, profile_js_1.timed)("price changed-file batch", () => this.putDeltaBatch(repositoryId, request.baseVersionId, taking, prior, packedBytes))
|
|
639
|
+
: null;
|
|
640
|
+
if (delta) {
|
|
641
|
+
(0, profile_js_1.counted)("files sent as delta batch", taking.length);
|
|
642
|
+
exactWireBytes = delta.sentBytes;
|
|
643
|
+
for (const [digest, one] of delta.objects)
|
|
644
|
+
batched.set(digest, one);
|
|
645
|
+
}
|
|
646
|
+
const placed = !delta && built
|
|
647
|
+
? await (0, profile_js_1.timed)("send packs", () => this.putPack(repositoryId, packItems, built))
|
|
585
648
|
: null;
|
|
586
649
|
if (placed) {
|
|
587
650
|
(0, profile_js_1.counted)("files sent in packs", taking.length);
|
|
651
|
+
exactWireBytes = packedBytes;
|
|
588
652
|
for (const [digest, one] of placed)
|
|
589
653
|
packedInto.set(digest, one);
|
|
590
654
|
}
|
|
591
|
-
else {
|
|
655
|
+
else if (!delta) {
|
|
592
656
|
/*
|
|
593
657
|
Not worth packing — already-compressed content, or too few files.
|
|
594
658
|
They still travel together, just as separate objects.
|
|
@@ -617,8 +681,11 @@ class Uploader {
|
|
|
617
681
|
if (!batched.has(digest) && !packedInto.has(digest))
|
|
618
682
|
continue;
|
|
619
683
|
sentBytes += item.declaration.size;
|
|
620
|
-
|
|
684
|
+
if (exactWireBytes === null)
|
|
685
|
+
transferred += item.encoded.body.byteLength;
|
|
621
686
|
}
|
|
687
|
+
if (exactWireBytes !== null)
|
|
688
|
+
transferred += exactWireBytes;
|
|
622
689
|
report({
|
|
623
690
|
stage: "upload",
|
|
624
691
|
files: 0,
|
|
@@ -634,7 +701,7 @@ class Uploader {
|
|
|
634
701
|
if (flying.size >= BATCH_LANES)
|
|
635
702
|
await Promise.race(flying);
|
|
636
703
|
};
|
|
637
|
-
for (const declaration of
|
|
704
|
+
for (const declaration of smallForPacking) {
|
|
638
705
|
this.check();
|
|
639
706
|
let body;
|
|
640
707
|
try {
|
|
@@ -718,7 +785,14 @@ class Uploader {
|
|
|
718
785
|
});
|
|
719
786
|
});
|
|
720
787
|
inFlight.delete(index);
|
|
721
|
-
|
|
788
|
+
/*
|
|
789
|
+
A chunk is not a file. Counting every reused piece here made a
|
|
790
|
+
one-file save report a negative number of sent files. The selected
|
|
791
|
+
file counts as already stored only when every one of its pieces was
|
|
792
|
+
reused and no payload crossed the wire.
|
|
793
|
+
*/
|
|
794
|
+
if (pieces.sentBytes === 0)
|
|
795
|
+
alreadyOnAccount += 1;
|
|
722
796
|
uploaded.push({
|
|
723
797
|
path: declaration.path,
|
|
724
798
|
sha256: declaration.sha256,
|
|
@@ -988,7 +1062,7 @@ class Uploader {
|
|
|
988
1062
|
...(completed.repeated ? { repeated: true } : {}),
|
|
989
1063
|
sourceBytes: completed.version.sourceSize ?? sourceBytes,
|
|
990
1064
|
storedBytes: completed.version.storedSize ?? storedBytes,
|
|
991
|
-
sentBytes,
|
|
1065
|
+
sentBytes: transferred,
|
|
992
1066
|
sentFiles: uploaded.length - alreadyOnAccount,
|
|
993
1067
|
reusedFiles: reused.length,
|
|
994
1068
|
/** Selected, but the service already held the content. */
|
|
@@ -1121,24 +1195,14 @@ class Uploader {
|
|
|
1121
1195
|
* service would not take it — in which case the caller falls back to sending
|
|
1122
1196
|
* them separately, which is slower and larger but always works.
|
|
1123
1197
|
*/
|
|
1124
|
-
async putPack(repositoryId, items) {
|
|
1125
|
-
const built = await (0, solid_js_1.buildSolidPack)(items.map((item) => ({
|
|
1198
|
+
async putPack(repositoryId, items, alreadyBuilt) {
|
|
1199
|
+
const built = alreadyBuilt ?? await (0, solid_js_1.buildSolidPack)(items.map((item) => ({
|
|
1126
1200
|
sha256: item.declaration.sha256,
|
|
1127
1201
|
body: item.body,
|
|
1128
1202
|
})));
|
|
1129
1203
|
if (!built)
|
|
1130
1204
|
return null;
|
|
1131
|
-
const
|
|
1132
|
-
sha256: built.sha256,
|
|
1133
|
-
storedSha256: built.storedSha256,
|
|
1134
|
-
size: built.size,
|
|
1135
|
-
members: built.members,
|
|
1136
|
-
});
|
|
1137
|
-
const manifestBytes = new TextEncoder().encode(manifest);
|
|
1138
|
-
const packed = new Uint8Array(4 + manifestBytes.byteLength + built.body.byteLength);
|
|
1139
|
-
new DataView(packed.buffer).setUint32(0, manifestBytes.byteLength, false);
|
|
1140
|
-
packed.set(manifestBytes, 4);
|
|
1141
|
-
packed.set(built.body, 4 + manifestBytes.byteLength);
|
|
1205
|
+
const packed = this.frameSolidPack(built);
|
|
1142
1206
|
const answer = await this.call(`/v1/repositories/${repositoryId}/objects/pack`, {
|
|
1143
1207
|
method: "POST",
|
|
1144
1208
|
contentType: "application/octet-stream",
|
|
@@ -1167,6 +1231,216 @@ class Uploader {
|
|
|
1167
1231
|
}
|
|
1168
1232
|
return placed;
|
|
1169
1233
|
}
|
|
1234
|
+
/** The exact HTTP body used for the solid-pack alternative. */
|
|
1235
|
+
frameSolidPack(built) {
|
|
1236
|
+
const manifestBytes = new TextEncoder().encode(JSON.stringify({
|
|
1237
|
+
sha256: built.sha256,
|
|
1238
|
+
storedSha256: built.storedSha256,
|
|
1239
|
+
size: built.size,
|
|
1240
|
+
members: built.members,
|
|
1241
|
+
}));
|
|
1242
|
+
const packed = new Uint8Array(4 + manifestBytes.byteLength + built.body.byteLength);
|
|
1243
|
+
new DataView(packed.buffer).setUint32(0, manifestBytes.byteLength, false);
|
|
1244
|
+
packed.set(manifestBytes, 4);
|
|
1245
|
+
packed.set(built.body, 4 + manifestBytes.byteLength);
|
|
1246
|
+
return packed;
|
|
1247
|
+
}
|
|
1248
|
+
/**
|
|
1249
|
+
* Price several independent file deltas against the already-built solid
|
|
1250
|
+
* pack. The batch is optional and may only replace the pack when every
|
|
1251
|
+
* member has an immutable base and the complete two-way wire cost wins by
|
|
1252
|
+
* the same twenty-percent margin used by the single-file transport.
|
|
1253
|
+
*/
|
|
1254
|
+
async putDeltaBatch(repositoryId, baseVersionId, items, prior, solidPackBytes) {
|
|
1255
|
+
if (!baseVersionId ||
|
|
1256
|
+
solidPackBytes <= 0 ||
|
|
1257
|
+
items.length < 2 ||
|
|
1258
|
+
items.length > delta_js_1.DELTA_BATCH_MAX_FILES ||
|
|
1259
|
+
!(await this.deltaBatchAllowed()))
|
|
1260
|
+
return null;
|
|
1261
|
+
let sourceBytes = 0;
|
|
1262
|
+
let signatureEstimate = 6 + items.length * 4;
|
|
1263
|
+
const bases = [];
|
|
1264
|
+
for (const item of items) {
|
|
1265
|
+
const base = prior.get(item.declaration.path);
|
|
1266
|
+
if (!base ||
|
|
1267
|
+
!base.sha256 ||
|
|
1268
|
+
item.declaration.size > delta_js_1.DELTA_MAX_FILE_BYTES ||
|
|
1269
|
+
base.sourceSize > delta_js_1.DELTA_MAX_FILE_BYTES)
|
|
1270
|
+
return null;
|
|
1271
|
+
if ((0, node_crypto_1.createHash)("sha256").update(item.body).digest("hex") !==
|
|
1272
|
+
item.declaration.sha256)
|
|
1273
|
+
return null;
|
|
1274
|
+
sourceBytes += item.declaration.size;
|
|
1275
|
+
if (sourceBytes > delta_js_1.DELTA_BATCH_MAX_SOURCE_BYTES)
|
|
1276
|
+
return null;
|
|
1277
|
+
signatureEstimate += (0, delta_js_1.estimateDeltaSignatureBytes)(base.sourceSize, 2);
|
|
1278
|
+
bases.push(base);
|
|
1279
|
+
}
|
|
1280
|
+
if (signatureEstimate >= solidPackBytes * 0.8)
|
|
1281
|
+
return null;
|
|
1282
|
+
try {
|
|
1283
|
+
const token = await this.credentials.token();
|
|
1284
|
+
if (!token)
|
|
1285
|
+
return null;
|
|
1286
|
+
const response = await fetch(`${this.credentials.origin()}/v1/repositories/${repositoryId}/objects/delta/signatures`, {
|
|
1287
|
+
method: "POST",
|
|
1288
|
+
headers: {
|
|
1289
|
+
accept: "application/vnd.coderook.delta-signature-batch; version=2",
|
|
1290
|
+
authorization: `Bearer ${token}`,
|
|
1291
|
+
"content-type": "application/json",
|
|
1292
|
+
"user-agent": "CodeRook/0.1",
|
|
1293
|
+
...(0, identify_js_1.clientHeaders)(),
|
|
1294
|
+
},
|
|
1295
|
+
body: JSON.stringify({
|
|
1296
|
+
baseVersionId,
|
|
1297
|
+
files: items.map((item, index) => ({
|
|
1298
|
+
path: item.declaration.path,
|
|
1299
|
+
baseSha256: bases[index].sha256,
|
|
1300
|
+
})),
|
|
1301
|
+
}),
|
|
1302
|
+
signal: this.controller.signal,
|
|
1303
|
+
});
|
|
1304
|
+
if (!response.ok)
|
|
1305
|
+
return null;
|
|
1306
|
+
const signatureBytes = new Uint8Array(await response.arrayBuffer());
|
|
1307
|
+
const signatures = (0, delta_js_1.decodeDeltaSignatureBatch)(signatureBytes);
|
|
1308
|
+
if (signatures.length !== items.length)
|
|
1309
|
+
return null;
|
|
1310
|
+
const framed = (0, delta_js_1.encodeDeltaBatchEnvelope)({
|
|
1311
|
+
baseVersionId,
|
|
1312
|
+
entries: items.map((item, index) => ({
|
|
1313
|
+
path: item.declaration.path,
|
|
1314
|
+
sha256: item.declaration.sha256,
|
|
1315
|
+
mediaType: item.declaration.mediaType,
|
|
1316
|
+
patch: (0, delta_js_1.createDeltaPatch)(item.body, signatures[index]),
|
|
1317
|
+
})),
|
|
1318
|
+
});
|
|
1319
|
+
const wireBytes = signatureBytes.byteLength + framed.byteLength;
|
|
1320
|
+
if (wireBytes >= solidPackBytes * 0.8)
|
|
1321
|
+
return null;
|
|
1322
|
+
const answer = await this.call(`/v1/repositories/${repositoryId}/objects/delta/batch`, {
|
|
1323
|
+
method: "POST",
|
|
1324
|
+
contentType: "application/octet-stream",
|
|
1325
|
+
body: framed,
|
|
1326
|
+
});
|
|
1327
|
+
if (answer.objects.length !== items.length)
|
|
1328
|
+
return null;
|
|
1329
|
+
const objects = new Map();
|
|
1330
|
+
for (const object of answer.objects) {
|
|
1331
|
+
if (!items.some((item) => item.declaration.sha256 === object.sha256)) {
|
|
1332
|
+
return null;
|
|
1333
|
+
}
|
|
1334
|
+
objects.set(object.sha256, {
|
|
1335
|
+
objectId: object.objectId,
|
|
1336
|
+
size: object.storedSize,
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
const expectedDigests = new Set(items.map((item) => item.declaration.sha256)).size;
|
|
1340
|
+
return objects.size === expectedDigests
|
|
1341
|
+
? { objects, sentBytes: wireBytes }
|
|
1342
|
+
: null;
|
|
1343
|
+
}
|
|
1344
|
+
catch {
|
|
1345
|
+
return null;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* Try the negotiated receiver-signature transport for one changed file.
|
|
1350
|
+
*
|
|
1351
|
+
* Any refusal, old deployment, stale base, or poor result returns null and
|
|
1352
|
+
* the caller sends the complete object exactly as it did before. The patch
|
|
1353
|
+
* is never a repository object: the service reconstructs and verifies the
|
|
1354
|
+
* complete target before returning the ordinary object id used below.
|
|
1355
|
+
*/
|
|
1356
|
+
async putDelta(repositoryId, baseVersionId, prior, declaration) {
|
|
1357
|
+
const deltaVersion = await this.deltaVersion();
|
|
1358
|
+
if (deltaVersion === 0)
|
|
1359
|
+
return null;
|
|
1360
|
+
let target;
|
|
1361
|
+
try {
|
|
1362
|
+
target = new Uint8Array(await (0, promises_1.readFile)(declaration.full));
|
|
1363
|
+
}
|
|
1364
|
+
catch {
|
|
1365
|
+
return null;
|
|
1366
|
+
}
|
|
1367
|
+
const digest = (0, node_crypto_1.createHash)("sha256").update(target).digest("hex");
|
|
1368
|
+
if (digest !== declaration.sha256)
|
|
1369
|
+
return null;
|
|
1370
|
+
const ordinary = await (0, compress_js_1.encodeForUpload)(target, declaration.path, await this.gzipAllowed());
|
|
1371
|
+
const signatureEstimate = (0, delta_js_1.estimateDeltaSignatureBytes)(prior.sourceSize, deltaVersion);
|
|
1372
|
+
if (signatureEstimate >= ordinary.body.byteLength * 0.8)
|
|
1373
|
+
return null;
|
|
1374
|
+
try {
|
|
1375
|
+
const signatureBytes = await this.deltaSignature(repositoryId, baseVersionId, declaration.path, prior.sha256, deltaVersion);
|
|
1376
|
+
const signature = (0, delta_js_1.parseDeltaSignature)(signatureBytes);
|
|
1377
|
+
const patch = (0, delta_js_1.createDeltaPatch)(target, signature);
|
|
1378
|
+
let framed;
|
|
1379
|
+
if (signature.version === 2) {
|
|
1380
|
+
framed = (0, delta_js_1.encodeDeltaEnvelope)({
|
|
1381
|
+
baseVersionId,
|
|
1382
|
+
path: declaration.path,
|
|
1383
|
+
sha256: declaration.sha256,
|
|
1384
|
+
mediaType: declaration.mediaType,
|
|
1385
|
+
patch,
|
|
1386
|
+
});
|
|
1387
|
+
}
|
|
1388
|
+
else {
|
|
1389
|
+
const manifest = new TextEncoder().encode(JSON.stringify({
|
|
1390
|
+
baseVersionId,
|
|
1391
|
+
path: declaration.path,
|
|
1392
|
+
baseSha256: prior.sha256,
|
|
1393
|
+
sha256: declaration.sha256,
|
|
1394
|
+
size: target.byteLength,
|
|
1395
|
+
mediaType: declaration.mediaType,
|
|
1396
|
+
}));
|
|
1397
|
+
framed = new Uint8Array(4 + manifest.byteLength + patch.byteLength);
|
|
1398
|
+
new DataView(framed.buffer).setUint32(0, manifest.byteLength, false);
|
|
1399
|
+
framed.set(manifest, 4);
|
|
1400
|
+
framed.set(patch, 4 + manifest.byteLength);
|
|
1401
|
+
}
|
|
1402
|
+
// Count both directions. A patch that only looks small because its
|
|
1403
|
+
// receiver signature was ignored is not an improvement.
|
|
1404
|
+
const wireBytes = signatureBytes.byteLength + framed.byteLength;
|
|
1405
|
+
if (wireBytes >= ordinary.body.byteLength * 0.8)
|
|
1406
|
+
return null;
|
|
1407
|
+
const answer = await this.call(`/v1/repositories/${repositoryId}/objects/delta`, {
|
|
1408
|
+
method: "POST",
|
|
1409
|
+
contentType: "application/octet-stream",
|
|
1410
|
+
body: framed,
|
|
1411
|
+
});
|
|
1412
|
+
return {
|
|
1413
|
+
objectId: answer.objectId,
|
|
1414
|
+
storedSize: answer.storedSize,
|
|
1415
|
+
sentBytes: wireBytes,
|
|
1416
|
+
};
|
|
1417
|
+
}
|
|
1418
|
+
catch {
|
|
1419
|
+
// This is an optional transport. The complete-object path is authority.
|
|
1420
|
+
return null;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
async deltaSignature(repositoryId, baseVersionId, filePath, baseSha256, version) {
|
|
1424
|
+
const token = await this.credentials.token();
|
|
1425
|
+
if (!token)
|
|
1426
|
+
throw new Error("Sign in again before uploading");
|
|
1427
|
+
const response = await fetch(`${this.credentials.origin()}/v1/repositories/${repositoryId}/objects/delta/signature`, {
|
|
1428
|
+
method: "POST",
|
|
1429
|
+
headers: {
|
|
1430
|
+
accept: `application/vnd.coderook.delta-signature; version=${version}`,
|
|
1431
|
+
authorization: `Bearer ${token}`,
|
|
1432
|
+
"content-type": "application/json",
|
|
1433
|
+
"user-agent": "CodeRook/0.1",
|
|
1434
|
+
...(0, identify_js_1.clientHeaders)(),
|
|
1435
|
+
},
|
|
1436
|
+
body: JSON.stringify({ baseVersionId, path: filePath, baseSha256 }),
|
|
1437
|
+
signal: this.controller.signal,
|
|
1438
|
+
});
|
|
1439
|
+
if (!response.ok) {
|
|
1440
|
+
throw new Error(`Delta signature failed with HTTP ${response.status}`);
|
|
1441
|
+
}
|
|
1442
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
1443
|
+
}
|
|
1170
1444
|
/**
|
|
1171
1445
|
* Send many small objects in one request.
|
|
1172
1446
|
*
|
|
@@ -1301,8 +1575,8 @@ class Uploader {
|
|
|
1301
1575
|
*/
|
|
1302
1576
|
const CHUNK_LANES = 4;
|
|
1303
1577
|
const active = new Set();
|
|
1304
|
-
const dispatch = async (piece, at) => {
|
|
1305
|
-
const task = this.sendChunk(repositoryId, declaration, piece, chunks, at, note).finally(() => {
|
|
1578
|
+
const dispatch = async (piece, at, digest) => {
|
|
1579
|
+
const task = this.sendChunk(repositoryId, declaration, piece, digest, chunks, at, note).finally(() => {
|
|
1306
1580
|
active.delete(task);
|
|
1307
1581
|
done += piece.length;
|
|
1308
1582
|
onOffset(done);
|
|
@@ -1311,6 +1585,51 @@ class Uploader {
|
|
|
1311
1585
|
if (active.size >= CHUNK_LANES)
|
|
1312
1586
|
await Promise.race(active);
|
|
1313
1587
|
};
|
|
1588
|
+
/*
|
|
1589
|
+
Ask about a bounded group of chunks before sending any of them.
|
|
1590
|
+
|
|
1591
|
+
The old path asked once per chunk, with four GETs in flight. Besides
|
|
1592
|
+
spending one network round trip per piece, an unavailable lookup was
|
|
1593
|
+
deliberately treated as a miss, so a transient failure retransmitted
|
|
1594
|
+
every otherwise reusable chunk. The website already uses the bulk
|
|
1595
|
+
endpoint. Reusing it here gives Desktop and CLI the same wire behaviour
|
|
1596
|
+
while retaining the safe "send it if unsure" fallback.
|
|
1597
|
+
|
|
1598
|
+
A batch is no larger than the existing chunk lane count, so this does
|
|
1599
|
+
not increase the memory ceiling: at most four bounded pieces are held.
|
|
1600
|
+
*/
|
|
1601
|
+
const batch = [];
|
|
1602
|
+
const flush = async () => {
|
|
1603
|
+
if (!batch.length)
|
|
1604
|
+
return;
|
|
1605
|
+
const ready = batch.splice(0, batch.length);
|
|
1606
|
+
const held = await this.storedInBulk(repositoryId, ready.map((item) => item.digest));
|
|
1607
|
+
for (const item of ready) {
|
|
1608
|
+
const stored = held.get(item.digest);
|
|
1609
|
+
if (stored) {
|
|
1610
|
+
note("reused", 0);
|
|
1611
|
+
chunks[item.at] = {
|
|
1612
|
+
objectId: stored.objectId,
|
|
1613
|
+
sourceSize: item.piece.length,
|
|
1614
|
+
storedSize: stored.size,
|
|
1615
|
+
};
|
|
1616
|
+
done += item.piece.length;
|
|
1617
|
+
onOffset(done);
|
|
1618
|
+
continue;
|
|
1619
|
+
}
|
|
1620
|
+
await dispatch(item.piece, item.at, item.digest);
|
|
1621
|
+
}
|
|
1622
|
+
await Promise.all(active);
|
|
1623
|
+
};
|
|
1624
|
+
const queue = async (piece) => {
|
|
1625
|
+
batch.push({
|
|
1626
|
+
piece,
|
|
1627
|
+
at: ordinal++,
|
|
1628
|
+
digest: (0, node_crypto_1.createHash)("sha256").update(piece).digest("hex"),
|
|
1629
|
+
});
|
|
1630
|
+
if (batch.length >= CHUNK_LANES)
|
|
1631
|
+
await flush();
|
|
1632
|
+
};
|
|
1314
1633
|
let ordinal = 0;
|
|
1315
1634
|
let pending = Buffer.alloc(0);
|
|
1316
1635
|
for await (const block of (0, node_fs_1.createReadStream)(declaration.full, {
|
|
@@ -1321,7 +1640,7 @@ class Uploader {
|
|
|
1321
1640
|
pending = pending.length ? Buffer.concat([pending, incoming]) : incoming;
|
|
1322
1641
|
let from = 0;
|
|
1323
1642
|
for (const cut of (0, chunking_js_1.cutPoints)(pending, profile)) {
|
|
1324
|
-
await
|
|
1643
|
+
await queue(Buffer.from(pending.subarray(from, cut)));
|
|
1325
1644
|
from = cut;
|
|
1326
1645
|
}
|
|
1327
1646
|
pending = Buffer.from(pending.subarray(from));
|
|
@@ -1333,8 +1652,8 @@ class Uploader {
|
|
|
1333
1652
|
Dropping it truncates every chunked file by exactly its last piece.
|
|
1334
1653
|
*/
|
|
1335
1654
|
if (pending.length)
|
|
1336
|
-
await
|
|
1337
|
-
await
|
|
1655
|
+
await queue(pending);
|
|
1656
|
+
await flush();
|
|
1338
1657
|
/*
|
|
1339
1658
|
Order is the file. Every position must be filled: a hole would mean a
|
|
1340
1659
|
chunk that never landed, and publishing around it would produce a version
|
|
@@ -1351,7 +1670,7 @@ class Uploader {
|
|
|
1351
1670
|
};
|
|
1352
1671
|
}
|
|
1353
1672
|
/** Send one chunk, reusing it if the account already holds those bytes. */
|
|
1354
|
-
async sendChunk(repositoryId, declaration, piece,
|
|
1673
|
+
async sendChunk(repositoryId, declaration, piece, digest,
|
|
1355
1674
|
/*
|
|
1356
1675
|
Written at a known position rather than appended. The chunks are sent
|
|
1357
1676
|
several at a time and finish in whatever order the network decides, but
|
|
@@ -1362,48 +1681,27 @@ class Uploader {
|
|
|
1362
1681
|
chunks, at, note) {
|
|
1363
1682
|
{
|
|
1364
1683
|
this.check();
|
|
1365
|
-
const
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
const
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1684
|
+
const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, await this.gzipAllowed());
|
|
1685
|
+
const query = encoded.encoding === "gzip"
|
|
1686
|
+
? `?kind=chunk&role=chunk&encoding=gzip` +
|
|
1687
|
+
`&logicalSize=${piece.length}` +
|
|
1688
|
+
`&storedSha256=${encoded.storedSha256}`
|
|
1689
|
+
: `?kind=chunk&role=chunk`;
|
|
1690
|
+
const stored = await this.call(`/v1/repositories/${repositoryId}/objects/${digest}${query}`, {
|
|
1691
|
+
method: "PUT",
|
|
1692
|
+
contentType: "application/octet-stream",
|
|
1693
|
+
body: encoded.body,
|
|
1694
|
+
}).catch((error) => {
|
|
1695
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1696
|
+
throw new Error(`${declaration.path}: ${reason}`);
|
|
1375
1697
|
});
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
}
|
|
1384
|
-
else {
|
|
1385
|
-
const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, await this.gzipAllowed());
|
|
1386
|
-
const query = encoded.encoding === "gzip"
|
|
1387
|
-
? `?kind=chunk&role=chunk&encoding=gzip` +
|
|
1388
|
-
`&logicalSize=${piece.length}` +
|
|
1389
|
-
`&storedSha256=${encoded.storedSha256}`
|
|
1390
|
-
: `?kind=chunk&role=chunk`;
|
|
1391
|
-
const stored = await this.call(`/v1/repositories/${repositoryId}/objects/${digest}${query}`, {
|
|
1392
|
-
method: "PUT",
|
|
1393
|
-
contentType: "application/octet-stream",
|
|
1394
|
-
body: encoded.body,
|
|
1395
|
-
}).catch((error) => {
|
|
1396
|
-
const reason = error instanceof Error ? error.message : String(error);
|
|
1397
|
-
throw new Error(`${declaration.path}: ${reason}`);
|
|
1398
|
-
});
|
|
1399
|
-
note("sent", encoded.body.byteLength);
|
|
1400
|
-
chunks[at] = {
|
|
1401
|
-
objectId: stored.objectId,
|
|
1402
|
-
sourceSize: piece.length,
|
|
1403
|
-
/* What the service keeps, which is smaller when the piece gzipped. */
|
|
1404
|
-
storedSize: stored.storedSize ?? stored.size,
|
|
1405
|
-
};
|
|
1406
|
-
}
|
|
1698
|
+
note("sent", encoded.body.byteLength);
|
|
1699
|
+
chunks[at] = {
|
|
1700
|
+
objectId: stored.objectId,
|
|
1701
|
+
sourceSize: piece.length,
|
|
1702
|
+
/* What the service keeps, which is smaller when the piece gzipped. */
|
|
1703
|
+
storedSize: stored.storedSize ?? stored.size,
|
|
1704
|
+
};
|
|
1407
1705
|
}
|
|
1408
1706
|
}
|
|
1409
1707
|
async putDirect(repositoryId, declaration) {
|
|
@@ -1470,6 +1768,23 @@ class Uploader {
|
|
|
1470
1768
|
this.packingSupported = (await this.serviceFeatures()).includes("solid-packs");
|
|
1471
1769
|
return this.packingSupported;
|
|
1472
1770
|
}
|
|
1771
|
+
async deltaVersion() {
|
|
1772
|
+
if (this.deltaProtocol !== null)
|
|
1773
|
+
return this.deltaProtocol;
|
|
1774
|
+
const features = await this.serviceFeatures();
|
|
1775
|
+
this.deltaProtocol = features.includes("delta-transport-v2")
|
|
1776
|
+
? 2
|
|
1777
|
+
: features.includes("delta-transport")
|
|
1778
|
+
? 1
|
|
1779
|
+
: 0;
|
|
1780
|
+
return this.deltaProtocol;
|
|
1781
|
+
}
|
|
1782
|
+
async deltaBatchAllowed() {
|
|
1783
|
+
if (this.deltaBatchSupported !== null)
|
|
1784
|
+
return this.deltaBatchSupported;
|
|
1785
|
+
this.deltaBatchSupported = (await this.serviceFeatures()).includes("delta-batch-v2");
|
|
1786
|
+
return this.deltaBatchSupported;
|
|
1787
|
+
}
|
|
1473
1788
|
async chunkingAllowed() {
|
|
1474
1789
|
if (this.chunkingSupported !== null)
|
|
1475
1790
|
return this.chunkingSupported;
|