@coderook/cli 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/dist/cli/src/cli.js +20 -3
- package/dist/cli/src/config.js +12 -2
- package/dist/desktop-app/src/main/cbx.js +6 -40
- package/dist/desktop-app/src/main/staging.js +1 -1
- package/dist/desktop-app/src/main/upload.js +151 -93
- package/dist/desktop-app/src/shared/chunking.js +128 -0
- package/dist/desktop-app/src/shared/publication.js +21 -0
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "coderook",
|
|
3
3
|
"displayName": "CodeRook",
|
|
4
4
|
"description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.22.0",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "ACCA Gaming Productions",
|
|
8
8
|
"url": "https://coderook.com"
|
package/dist/cli/src/cli.js
CHANGED
|
@@ -167,11 +167,15 @@ async function reconcile(folder, options = {}) {
|
|
|
167
167
|
folder that has never been reconciled — or was linked before versions
|
|
168
168
|
were recorded — adopts the newest as its starting point.
|
|
169
169
|
*/
|
|
170
|
-
const known = link?.
|
|
171
|
-
const behind = known && known.
|
|
170
|
+
const known = link?.baseVersionId && link.manifest ? link : null;
|
|
171
|
+
const behind = known && known.baseVersionId !== latest.id
|
|
172
172
|
? { sequence: latest.sequence, id: latest.id }
|
|
173
173
|
: null;
|
|
174
174
|
if (known) {
|
|
175
|
+
if (known.observedHeadVersionId !== latest.id) {
|
|
176
|
+
known.observedHeadVersionId = latest.id;
|
|
177
|
+
await (0, config_js_1.writeLink)(folder, known);
|
|
178
|
+
}
|
|
175
179
|
if (behind && !options.quiet) {
|
|
176
180
|
console.log(dim(`The account is on v${behind.sequence}; this folder is based on v${known.sequence}.`));
|
|
177
181
|
}
|
|
@@ -196,6 +200,8 @@ async function reconcile(folder, options = {}) {
|
|
|
196
200
|
slug: project.slug,
|
|
197
201
|
sequence: latest.sequence,
|
|
198
202
|
versionId: latest.id,
|
|
203
|
+
baseVersionId: latest.id,
|
|
204
|
+
observedHeadVersionId: latest.id,
|
|
199
205
|
// Adopted whole, so the folder is taken to hold what the version holds.
|
|
200
206
|
local: Object.fromEntries(baseline),
|
|
201
207
|
manifest: Object.fromEntries(baseline),
|
|
@@ -479,9 +485,14 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
|
|
|
479
485
|
result = await uploader.run({
|
|
480
486
|
localPath: folder,
|
|
481
487
|
include: files.map((file) => file.path),
|
|
488
|
+
deletions: files.filter((file) => file.deleted).map((file) => file.path),
|
|
482
489
|
message,
|
|
483
490
|
projectName: link?.slug ?? node_path_1.default.basename(folder),
|
|
484
491
|
repositoryId: link?.repositoryId ?? null,
|
|
492
|
+
// Every current CLI publish states its ancestry. A brand-new project
|
|
493
|
+
// is explicitly based on an empty Track; a linked folder names the
|
|
494
|
+
// immutable Version it was last reconciled with.
|
|
495
|
+
baseVersionId: link?.baseVersionId ?? null,
|
|
485
496
|
/*
|
|
486
497
|
The line this save belongs on: named on the command, or whichever one
|
|
487
498
|
the folder is set to. Sent by name rather than resolved here, because
|
|
@@ -495,7 +506,9 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
|
|
|
495
506
|
known version. A folder linked before versions were recorded says
|
|
496
507
|
nothing rather than guessing, and publishes as it always did.
|
|
497
508
|
*/
|
|
498
|
-
...(link?.
|
|
509
|
+
...(link?.baseVersionId
|
|
510
|
+
? { expectedHeadVersionId: link.baseVersionId }
|
|
511
|
+
: {}),
|
|
499
512
|
/*
|
|
500
513
|
What this folder believed the project held. It is how the service
|
|
501
514
|
tells a file this person deleted from one they never had, and
|
|
@@ -570,6 +583,8 @@ The connection failed: ${text}`));
|
|
|
570
583
|
sequence: result.sequence,
|
|
571
584
|
// What this folder is now working from, so the next submit can say so.
|
|
572
585
|
versionId: result.versionId,
|
|
586
|
+
baseVersionId: result.versionId,
|
|
587
|
+
observedHeadVersionId: result.versionId,
|
|
573
588
|
local: result.local,
|
|
574
589
|
manifest: result.manifest,
|
|
575
590
|
});
|
|
@@ -795,6 +810,8 @@ held, mode = "replace") {
|
|
|
795
810
|
slug,
|
|
796
811
|
sequence: latest.sequence,
|
|
797
812
|
versionId: latest.id,
|
|
813
|
+
baseVersionId: latest.id,
|
|
814
|
+
observedHeadVersionId: latest.id,
|
|
798
815
|
// Fetching writes every file, so the folder now holds what the
|
|
799
816
|
// version holds and the two agree again.
|
|
800
817
|
local: result.manifest,
|
package/dist/cli/src/config.js
CHANGED
|
@@ -143,13 +143,23 @@ async function readLinks() {
|
|
|
143
143
|
}
|
|
144
144
|
async function readLink(localPath) {
|
|
145
145
|
const links = await readLinks();
|
|
146
|
-
|
|
146
|
+
const link = links[keyFor(localPath)] ?? links[legacyKeyFor(localPath)] ?? null;
|
|
147
|
+
if (!link)
|
|
148
|
+
return null;
|
|
149
|
+
return {
|
|
150
|
+
...link,
|
|
151
|
+
baseVersionId: link.baseVersionId ?? link.versionId,
|
|
152
|
+
};
|
|
147
153
|
}
|
|
148
154
|
async function writeLink(localPath, link) {
|
|
149
155
|
const links = await readLinks();
|
|
150
156
|
// A folder recorded under the old key moves to the new one rather than
|
|
151
157
|
// being left behind as a second connection to the same place.
|
|
152
158
|
delete links[legacyKeyFor(localPath)];
|
|
153
|
-
|
|
159
|
+
const baseVersionId = link.baseVersionId ?? link.versionId;
|
|
160
|
+
links[keyFor(localPath)] = {
|
|
161
|
+
...link,
|
|
162
|
+
...(baseVersionId ? { baseVersionId, versionId: baseVersionId } : {}),
|
|
163
|
+
};
|
|
154
164
|
await writePrivate(linksFile(), JSON.stringify(links, null, 2));
|
|
155
165
|
}
|
|
@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.FOOTER_MAGIC = exports.MANIFEST_MAGIC = exports.CHUNK_MAGIC = exports.HEADER_MAGIC = void 0;
|
|
7
|
-
exports.cutPoints = cutPoints;
|
|
6
|
+
exports.FOOTER_MAGIC = exports.MANIFEST_MAGIC = exports.CHUNK_MAGIC = exports.HEADER_MAGIC = exports.cutPoints = void 0;
|
|
8
7
|
exports.packBundle = packBundle;
|
|
9
8
|
exports.readManifest = readManifest;
|
|
10
9
|
exports.unpackBundle = unpackBundle;
|
|
@@ -32,6 +31,9 @@ const promises_1 = require("node:fs/promises");
|
|
|
32
31
|
const node_path_1 = __importDefault(require("node:path"));
|
|
33
32
|
const node_zlib_1 = require("node:zlib");
|
|
34
33
|
const download_js_1 = require("./download.js");
|
|
34
|
+
const chunking_js_1 = require("../shared/chunking.js");
|
|
35
|
+
var chunking_js_2 = require("../shared/chunking.js");
|
|
36
|
+
Object.defineProperty(exports, "cutPoints", { enumerable: true, get: function () { return chunking_js_2.cutPoints; } });
|
|
35
37
|
exports.HEADER_MAGIC = "CBX1";
|
|
36
38
|
exports.CHUNK_MAGIC = "CHNK";
|
|
37
39
|
exports.MANIFEST_MAGIC = "MANF";
|
|
@@ -48,9 +50,6 @@ const CODEC_ZSTD = 1;
|
|
|
48
50
|
const ZSTD_LEVEL = 3;
|
|
49
51
|
const MIN_COMPRESSION_SAVING = 0.02;
|
|
50
52
|
const READ_SIZE = 8 * 1024 * 1024;
|
|
51
|
-
const MIN_CHUNK_SIZE = 2 * 1024 * 1024;
|
|
52
|
-
const AVG_CHUNK_SIZE = 8 * 1024 * 1024;
|
|
53
|
-
const MAX_CHUNK_SIZE = 32 * 1024 * 1024;
|
|
54
53
|
const zstd = (raw) => (0, node_zlib_1.zstdCompressSync)(raw, {
|
|
55
54
|
params: { [node_zlib_1.constants.ZSTD_c_compressionLevel]: ZSTD_LEVEL },
|
|
56
55
|
});
|
|
@@ -64,36 +63,6 @@ function sha256(data) {
|
|
|
64
63
|
* the `fastcdc` package; cut positions do not have to agree between writers
|
|
65
64
|
* for an archive to be readable, because the manifest records the chunk list.
|
|
66
65
|
*/
|
|
67
|
-
const GEAR = (() => {
|
|
68
|
-
const table = new Uint32Array(256);
|
|
69
|
-
let seed = 0x9e3779b9;
|
|
70
|
-
for (let index = 0; index < 256; index += 1) {
|
|
71
|
-
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
|
72
|
-
table[index] = seed;
|
|
73
|
-
}
|
|
74
|
-
return table;
|
|
75
|
-
})();
|
|
76
|
-
const NORMAL_MASK = 0x0003_5000;
|
|
77
|
-
const SMALL_MASK = 0x0003_5403;
|
|
78
|
-
/** Cut positions inside `buffer`; the tail is left for the next read. */
|
|
79
|
-
function cutPoints(buffer) {
|
|
80
|
-
const cuts = [];
|
|
81
|
-
let from = 0;
|
|
82
|
-
let hash = 0;
|
|
83
|
-
for (let at = 0; at < buffer.length; at += 1) {
|
|
84
|
-
hash = ((hash << 1) + GEAR[buffer[at]]) >>> 0;
|
|
85
|
-
const length = at - from + 1;
|
|
86
|
-
if (length < MIN_CHUNK_SIZE)
|
|
87
|
-
continue;
|
|
88
|
-
const mask = length < AVG_CHUNK_SIZE ? SMALL_MASK : NORMAL_MASK;
|
|
89
|
-
if (length >= MAX_CHUNK_SIZE || (hash & mask) === 0) {
|
|
90
|
-
cuts.push(at + 1);
|
|
91
|
-
from = at + 1;
|
|
92
|
-
hash = 0;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
return cuts;
|
|
96
|
-
}
|
|
97
66
|
function chunkFrame(raw) {
|
|
98
67
|
const rawDigest = (0, node_crypto_1.createHash)("sha256").update(raw).digest();
|
|
99
68
|
let codec = CODEC_NONE;
|
|
@@ -199,7 +168,7 @@ async function packBundle(request, report) {
|
|
|
199
168
|
whole.update(block);
|
|
200
169
|
pending = pending.length ? Buffer.concat([pending, block]) : block;
|
|
201
170
|
let from = 0;
|
|
202
|
-
for (const cut of cutPoints(pending)) {
|
|
171
|
+
for (const cut of (0, chunking_js_1.cutPoints)(pending)) {
|
|
203
172
|
await emit(pending.subarray(from, cut));
|
|
204
173
|
from = cut;
|
|
205
174
|
}
|
|
@@ -225,10 +194,7 @@ async function packBundle(request, report) {
|
|
|
225
194
|
source_kind: request.sourceKind ?? "directory",
|
|
226
195
|
created_unix_ns: Date.now() * 1e6,
|
|
227
196
|
chunking: {
|
|
228
|
-
|
|
229
|
-
min_size: MIN_CHUNK_SIZE,
|
|
230
|
-
avg_size: AVG_CHUNK_SIZE,
|
|
231
|
-
max_size: MAX_CHUNK_SIZE,
|
|
197
|
+
...(0, chunking_js_1.manifestChunking)(chunking_js_1.DEFAULT_CHUNK_PROFILE),
|
|
232
198
|
read_size: READ_SIZE,
|
|
233
199
|
small_file_packing: { enabled: false },
|
|
234
200
|
},
|
|
@@ -69,7 +69,7 @@ exports.describePlan = describePlan;
|
|
|
69
69
|
disagreement.
|
|
70
70
|
*/
|
|
71
71
|
exports.PACKABLE_LIMIT = 1024 * 1024;
|
|
72
|
-
exports.CHUNK_THRESHOLD =
|
|
72
|
+
exports.CHUNK_THRESHOLD = 8 * 1024 * 1024;
|
|
73
73
|
exports.DIRECT_LIMIT = 95 * 1024 * 1024;
|
|
74
74
|
function bandOf(size) {
|
|
75
75
|
if (size <= exports.PACKABLE_LIMIT)
|
|
@@ -21,7 +21,8 @@ const identify_js_1 = require("./identify.js");
|
|
|
21
21
|
const worktree_js_1 = require("./worktree.js");
|
|
22
22
|
const publish_name_js_1 = require("./publish_name.js");
|
|
23
23
|
const compress_js_1 = require("./compress.js");
|
|
24
|
-
const
|
|
24
|
+
const chunking_js_1 = require("../shared/chunking.js");
|
|
25
|
+
const publication_js_1 = require("../shared/publication.js");
|
|
25
26
|
const profile_js_1 = require("./profile.js");
|
|
26
27
|
const solid_js_1 = require("./solid.js");
|
|
27
28
|
const retry_js_1 = require("./retry.js");
|
|
@@ -34,7 +35,7 @@ const DIRECT_LIMIT = 95 * 1024 * 1024;
|
|
|
34
35
|
chunker's own average chunk is eight megabytes, so a smaller file would
|
|
35
36
|
usually come out as one chunk anyway.
|
|
36
37
|
*/
|
|
37
|
-
const CHUNK_THRESHOLD =
|
|
38
|
+
const CHUNK_THRESHOLD = 8 * 1024 * 1024;
|
|
38
39
|
/*
|
|
39
40
|
How many files are sent at once. Six is the limit browsers settled on per
|
|
40
41
|
host for the same reason: enough to keep the link busy through the latency
|
|
@@ -249,12 +250,11 @@ class Uploader {
|
|
|
249
250
|
if (!ticked.size)
|
|
250
251
|
throw new Error("Nothing is selected to upload");
|
|
251
252
|
const prior = request.repositoryId
|
|
252
|
-
? await this.
|
|
253
|
+
? await this.versionFiles(request.repositoryId, request.baseVersionId)
|
|
253
254
|
: new Map();
|
|
254
255
|
// Ticked files are sent; everything else the project still has keeps the
|
|
255
256
|
// copy already stored, and a file that is new but unticked is left out.
|
|
256
257
|
const sending = everything.filter((file) => ticked.has(file.path));
|
|
257
|
-
const onDiskNow = new Set(everything.map((file) => file.path));
|
|
258
258
|
/*
|
|
259
259
|
Everything the version already holds is kept, except what this
|
|
260
260
|
workspace deliberately removed.
|
|
@@ -265,28 +265,21 @@ class Uploader {
|
|
|
265
265
|
next save. Reading this from the disk scan alone is what silently lost
|
|
266
266
|
other people's files.
|
|
267
267
|
*/
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
added: 0,
|
|
284
|
-
removed: 0,
|
|
285
|
-
included: true,
|
|
286
|
-
deleted: false,
|
|
287
|
-
binary: false,
|
|
288
|
-
lines: 0,
|
|
289
|
-
}));
|
|
268
|
+
const reused = (0, publication_js_1.retainedBasePaths)(prior.keys(), ticked).map((path) => {
|
|
269
|
+
const held = prior.get(path);
|
|
270
|
+
return {
|
|
271
|
+
path,
|
|
272
|
+
sourceSize: held.sourceSize,
|
|
273
|
+
storedSize: held.storedSize,
|
|
274
|
+
mediaType: held.mediaType,
|
|
275
|
+
added: 0,
|
|
276
|
+
removed: 0,
|
|
277
|
+
included: true,
|
|
278
|
+
deleted: false,
|
|
279
|
+
binary: false,
|
|
280
|
+
lines: 0,
|
|
281
|
+
};
|
|
282
|
+
});
|
|
290
283
|
/*
|
|
291
284
|
A ticked path that the rescan cannot see is either a deletion or a file
|
|
292
285
|
that has gone since the changes list was drawn. Deletions are meant to
|
|
@@ -294,7 +287,8 @@ class Uploader {
|
|
|
294
287
|
so it is collected and reported rather than quietly skipped.
|
|
295
288
|
*/
|
|
296
289
|
const onDisk = new Set(everything.map((file) => file.path));
|
|
297
|
-
const
|
|
290
|
+
const deletions = new Set(request.deletions ?? []);
|
|
291
|
+
const vanished = [...ticked].filter((chosen) => !onDisk.has(chosen) && !deletions.has(chosen));
|
|
298
292
|
// 1. Measure and hash. This is what makes an unchanged file free.
|
|
299
293
|
const declarations = [];
|
|
300
294
|
let totalBytes = 0;
|
|
@@ -705,8 +699,11 @@ class Uploader {
|
|
|
705
699
|
whole: cutting it up would trade one request for several and save
|
|
706
700
|
nothing.
|
|
707
701
|
*/
|
|
708
|
-
|
|
709
|
-
|
|
702
|
+
const chunkProfile = (0, chunking_js_1.profileForFileSize)(declaration.size);
|
|
703
|
+
if (chunkProfile &&
|
|
704
|
+
declaration.size >= CHUNK_THRESHOLD &&
|
|
705
|
+
(await this.chunkingAllowed())) {
|
|
706
|
+
const pieces = await this.putChunked(repositoryId, declaration, chunkProfile, (offset) => {
|
|
710
707
|
inFlight.set(index, offset);
|
|
711
708
|
report({
|
|
712
709
|
stage: "upload",
|
|
@@ -721,7 +718,14 @@ class Uploader {
|
|
|
721
718
|
});
|
|
722
719
|
});
|
|
723
720
|
inFlight.delete(index);
|
|
724
|
-
|
|
721
|
+
/*
|
|
722
|
+
A chunk is not a file. Counting every reused piece here made a
|
|
723
|
+
one-file save report a negative number of sent files. The selected
|
|
724
|
+
file counts as already stored only when every one of its pieces was
|
|
725
|
+
reused and no payload crossed the wire.
|
|
726
|
+
*/
|
|
727
|
+
if (pieces.sentBytes === 0)
|
|
728
|
+
alreadyOnAccount += 1;
|
|
725
729
|
uploaded.push({
|
|
726
730
|
path: declaration.path,
|
|
727
731
|
sha256: declaration.sha256,
|
|
@@ -872,7 +876,12 @@ class Uploader {
|
|
|
872
876
|
tool must never have.
|
|
873
877
|
*/
|
|
874
878
|
const named = new Set(contents.map((item) => item.path));
|
|
875
|
-
const dropped = [
|
|
879
|
+
const dropped = [
|
|
880
|
+
...new Set([
|
|
881
|
+
...vanished,
|
|
882
|
+
...[...ticked].filter((chosen) => !named.has(chosen) && !deletions.has(chosen)),
|
|
883
|
+
]),
|
|
884
|
+
];
|
|
876
885
|
if (dropped.length) {
|
|
877
886
|
const shown = dropped.slice(0, 5).join(", ");
|
|
878
887
|
const rest = dropped.length > 5 ? ` and ${dropped.length - 5} more` : "";
|
|
@@ -902,6 +911,12 @@ class Uploader {
|
|
|
902
911
|
contentType: "application/json",
|
|
903
912
|
body: JSON.stringify({
|
|
904
913
|
message: request.message.trim() || "Saved from the CodeRook desktop app",
|
|
914
|
+
...(request.baseVersionId === undefined
|
|
915
|
+
? {}
|
|
916
|
+
: {
|
|
917
|
+
ancestryMode: "required",
|
|
918
|
+
baseVersionId: request.baseVersionId,
|
|
919
|
+
}),
|
|
905
920
|
...(request.expectedHeadVersionId === undefined
|
|
906
921
|
? {}
|
|
907
922
|
: { expectedHeadVersionId: request.expectedHeadVersionId }),
|
|
@@ -951,6 +966,17 @@ class Uploader {
|
|
|
951
966
|
})),
|
|
952
967
|
}),
|
|
953
968
|
});
|
|
969
|
+
/*
|
|
970
|
+
A clean stale-base save may contain a server-side merge. The candidate
|
|
971
|
+
manifest built above is then not the Version that landed: it lacks the
|
|
972
|
+
other publisher's safe changes. Persist the service's accepted snapshot
|
|
973
|
+
instead. A repeated attempt predates manifest echoing, so read that
|
|
974
|
+
Version once rather than guessing.
|
|
975
|
+
*/
|
|
976
|
+
let acceptedManifest = (0, publication_js_1.acceptedManifestDigests)(completed.manifest?.files);
|
|
977
|
+
if (!completed.mergeTrack && !Object.keys(acceptedManifest).length) {
|
|
978
|
+
acceptedManifest = Object.fromEntries([...(await this.versionFiles(repositoryId, completed.version.id))].map(([file, descriptor]) => [file, descriptor.sha256]));
|
|
979
|
+
}
|
|
954
980
|
report({
|
|
955
981
|
stage: "done",
|
|
956
982
|
files: declarations.length,
|
|
@@ -967,9 +993,9 @@ class Uploader {
|
|
|
967
993
|
sequence: completed.version.sequence,
|
|
968
994
|
...(completed.mergeTrack ? { mergeTrack: completed.mergeTrack } : {}),
|
|
969
995
|
...(completed.repeated ? { repeated: true } : {}),
|
|
970
|
-
sourceBytes,
|
|
971
|
-
storedBytes,
|
|
972
|
-
sentBytes,
|
|
996
|
+
sourceBytes: completed.version.sourceSize ?? sourceBytes,
|
|
997
|
+
storedBytes: completed.version.storedSize ?? storedBytes,
|
|
998
|
+
sentBytes: transferred,
|
|
973
999
|
sentFiles: uploaded.length - alreadyOnAccount,
|
|
974
1000
|
reusedFiles: reused.length,
|
|
975
1001
|
/** Selected, but the service already held the content. */
|
|
@@ -991,22 +1017,30 @@ class Uploader {
|
|
|
991
1017
|
const before = request.known?.[file.path] ?? prior.get(file.path)?.sha256;
|
|
992
1018
|
return before ? [[file.path, before]] : [];
|
|
993
1019
|
})),
|
|
994
|
-
manifest:
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1020
|
+
manifest: Object.keys(acceptedManifest).length
|
|
1021
|
+
? acceptedManifest
|
|
1022
|
+
: {
|
|
1023
|
+
...Object.fromEntries(reused.map((file) => [file.path, prior.get(file.path).sha256])),
|
|
1024
|
+
...Object.fromEntries(declarations.map((declaration) => [declaration.path, declaration.sha256])),
|
|
1025
|
+
},
|
|
998
1026
|
};
|
|
999
1027
|
}
|
|
1000
1028
|
/** What the newest version holds, so unchanged files can point at it. */
|
|
1001
|
-
async
|
|
1029
|
+
async versionFiles(repositoryId, versionId) {
|
|
1002
1030
|
const held = new Map();
|
|
1003
1031
|
try {
|
|
1004
|
-
|
|
1005
|
-
//
|
|
1006
|
-
|
|
1007
|
-
|
|
1032
|
+
// Explicit null is a genuinely empty base. Undefined belongs only to
|
|
1033
|
+
// legacy callers, which retain their historical newest-Version lookup.
|
|
1034
|
+
if (versionId === null)
|
|
1035
|
+
return held;
|
|
1036
|
+
let wanted = versionId;
|
|
1037
|
+
if (wanted === undefined) {
|
|
1038
|
+
const versions = await this.call(`/v1/repositories/${repositoryId}/versions`, { method: "GET" });
|
|
1039
|
+
wanted = (versions.versions ?? [])[0]?.id;
|
|
1040
|
+
}
|
|
1041
|
+
if (!wanted)
|
|
1008
1042
|
return held;
|
|
1009
|
-
const body = await this.call(`/v1/repositories/${repositoryId}/versions/${
|
|
1043
|
+
const body = await this.call(`/v1/repositories/${repositoryId}/versions/${wanted}/files`, {
|
|
1010
1044
|
method: "GET",
|
|
1011
1045
|
});
|
|
1012
1046
|
for (const row of body.files ?? []) {
|
|
@@ -1237,7 +1271,7 @@ class Uploader {
|
|
|
1237
1271
|
* content-addressed object, which is what makes "the service already has
|
|
1238
1272
|
* this one" a question the existing protocol can already answer.
|
|
1239
1273
|
*/
|
|
1240
|
-
async putChunked(repositoryId, declaration, onOffset) {
|
|
1274
|
+
async putChunked(repositoryId, declaration, profile, onOffset) {
|
|
1241
1275
|
/*
|
|
1242
1276
|
Read in windows, not all at once.
|
|
1243
1277
|
|
|
@@ -1274,8 +1308,8 @@ class Uploader {
|
|
|
1274
1308
|
*/
|
|
1275
1309
|
const CHUNK_LANES = 4;
|
|
1276
1310
|
const active = new Set();
|
|
1277
|
-
const dispatch = async (piece, at) => {
|
|
1278
|
-
const task = this.sendChunk(repositoryId, declaration, piece, chunks, at, note).finally(() => {
|
|
1311
|
+
const dispatch = async (piece, at, digest) => {
|
|
1312
|
+
const task = this.sendChunk(repositoryId, declaration, piece, digest, chunks, at, note).finally(() => {
|
|
1279
1313
|
active.delete(task);
|
|
1280
1314
|
done += piece.length;
|
|
1281
1315
|
onOffset(done);
|
|
@@ -1284,6 +1318,51 @@ class Uploader {
|
|
|
1284
1318
|
if (active.size >= CHUNK_LANES)
|
|
1285
1319
|
await Promise.race(active);
|
|
1286
1320
|
};
|
|
1321
|
+
/*
|
|
1322
|
+
Ask about a bounded group of chunks before sending any of them.
|
|
1323
|
+
|
|
1324
|
+
The old path asked once per chunk, with four GETs in flight. Besides
|
|
1325
|
+
spending one network round trip per piece, an unavailable lookup was
|
|
1326
|
+
deliberately treated as a miss, so a transient failure retransmitted
|
|
1327
|
+
every otherwise reusable chunk. The website already uses the bulk
|
|
1328
|
+
endpoint. Reusing it here gives Desktop and CLI the same wire behaviour
|
|
1329
|
+
while retaining the safe "send it if unsure" fallback.
|
|
1330
|
+
|
|
1331
|
+
A batch is no larger than the existing chunk lane count, so this does
|
|
1332
|
+
not increase the memory ceiling: at most four bounded pieces are held.
|
|
1333
|
+
*/
|
|
1334
|
+
const batch = [];
|
|
1335
|
+
const flush = async () => {
|
|
1336
|
+
if (!batch.length)
|
|
1337
|
+
return;
|
|
1338
|
+
const ready = batch.splice(0, batch.length);
|
|
1339
|
+
const held = await this.storedInBulk(repositoryId, ready.map((item) => item.digest));
|
|
1340
|
+
for (const item of ready) {
|
|
1341
|
+
const stored = held.get(item.digest);
|
|
1342
|
+
if (stored) {
|
|
1343
|
+
note("reused", 0);
|
|
1344
|
+
chunks[item.at] = {
|
|
1345
|
+
objectId: stored.objectId,
|
|
1346
|
+
sourceSize: item.piece.length,
|
|
1347
|
+
storedSize: stored.size,
|
|
1348
|
+
};
|
|
1349
|
+
done += item.piece.length;
|
|
1350
|
+
onOffset(done);
|
|
1351
|
+
continue;
|
|
1352
|
+
}
|
|
1353
|
+
await dispatch(item.piece, item.at, item.digest);
|
|
1354
|
+
}
|
|
1355
|
+
await Promise.all(active);
|
|
1356
|
+
};
|
|
1357
|
+
const queue = async (piece) => {
|
|
1358
|
+
batch.push({
|
|
1359
|
+
piece,
|
|
1360
|
+
at: ordinal++,
|
|
1361
|
+
digest: (0, node_crypto_1.createHash)("sha256").update(piece).digest("hex"),
|
|
1362
|
+
});
|
|
1363
|
+
if (batch.length >= CHUNK_LANES)
|
|
1364
|
+
await flush();
|
|
1365
|
+
};
|
|
1287
1366
|
let ordinal = 0;
|
|
1288
1367
|
let pending = Buffer.alloc(0);
|
|
1289
1368
|
for await (const block of (0, node_fs_1.createReadStream)(declaration.full, {
|
|
@@ -1293,8 +1372,8 @@ class Uploader {
|
|
|
1293
1372
|
const incoming = Buffer.from(block);
|
|
1294
1373
|
pending = pending.length ? Buffer.concat([pending, incoming]) : incoming;
|
|
1295
1374
|
let from = 0;
|
|
1296
|
-
for (const cut of (0,
|
|
1297
|
-
await
|
|
1375
|
+
for (const cut of (0, chunking_js_1.cutPoints)(pending, profile)) {
|
|
1376
|
+
await queue(Buffer.from(pending.subarray(from, cut)));
|
|
1298
1377
|
from = cut;
|
|
1299
1378
|
}
|
|
1300
1379
|
pending = Buffer.from(pending.subarray(from));
|
|
@@ -1306,8 +1385,8 @@ class Uploader {
|
|
|
1306
1385
|
Dropping it truncates every chunked file by exactly its last piece.
|
|
1307
1386
|
*/
|
|
1308
1387
|
if (pending.length)
|
|
1309
|
-
await
|
|
1310
|
-
await
|
|
1388
|
+
await queue(pending);
|
|
1389
|
+
await flush();
|
|
1311
1390
|
/*
|
|
1312
1391
|
Order is the file. Every position must be filled: a hole would mean a
|
|
1313
1392
|
chunk that never landed, and publishing around it would produce a version
|
|
@@ -1324,7 +1403,7 @@ class Uploader {
|
|
|
1324
1403
|
};
|
|
1325
1404
|
}
|
|
1326
1405
|
/** Send one chunk, reusing it if the account already holds those bytes. */
|
|
1327
|
-
async sendChunk(repositoryId, declaration, piece,
|
|
1406
|
+
async sendChunk(repositoryId, declaration, piece, digest,
|
|
1328
1407
|
/*
|
|
1329
1408
|
Written at a known position rather than appended. The chunks are sent
|
|
1330
1409
|
several at a time and finish in whatever order the network decides, but
|
|
@@ -1335,48 +1414,27 @@ class Uploader {
|
|
|
1335
1414
|
chunks, at, note) {
|
|
1336
1415
|
{
|
|
1337
1416
|
this.check();
|
|
1338
|
-
const
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
const
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1417
|
+
const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, await this.gzipAllowed());
|
|
1418
|
+
const query = encoded.encoding === "gzip"
|
|
1419
|
+
? `?kind=chunk&role=chunk&encoding=gzip` +
|
|
1420
|
+
`&logicalSize=${piece.length}` +
|
|
1421
|
+
`&storedSha256=${encoded.storedSha256}`
|
|
1422
|
+
: `?kind=chunk&role=chunk`;
|
|
1423
|
+
const stored = await this.call(`/v1/repositories/${repositoryId}/objects/${digest}${query}`, {
|
|
1424
|
+
method: "PUT",
|
|
1425
|
+
contentType: "application/octet-stream",
|
|
1426
|
+
body: encoded.body,
|
|
1427
|
+
}).catch((error) => {
|
|
1428
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1429
|
+
throw new Error(`${declaration.path}: ${reason}`);
|
|
1348
1430
|
});
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
}
|
|
1357
|
-
else {
|
|
1358
|
-
const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, await this.gzipAllowed());
|
|
1359
|
-
const query = encoded.encoding === "gzip"
|
|
1360
|
-
? `?kind=chunk&role=chunk&encoding=gzip` +
|
|
1361
|
-
`&logicalSize=${piece.length}` +
|
|
1362
|
-
`&storedSha256=${encoded.storedSha256}`
|
|
1363
|
-
: `?kind=chunk&role=chunk`;
|
|
1364
|
-
const stored = await this.call(`/v1/repositories/${repositoryId}/objects/${digest}${query}`, {
|
|
1365
|
-
method: "PUT",
|
|
1366
|
-
contentType: "application/octet-stream",
|
|
1367
|
-
body: encoded.body,
|
|
1368
|
-
}).catch((error) => {
|
|
1369
|
-
const reason = error instanceof Error ? error.message : String(error);
|
|
1370
|
-
throw new Error(`${declaration.path}: ${reason}`);
|
|
1371
|
-
});
|
|
1372
|
-
note("sent", encoded.body.byteLength);
|
|
1373
|
-
chunks[at] = {
|
|
1374
|
-
objectId: stored.objectId,
|
|
1375
|
-
sourceSize: piece.length,
|
|
1376
|
-
/* What the service keeps, which is smaller when the piece gzipped. */
|
|
1377
|
-
storedSize: stored.storedSize ?? stored.size,
|
|
1378
|
-
};
|
|
1379
|
-
}
|
|
1431
|
+
note("sent", encoded.body.byteLength);
|
|
1432
|
+
chunks[at] = {
|
|
1433
|
+
objectId: stored.objectId,
|
|
1434
|
+
sourceSize: piece.length,
|
|
1435
|
+
/* What the service keeps, which is smaller when the piece gzipped. */
|
|
1436
|
+
storedSize: stored.storedSize ?? stored.size,
|
|
1437
|
+
};
|
|
1380
1438
|
}
|
|
1381
1439
|
}
|
|
1382
1440
|
async putDirect(repositoryId, declaration) {
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* CodeRook's portable content-defined chunking contract.
|
|
4
|
+
*
|
|
5
|
+
* This file deliberately depends on no Node, Electron or browser APIs. The
|
|
6
|
+
* CLI, Desktop app and website can therefore cut the same bytes at the same
|
|
7
|
+
* positions. Storage manifests record the profile id; changing a profile
|
|
8
|
+
* means adding a new id, never changing the meaning of an existing one.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.DEFAULT_CHUNK_PROFILE = exports.HUGE_CHUNK_PROFILE = exports.LARGE_CHUNK_PROFILE = exports.MEDIUM_CHUNK_PROFILE = exports.LEGACY_CHUNK_PROFILE = void 0;
|
|
12
|
+
exports.profileForFileSize = profileForFileSize;
|
|
13
|
+
exports.cutPoints = cutPoints;
|
|
14
|
+
exports.chunkBoundaries = chunkBoundaries;
|
|
15
|
+
exports.manifestChunking = manifestChunking;
|
|
16
|
+
const KIB = 1024;
|
|
17
|
+
const MIB = 1024 * KIB;
|
|
18
|
+
/*
|
|
19
|
+
* Kept byte-for-byte for compatibility tests and for diagnosing old bundles.
|
|
20
|
+
* The seven-bit pre-target mask is why this profile cut shortly after its
|
|
21
|
+
* minimum instead of averaging eight MiB.
|
|
22
|
+
*/
|
|
23
|
+
exports.LEGACY_CHUNK_PROFILE = Object.freeze({
|
|
24
|
+
id: "gear-legacy-v1",
|
|
25
|
+
algorithm: "gear-streaming",
|
|
26
|
+
minSize: 2 * MIB,
|
|
27
|
+
targetSize: 8 * MIB,
|
|
28
|
+
maxSize: 32 * MIB,
|
|
29
|
+
strictMask: 0x0003_5403,
|
|
30
|
+
looseMask: 0x0003_5000,
|
|
31
|
+
});
|
|
32
|
+
/*
|
|
33
|
+
* FastCDC normalisation uses a boundary that is half as likely before the
|
|
34
|
+
* target and twice as likely afterwards. Contiguous low-bit masks contain
|
|
35
|
+
* exactly the documented number of bits; the rolling gear hash mixes the
|
|
36
|
+
* current byte into those bits on every step.
|
|
37
|
+
*/
|
|
38
|
+
exports.MEDIUM_CHUNK_PROFILE = Object.freeze({
|
|
39
|
+
id: "fastcdc-v2-medium",
|
|
40
|
+
algorithm: "fastcdc-v2",
|
|
41
|
+
minSize: 256 * KIB,
|
|
42
|
+
targetSize: 1 * MIB,
|
|
43
|
+
maxSize: 4 * MIB,
|
|
44
|
+
strictMask: 0x001f_ffff,
|
|
45
|
+
looseMask: 0x0007_ffff,
|
|
46
|
+
});
|
|
47
|
+
exports.LARGE_CHUNK_PROFILE = Object.freeze({
|
|
48
|
+
id: "fastcdc-v2-large",
|
|
49
|
+
algorithm: "fastcdc-v2",
|
|
50
|
+
minSize: 1 * MIB,
|
|
51
|
+
targetSize: 4 * MIB,
|
|
52
|
+
maxSize: 16 * MIB,
|
|
53
|
+
strictMask: 0x007f_ffff,
|
|
54
|
+
looseMask: 0x001f_ffff,
|
|
55
|
+
});
|
|
56
|
+
exports.HUGE_CHUNK_PROFILE = Object.freeze({
|
|
57
|
+
id: "fastcdc-v2-huge",
|
|
58
|
+
algorithm: "fastcdc-v2",
|
|
59
|
+
minSize: 2 * MIB,
|
|
60
|
+
targetSize: 8 * MIB,
|
|
61
|
+
maxSize: 32 * MIB,
|
|
62
|
+
strictMask: 0x00ff_ffff,
|
|
63
|
+
looseMask: 0x003f_ffff,
|
|
64
|
+
});
|
|
65
|
+
exports.DEFAULT_CHUNK_PROFILE = exports.HUGE_CHUNK_PROFILE;
|
|
66
|
+
/** The profile repository uploads use for a file of `size` bytes. */
|
|
67
|
+
function profileForFileSize(size) {
|
|
68
|
+
if (!Number.isFinite(size) || size < 0) {
|
|
69
|
+
throw new RangeError("File size must be a finite non-negative number");
|
|
70
|
+
}
|
|
71
|
+
if (size < 8 * MIB)
|
|
72
|
+
return null;
|
|
73
|
+
if (size < 128 * MIB)
|
|
74
|
+
return exports.MEDIUM_CHUNK_PROFILE;
|
|
75
|
+
if (size < 2 * 1024 * MIB)
|
|
76
|
+
return exports.LARGE_CHUNK_PROFILE;
|
|
77
|
+
return exports.HUGE_CHUNK_PROFILE;
|
|
78
|
+
}
|
|
79
|
+
const GEAR = (() => {
|
|
80
|
+
const table = new Uint32Array(256);
|
|
81
|
+
let seed = 0x9e3779b9;
|
|
82
|
+
for (let index = 0; index < 256; index += 1) {
|
|
83
|
+
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
|
84
|
+
table[index] = seed;
|
|
85
|
+
}
|
|
86
|
+
return table;
|
|
87
|
+
})();
|
|
88
|
+
/**
|
|
89
|
+
* Cut positions inside `bytes`; the final tail is deliberately left to the
|
|
90
|
+
* streaming caller. Supplying the same bytes and profile on any platform
|
|
91
|
+
* must always return the same positions.
|
|
92
|
+
*/
|
|
93
|
+
function cutPoints(bytes, profile = exports.DEFAULT_CHUNK_PROFILE) {
|
|
94
|
+
const cuts = [];
|
|
95
|
+
let from = 0;
|
|
96
|
+
let hash = 0;
|
|
97
|
+
for (let at = 0; at < bytes.length; at += 1) {
|
|
98
|
+
hash = ((hash << 1) + GEAR[bytes[at]]) >>> 0;
|
|
99
|
+
const length = at - from + 1;
|
|
100
|
+
if (length < profile.minSize)
|
|
101
|
+
continue;
|
|
102
|
+
const mask = length < profile.targetSize
|
|
103
|
+
? profile.strictMask
|
|
104
|
+
: profile.looseMask;
|
|
105
|
+
if (length >= profile.maxSize || (hash & mask) === 0) {
|
|
106
|
+
cuts.push(at + 1);
|
|
107
|
+
from = at + 1;
|
|
108
|
+
hash = 0;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return cuts;
|
|
112
|
+
}
|
|
113
|
+
/** Full piece boundaries, including the final tail. */
|
|
114
|
+
function chunkBoundaries(bytes, profile = exports.DEFAULT_CHUNK_PROFILE) {
|
|
115
|
+
const bounds = cutPoints(bytes, profile);
|
|
116
|
+
if (bounds[bounds.length - 1] !== bytes.length)
|
|
117
|
+
bounds.push(bytes.length);
|
|
118
|
+
return bounds;
|
|
119
|
+
}
|
|
120
|
+
function manifestChunking(profile) {
|
|
121
|
+
return {
|
|
122
|
+
algorithm: profile.algorithm,
|
|
123
|
+
profile: profile.id,
|
|
124
|
+
min_size: profile.minSize,
|
|
125
|
+
avg_size: profile.targetSize,
|
|
126
|
+
max_size: profile.maxSize,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.retainedBasePaths = retainedBasePaths;
|
|
4
|
+
exports.acceptedManifestDigests = acceptedManifestDigests;
|
|
5
|
+
/**
|
|
6
|
+
* Paths from the base Version that remain in a candidate snapshot.
|
|
7
|
+
*
|
|
8
|
+
* A Version is a complete immutable view, never a list of changed files.
|
|
9
|
+
* Selected paths are replaced by freshly read declarations, or removed when
|
|
10
|
+
* explicitly marked as deletions. Everything unselected retains the exact
|
|
11
|
+
* object, chunk list, or pack slice from the base.
|
|
12
|
+
*/
|
|
13
|
+
function retainedBasePaths(basePaths, selectedPaths) {
|
|
14
|
+
return [...basePaths]
|
|
15
|
+
.filter((path) => !selectedPaths.has(path))
|
|
16
|
+
.sort((left, right) => left.localeCompare(right));
|
|
17
|
+
}
|
|
18
|
+
/** Digest map returned by the accepted server manifest after an auto-merge. */
|
|
19
|
+
function acceptedManifestDigests(files) {
|
|
20
|
+
return Object.fromEntries([...(files ?? [])].flatMap((file) => file.path && file.sha256 ? [[file.path, file.sha256]] : []));
|
|
21
|
+
}
|