@coderook/cli 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/dist/cli/src/cli.js +20 -3
- package/dist/cli/src/config.js +12 -2
- package/dist/desktop-app/src/main/cbx.js +6 -40
- package/dist/desktop-app/src/main/staging.js +1 -1
- package/dist/desktop-app/src/main/upload.js +71 -44
- package/dist/desktop-app/src/shared/chunking.js +128 -0
- package/dist/desktop-app/src/shared/publication.js +21 -0
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "coderook",
|
|
3
3
|
"displayName": "CodeRook",
|
|
4
4
|
"description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.21.0",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "ACCA Gaming Productions",
|
|
8
8
|
"url": "https://coderook.com"
|
package/dist/cli/src/cli.js
CHANGED
|
@@ -167,11 +167,15 @@ async function reconcile(folder, options = {}) {
|
|
|
167
167
|
folder that has never been reconciled — or was linked before versions
|
|
168
168
|
were recorded — adopts the newest as its starting point.
|
|
169
169
|
*/
|
|
170
|
-
const known = link?.
|
|
171
|
-
const behind = known && known.
|
|
170
|
+
const known = link?.baseVersionId && link.manifest ? link : null;
|
|
171
|
+
const behind = known && known.baseVersionId !== latest.id
|
|
172
172
|
? { sequence: latest.sequence, id: latest.id }
|
|
173
173
|
: null;
|
|
174
174
|
if (known) {
|
|
175
|
+
if (known.observedHeadVersionId !== latest.id) {
|
|
176
|
+
known.observedHeadVersionId = latest.id;
|
|
177
|
+
await (0, config_js_1.writeLink)(folder, known);
|
|
178
|
+
}
|
|
175
179
|
if (behind && !options.quiet) {
|
|
176
180
|
console.log(dim(`The account is on v${behind.sequence}; this folder is based on v${known.sequence}.`));
|
|
177
181
|
}
|
|
@@ -196,6 +200,8 @@ async function reconcile(folder, options = {}) {
|
|
|
196
200
|
slug: project.slug,
|
|
197
201
|
sequence: latest.sequence,
|
|
198
202
|
versionId: latest.id,
|
|
203
|
+
baseVersionId: latest.id,
|
|
204
|
+
observedHeadVersionId: latest.id,
|
|
199
205
|
// Adopted whole, so the folder is taken to hold what the version holds.
|
|
200
206
|
local: Object.fromEntries(baseline),
|
|
201
207
|
manifest: Object.fromEntries(baseline),
|
|
@@ -479,9 +485,14 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
|
|
|
479
485
|
result = await uploader.run({
|
|
480
486
|
localPath: folder,
|
|
481
487
|
include: files.map((file) => file.path),
|
|
488
|
+
deletions: files.filter((file) => file.deleted).map((file) => file.path),
|
|
482
489
|
message,
|
|
483
490
|
projectName: link?.slug ?? node_path_1.default.basename(folder),
|
|
484
491
|
repositoryId: link?.repositoryId ?? null,
|
|
492
|
+
// Every current CLI publish states its ancestry. A brand-new project
|
|
493
|
+
// is explicitly based on an empty Track; a linked folder names the
|
|
494
|
+
// immutable Version it was last reconciled with.
|
|
495
|
+
baseVersionId: link?.baseVersionId ?? null,
|
|
485
496
|
/*
|
|
486
497
|
The line this save belongs on: named on the command, or whichever one
|
|
487
498
|
the folder is set to. Sent by name rather than resolved here, because
|
|
@@ -495,7 +506,9 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
|
|
|
495
506
|
known version. A folder linked before versions were recorded says
|
|
496
507
|
nothing rather than guessing, and publishes as it always did.
|
|
497
508
|
*/
|
|
498
|
-
...(link?.
|
|
509
|
+
...(link?.baseVersionId
|
|
510
|
+
? { expectedHeadVersionId: link.baseVersionId }
|
|
511
|
+
: {}),
|
|
499
512
|
/*
|
|
500
513
|
What this folder believed the project held. It is how the service
|
|
501
514
|
tells a file this person deleted from one they never had, and
|
|
@@ -570,6 +583,8 @@ The connection failed: ${text}`));
|
|
|
570
583
|
sequence: result.sequence,
|
|
571
584
|
// What this folder is now working from, so the next submit can say so.
|
|
572
585
|
versionId: result.versionId,
|
|
586
|
+
baseVersionId: result.versionId,
|
|
587
|
+
observedHeadVersionId: result.versionId,
|
|
573
588
|
local: result.local,
|
|
574
589
|
manifest: result.manifest,
|
|
575
590
|
});
|
|
@@ -795,6 +810,8 @@ held, mode = "replace") {
|
|
|
795
810
|
slug,
|
|
796
811
|
sequence: latest.sequence,
|
|
797
812
|
versionId: latest.id,
|
|
813
|
+
baseVersionId: latest.id,
|
|
814
|
+
observedHeadVersionId: latest.id,
|
|
798
815
|
// Fetching writes every file, so the folder now holds what the
|
|
799
816
|
// version holds and the two agree again.
|
|
800
817
|
local: result.manifest,
|
package/dist/cli/src/config.js
CHANGED
|
@@ -143,13 +143,23 @@ async function readLinks() {
|
|
|
143
143
|
}
|
|
144
144
|
async function readLink(localPath) {
|
|
145
145
|
const links = await readLinks();
|
|
146
|
-
|
|
146
|
+
const link = links[keyFor(localPath)] ?? links[legacyKeyFor(localPath)] ?? null;
|
|
147
|
+
if (!link)
|
|
148
|
+
return null;
|
|
149
|
+
return {
|
|
150
|
+
...link,
|
|
151
|
+
baseVersionId: link.baseVersionId ?? link.versionId,
|
|
152
|
+
};
|
|
147
153
|
}
|
|
148
154
|
async function writeLink(localPath, link) {
|
|
149
155
|
const links = await readLinks();
|
|
150
156
|
// A folder recorded under the old key moves to the new one rather than
|
|
151
157
|
// being left behind as a second connection to the same place.
|
|
152
158
|
delete links[legacyKeyFor(localPath)];
|
|
153
|
-
|
|
159
|
+
const baseVersionId = link.baseVersionId ?? link.versionId;
|
|
160
|
+
links[keyFor(localPath)] = {
|
|
161
|
+
...link,
|
|
162
|
+
...(baseVersionId ? { baseVersionId, versionId: baseVersionId } : {}),
|
|
163
|
+
};
|
|
154
164
|
await writePrivate(linksFile(), JSON.stringify(links, null, 2));
|
|
155
165
|
}
|
|
@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.FOOTER_MAGIC = exports.MANIFEST_MAGIC = exports.CHUNK_MAGIC = exports.HEADER_MAGIC = void 0;
|
|
7
|
-
exports.cutPoints = cutPoints;
|
|
6
|
+
exports.FOOTER_MAGIC = exports.MANIFEST_MAGIC = exports.CHUNK_MAGIC = exports.HEADER_MAGIC = exports.cutPoints = void 0;
|
|
8
7
|
exports.packBundle = packBundle;
|
|
9
8
|
exports.readManifest = readManifest;
|
|
10
9
|
exports.unpackBundle = unpackBundle;
|
|
@@ -32,6 +31,9 @@ const promises_1 = require("node:fs/promises");
|
|
|
32
31
|
const node_path_1 = __importDefault(require("node:path"));
|
|
33
32
|
const node_zlib_1 = require("node:zlib");
|
|
34
33
|
const download_js_1 = require("./download.js");
|
|
34
|
+
const chunking_js_1 = require("../shared/chunking.js");
|
|
35
|
+
var chunking_js_2 = require("../shared/chunking.js");
|
|
36
|
+
Object.defineProperty(exports, "cutPoints", { enumerable: true, get: function () { return chunking_js_2.cutPoints; } });
|
|
35
37
|
exports.HEADER_MAGIC = "CBX1";
|
|
36
38
|
exports.CHUNK_MAGIC = "CHNK";
|
|
37
39
|
exports.MANIFEST_MAGIC = "MANF";
|
|
@@ -48,9 +50,6 @@ const CODEC_ZSTD = 1;
|
|
|
48
50
|
const ZSTD_LEVEL = 3;
|
|
49
51
|
const MIN_COMPRESSION_SAVING = 0.02;
|
|
50
52
|
const READ_SIZE = 8 * 1024 * 1024;
|
|
51
|
-
const MIN_CHUNK_SIZE = 2 * 1024 * 1024;
|
|
52
|
-
const AVG_CHUNK_SIZE = 8 * 1024 * 1024;
|
|
53
|
-
const MAX_CHUNK_SIZE = 32 * 1024 * 1024;
|
|
54
53
|
const zstd = (raw) => (0, node_zlib_1.zstdCompressSync)(raw, {
|
|
55
54
|
params: { [node_zlib_1.constants.ZSTD_c_compressionLevel]: ZSTD_LEVEL },
|
|
56
55
|
});
|
|
@@ -64,36 +63,6 @@ function sha256(data) {
|
|
|
64
63
|
* the `fastcdc` package; cut positions do not have to agree between writers
|
|
65
64
|
* for an archive to be readable, because the manifest records the chunk list.
|
|
66
65
|
*/
|
|
67
|
-
const GEAR = (() => {
|
|
68
|
-
const table = new Uint32Array(256);
|
|
69
|
-
let seed = 0x9e3779b9;
|
|
70
|
-
for (let index = 0; index < 256; index += 1) {
|
|
71
|
-
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
|
72
|
-
table[index] = seed;
|
|
73
|
-
}
|
|
74
|
-
return table;
|
|
75
|
-
})();
|
|
76
|
-
const NORMAL_MASK = 0x0003_5000;
|
|
77
|
-
const SMALL_MASK = 0x0003_5403;
|
|
78
|
-
/** Cut positions inside `buffer`; the tail is left for the next read. */
|
|
79
|
-
function cutPoints(buffer) {
|
|
80
|
-
const cuts = [];
|
|
81
|
-
let from = 0;
|
|
82
|
-
let hash = 0;
|
|
83
|
-
for (let at = 0; at < buffer.length; at += 1) {
|
|
84
|
-
hash = ((hash << 1) + GEAR[buffer[at]]) >>> 0;
|
|
85
|
-
const length = at - from + 1;
|
|
86
|
-
if (length < MIN_CHUNK_SIZE)
|
|
87
|
-
continue;
|
|
88
|
-
const mask = length < AVG_CHUNK_SIZE ? SMALL_MASK : NORMAL_MASK;
|
|
89
|
-
if (length >= MAX_CHUNK_SIZE || (hash & mask) === 0) {
|
|
90
|
-
cuts.push(at + 1);
|
|
91
|
-
from = at + 1;
|
|
92
|
-
hash = 0;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
return cuts;
|
|
96
|
-
}
|
|
97
66
|
function chunkFrame(raw) {
|
|
98
67
|
const rawDigest = (0, node_crypto_1.createHash)("sha256").update(raw).digest();
|
|
99
68
|
let codec = CODEC_NONE;
|
|
@@ -199,7 +168,7 @@ async function packBundle(request, report) {
|
|
|
199
168
|
whole.update(block);
|
|
200
169
|
pending = pending.length ? Buffer.concat([pending, block]) : block;
|
|
201
170
|
let from = 0;
|
|
202
|
-
for (const cut of cutPoints(pending)) {
|
|
171
|
+
for (const cut of (0, chunking_js_1.cutPoints)(pending)) {
|
|
203
172
|
await emit(pending.subarray(from, cut));
|
|
204
173
|
from = cut;
|
|
205
174
|
}
|
|
@@ -225,10 +194,7 @@ async function packBundle(request, report) {
|
|
|
225
194
|
source_kind: request.sourceKind ?? "directory",
|
|
226
195
|
created_unix_ns: Date.now() * 1e6,
|
|
227
196
|
chunking: {
|
|
228
|
-
|
|
229
|
-
min_size: MIN_CHUNK_SIZE,
|
|
230
|
-
avg_size: AVG_CHUNK_SIZE,
|
|
231
|
-
max_size: MAX_CHUNK_SIZE,
|
|
197
|
+
...(0, chunking_js_1.manifestChunking)(chunking_js_1.DEFAULT_CHUNK_PROFILE),
|
|
232
198
|
read_size: READ_SIZE,
|
|
233
199
|
small_file_packing: { enabled: false },
|
|
234
200
|
},
|
|
@@ -69,7 +69,7 @@ exports.describePlan = describePlan;
|
|
|
69
69
|
disagreement.
|
|
70
70
|
*/
|
|
71
71
|
exports.PACKABLE_LIMIT = 1024 * 1024;
|
|
72
|
-
exports.CHUNK_THRESHOLD =
|
|
72
|
+
exports.CHUNK_THRESHOLD = 8 * 1024 * 1024;
|
|
73
73
|
exports.DIRECT_LIMIT = 95 * 1024 * 1024;
|
|
74
74
|
function bandOf(size) {
|
|
75
75
|
if (size <= exports.PACKABLE_LIMIT)
|
|
@@ -21,7 +21,8 @@ const identify_js_1 = require("./identify.js");
|
|
|
21
21
|
const worktree_js_1 = require("./worktree.js");
|
|
22
22
|
const publish_name_js_1 = require("./publish_name.js");
|
|
23
23
|
const compress_js_1 = require("./compress.js");
|
|
24
|
-
const
|
|
24
|
+
const chunking_js_1 = require("../shared/chunking.js");
|
|
25
|
+
const publication_js_1 = require("../shared/publication.js");
|
|
25
26
|
const profile_js_1 = require("./profile.js");
|
|
26
27
|
const solid_js_1 = require("./solid.js");
|
|
27
28
|
const retry_js_1 = require("./retry.js");
|
|
@@ -34,7 +35,7 @@ const DIRECT_LIMIT = 95 * 1024 * 1024;
|
|
|
34
35
|
chunker's own average chunk is eight megabytes, so a smaller file would
|
|
35
36
|
usually come out as one chunk anyway.
|
|
36
37
|
*/
|
|
37
|
-
const CHUNK_THRESHOLD =
|
|
38
|
+
const CHUNK_THRESHOLD = 8 * 1024 * 1024;
|
|
38
39
|
/*
|
|
39
40
|
How many files are sent at once. Six is the limit browsers settled on per
|
|
40
41
|
host for the same reason: enough to keep the link busy through the latency
|
|
@@ -249,12 +250,11 @@ class Uploader {
|
|
|
249
250
|
if (!ticked.size)
|
|
250
251
|
throw new Error("Nothing is selected to upload");
|
|
251
252
|
const prior = request.repositoryId
|
|
252
|
-
? await this.
|
|
253
|
+
? await this.versionFiles(request.repositoryId, request.baseVersionId)
|
|
253
254
|
: new Map();
|
|
254
255
|
// Ticked files are sent; everything else the project still has keeps the
|
|
255
256
|
// copy already stored, and a file that is new but unticked is left out.
|
|
256
257
|
const sending = everything.filter((file) => ticked.has(file.path));
|
|
257
|
-
const onDiskNow = new Set(everything.map((file) => file.path));
|
|
258
258
|
/*
|
|
259
259
|
Everything the version already holds is kept, except what this
|
|
260
260
|
workspace deliberately removed.
|
|
@@ -265,28 +265,21 @@ class Uploader {
|
|
|
265
265
|
next save. Reading this from the disk scan alone is what silently lost
|
|
266
266
|
other people's files.
|
|
267
267
|
*/
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
added: 0,
|
|
284
|
-
removed: 0,
|
|
285
|
-
included: true,
|
|
286
|
-
deleted: false,
|
|
287
|
-
binary: false,
|
|
288
|
-
lines: 0,
|
|
289
|
-
}));
|
|
268
|
+
const reused = (0, publication_js_1.retainedBasePaths)(prior.keys(), ticked).map((path) => {
|
|
269
|
+
const held = prior.get(path);
|
|
270
|
+
return {
|
|
271
|
+
path,
|
|
272
|
+
sourceSize: held.sourceSize,
|
|
273
|
+
storedSize: held.storedSize,
|
|
274
|
+
mediaType: held.mediaType,
|
|
275
|
+
added: 0,
|
|
276
|
+
removed: 0,
|
|
277
|
+
included: true,
|
|
278
|
+
deleted: false,
|
|
279
|
+
binary: false,
|
|
280
|
+
lines: 0,
|
|
281
|
+
};
|
|
282
|
+
});
|
|
290
283
|
/*
|
|
291
284
|
A ticked path that the rescan cannot see is either a deletion or a file
|
|
292
285
|
that has gone since the changes list was drawn. Deletions are meant to
|
|
@@ -294,7 +287,8 @@ class Uploader {
|
|
|
294
287
|
so it is collected and reported rather than quietly skipped.
|
|
295
288
|
*/
|
|
296
289
|
const onDisk = new Set(everything.map((file) => file.path));
|
|
297
|
-
const
|
|
290
|
+
const deletions = new Set(request.deletions ?? []);
|
|
291
|
+
const vanished = [...ticked].filter((chosen) => !onDisk.has(chosen) && !deletions.has(chosen));
|
|
298
292
|
// 1. Measure and hash. This is what makes an unchanged file free.
|
|
299
293
|
const declarations = [];
|
|
300
294
|
let totalBytes = 0;
|
|
@@ -705,8 +699,11 @@ class Uploader {
|
|
|
705
699
|
whole: cutting it up would trade one request for several and save
|
|
706
700
|
nothing.
|
|
707
701
|
*/
|
|
708
|
-
|
|
709
|
-
|
|
702
|
+
const chunkProfile = (0, chunking_js_1.profileForFileSize)(declaration.size);
|
|
703
|
+
if (chunkProfile &&
|
|
704
|
+
declaration.size >= CHUNK_THRESHOLD &&
|
|
705
|
+
(await this.chunkingAllowed())) {
|
|
706
|
+
const pieces = await this.putChunked(repositoryId, declaration, chunkProfile, (offset) => {
|
|
710
707
|
inFlight.set(index, offset);
|
|
711
708
|
report({
|
|
712
709
|
stage: "upload",
|
|
@@ -872,7 +869,12 @@ class Uploader {
|
|
|
872
869
|
tool must never have.
|
|
873
870
|
*/
|
|
874
871
|
const named = new Set(contents.map((item) => item.path));
|
|
875
|
-
const dropped = [
|
|
872
|
+
const dropped = [
|
|
873
|
+
...new Set([
|
|
874
|
+
...vanished,
|
|
875
|
+
...[...ticked].filter((chosen) => !named.has(chosen) && !deletions.has(chosen)),
|
|
876
|
+
]),
|
|
877
|
+
];
|
|
876
878
|
if (dropped.length) {
|
|
877
879
|
const shown = dropped.slice(0, 5).join(", ");
|
|
878
880
|
const rest = dropped.length > 5 ? ` and ${dropped.length - 5} more` : "";
|
|
@@ -902,6 +904,12 @@ class Uploader {
|
|
|
902
904
|
contentType: "application/json",
|
|
903
905
|
body: JSON.stringify({
|
|
904
906
|
message: request.message.trim() || "Saved from the CodeRook desktop app",
|
|
907
|
+
...(request.baseVersionId === undefined
|
|
908
|
+
? {}
|
|
909
|
+
: {
|
|
910
|
+
ancestryMode: "required",
|
|
911
|
+
baseVersionId: request.baseVersionId,
|
|
912
|
+
}),
|
|
905
913
|
...(request.expectedHeadVersionId === undefined
|
|
906
914
|
? {}
|
|
907
915
|
: { expectedHeadVersionId: request.expectedHeadVersionId }),
|
|
@@ -951,6 +959,17 @@ class Uploader {
|
|
|
951
959
|
})),
|
|
952
960
|
}),
|
|
953
961
|
});
|
|
962
|
+
/*
|
|
963
|
+
A clean stale-base save may contain a server-side merge. The candidate
|
|
964
|
+
manifest built above is then not the Version that landed: it lacks the
|
|
965
|
+
other publisher's safe changes. Persist the service's accepted snapshot
|
|
966
|
+
instead. A repeated attempt predates manifest echoing, so read that
|
|
967
|
+
Version once rather than guessing.
|
|
968
|
+
*/
|
|
969
|
+
let acceptedManifest = (0, publication_js_1.acceptedManifestDigests)(completed.manifest?.files);
|
|
970
|
+
if (!completed.mergeTrack && !Object.keys(acceptedManifest).length) {
|
|
971
|
+
acceptedManifest = Object.fromEntries([...(await this.versionFiles(repositoryId, completed.version.id))].map(([file, descriptor]) => [file, descriptor.sha256]));
|
|
972
|
+
}
|
|
954
973
|
report({
|
|
955
974
|
stage: "done",
|
|
956
975
|
files: declarations.length,
|
|
@@ -967,8 +986,8 @@ class Uploader {
|
|
|
967
986
|
sequence: completed.version.sequence,
|
|
968
987
|
...(completed.mergeTrack ? { mergeTrack: completed.mergeTrack } : {}),
|
|
969
988
|
...(completed.repeated ? { repeated: true } : {}),
|
|
970
|
-
sourceBytes,
|
|
971
|
-
storedBytes,
|
|
989
|
+
sourceBytes: completed.version.sourceSize ?? sourceBytes,
|
|
990
|
+
storedBytes: completed.version.storedSize ?? storedBytes,
|
|
972
991
|
sentBytes,
|
|
973
992
|
sentFiles: uploaded.length - alreadyOnAccount,
|
|
974
993
|
reusedFiles: reused.length,
|
|
@@ -991,22 +1010,30 @@ class Uploader {
|
|
|
991
1010
|
const before = request.known?.[file.path] ?? prior.get(file.path)?.sha256;
|
|
992
1011
|
return before ? [[file.path, before]] : [];
|
|
993
1012
|
})),
|
|
994
|
-
manifest:
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1013
|
+
manifest: Object.keys(acceptedManifest).length
|
|
1014
|
+
? acceptedManifest
|
|
1015
|
+
: {
|
|
1016
|
+
...Object.fromEntries(reused.map((file) => [file.path, prior.get(file.path).sha256])),
|
|
1017
|
+
...Object.fromEntries(declarations.map((declaration) => [declaration.path, declaration.sha256])),
|
|
1018
|
+
},
|
|
998
1019
|
};
|
|
999
1020
|
}
|
|
1000
1021
|
/** What the newest version holds, so unchanged files can point at it. */
|
|
1001
|
-
async
|
|
1022
|
+
async versionFiles(repositoryId, versionId) {
|
|
1002
1023
|
const held = new Map();
|
|
1003
1024
|
try {
|
|
1004
|
-
|
|
1005
|
-
//
|
|
1006
|
-
|
|
1007
|
-
|
|
1025
|
+
// Explicit null is a genuinely empty base. Undefined belongs only to
|
|
1026
|
+
// legacy callers, which retain their historical newest-Version lookup.
|
|
1027
|
+
if (versionId === null)
|
|
1028
|
+
return held;
|
|
1029
|
+
let wanted = versionId;
|
|
1030
|
+
if (wanted === undefined) {
|
|
1031
|
+
const versions = await this.call(`/v1/repositories/${repositoryId}/versions`, { method: "GET" });
|
|
1032
|
+
wanted = (versions.versions ?? [])[0]?.id;
|
|
1033
|
+
}
|
|
1034
|
+
if (!wanted)
|
|
1008
1035
|
return held;
|
|
1009
|
-
const body = await this.call(`/v1/repositories/${repositoryId}/versions/${
|
|
1036
|
+
const body = await this.call(`/v1/repositories/${repositoryId}/versions/${wanted}/files`, {
|
|
1010
1037
|
method: "GET",
|
|
1011
1038
|
});
|
|
1012
1039
|
for (const row of body.files ?? []) {
|
|
@@ -1237,7 +1264,7 @@ class Uploader {
|
|
|
1237
1264
|
* content-addressed object, which is what makes "the service already has
|
|
1238
1265
|
* this one" a question the existing protocol can already answer.
|
|
1239
1266
|
*/
|
|
1240
|
-
async putChunked(repositoryId, declaration, onOffset) {
|
|
1267
|
+
async putChunked(repositoryId, declaration, profile, onOffset) {
|
|
1241
1268
|
/*
|
|
1242
1269
|
Read in windows, not all at once.
|
|
1243
1270
|
|
|
@@ -1293,7 +1320,7 @@ class Uploader {
|
|
|
1293
1320
|
const incoming = Buffer.from(block);
|
|
1294
1321
|
pending = pending.length ? Buffer.concat([pending, incoming]) : incoming;
|
|
1295
1322
|
let from = 0;
|
|
1296
|
-
for (const cut of (0,
|
|
1323
|
+
for (const cut of (0, chunking_js_1.cutPoints)(pending, profile)) {
|
|
1297
1324
|
await dispatch(Buffer.from(pending.subarray(from, cut)), ordinal++);
|
|
1298
1325
|
from = cut;
|
|
1299
1326
|
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* CodeRook's portable content-defined chunking contract.
|
|
4
|
+
*
|
|
5
|
+
* This file deliberately depends on no Node, Electron or browser APIs. The
|
|
6
|
+
* CLI, Desktop app and website can therefore cut the same bytes at the same
|
|
7
|
+
* positions. Storage manifests record the profile id; changing a profile
|
|
8
|
+
* means adding a new id, never changing the meaning of an existing one.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.DEFAULT_CHUNK_PROFILE = exports.HUGE_CHUNK_PROFILE = exports.LARGE_CHUNK_PROFILE = exports.MEDIUM_CHUNK_PROFILE = exports.LEGACY_CHUNK_PROFILE = void 0;
|
|
12
|
+
exports.profileForFileSize = profileForFileSize;
|
|
13
|
+
exports.cutPoints = cutPoints;
|
|
14
|
+
exports.chunkBoundaries = chunkBoundaries;
|
|
15
|
+
exports.manifestChunking = manifestChunking;
|
|
16
|
+
const KIB = 1024;
|
|
17
|
+
const MIB = 1024 * KIB;
|
|
18
|
+
/*
|
|
19
|
+
* Kept byte-for-byte for compatibility tests and for diagnosing old bundles.
|
|
20
|
+
* The seven-bit pre-target mask is why this profile cut shortly after its
|
|
21
|
+
* minimum instead of averaging eight MiB.
|
|
22
|
+
*/
|
|
23
|
+
exports.LEGACY_CHUNK_PROFILE = Object.freeze({
|
|
24
|
+
id: "gear-legacy-v1",
|
|
25
|
+
algorithm: "gear-streaming",
|
|
26
|
+
minSize: 2 * MIB,
|
|
27
|
+
targetSize: 8 * MIB,
|
|
28
|
+
maxSize: 32 * MIB,
|
|
29
|
+
strictMask: 0x0003_5403,
|
|
30
|
+
looseMask: 0x0003_5000,
|
|
31
|
+
});
|
|
32
|
+
/*
|
|
33
|
+
* FastCDC normalisation uses a boundary that is half as likely before the
|
|
34
|
+
* target and twice as likely afterwards. Contiguous low-bit masks contain
|
|
35
|
+
* exactly the documented number of bits; the rolling gear hash mixes the
|
|
36
|
+
* current byte into those bits on every step.
|
|
37
|
+
*/
|
|
38
|
+
exports.MEDIUM_CHUNK_PROFILE = Object.freeze({
|
|
39
|
+
id: "fastcdc-v2-medium",
|
|
40
|
+
algorithm: "fastcdc-v2",
|
|
41
|
+
minSize: 256 * KIB,
|
|
42
|
+
targetSize: 1 * MIB,
|
|
43
|
+
maxSize: 4 * MIB,
|
|
44
|
+
strictMask: 0x001f_ffff,
|
|
45
|
+
looseMask: 0x0007_ffff,
|
|
46
|
+
});
|
|
47
|
+
exports.LARGE_CHUNK_PROFILE = Object.freeze({
|
|
48
|
+
id: "fastcdc-v2-large",
|
|
49
|
+
algorithm: "fastcdc-v2",
|
|
50
|
+
minSize: 1 * MIB,
|
|
51
|
+
targetSize: 4 * MIB,
|
|
52
|
+
maxSize: 16 * MIB,
|
|
53
|
+
strictMask: 0x007f_ffff,
|
|
54
|
+
looseMask: 0x001f_ffff,
|
|
55
|
+
});
|
|
56
|
+
exports.HUGE_CHUNK_PROFILE = Object.freeze({
|
|
57
|
+
id: "fastcdc-v2-huge",
|
|
58
|
+
algorithm: "fastcdc-v2",
|
|
59
|
+
minSize: 2 * MIB,
|
|
60
|
+
targetSize: 8 * MIB,
|
|
61
|
+
maxSize: 32 * MIB,
|
|
62
|
+
strictMask: 0x00ff_ffff,
|
|
63
|
+
looseMask: 0x003f_ffff,
|
|
64
|
+
});
|
|
65
|
+
exports.DEFAULT_CHUNK_PROFILE = exports.HUGE_CHUNK_PROFILE;
|
|
66
|
+
/** The profile repository uploads use for a file of `size` bytes. */
|
|
67
|
+
function profileForFileSize(size) {
|
|
68
|
+
if (!Number.isFinite(size) || size < 0) {
|
|
69
|
+
throw new RangeError("File size must be a finite non-negative number");
|
|
70
|
+
}
|
|
71
|
+
if (size < 8 * MIB)
|
|
72
|
+
return null;
|
|
73
|
+
if (size < 128 * MIB)
|
|
74
|
+
return exports.MEDIUM_CHUNK_PROFILE;
|
|
75
|
+
if (size < 2 * 1024 * MIB)
|
|
76
|
+
return exports.LARGE_CHUNK_PROFILE;
|
|
77
|
+
return exports.HUGE_CHUNK_PROFILE;
|
|
78
|
+
}
|
|
79
|
+
const GEAR = (() => {
|
|
80
|
+
const table = new Uint32Array(256);
|
|
81
|
+
let seed = 0x9e3779b9;
|
|
82
|
+
for (let index = 0; index < 256; index += 1) {
|
|
83
|
+
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
|
84
|
+
table[index] = seed;
|
|
85
|
+
}
|
|
86
|
+
return table;
|
|
87
|
+
})();
|
|
88
|
+
/**
|
|
89
|
+
* Cut positions inside `bytes`; the final tail is deliberately left to the
|
|
90
|
+
* streaming caller. Supplying the same bytes and profile on any platform
|
|
91
|
+
* must always return the same positions.
|
|
92
|
+
*/
|
|
93
|
+
function cutPoints(bytes, profile = exports.DEFAULT_CHUNK_PROFILE) {
|
|
94
|
+
const cuts = [];
|
|
95
|
+
let from = 0;
|
|
96
|
+
let hash = 0;
|
|
97
|
+
for (let at = 0; at < bytes.length; at += 1) {
|
|
98
|
+
hash = ((hash << 1) + GEAR[bytes[at]]) >>> 0;
|
|
99
|
+
const length = at - from + 1;
|
|
100
|
+
if (length < profile.minSize)
|
|
101
|
+
continue;
|
|
102
|
+
const mask = length < profile.targetSize
|
|
103
|
+
? profile.strictMask
|
|
104
|
+
: profile.looseMask;
|
|
105
|
+
if (length >= profile.maxSize || (hash & mask) === 0) {
|
|
106
|
+
cuts.push(at + 1);
|
|
107
|
+
from = at + 1;
|
|
108
|
+
hash = 0;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return cuts;
|
|
112
|
+
}
|
|
113
|
+
/** Full piece boundaries, including the final tail. */
|
|
114
|
+
function chunkBoundaries(bytes, profile = exports.DEFAULT_CHUNK_PROFILE) {
|
|
115
|
+
const bounds = cutPoints(bytes, profile);
|
|
116
|
+
if (bounds[bounds.length - 1] !== bytes.length)
|
|
117
|
+
bounds.push(bytes.length);
|
|
118
|
+
return bounds;
|
|
119
|
+
}
|
|
120
|
+
function manifestChunking(profile) {
|
|
121
|
+
return {
|
|
122
|
+
algorithm: profile.algorithm,
|
|
123
|
+
profile: profile.id,
|
|
124
|
+
min_size: profile.minSize,
|
|
125
|
+
avg_size: profile.targetSize,
|
|
126
|
+
max_size: profile.maxSize,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.retainedBasePaths = retainedBasePaths;
|
|
4
|
+
exports.acceptedManifestDigests = acceptedManifestDigests;
|
|
5
|
+
/**
|
|
6
|
+
* Paths from the base Version that remain in a candidate snapshot.
|
|
7
|
+
*
|
|
8
|
+
* A Version is a complete immutable view, never a list of changed files.
|
|
9
|
+
* Selected paths are replaced by freshly read declarations, or removed when
|
|
10
|
+
* explicitly marked as deletions. Everything unselected retains the exact
|
|
11
|
+
* object, chunk list, or pack slice from the base.
|
|
12
|
+
*/
|
|
13
|
+
function retainedBasePaths(basePaths, selectedPaths) {
|
|
14
|
+
return [...basePaths]
|
|
15
|
+
.filter((path) => !selectedPaths.has(path))
|
|
16
|
+
.sort((left, right) => left.localeCompare(right));
|
|
17
|
+
}
|
|
18
|
+
/** Digest map returned by the accepted server manifest after an auto-merge. */
|
|
19
|
+
function acceptedManifestDigests(files) {
|
|
20
|
+
return Object.fromEntries([...(files ?? [])].flatMap((file) => file.path && file.sha256 ? [[file.path, file.sha256]] : []));
|
|
21
|
+
}
|