@coderook/cli 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
  "name": "coderook",
3
3
  "displayName": "CodeRook",
4
4
  "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
5
- "version": "0.20.0",
5
+ "version": "0.22.0",
6
6
  "author": {
7
7
  "name": "ACCA Gaming Productions",
8
8
  "url": "https://coderook.com"
@@ -167,11 +167,15 @@ async function reconcile(folder, options = {}) {
167
167
  folder that has never been reconciled — or was linked before versions
168
168
  were recorded — adopts the newest as its starting point.
169
169
  */
170
- const known = link?.versionId && link.manifest ? link : null;
171
- const behind = known && known.versionId !== latest.id
170
+ const known = link?.baseVersionId && link.manifest ? link : null;
171
+ const behind = known && known.baseVersionId !== latest.id
172
172
  ? { sequence: latest.sequence, id: latest.id }
173
173
  : null;
174
174
  if (known) {
175
+ if (known.observedHeadVersionId !== latest.id) {
176
+ known.observedHeadVersionId = latest.id;
177
+ await (0, config_js_1.writeLink)(folder, known);
178
+ }
175
179
  if (behind && !options.quiet) {
176
180
  console.log(dim(`The account is on v${behind.sequence}; this folder is based on v${known.sequence}.`));
177
181
  }
@@ -196,6 +200,8 @@ async function reconcile(folder, options = {}) {
196
200
  slug: project.slug,
197
201
  sequence: latest.sequence,
198
202
  versionId: latest.id,
203
+ baseVersionId: latest.id,
204
+ observedHeadVersionId: latest.id,
199
205
  // Adopted whole, so the folder is taken to hold what the version holds.
200
206
  local: Object.fromEntries(baseline),
201
207
  manifest: Object.fromEntries(baseline),
@@ -479,9 +485,14 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
479
485
  result = await uploader.run({
480
486
  localPath: folder,
481
487
  include: files.map((file) => file.path),
488
+ deletions: files.filter((file) => file.deleted).map((file) => file.path),
482
489
  message,
483
490
  projectName: link?.slug ?? node_path_1.default.basename(folder),
484
491
  repositoryId: link?.repositoryId ?? null,
492
+ // Every current CLI publish states its ancestry. A brand-new project
493
+ // is explicitly based on an empty Track; a linked folder names the
494
+ // immutable Version it was last reconciled with.
495
+ baseVersionId: link?.baseVersionId ?? null,
485
496
  /*
486
497
  The line this save belongs on: named on the command, or whichever one
487
498
  the folder is set to. Sent by name rather than resolved here, because
@@ -495,7 +506,9 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
495
506
  known version. A folder linked before versions were recorded says
496
507
  nothing rather than guessing, and publishes as it always did.
497
508
  */
498
- ...(link?.versionId ? { expectedHeadVersionId: link.versionId } : {}),
509
+ ...(link?.baseVersionId
510
+ ? { expectedHeadVersionId: link.baseVersionId }
511
+ : {}),
499
512
  /*
500
513
  What this folder believed the project held. It is how the service
501
514
  tells a file this person deleted from one they never had, and
@@ -570,6 +583,8 @@ The connection failed: ${text}`));
570
583
  sequence: result.sequence,
571
584
  // What this folder is now working from, so the next submit can say so.
572
585
  versionId: result.versionId,
586
+ baseVersionId: result.versionId,
587
+ observedHeadVersionId: result.versionId,
573
588
  local: result.local,
574
589
  manifest: result.manifest,
575
590
  });
@@ -795,6 +810,8 @@ held, mode = "replace") {
795
810
  slug,
796
811
  sequence: latest.sequence,
797
812
  versionId: latest.id,
813
+ baseVersionId: latest.id,
814
+ observedHeadVersionId: latest.id,
798
815
  // Fetching writes every file, so the folder now holds what the
799
816
  // version holds and the two agree again.
800
817
  local: result.manifest,
@@ -143,13 +143,23 @@ async function readLinks() {
143
143
  }
144
144
  async function readLink(localPath) {
145
145
  const links = await readLinks();
146
- return links[keyFor(localPath)] ?? links[legacyKeyFor(localPath)] ?? null;
146
+ const link = links[keyFor(localPath)] ?? links[legacyKeyFor(localPath)] ?? null;
147
+ if (!link)
148
+ return null;
149
+ return {
150
+ ...link,
151
+ baseVersionId: link.baseVersionId ?? link.versionId,
152
+ };
147
153
  }
148
154
  async function writeLink(localPath, link) {
149
155
  const links = await readLinks();
150
156
  // A folder recorded under the old key moves to the new one rather than
151
157
  // being left behind as a second connection to the same place.
152
158
  delete links[legacyKeyFor(localPath)];
153
- links[keyFor(localPath)] = link;
159
+ const baseVersionId = link.baseVersionId ?? link.versionId;
160
+ links[keyFor(localPath)] = {
161
+ ...link,
162
+ ...(baseVersionId ? { baseVersionId, versionId: baseVersionId } : {}),
163
+ };
154
164
  await writePrivate(linksFile(), JSON.stringify(links, null, 2));
155
165
  }
@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.FOOTER_MAGIC = exports.MANIFEST_MAGIC = exports.CHUNK_MAGIC = exports.HEADER_MAGIC = void 0;
7
- exports.cutPoints = cutPoints;
6
+ exports.FOOTER_MAGIC = exports.MANIFEST_MAGIC = exports.CHUNK_MAGIC = exports.HEADER_MAGIC = exports.cutPoints = void 0;
8
7
  exports.packBundle = packBundle;
9
8
  exports.readManifest = readManifest;
10
9
  exports.unpackBundle = unpackBundle;
@@ -32,6 +31,9 @@ const promises_1 = require("node:fs/promises");
32
31
  const node_path_1 = __importDefault(require("node:path"));
33
32
  const node_zlib_1 = require("node:zlib");
34
33
  const download_js_1 = require("./download.js");
34
+ const chunking_js_1 = require("../shared/chunking.js");
35
+ var chunking_js_2 = require("../shared/chunking.js");
36
+ Object.defineProperty(exports, "cutPoints", { enumerable: true, get: function () { return chunking_js_2.cutPoints; } });
35
37
  exports.HEADER_MAGIC = "CBX1";
36
38
  exports.CHUNK_MAGIC = "CHNK";
37
39
  exports.MANIFEST_MAGIC = "MANF";
@@ -48,9 +50,6 @@ const CODEC_ZSTD = 1;
48
50
  const ZSTD_LEVEL = 3;
49
51
  const MIN_COMPRESSION_SAVING = 0.02;
50
52
  const READ_SIZE = 8 * 1024 * 1024;
51
- const MIN_CHUNK_SIZE = 2 * 1024 * 1024;
52
- const AVG_CHUNK_SIZE = 8 * 1024 * 1024;
53
- const MAX_CHUNK_SIZE = 32 * 1024 * 1024;
54
53
  const zstd = (raw) => (0, node_zlib_1.zstdCompressSync)(raw, {
55
54
  params: { [node_zlib_1.constants.ZSTD_c_compressionLevel]: ZSTD_LEVEL },
56
55
  });
@@ -64,36 +63,6 @@ function sha256(data) {
64
63
  * the `fastcdc` package; cut positions do not have to agree between writers
65
64
  * for an archive to be readable, because the manifest records the chunk list.
66
65
  */
67
- const GEAR = (() => {
68
- const table = new Uint32Array(256);
69
- let seed = 0x9e3779b9;
70
- for (let index = 0; index < 256; index += 1) {
71
- seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
72
- table[index] = seed;
73
- }
74
- return table;
75
- })();
76
- const NORMAL_MASK = 0x0003_5000;
77
- const SMALL_MASK = 0x0003_5403;
78
- /** Cut positions inside `buffer`; the tail is left for the next read. */
79
- function cutPoints(buffer) {
80
- const cuts = [];
81
- let from = 0;
82
- let hash = 0;
83
- for (let at = 0; at < buffer.length; at += 1) {
84
- hash = ((hash << 1) + GEAR[buffer[at]]) >>> 0;
85
- const length = at - from + 1;
86
- if (length < MIN_CHUNK_SIZE)
87
- continue;
88
- const mask = length < AVG_CHUNK_SIZE ? SMALL_MASK : NORMAL_MASK;
89
- if (length >= MAX_CHUNK_SIZE || (hash & mask) === 0) {
90
- cuts.push(at + 1);
91
- from = at + 1;
92
- hash = 0;
93
- }
94
- }
95
- return cuts;
96
- }
97
66
  function chunkFrame(raw) {
98
67
  const rawDigest = (0, node_crypto_1.createHash)("sha256").update(raw).digest();
99
68
  let codec = CODEC_NONE;
@@ -199,7 +168,7 @@ async function packBundle(request, report) {
199
168
  whole.update(block);
200
169
  pending = pending.length ? Buffer.concat([pending, block]) : block;
201
170
  let from = 0;
202
- for (const cut of cutPoints(pending)) {
171
+ for (const cut of (0, chunking_js_1.cutPoints)(pending)) {
203
172
  await emit(pending.subarray(from, cut));
204
173
  from = cut;
205
174
  }
@@ -225,10 +194,7 @@ async function packBundle(request, report) {
225
194
  source_kind: request.sourceKind ?? "directory",
226
195
  created_unix_ns: Date.now() * 1e6,
227
196
  chunking: {
228
- algorithm: "gear-streaming",
229
- min_size: MIN_CHUNK_SIZE,
230
- avg_size: AVG_CHUNK_SIZE,
231
- max_size: MAX_CHUNK_SIZE,
197
+ ...(0, chunking_js_1.manifestChunking)(chunking_js_1.DEFAULT_CHUNK_PROFILE),
232
198
  read_size: READ_SIZE,
233
199
  small_file_packing: { enabled: false },
234
200
  },
@@ -69,7 +69,7 @@ exports.describePlan = describePlan;
69
69
  disagreement.
70
70
  */
71
71
  exports.PACKABLE_LIMIT = 1024 * 1024;
72
- exports.CHUNK_THRESHOLD = 16 * 1024 * 1024;
72
+ exports.CHUNK_THRESHOLD = 8 * 1024 * 1024;
73
73
  exports.DIRECT_LIMIT = 95 * 1024 * 1024;
74
74
  function bandOf(size) {
75
75
  if (size <= exports.PACKABLE_LIMIT)
@@ -21,7 +21,8 @@ const identify_js_1 = require("./identify.js");
21
21
  const worktree_js_1 = require("./worktree.js");
22
22
  const publish_name_js_1 = require("./publish_name.js");
23
23
  const compress_js_1 = require("./compress.js");
24
- const cbx_js_1 = require("./cbx.js");
24
+ const chunking_js_1 = require("../shared/chunking.js");
25
+ const publication_js_1 = require("../shared/publication.js");
25
26
  const profile_js_1 = require("./profile.js");
26
27
  const solid_js_1 = require("./solid.js");
27
28
  const retry_js_1 = require("./retry.js");
@@ -34,7 +35,7 @@ const DIRECT_LIMIT = 95 * 1024 * 1024;
34
35
  chunker's own average chunk is eight megabytes, so a smaller file would
35
36
  usually come out as one chunk anyway.
36
37
  */
37
- const CHUNK_THRESHOLD = 16 * 1024 * 1024;
38
+ const CHUNK_THRESHOLD = 8 * 1024 * 1024;
38
39
  /*
39
40
  How many files are sent at once. Six is the limit browsers settled on per
40
41
  host for the same reason: enough to keep the link busy through the latency
@@ -249,12 +250,11 @@ class Uploader {
249
250
  if (!ticked.size)
250
251
  throw new Error("Nothing is selected to upload");
251
252
  const prior = request.repositoryId
252
- ? await this.latestVersionFiles(request.repositoryId)
253
+ ? await this.versionFiles(request.repositoryId, request.baseVersionId)
253
254
  : new Map();
254
255
  // Ticked files are sent; everything else the project still has keeps the
255
256
  // copy already stored, and a file that is new but unticked is left out.
256
257
  const sending = everything.filter((file) => ticked.has(file.path));
257
- const onDiskNow = new Set(everything.map((file) => file.path));
258
258
  /*
259
259
  Everything the version already holds is kept, except what this
260
260
  workspace deliberately removed.
@@ -265,28 +265,21 @@ class Uploader {
265
265
  next save. Reading this from the disk scan alone is what silently lost
266
266
  other people's files.
267
267
  */
268
- const knew = request.known ? new Set(Object.keys(request.known)) : null;
269
- const reused = [...prior.entries()]
270
- .filter(([path]) => {
271
- if (ticked.has(path))
272
- return false;
273
- if (onDiskNow.has(path))
274
- return true;
275
- // Missing here. Only a deletion if this workspace ever had it.
276
- return knew ? !knew.has(path) : true;
277
- })
278
- .map(([path, held]) => ({
279
- path,
280
- sourceSize: held.sourceSize,
281
- storedSize: held.storedSize,
282
- mediaType: held.mediaType,
283
- added: 0,
284
- removed: 0,
285
- included: true,
286
- deleted: false,
287
- binary: false,
288
- lines: 0,
289
- }));
268
+ const reused = (0, publication_js_1.retainedBasePaths)(prior.keys(), ticked).map((path) => {
269
+ const held = prior.get(path);
270
+ return {
271
+ path,
272
+ sourceSize: held.sourceSize,
273
+ storedSize: held.storedSize,
274
+ mediaType: held.mediaType,
275
+ added: 0,
276
+ removed: 0,
277
+ included: true,
278
+ deleted: false,
279
+ binary: false,
280
+ lines: 0,
281
+ };
282
+ });
290
283
  /*
291
284
  A ticked path that the rescan cannot see is either a deletion or a file
292
285
  that has gone since the changes list was drawn. Deletions are meant to
@@ -294,7 +287,8 @@ class Uploader {
294
287
  so it is collected and reported rather than quietly skipped.
295
288
  */
296
289
  const onDisk = new Set(everything.map((file) => file.path));
297
- const vanished = [...ticked].filter((chosen) => !onDisk.has(chosen) && !prior.has(chosen));
290
+ const deletions = new Set(request.deletions ?? []);
291
+ const vanished = [...ticked].filter((chosen) => !onDisk.has(chosen) && !deletions.has(chosen));
298
292
  // 1. Measure and hash. This is what makes an unchanged file free.
299
293
  const declarations = [];
300
294
  let totalBytes = 0;
@@ -705,8 +699,11 @@ class Uploader {
705
699
  whole: cutting it up would trade one request for several and save
706
700
  nothing.
707
701
  */
708
- if (declaration.size > CHUNK_THRESHOLD && (await this.chunkingAllowed())) {
709
- const pieces = await this.putChunked(repositoryId, declaration, (offset) => {
702
+ const chunkProfile = (0, chunking_js_1.profileForFileSize)(declaration.size);
703
+ if (chunkProfile &&
704
+ declaration.size >= CHUNK_THRESHOLD &&
705
+ (await this.chunkingAllowed())) {
706
+ const pieces = await this.putChunked(repositoryId, declaration, chunkProfile, (offset) => {
710
707
  inFlight.set(index, offset);
711
708
  report({
712
709
  stage: "upload",
@@ -721,7 +718,14 @@ class Uploader {
721
718
  });
722
719
  });
723
720
  inFlight.delete(index);
724
- alreadyOnAccount += pieces.reused;
721
+ /*
722
+ A chunk is not a file. Counting every reused piece here made a
723
+ one-file save report a negative number of sent files. The selected
724
+ file counts as already stored only when every one of its pieces was
725
+ reused and no payload crossed the wire.
726
+ */
727
+ if (pieces.sentBytes === 0)
728
+ alreadyOnAccount += 1;
725
729
  uploaded.push({
726
730
  path: declaration.path,
727
731
  sha256: declaration.sha256,
@@ -872,7 +876,12 @@ class Uploader {
872
876
  tool must never have.
873
877
  */
874
878
  const named = new Set(contents.map((item) => item.path));
875
- const dropped = [...new Set([...vanished, ...[...ticked].filter((chosen) => !named.has(chosen))])];
879
+ const dropped = [
880
+ ...new Set([
881
+ ...vanished,
882
+ ...[...ticked].filter((chosen) => !named.has(chosen) && !deletions.has(chosen)),
883
+ ]),
884
+ ];
876
885
  if (dropped.length) {
877
886
  const shown = dropped.slice(0, 5).join(", ");
878
887
  const rest = dropped.length > 5 ? ` and ${dropped.length - 5} more` : "";
@@ -902,6 +911,12 @@ class Uploader {
902
911
  contentType: "application/json",
903
912
  body: JSON.stringify({
904
913
  message: request.message.trim() || "Saved from the CodeRook desktop app",
914
+ ...(request.baseVersionId === undefined
915
+ ? {}
916
+ : {
917
+ ancestryMode: "required",
918
+ baseVersionId: request.baseVersionId,
919
+ }),
905
920
  ...(request.expectedHeadVersionId === undefined
906
921
  ? {}
907
922
  : { expectedHeadVersionId: request.expectedHeadVersionId }),
@@ -951,6 +966,17 @@ class Uploader {
951
966
  })),
952
967
  }),
953
968
  });
969
+ /*
970
+ A clean stale-base save may contain a server-side merge. The candidate
971
+ manifest built above is then not the Version that landed: it lacks the
972
+ other publisher's safe changes. Persist the service's accepted snapshot
973
+ instead. A repeated attempt predates manifest echoing, so read that
974
+ Version once rather than guessing.
975
+ */
976
+ let acceptedManifest = (0, publication_js_1.acceptedManifestDigests)(completed.manifest?.files);
977
+ if (!completed.mergeTrack && !Object.keys(acceptedManifest).length) {
978
+ acceptedManifest = Object.fromEntries([...(await this.versionFiles(repositoryId, completed.version.id))].map(([file, descriptor]) => [file, descriptor.sha256]));
979
+ }
954
980
  report({
955
981
  stage: "done",
956
982
  files: declarations.length,
@@ -967,9 +993,9 @@ class Uploader {
967
993
  sequence: completed.version.sequence,
968
994
  ...(completed.mergeTrack ? { mergeTrack: completed.mergeTrack } : {}),
969
995
  ...(completed.repeated ? { repeated: true } : {}),
970
- sourceBytes,
971
- storedBytes,
972
- sentBytes,
996
+ sourceBytes: completed.version.sourceSize ?? sourceBytes,
997
+ storedBytes: completed.version.storedSize ?? storedBytes,
998
+ sentBytes: transferred,
973
999
  sentFiles: uploaded.length - alreadyOnAccount,
974
1000
  reusedFiles: reused.length,
975
1001
  /** Selected, but the service already held the content. */
@@ -991,22 +1017,30 @@ class Uploader {
991
1017
  const before = request.known?.[file.path] ?? prior.get(file.path)?.sha256;
992
1018
  return before ? [[file.path, before]] : [];
993
1019
  })),
994
- manifest: {
995
- ...Object.fromEntries(reused.map((file) => [file.path, prior.get(file.path).sha256])),
996
- ...Object.fromEntries(declarations.map((declaration) => [declaration.path, declaration.sha256])),
997
- },
1020
+ manifest: Object.keys(acceptedManifest).length
1021
+ ? acceptedManifest
1022
+ : {
1023
+ ...Object.fromEntries(reused.map((file) => [file.path, prior.get(file.path).sha256])),
1024
+ ...Object.fromEntries(declarations.map((declaration) => [declaration.path, declaration.sha256])),
1025
+ },
998
1026
  };
999
1027
  }
1000
1028
  /** What the newest version holds, so unchanged files can point at it. */
1001
- async latestVersionFiles(repositoryId) {
1029
+ async versionFiles(repositoryId, versionId) {
1002
1030
  const held = new Map();
1003
1031
  try {
1004
- const versions = await this.call(`/v1/repositories/${repositoryId}/versions`, { method: "GET" });
1005
- // The API returns them newest first.
1006
- const latest = (versions.versions ?? [])[0];
1007
- if (!latest?.id)
1032
+ // Explicit null is a genuinely empty base. Undefined belongs only to
1033
+ // legacy callers, which retain their historical newest-Version lookup.
1034
+ if (versionId === null)
1035
+ return held;
1036
+ let wanted = versionId;
1037
+ if (wanted === undefined) {
1038
+ const versions = await this.call(`/v1/repositories/${repositoryId}/versions`, { method: "GET" });
1039
+ wanted = (versions.versions ?? [])[0]?.id;
1040
+ }
1041
+ if (!wanted)
1008
1042
  return held;
1009
- const body = await this.call(`/v1/repositories/${repositoryId}/versions/${latest.id}/files`, {
1043
+ const body = await this.call(`/v1/repositories/${repositoryId}/versions/${wanted}/files`, {
1010
1044
  method: "GET",
1011
1045
  });
1012
1046
  for (const row of body.files ?? []) {
@@ -1237,7 +1271,7 @@ class Uploader {
1237
1271
  * content-addressed object, which is what makes "the service already has
1238
1272
  * this one" a question the existing protocol can already answer.
1239
1273
  */
1240
- async putChunked(repositoryId, declaration, onOffset) {
1274
+ async putChunked(repositoryId, declaration, profile, onOffset) {
1241
1275
  /*
1242
1276
  Read in windows, not all at once.
1243
1277
 
@@ -1274,8 +1308,8 @@ class Uploader {
1274
1308
  */
1275
1309
  const CHUNK_LANES = 4;
1276
1310
  const active = new Set();
1277
- const dispatch = async (piece, at) => {
1278
- const task = this.sendChunk(repositoryId, declaration, piece, chunks, at, note).finally(() => {
1311
+ const dispatch = async (piece, at, digest) => {
1312
+ const task = this.sendChunk(repositoryId, declaration, piece, digest, chunks, at, note).finally(() => {
1279
1313
  active.delete(task);
1280
1314
  done += piece.length;
1281
1315
  onOffset(done);
@@ -1284,6 +1318,51 @@ class Uploader {
1284
1318
  if (active.size >= CHUNK_LANES)
1285
1319
  await Promise.race(active);
1286
1320
  };
1321
+ /*
1322
+ Ask about a bounded group of chunks before sending any of them.
1323
+
1324
+ The old path asked once per chunk, with four GETs in flight. Besides
1325
+ spending one network round trip per piece, an unavailable lookup was
1326
+ deliberately treated as a miss, so a transient failure retransmitted
1327
+ every otherwise reusable chunk. The website already uses the bulk
1328
+ endpoint. Reusing it here gives Desktop and CLI the same wire behaviour
1329
+ while retaining the safe "send it if unsure" fallback.
1330
+
1331
+ A batch is no larger than the existing chunk lane count, so this does
1332
+ not increase the memory ceiling: at most four bounded pieces are held.
1333
+ */
1334
+ const batch = [];
1335
+ const flush = async () => {
1336
+ if (!batch.length)
1337
+ return;
1338
+ const ready = batch.splice(0, batch.length);
1339
+ const held = await this.storedInBulk(repositoryId, ready.map((item) => item.digest));
1340
+ for (const item of ready) {
1341
+ const stored = held.get(item.digest);
1342
+ if (stored) {
1343
+ note("reused", 0);
1344
+ chunks[item.at] = {
1345
+ objectId: stored.objectId,
1346
+ sourceSize: item.piece.length,
1347
+ storedSize: stored.size,
1348
+ };
1349
+ done += item.piece.length;
1350
+ onOffset(done);
1351
+ continue;
1352
+ }
1353
+ await dispatch(item.piece, item.at, item.digest);
1354
+ }
1355
+ await Promise.all(active);
1356
+ };
1357
+ const queue = async (piece) => {
1358
+ batch.push({
1359
+ piece,
1360
+ at: ordinal++,
1361
+ digest: (0, node_crypto_1.createHash)("sha256").update(piece).digest("hex"),
1362
+ });
1363
+ if (batch.length >= CHUNK_LANES)
1364
+ await flush();
1365
+ };
1287
1366
  let ordinal = 0;
1288
1367
  let pending = Buffer.alloc(0);
1289
1368
  for await (const block of (0, node_fs_1.createReadStream)(declaration.full, {
@@ -1293,8 +1372,8 @@ class Uploader {
1293
1372
  const incoming = Buffer.from(block);
1294
1373
  pending = pending.length ? Buffer.concat([pending, incoming]) : incoming;
1295
1374
  let from = 0;
1296
- for (const cut of (0, cbx_js_1.cutPoints)(pending)) {
1297
- await dispatch(Buffer.from(pending.subarray(from, cut)), ordinal++);
1375
+ for (const cut of (0, chunking_js_1.cutPoints)(pending, profile)) {
1376
+ await queue(Buffer.from(pending.subarray(from, cut)));
1298
1377
  from = cut;
1299
1378
  }
1300
1379
  pending = Buffer.from(pending.subarray(from));
@@ -1306,8 +1385,8 @@ class Uploader {
1306
1385
  Dropping it truncates every chunked file by exactly its last piece.
1307
1386
  */
1308
1387
  if (pending.length)
1309
- await dispatch(pending, ordinal++);
1310
- await Promise.all(active);
1388
+ await queue(pending);
1389
+ await flush();
1311
1390
  /*
1312
1391
  Order is the file. Every position must be filled: a hole would mean a
1313
1392
  chunk that never landed, and publishing around it would produce a version
@@ -1324,7 +1403,7 @@ class Uploader {
1324
1403
  };
1325
1404
  }
1326
1405
  /** Send one chunk, reusing it if the account already holds those bytes. */
1327
- async sendChunk(repositoryId, declaration, piece,
1406
+ async sendChunk(repositoryId, declaration, piece, digest,
1328
1407
  /*
1329
1408
  Written at a known position rather than appended. The chunks are sent
1330
1409
  several at a time and finish in whatever order the network decides, but
@@ -1335,48 +1414,27 @@ class Uploader {
1335
1414
  chunks, at, note) {
1336
1415
  {
1337
1416
  this.check();
1338
- const digest = (0, node_crypto_1.createHash)("sha256").update(piece).digest("hex");
1339
- /*
1340
- Asked per chunk rather than per file. This is the whole saving: the
1341
- parts of a large file that did not change answer "already held" and
1342
- never travel.
1343
- */
1344
- const held = await this.findStored(repositoryId, {
1345
- ...declaration,
1346
- sha256: digest,
1347
- size: piece.length,
1417
+ const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, await this.gzipAllowed());
1418
+ const query = encoded.encoding === "gzip"
1419
+ ? `?kind=chunk&role=chunk&encoding=gzip` +
1420
+ `&logicalSize=${piece.length}` +
1421
+ `&storedSha256=${encoded.storedSha256}`
1422
+ : `?kind=chunk&role=chunk`;
1423
+ const stored = await this.call(`/v1/repositories/${repositoryId}/objects/${digest}${query}`, {
1424
+ method: "PUT",
1425
+ contentType: "application/octet-stream",
1426
+ body: encoded.body,
1427
+ }).catch((error) => {
1428
+ const reason = error instanceof Error ? error.message : String(error);
1429
+ throw new Error(`${declaration.path}: ${reason}`);
1348
1430
  });
1349
- if (held) {
1350
- note("reused", 0);
1351
- chunks[at] = {
1352
- objectId: held.objectId,
1353
- sourceSize: piece.length,
1354
- storedSize: held.size,
1355
- };
1356
- }
1357
- else {
1358
- const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, await this.gzipAllowed());
1359
- const query = encoded.encoding === "gzip"
1360
- ? `?kind=chunk&role=chunk&encoding=gzip` +
1361
- `&logicalSize=${piece.length}` +
1362
- `&storedSha256=${encoded.storedSha256}`
1363
- : `?kind=chunk&role=chunk`;
1364
- const stored = await this.call(`/v1/repositories/${repositoryId}/objects/${digest}${query}`, {
1365
- method: "PUT",
1366
- contentType: "application/octet-stream",
1367
- body: encoded.body,
1368
- }).catch((error) => {
1369
- const reason = error instanceof Error ? error.message : String(error);
1370
- throw new Error(`${declaration.path}: ${reason}`);
1371
- });
1372
- note("sent", encoded.body.byteLength);
1373
- chunks[at] = {
1374
- objectId: stored.objectId,
1375
- sourceSize: piece.length,
1376
- /* What the service keeps, which is smaller when the piece gzipped. */
1377
- storedSize: stored.storedSize ?? stored.size,
1378
- };
1379
- }
1431
+ note("sent", encoded.body.byteLength);
1432
+ chunks[at] = {
1433
+ objectId: stored.objectId,
1434
+ sourceSize: piece.length,
1435
+ /* What the service keeps, which is smaller when the piece gzipped. */
1436
+ storedSize: stored.storedSize ?? stored.size,
1437
+ };
1380
1438
  }
1381
1439
  }
1382
1440
  async putDirect(repositoryId, declaration) {
@@ -0,0 +1,128 @@
1
+ "use strict";
2
+ /**
3
+ * CodeRook's portable content-defined chunking contract.
4
+ *
5
+ * This file deliberately depends on no Node, Electron or browser APIs. The
6
+ * CLI, Desktop app and website can therefore cut the same bytes at the same
7
+ * positions. Storage manifests record the profile id; changing a profile
8
+ * means adding a new id, never changing the meaning of an existing one.
9
+ */
10
+ Object.defineProperty(exports, "__esModule", { value: true });
11
+ exports.DEFAULT_CHUNK_PROFILE = exports.HUGE_CHUNK_PROFILE = exports.LARGE_CHUNK_PROFILE = exports.MEDIUM_CHUNK_PROFILE = exports.LEGACY_CHUNK_PROFILE = void 0;
12
+ exports.profileForFileSize = profileForFileSize;
13
+ exports.cutPoints = cutPoints;
14
+ exports.chunkBoundaries = chunkBoundaries;
15
+ exports.manifestChunking = manifestChunking;
16
+ const KIB = 1024;
17
+ const MIB = 1024 * KIB;
18
+ /*
19
+ * Kept byte-for-byte for compatibility tests and for diagnosing old bundles.
20
+ * The seven-bit pre-target mask is why this profile cut shortly after its
21
+ * minimum instead of averaging eight MiB.
22
+ */
23
+ exports.LEGACY_CHUNK_PROFILE = Object.freeze({
24
+ id: "gear-legacy-v1",
25
+ algorithm: "gear-streaming",
26
+ minSize: 2 * MIB,
27
+ targetSize: 8 * MIB,
28
+ maxSize: 32 * MIB,
29
+ strictMask: 0x0003_5403,
30
+ looseMask: 0x0003_5000,
31
+ });
32
+ /*
33
+ * FastCDC normalisation uses a boundary that is half as likely before the
34
+ * target and twice as likely afterwards. Contiguous low-bit masks contain
35
+ * exactly the documented number of bits; the rolling gear hash mixes the
36
+ * current byte into those bits on every step.
37
+ */
38
+ exports.MEDIUM_CHUNK_PROFILE = Object.freeze({
39
+ id: "fastcdc-v2-medium",
40
+ algorithm: "fastcdc-v2",
41
+ minSize: 256 * KIB,
42
+ targetSize: 1 * MIB,
43
+ maxSize: 4 * MIB,
44
+ strictMask: 0x001f_ffff,
45
+ looseMask: 0x0007_ffff,
46
+ });
47
+ exports.LARGE_CHUNK_PROFILE = Object.freeze({
48
+ id: "fastcdc-v2-large",
49
+ algorithm: "fastcdc-v2",
50
+ minSize: 1 * MIB,
51
+ targetSize: 4 * MIB,
52
+ maxSize: 16 * MIB,
53
+ strictMask: 0x007f_ffff,
54
+ looseMask: 0x001f_ffff,
55
+ });
56
+ exports.HUGE_CHUNK_PROFILE = Object.freeze({
57
+ id: "fastcdc-v2-huge",
58
+ algorithm: "fastcdc-v2",
59
+ minSize: 2 * MIB,
60
+ targetSize: 8 * MIB,
61
+ maxSize: 32 * MIB,
62
+ strictMask: 0x00ff_ffff,
63
+ looseMask: 0x003f_ffff,
64
+ });
65
+ exports.DEFAULT_CHUNK_PROFILE = exports.HUGE_CHUNK_PROFILE;
66
+ /** The profile repository uploads use for a file of `size` bytes. */
67
+ function profileForFileSize(size) {
68
+ if (!Number.isFinite(size) || size < 0) {
69
+ throw new RangeError("File size must be a finite non-negative number");
70
+ }
71
+ if (size < 8 * MIB)
72
+ return null;
73
+ if (size < 128 * MIB)
74
+ return exports.MEDIUM_CHUNK_PROFILE;
75
+ if (size < 2 * 1024 * MIB)
76
+ return exports.LARGE_CHUNK_PROFILE;
77
+ return exports.HUGE_CHUNK_PROFILE;
78
+ }
79
+ const GEAR = (() => {
80
+ const table = new Uint32Array(256);
81
+ let seed = 0x9e3779b9;
82
+ for (let index = 0; index < 256; index += 1) {
83
+ seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
84
+ table[index] = seed;
85
+ }
86
+ return table;
87
+ })();
88
+ /**
89
+ * Cut positions inside `bytes`; the final tail is deliberately left to the
90
+ * streaming caller. Supplying the same bytes and profile on any platform
91
+ * must always return the same positions.
92
+ */
93
+ function cutPoints(bytes, profile = exports.DEFAULT_CHUNK_PROFILE) {
94
+ const cuts = [];
95
+ let from = 0;
96
+ let hash = 0;
97
+ for (let at = 0; at < bytes.length; at += 1) {
98
+ hash = ((hash << 1) + GEAR[bytes[at]]) >>> 0;
99
+ const length = at - from + 1;
100
+ if (length < profile.minSize)
101
+ continue;
102
+ const mask = length < profile.targetSize
103
+ ? profile.strictMask
104
+ : profile.looseMask;
105
+ if (length >= profile.maxSize || (hash & mask) === 0) {
106
+ cuts.push(at + 1);
107
+ from = at + 1;
108
+ hash = 0;
109
+ }
110
+ }
111
+ return cuts;
112
+ }
113
+ /** Full piece boundaries, including the final tail. */
114
+ function chunkBoundaries(bytes, profile = exports.DEFAULT_CHUNK_PROFILE) {
115
+ const bounds = cutPoints(bytes, profile);
116
+ if (bounds[bounds.length - 1] !== bytes.length)
117
+ bounds.push(bytes.length);
118
+ return bounds;
119
+ }
120
+ function manifestChunking(profile) {
121
+ return {
122
+ algorithm: profile.algorithm,
123
+ profile: profile.id,
124
+ min_size: profile.minSize,
125
+ avg_size: profile.targetSize,
126
+ max_size: profile.maxSize,
127
+ };
128
+ }
@@ -0,0 +1,21 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.retainedBasePaths = retainedBasePaths;
4
+ exports.acceptedManifestDigests = acceptedManifestDigests;
5
+ /**
6
+ * Paths from the base Version that remain in a candidate snapshot.
7
+ *
8
+ * A Version is a complete immutable view, never a list of changed files.
9
+ * Selected paths are replaced by freshly read declarations, or removed when
10
+ * explicitly marked as deletions. Everything unselected retains the exact
11
+ * object, chunk list, or pack slice from the base.
12
+ */
13
+ function retainedBasePaths(basePaths, selectedPaths) {
14
+ return [...basePaths]
15
+ .filter((path) => !selectedPaths.has(path))
16
+ .sort((left, right) => left.localeCompare(right));
17
+ }
18
+ /** Digest map returned by the accepted server manifest after an auto-merge. */
19
+ function acceptedManifestDigests(files) {
20
+ return Object.fromEntries([...(files ?? [])].flatMap((file) => file.path && file.sha256 ? [[file.path, file.sha256]] : []));
21
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@coderook/cli",
3
- "version": "0.20.0",
3
+ "version": "0.22.0",
4
4
  "description": "CodeRook from the command line, on any operating system",
5
5
  "license": "SEE LICENSE IN LICENSE.txt",
6
6
  "homepage": "https://coderook.com",