@coderook/cli 0.30.2 → 0.31.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/dist/cli/src/cli.js +25 -1
- package/dist/cli/src/config.js +209 -22
- package/dist/cli/src/git_remote.js +105 -16
- package/dist/cli/src/publish.js +5 -0
- package/dist/cli/src/stored_zip.js +85 -0
- package/dist/desktop-app/src/main/compress.js +21 -0
- package/dist/desktop-app/src/main/detect.js +84 -15
- package/dist/desktop-app/src/main/download.js +58 -2
- package/dist/desktop-app/src/main/upload.js +187 -83
- package/dist/desktop-app/src/main/worktree.js +193 -5
- package/dist/desktop-app/src/shared/compression_policy.js +122 -5
- package/dist/desktop-app/src/shared/format.js +31 -0
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "coderook",
|
|
3
3
|
"displayName": "CodeRook",
|
|
4
4
|
"description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.31.1",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "ACCA Gaming Productions",
|
|
8
8
|
"url": "https://coderook.com"
|
package/dist/cli/src/cli.js
CHANGED
|
@@ -623,7 +623,21 @@ async function commandSubmit(parsed) {
|
|
|
623
623
|
which is the wrong way round — so this is that capability, asked for
|
|
624
624
|
plainly.
|
|
625
625
|
*/
|
|
626
|
-
|
|
626
|
+
/*
|
|
627
|
+
Digests this machine has taken before, read before the scan rather than
|
|
628
|
+
before the upload.
|
|
629
|
+
|
|
630
|
+
The uploader has always accepted these and the desktop has always passed
|
|
631
|
+
them, but on the command line the expensive part happens earlier: the
|
|
632
|
+
scan below reads and hashes the whole folder to decide what changed, and
|
|
633
|
+
that is where a 21,071-file project spent 169 of its 299 seconds. The
|
|
634
|
+
same map serves both — `changedFiles` fills it in place, so what the scan
|
|
635
|
+
learns is what the upload is handed.
|
|
636
|
+
*/
|
|
637
|
+
const knownDigests = link?.repositoryId
|
|
638
|
+
? await (0, config_js_1.readDigests)(link.repositoryId)
|
|
639
|
+
: new Map();
|
|
640
|
+
const files = await (0, worktree_js_1.changedFiles)(folder, rules, baseline, hasFlag(parsed, "sync") ? "synchronize" : "add-and-update", knownDigests);
|
|
627
641
|
/*
|
|
628
642
|
An upgraded folder with no materialisation record and something that
|
|
629
643
|
differs from its recorded version is ambiguous in the one way that
|
|
@@ -788,6 +802,7 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
|
|
|
788
802
|
*/
|
|
789
803
|
const uploadRequest = (0, publish_js_1.uploadRequestFor)({
|
|
790
804
|
localPath: folder,
|
|
805
|
+
...(knownDigests.size ? { knownDigests } : {}),
|
|
791
806
|
changed: files.filter((file) => !file.deleted).map((file) => file.path),
|
|
792
807
|
deleted: files.filter((file) => file.deleted).map((file) => file.path),
|
|
793
808
|
/*
|
|
@@ -994,6 +1009,15 @@ The connection failed: ${text}`));
|
|
|
994
1009
|
local: result.local,
|
|
995
1010
|
manifest: result.manifest,
|
|
996
1011
|
});
|
|
1012
|
+
/*
|
|
1013
|
+
What this save hashed, kept for the next one.
|
|
1014
|
+
|
|
1015
|
+
Merged rather than replaced: `digestsTaken` holds only the files actually
|
|
1016
|
+
read this time, and everything served from the existing record has to
|
|
1017
|
+
survive or the next save reads it all again — which is precisely the cost
|
|
1018
|
+
this exists to avoid.
|
|
1019
|
+
*/
|
|
1020
|
+
await (0, config_js_1.writeDigests)(result.repositoryId, new Map([...knownDigests, ...uploader.digestsTaken()]));
|
|
997
1021
|
if (result.repeated) {
|
|
998
1022
|
console.log(`Already saved as ${accent(`v${result.sequence}`)} by an earlier attempt;` +
|
|
999
1023
|
` nothing was sent again.`);
|
package/dist/cli/src/config.js
CHANGED
|
@@ -3,16 +3,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.credentials = exports.DEFAULT_API = void 0;
|
|
6
|
+
exports.DamagedLinkError = exports.credentials = exports.DEFAULT_API = void 0;
|
|
7
7
|
exports.apiOrigin = apiOrigin;
|
|
8
8
|
exports.configDirectory = configDirectory;
|
|
9
9
|
exports.storeToken = storeToken;
|
|
10
10
|
exports.loadToken = loadToken;
|
|
11
11
|
exports.clearToken = clearToken;
|
|
12
12
|
exports.keyFor = keyFor;
|
|
13
|
+
exports.linkFileFor = linkFileFor;
|
|
13
14
|
exports.readLink = readLink;
|
|
14
15
|
exports.forgetLink = forgetLink;
|
|
15
16
|
exports.writeLink = writeLink;
|
|
17
|
+
exports.readDigests = readDigests;
|
|
18
|
+
exports.writeDigests = writeDigests;
|
|
16
19
|
/**
|
|
17
20
|
* Where the command-line tool keeps its credential and its project links.
|
|
18
21
|
*
|
|
@@ -21,6 +24,7 @@ exports.writeLink = writeLink;
|
|
|
21
24
|
* location follows each platform's convention rather than scattering dot
|
|
22
25
|
* directories about.
|
|
23
26
|
*/
|
|
27
|
+
const node_crypto_1 = require("node:crypto");
|
|
24
28
|
const node_fs_1 = require("node:fs");
|
|
25
29
|
const promises_1 = require("node:fs/promises");
|
|
26
30
|
const node_os_1 = require("node:os");
|
|
@@ -42,7 +46,21 @@ function configDirectory() {
|
|
|
42
46
|
return node_path_1.default.join(process.env.XDG_CONFIG_HOME ?? node_path_1.default.join((0, node_os_1.homedir)(), ".config"), "coderook");
|
|
43
47
|
}
|
|
44
48
|
const tokenFile = () => node_path_1.default.join(configDirectory(), "token");
|
|
45
|
-
|
|
49
|
+
/*
|
|
50
|
+
One small file per folder, under `links/`.
|
|
51
|
+
|
|
52
|
+
It was one file for every folder this machine had ever linked, read whole and
|
|
53
|
+
rewritten whole — pretty-printed — by every command. On a real machine that
|
|
54
|
+
file reached 419 MB across 549 folders, 536 of which no longer existed, and
|
|
55
|
+
every `cbx` paid about three seconds to read it before doing anything and
|
|
56
|
+
another two and a half to write it back after a save. It also lost updates:
|
|
57
|
+
two commands in two folders each rewrote the lot, and the later one erased
|
|
58
|
+
the earlier one's change.
|
|
59
|
+
|
|
60
|
+
The old file is read once, split into these, and renamed out of the way.
|
|
61
|
+
*/
|
|
62
|
+
const linksDirectory = () => node_path_1.default.join(configDirectory(), "links");
|
|
63
|
+
const legacyLinksFile = () => node_path_1.default.join(configDirectory(), "links.json");
|
|
46
64
|
/**
|
|
47
65
|
* Write a file only its owner can read, and atomically.
|
|
48
66
|
*
|
|
@@ -134,17 +152,165 @@ function keyFor(localPath) {
|
|
|
134
152
|
}
|
|
135
153
|
/** How the key used to be worked out, so existing links keep working. */
|
|
136
154
|
const legacyKeyFor = (localPath) => node_path_1.default.resolve(localPath).toLowerCase();
|
|
137
|
-
|
|
155
|
+
/**
|
|
156
|
+
* A link record that exists and cannot be read.
|
|
157
|
+
*
|
|
158
|
+
* Deliberately not the same as "no link". Reading a damaged record as absent
|
|
159
|
+
* makes a linked folder look new, and the next save creates a second project
|
|
160
|
+
* beside the real one — so this stops and names the file instead.
|
|
161
|
+
*/
|
|
162
|
+
class DamagedLinkError extends Error {
|
|
163
|
+
file;
|
|
164
|
+
constructor(file, cause) {
|
|
165
|
+
super(`CodeRook's record of which project a folder belongs to is damaged: ${file}\n` +
|
|
166
|
+
"Nothing was changed. Restore that file from a backup, or move it aside " +
|
|
167
|
+
"to treat the folder as not yet linked.");
|
|
168
|
+
this.file = file;
|
|
169
|
+
this.name = "DamagedLinkError";
|
|
170
|
+
this.cause = cause;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
exports.DamagedLinkError = DamagedLinkError;
|
|
174
|
+
/** The file one folder's link lives in. Hashed, because a key is a path. */
|
|
175
|
+
function linkFileFor(localPath) {
|
|
176
|
+
return storedFileFor(keyFor(localPath));
|
|
177
|
+
}
|
|
178
|
+
const storedFileFor = (key) => node_path_1.default.join(linksDirectory(), `${(0, node_crypto_1.createHash)("sha256").update(key).digest("hex").slice(0, 32)}.json`);
|
|
179
|
+
function sameMap(left, right) {
|
|
180
|
+
const keys = Object.keys(left);
|
|
181
|
+
if (keys.length !== Object.keys(right).length)
|
|
182
|
+
return false;
|
|
183
|
+
for (const key of keys)
|
|
184
|
+
if (left[key] !== right[key])
|
|
185
|
+
return false;
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
function pack(key, link) {
|
|
189
|
+
const { local, ...rest } = link;
|
|
190
|
+
if (local && sameMap(local, link.manifest)) {
|
|
191
|
+
return { key, link: { ...rest, localIsManifest: true } };
|
|
192
|
+
}
|
|
193
|
+
return { key, link: local ? { ...rest, local } : rest };
|
|
194
|
+
}
|
|
195
|
+
function unpack(stored) {
|
|
196
|
+
const { localIsManifest, ...link } = stored.link;
|
|
197
|
+
/* A copy, so a caller changing one cannot quietly change the other. */
|
|
198
|
+
return localIsManifest ? { ...link, local: { ...link.manifest } } : link;
|
|
199
|
+
}
|
|
200
|
+
async function exists(file) {
|
|
201
|
+
return (0, promises_1.access)(file).then(() => true, () => false);
|
|
202
|
+
}
|
|
203
|
+
/** One folder's record: undefined when there is none, and loud when damaged. */
|
|
204
|
+
async function readStored(key) {
|
|
205
|
+
const file = storedFileFor(key);
|
|
206
|
+
let text;
|
|
138
207
|
try {
|
|
139
|
-
|
|
208
|
+
text = await (0, promises_1.readFile)(file, "utf8");
|
|
140
209
|
}
|
|
141
|
-
catch {
|
|
142
|
-
|
|
210
|
+
catch (error) {
|
|
211
|
+
if (error.code === "ENOENT")
|
|
212
|
+
return undefined;
|
|
213
|
+
throw error;
|
|
214
|
+
}
|
|
215
|
+
try {
|
|
216
|
+
const stored = JSON.parse(text);
|
|
217
|
+
if (stored?.key !== key)
|
|
218
|
+
throw new Error("the record is for another folder");
|
|
219
|
+
return stored;
|
|
220
|
+
}
|
|
221
|
+
catch (error) {
|
|
222
|
+
throw new DamagedLinkError(file, error);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Write a record only if there is not one already.
|
|
227
|
+
*
|
|
228
|
+
* The migration's writer. A link written by a command running at the same
|
|
229
|
+
* moment is newer than anything in the old file, and must win. A hard link
|
|
230
|
+
* makes "create only if absent" one atomic step, and a reader never sees a
|
|
231
|
+
* half-written file; where the disk cannot do hard links the check and the
|
|
232
|
+
* move are two steps, which is only a race against a command in that same
|
|
233
|
+
* folder in the same instant.
|
|
234
|
+
*/
|
|
235
|
+
async function writeStoredIfAbsent(stored) {
|
|
236
|
+
const target = storedFileFor(stored.key);
|
|
237
|
+
if (await exists(target))
|
|
238
|
+
return;
|
|
239
|
+
await (0, promises_1.mkdir)(node_path_1.default.dirname(target), { recursive: true });
|
|
240
|
+
const temporary = `${target}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
241
|
+
await (0, promises_1.writeFile)(temporary, JSON.stringify(stored), "utf8");
|
|
242
|
+
await (0, promises_1.chmod)(temporary, 0o600).catch(() => undefined);
|
|
243
|
+
try {
|
|
244
|
+
await (0, promises_1.link)(temporary, target);
|
|
245
|
+
}
|
|
246
|
+
catch (error) {
|
|
247
|
+
const code = error.code;
|
|
248
|
+
if (code !== "EEXIST" && !(await exists(target))) {
|
|
249
|
+
await (0, promises_1.rename)(temporary, target);
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
finally {
|
|
254
|
+
await (0, promises_1.rm)(temporary, { force: true }).catch(() => undefined);
|
|
143
255
|
}
|
|
144
256
|
}
|
|
257
|
+
/**
|
|
258
|
+
* Split the old single file into one per folder, once.
|
|
259
|
+
*
|
|
260
|
+
* Whoever renames it to `.migrating` first does the work, and anyone arriving
|
|
261
|
+
* while that is going on — or after a run that was interrupted — works from
|
|
262
|
+
* the `.migrating` copy too. Every write here is "only if absent", so running
|
|
263
|
+
* it twice is harmless and a newer link written meanwhile is never overwritten.
|
|
264
|
+
*
|
|
265
|
+
* A damaged old file is put back and reported rather than migrated as empty:
|
|
266
|
+
* that would silently unlink every folder on the machine.
|
|
267
|
+
*/
|
|
268
|
+
async function migrateLegacyLinks() {
|
|
269
|
+
const legacy = legacyLinksFile();
|
|
270
|
+
const claimed = `${legacy}.migrating`;
|
|
271
|
+
await (0, promises_1.rename)(legacy, claimed).catch(() => undefined);
|
|
272
|
+
let text;
|
|
273
|
+
try {
|
|
274
|
+
text = await (0, promises_1.readFile)(claimed, "utf8");
|
|
275
|
+
}
|
|
276
|
+
catch (error) {
|
|
277
|
+
if (error.code === "ENOENT")
|
|
278
|
+
return;
|
|
279
|
+
throw error;
|
|
280
|
+
}
|
|
281
|
+
let links;
|
|
282
|
+
try {
|
|
283
|
+
links = JSON.parse(text);
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
await (0, promises_1.rename)(claimed, legacy).catch(() => undefined);
|
|
287
|
+
throw new DamagedLinkError(legacy, error);
|
|
288
|
+
}
|
|
289
|
+
for (const [key, link] of Object.entries(links)) {
|
|
290
|
+
await writeStoredIfAbsent(pack(key, link));
|
|
291
|
+
}
|
|
292
|
+
/* Kept rather than deleted: it is somebody's data, and moving it is enough. */
|
|
293
|
+
await (0, promises_1.rename)(claimed, `${legacy}.migrated`).catch(() => undefined);
|
|
294
|
+
}
|
|
295
|
+
async function lookUp(localPath) {
|
|
296
|
+
for (const key of new Set([keyFor(localPath), legacyKeyFor(localPath)])) {
|
|
297
|
+
const stored = await readStored(key);
|
|
298
|
+
if (!stored)
|
|
299
|
+
continue;
|
|
300
|
+
if ("forgotten" in stored)
|
|
301
|
+
return null;
|
|
302
|
+
return unpack(stored);
|
|
303
|
+
}
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
145
306
|
async function readLink(localPath) {
|
|
146
|
-
|
|
147
|
-
|
|
307
|
+
let link = await lookUp(localPath);
|
|
308
|
+
if (!link &&
|
|
309
|
+
((await exists(legacyLinksFile())) ||
|
|
310
|
+
(await exists(`${legacyLinksFile()}.migrating`)))) {
|
|
311
|
+
await migrateLegacyLinks();
|
|
312
|
+
link = await lookUp(localPath);
|
|
313
|
+
}
|
|
148
314
|
if (!link)
|
|
149
315
|
return null;
|
|
150
316
|
return {
|
|
@@ -160,25 +326,46 @@ async function readLink(localPath) {
|
|
|
160
326
|
* afterwards — which is the one thing this is for.
|
|
161
327
|
*/
|
|
162
328
|
async function forgetLink(localPath) {
|
|
163
|
-
|
|
164
|
-
const had = links[keyFor(localPath)] !== undefined ||
|
|
165
|
-
links[legacyKeyFor(localPath)] !== undefined;
|
|
166
|
-
if (!had)
|
|
329
|
+
if (!(await readLink(localPath)))
|
|
167
330
|
return false;
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
331
|
+
for (const key of new Set([keyFor(localPath), legacyKeyFor(localPath)])) {
|
|
332
|
+
const forgotten = { key, forgotten: true };
|
|
333
|
+
await writePrivate(storedFileFor(key), JSON.stringify(forgotten));
|
|
334
|
+
}
|
|
171
335
|
return true;
|
|
172
336
|
}
|
|
173
337
|
async function writeLink(localPath, link) {
|
|
174
|
-
const
|
|
175
|
-
// A folder recorded under the old key moves to the new one rather than
|
|
176
|
-
// being left behind as a second connection to the same place.
|
|
177
|
-
delete links[legacyKeyFor(localPath)];
|
|
338
|
+
const key = keyFor(localPath);
|
|
178
339
|
const baseVersionId = link.baseVersionId ?? link.versionId;
|
|
179
|
-
|
|
340
|
+
const stored = pack(key, {
|
|
180
341
|
...link,
|
|
181
342
|
...(baseVersionId ? { baseVersionId, versionId: baseVersionId } : {}),
|
|
182
|
-
};
|
|
183
|
-
await writePrivate(
|
|
343
|
+
});
|
|
344
|
+
await writePrivate(storedFileFor(key), JSON.stringify(stored));
|
|
345
|
+
// A folder recorded under the old key moves to the new one rather than
|
|
346
|
+
// being left behind as a second connection to the same place.
|
|
347
|
+
const legacy = legacyKeyFor(localPath);
|
|
348
|
+
if (legacy !== key)
|
|
349
|
+
await (0, promises_1.rm)(storedFileFor(legacy), { force: true });
|
|
350
|
+
}
|
|
351
|
+
const digestsFile = (projectId) => node_path_1.default.join(configDirectory(), "digests", `${projectId}.json`);
|
|
352
|
+
async function readDigests(projectId) {
|
|
353
|
+
try {
|
|
354
|
+
const raw = await (0, promises_1.readFile)(digestsFile(projectId), "utf8");
|
|
355
|
+
return new Map(Object.entries(JSON.parse(raw)));
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
/* No record yet, or one this build cannot read: hash everything. */
|
|
359
|
+
return new Map();
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
async function writeDigests(projectId, digests) {
|
|
363
|
+
if (!digests.size)
|
|
364
|
+
return;
|
|
365
|
+
try {
|
|
366
|
+
await writePrivate(digestsFile(projectId), JSON.stringify(Object.fromEntries(digests)));
|
|
367
|
+
}
|
|
368
|
+
catch {
|
|
369
|
+
/* The next save hashes again, which is where this started. */
|
|
370
|
+
}
|
|
184
371
|
}
|
|
@@ -114,6 +114,7 @@ const node_os_1 = __importDefault(require("node:os"));
|
|
|
114
114
|
const node_path_1 = __importDefault(require("node:path"));
|
|
115
115
|
const node_process_1 = __importDefault(require("node:process"));
|
|
116
116
|
const upload_js_1 = require("../../desktop-app/src/main/upload.js");
|
|
117
|
+
const stored_zip_js_1 = require("./stored_zip.js");
|
|
117
118
|
const tracks_js_1 = require("../../desktop-app/src/main/tracks.js");
|
|
118
119
|
const identify_js_1 = require("../../desktop-app/src/main/identify.js");
|
|
119
120
|
const config_js_1 = require("./config.js");
|
|
@@ -533,6 +534,17 @@ async function recordPushedCommits(marks) {
|
|
|
533
534
|
* Contents are deduplicated by SHA-256 across the whole history, so a file
|
|
534
535
|
* unchanged across a thousand versions is downloaded and sent to git once.
|
|
535
536
|
*/
|
|
537
|
+
/*
|
|
538
|
+
How many blobs a clone fetches at once, and how much it will hold doing it.
|
|
539
|
+
|
|
540
|
+
Eight lanes for the same reason the folder walks use eight: these are
|
|
541
|
+
latency, not work, and the measured curve flattens there. The byte ceiling
|
|
542
|
+
is what stops a window of large files becoming a window of memory — one
|
|
543
|
+
oversized file still goes through on its own, because the count check comes
|
|
544
|
+
first and an empty window always accepts one more.
|
|
545
|
+
*/
|
|
546
|
+
const BLOB_LANES = 8;
|
|
547
|
+
const BLOB_WINDOW_BYTES = 64 * 1024 * 1024;
|
|
536
548
|
async function doImport(refs, url) {
|
|
537
549
|
const { owner, slug } = (0, git_history_js_1.parseRemoteUrl)(url);
|
|
538
550
|
const project = await (0, api_js_1.findProject)(owner ? `${owner}/${slug}` : slug);
|
|
@@ -639,22 +651,99 @@ async function doImport(refs, url) {
|
|
|
639
651
|
let done = 0;
|
|
640
652
|
for (const version of fresh) {
|
|
641
653
|
const files = await downloader.files(repositoryId, version.id);
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
654
|
+
/*
|
|
655
|
+
Blobs first: fast-import requires a mark to exist before a commit
|
|
656
|
+
names it. Fetched a window at a time and written out in order.
|
|
657
|
+
|
|
658
|
+
This asked for one file, waited for it, wrote it to a scratch path,
|
|
659
|
+
read it back, and only then asked for the next — so a clone cost one
|
|
660
|
+
network round trip per file, in series. Against a 21,071-file project
|
|
661
|
+
that is 21,071 round trips at about seven tenths of a second each,
|
|
662
|
+
and a clone that never finished: measured over twenty seconds it was
|
|
663
|
+
using 0.17s of processor and moving two kilobytes.
|
|
664
|
+
|
|
665
|
+
The stream fast-import reads has to stay ordered, so the fetching is
|
|
666
|
+
what overlaps, not the writing. The window is bounded by count and by
|
|
667
|
+
bytes together, because sixteen source files and sixteen video files
|
|
668
|
+
are not the same amount to be holding at once.
|
|
669
|
+
*/
|
|
670
|
+
/*
|
|
671
|
+
The whole version in one request, where that works.
|
|
672
|
+
|
|
673
|
+
Fetching per file is one round trip each: 21,071 of them on a real
|
|
674
|
+
Unity project, which is four hours in series and about half an hour
|
|
675
|
+
even eight at a time. The archive is the same bytes in one request.
|
|
676
|
+
|
|
677
|
+
Anything unexpected about it — an entry this cannot read, a short
|
|
678
|
+
download, a service that does not offer the route — falls through to
|
|
679
|
+
the per-file path below, which is slow and known to work. A clone
|
|
680
|
+
that is slow is a nuisance; a clone that writes the wrong bytes is
|
|
681
|
+
not recoverable by trying again.
|
|
682
|
+
*/
|
|
683
|
+
const fromArchive = new Map();
|
|
684
|
+
try {
|
|
685
|
+
const packed = await downloader.archive(repositoryId, version.id);
|
|
686
|
+
for (const entry of (0, stored_zip_js_1.readStoredZip)(packed)) {
|
|
687
|
+
fromArchive.set(entry.path, entry.bytes);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
catch (error) {
|
|
691
|
+
/*
|
|
692
|
+
Said out loud, not swallowed.
|
|
693
|
+
|
|
694
|
+
This fell back silently, and the silence cost an hour: a deadline
|
|
695
|
+
meant for ordinary requests was aborting the archive, the import
|
|
696
|
+
dropped to fetching every file on its own, and from the outside it
|
|
697
|
+
looked exactly like the slow clone the archive was added to fix.
|
|
698
|
+
A fallback nobody can see is a fallback nobody can diagnose.
|
|
699
|
+
*/
|
|
700
|
+
say(` the archive for v${version.sequence} could not be used ` +
|
|
701
|
+
`(${error instanceof Error ? error.message : String(error)}); ` +
|
|
702
|
+
`fetching its files one by one`);
|
|
703
|
+
fromArchive.clear();
|
|
704
|
+
}
|
|
705
|
+
const seenBlob = new Set();
|
|
706
|
+
const wantedBlobs = files.filter((file) => {
|
|
707
|
+
if (blobMark.has(file.sha256) || seenBlob.has(file.sha256))
|
|
708
|
+
return false;
|
|
709
|
+
seenBlob.add(file.sha256);
|
|
710
|
+
return true;
|
|
711
|
+
});
|
|
712
|
+
for (let at = 0; at < wantedBlobs.length;) {
|
|
713
|
+
const window = [];
|
|
714
|
+
let held = 0;
|
|
715
|
+
while (at < wantedBlobs.length && window.length < BLOB_LANES) {
|
|
716
|
+
const size = Number(wantedBlobs[at].sourceSize ?? 0);
|
|
717
|
+
if (window.length > 0 && held + size > BLOB_WINDOW_BYTES)
|
|
718
|
+
break;
|
|
719
|
+
held += size;
|
|
720
|
+
window.push(wantedBlobs[at]);
|
|
721
|
+
at += 1;
|
|
722
|
+
}
|
|
723
|
+
const fetched = await Promise.all(window.map(async (file, lane) => {
|
|
724
|
+
/* Already in hand, if the archive carried it. */
|
|
725
|
+
const packed = fromArchive.get(file.path);
|
|
726
|
+
if (packed)
|
|
727
|
+
return { file, bytes: packed };
|
|
728
|
+
/*
|
|
729
|
+
A scratch path per lane. They shared one name, which is fine
|
|
730
|
+
while only one download is ever in flight and silently wrong
|
|
731
|
+
the moment two are.
|
|
732
|
+
*/
|
|
733
|
+
const target = node_path_1.default.join(scratch, `blob-${lane}.bin`);
|
|
734
|
+
await (0, promises_1.rm)(target, { force: true });
|
|
735
|
+
await downloader.fileTo(repositoryId, version.id, file, target);
|
|
736
|
+
return { file, bytes: await (0, promises_1.readFile)(target) };
|
|
737
|
+
}));
|
|
738
|
+
for (const { file, bytes } of fetched) {
|
|
739
|
+
const mark = nextMark++;
|
|
740
|
+
blobMark.set(file.sha256, mark);
|
|
741
|
+
send("blob");
|
|
742
|
+
send(`mark :${mark}`);
|
|
743
|
+
send(`data ${bytes.length}`);
|
|
744
|
+
node_process_1.default.stdout.write(bytes);
|
|
745
|
+
node_process_1.default.stdout.write("\n");
|
|
746
|
+
}
|
|
658
747
|
}
|
|
659
748
|
const mark = nextMark++;
|
|
660
749
|
commitMark.set(version.id, mark);
|
package/dist/cli/src/publish.js
CHANGED
|
@@ -23,6 +23,11 @@ function uploadRequestFor(input) {
|
|
|
23
23
|
? { expectedHeadVersionId: input.baseVersionId }
|
|
24
24
|
: {}),
|
|
25
25
|
...(input.known ? { known: input.known } : {}),
|
|
26
|
+
/*
|
|
27
|
+
Digests this machine has taken before, so an unchanged file is not read
|
|
28
|
+
again. The desktop has always passed these; the command line never did.
|
|
29
|
+
*/
|
|
30
|
+
...(input.knownDigests ? { knownDigests: input.knownDigests } : {}),
|
|
26
31
|
};
|
|
27
32
|
}
|
|
28
33
|
function classifyPublishFailure(error) {
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Reading the archive CodeRook builds, without a compression library.
|
|
4
|
+
*
|
|
5
|
+
* A clone fetched every file individually: one HTTP round trip per blob, in
|
|
6
|
+
* series. Against a 21,071-file project that is 21,071 round trips, which at
|
|
7
|
+
* seven tenths of a second each is four hours — and the clone that exposed
|
|
8
|
+
* this simply never finished. Fetching eight at a time cut it to about half
|
|
9
|
+
* an hour; asking for the version as one archive cuts it to one request.
|
|
10
|
+
*
|
|
11
|
+
* Written by hand for the same reason `backend/src/archive.ts` writes it by
|
|
12
|
+
* hand: the entries are *stored*, never deflated, so the whole job is a
|
|
13
|
+
* header, a body, and skipping the directory at the end. Pulling in a zip
|
|
14
|
+
* library to not decompress anything would be a poor trade.
|
|
15
|
+
*
|
|
16
|
+
* Only the format that writer produces is accepted. Anything else — a
|
|
17
|
+
* deflated entry, a size deferred to a data descriptor, a Zip64 field — is
|
|
18
|
+
* refused rather than guessed at, and the caller falls back to fetching the
|
|
19
|
+
* files one by one. A reader that quietly mangles a project is worse than a
|
|
20
|
+
* slow one.
|
|
21
|
+
*/
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.readStoredZip = readStoredZip;
|
|
24
|
+
const LOCAL_HEADER = 0x04034b50;
|
|
25
|
+
const CENTRAL_HEADER = 0x02014b50;
|
|
26
|
+
/* An archive of nothing has no entries and no directory records — it opens
|
|
27
|
+
with this, and a reader that only knew the other two would refuse it. */
|
|
28
|
+
const END_OF_DIRECTORY = 0x06054b50;
|
|
29
|
+
/** Set in the general-purpose flags when sizes come after the body. */
|
|
30
|
+
const SIZES_DEFERRED = 0x0008;
|
|
31
|
+
const STORED = 0;
|
|
32
|
+
/**
|
|
33
|
+
* Every entry in a stored archive, in the order it was written.
|
|
34
|
+
*
|
|
35
|
+
* Deliberately whole-buffer rather than streaming: the caller is holding one
|
|
36
|
+
* version's files to hand to fast-import, which needs them in order anyway,
|
|
37
|
+
* and a version large enough for this to matter is one the archive path
|
|
38
|
+
* should not be taking.
|
|
39
|
+
*/
|
|
40
|
+
function readStoredZip(archive) {
|
|
41
|
+
const view = new DataView(archive.buffer, archive.byteOffset, archive.byteLength);
|
|
42
|
+
const names = new TextDecoder("utf-8", { fatal: false });
|
|
43
|
+
const entries = [];
|
|
44
|
+
let at = 0;
|
|
45
|
+
while (at + 4 <= archive.length) {
|
|
46
|
+
const signature = view.getUint32(at, true);
|
|
47
|
+
/* Either record marks the end of the entries. */
|
|
48
|
+
if (signature === CENTRAL_HEADER || signature === END_OF_DIRECTORY)
|
|
49
|
+
break;
|
|
50
|
+
if (signature !== LOCAL_HEADER) {
|
|
51
|
+
throw new Error(`Not a stored archive: unexpected record at byte ${at}`);
|
|
52
|
+
}
|
|
53
|
+
if (at + 30 > archive.length)
|
|
54
|
+
throw new Error("Archive ends inside a header");
|
|
55
|
+
const flags = view.getUint16(at + 6, true);
|
|
56
|
+
const method = view.getUint16(at + 8, true);
|
|
57
|
+
const compressed = view.getUint32(at + 18, true);
|
|
58
|
+
const nameLength = view.getUint16(at + 26, true);
|
|
59
|
+
const extraLength = view.getUint16(at + 28, true);
|
|
60
|
+
if (method !== STORED) {
|
|
61
|
+
throw new Error(`Archive entry is compressed (method ${method})`);
|
|
62
|
+
}
|
|
63
|
+
if (flags & SIZES_DEFERRED) {
|
|
64
|
+
throw new Error("Archive entry defers its size to a data descriptor");
|
|
65
|
+
}
|
|
66
|
+
/*
|
|
67
|
+
0xffffffff is Zip64's marker for "the real size is in an extra field".
|
|
68
|
+
This writer never emits one, so seeing it means the archive did not come
|
|
69
|
+
from where it is assumed to have come from.
|
|
70
|
+
*/
|
|
71
|
+
if (compressed === 0xffffffff)
|
|
72
|
+
throw new Error("Archive entry needs Zip64");
|
|
73
|
+
const nameAt = at + 30;
|
|
74
|
+
const bodyAt = nameAt + nameLength + extraLength;
|
|
75
|
+
const end = bodyAt + compressed;
|
|
76
|
+
if (end > archive.length)
|
|
77
|
+
throw new Error("Archive ends inside an entry");
|
|
78
|
+
entries.push({
|
|
79
|
+
path: names.decode(archive.subarray(nameAt, nameAt + nameLength)),
|
|
80
|
+
bytes: archive.subarray(bodyAt, end),
|
|
81
|
+
});
|
|
82
|
+
at = end;
|
|
83
|
+
}
|
|
84
|
+
return entries;
|
|
85
|
+
}
|
|
@@ -55,6 +55,27 @@ async function encodeForUpload(original, relativePath, allowGzip) {
|
|
|
55
55
|
if (!(0, compression_policy_1.shouldTryGzip)(relativePath, original.byteLength, allowGzip))
|
|
56
56
|
return plain;
|
|
57
57
|
try {
|
|
58
|
+
/*
|
|
59
|
+
Ask a little of it before asking all of it.
|
|
60
|
+
|
|
61
|
+
The extension list is a guess and the cost of a miss used to be the whole
|
|
62
|
+
file. Two sampled windows make it an eighth of that, which is what lets
|
|
63
|
+
the decision stay measured — the principle above — for a format nothing
|
|
64
|
+
here has heard of, instead of measured-at-any-price.
|
|
65
|
+
*/
|
|
66
|
+
if ((0, compression_policy_1.worthSampling)(original.byteLength)) {
|
|
67
|
+
let anyWindowArguesFor = false;
|
|
68
|
+
for (const window of (0, compression_policy_1.sampleWindows)(original.byteLength)) {
|
|
69
|
+
const sample = original.subarray(window.at, window.at + window.size);
|
|
70
|
+
const tried = await deflate(sample, { level: 6 });
|
|
71
|
+
if (!(0, compression_policy_1.sampleSaysDoNotBother)(sample.byteLength, tried.byteLength)) {
|
|
72
|
+
anyWindowArguesFor = true;
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (!anyWindowArguesFor)
|
|
77
|
+
return plain;
|
|
78
|
+
}
|
|
58
79
|
const packed = await deflate(original, { level: 6 });
|
|
59
80
|
if (!(0, compression_policy_1.gzipIsWorthKeeping)(original.byteLength, packed.byteLength))
|
|
60
81
|
return plain;
|