@indigoai-us/hq-cli 5.345.137 → 5.345.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/commands/storage-offload.d.ts +154 -42
- package/dist/commands/storage-offload.js +469 -111
- package/dist/commands/storage.d.ts +12 -1
- package/dist/commands/storage.js +97 -17
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +577 -441
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/package.json +2 -2
|
@@ -1,45 +1,56 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `hq storage offload` / `hq storage fetch` — move big files
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* `hq storage offload` / `hq storage fetch` — move big files out of the HQ
|
|
3
|
+
* folder into HQ cloud storage and leave small linked placeholders behind.
|
|
4
|
+
* Shared contract: owner-approved offload design, 2026-10-09; company upload
|
|
5
|
+
* for files over the sync limit approved 2026-10-15.
|
|
5
6
|
*
|
|
6
7
|
* Two kinds of big file:
|
|
7
8
|
* - History copies (old versions inside the HQ root git history). The CLI
|
|
8
|
-
* uploads each blob, verifies size + sha256
|
|
9
|
-
* verified blobs to core `hq-storage.sh
|
|
10
|
-
* to the pointer text. Working-tree
|
|
9
|
+
* uploads each blob to the caller's PERSONAL vault, verifies size + sha256
|
|
10
|
+
* + git blob id, then hands only verified blobs to core `hq-storage.sh
|
|
11
|
+
* strip-large`, which rewrites history to the pointer text. Working-tree
|
|
12
|
+
* files are never touched.
|
|
11
13
|
* - Current files (`--current [paths...]`, idle big files in the working
|
|
12
|
-
* tree). Each file goes to the vault that owns its path
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
14
|
+
* tree). Each file goes to the vault that owns its path:
|
|
15
|
+
* - A file in a company folder belongs to that company's vault, and only
|
|
16
|
+
* an active member can offload it. When HQ sync already stored this
|
|
17
|
+
* exact version (journal hash + size, and the vault object still has
|
|
18
|
+
* the journal's ETag), the file is replaced by a placeholder pointing
|
|
19
|
+
* at that object. A file over the sync limit (SYNC_MAX_FILE_BYTES),
|
|
20
|
+
* which sync never uploads, is uploaded to the company vault at its
|
|
21
|
+
* sync key by a resumable, checksum-verified multipart upload
|
|
22
|
+
* (`upload+offload`). Once the vault copy is verified the file is
|
|
23
|
+
* recorded in the sync journal as synced and cloud-only, then the
|
|
24
|
+
* placeholder is written, then the local file is removed. A path the
|
|
25
|
+
* sync ignore rules exclude is refused and the excluding rule is named
|
|
26
|
+
* (for `.hqinclude`, with the exact line to add). Other unsynced
|
|
27
|
+
* company files are skipped with "run hq sync first".
|
|
28
|
+
* - Any other file points at its personal-vault sync object the same
|
|
29
|
+
* way, or is uploaded to the personal vault's `.hq-offload/blobs/`.
|
|
30
|
+
* Current-file offload runs only when the installed HQ sync holds every
|
|
31
|
+
* `.hqcloud` sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and
|
|
32
|
+
* skips `.hq-offload/`; otherwise it reports "update HQ sync".
|
|
23
33
|
*
|
|
24
|
-
* Fetch also reads placeholders that HQ sync
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* it can never sync back.
|
|
34
|
+
* Fetch also reads placeholders that HQ sync wrote on a teammate's machine
|
|
35
|
+
* (`vault: company`, the company vault key, `sha256: unknown`, `remote-etag`).
|
|
36
|
+
* Those download from the company vault, only for an active member, and
|
|
37
|
+
* verify by size, ETag and, when the object carries it, its `hq-sha256`
|
|
38
|
+
* metadata. A placeholder marked `orphaned: true` (a teammate deleted the
|
|
39
|
+
* file) is refused unless `--force-orphan` is given, which writes the caller's
|
|
40
|
+
* own personal-vault copy to a file outside HQ so it can never sync back.
|
|
31
41
|
*
|
|
32
|
-
* Tenant isolation: uploads only ever go to the caller's
|
|
33
|
-
* the personal store refuses a company vault bucket. A
|
|
34
|
-
*
|
|
35
|
-
*
|
|
42
|
+
* Tenant isolation: history and personal uploads only ever go to the caller's
|
|
43
|
+
* personal vault, and the personal store refuses a company vault bucket. A
|
|
44
|
+
* company vault is opened only for a company the caller is an active member
|
|
45
|
+
* of, and only for a file in that company's folder; the company uploader must
|
|
46
|
+
* name that company and refuses a personal bucket.
|
|
36
47
|
*/
|
|
37
48
|
import * as crypto from "node:crypto";
|
|
38
49
|
import * as fs from "node:fs";
|
|
39
50
|
import * as os from "node:os";
|
|
40
51
|
import * as path from "node:path";
|
|
41
52
|
import { spawn, execFile } from "node:child_process";
|
|
42
|
-
import {
|
|
53
|
+
import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, } from "@aws-sdk/client-s3";
|
|
43
54
|
import { checkPreconditions, coreHasFlag, coreSupports, dropExpiredBackups, PRECONDITION_FALLBACK, coreKeepsUndo, reclaimFields, reclaimWindowHours, runCoreScript, } from "./storage-core.js";
|
|
44
55
|
// ── Constants ───────────────────────────────────────────────────────────────
|
|
45
56
|
export const PLACEHOLDER_SUFFIX = ".hqcloud";
|
|
@@ -53,6 +64,13 @@ const UPDATE_HQ = "update HQ";
|
|
|
53
64
|
export const UPDATE_SYNC = "update HQ sync";
|
|
54
65
|
/** Reason on company files HQ sync has not stored in the company vault yet. */
|
|
55
66
|
export const RUN_SYNC_FIRST = "run hq sync first";
|
|
67
|
+
/**
|
|
68
|
+
* HQ sync's per-file limit (hq-cloud SYNC_MAX_FILE_BYTES; sync skips files
|
|
69
|
+
* strictly larger). storage.ts passes hq-cloud's value when it exports one.
|
|
70
|
+
*/
|
|
71
|
+
export const SYNC_MAX_FILE_BYTES = 50 * 1024 * 1024;
|
|
72
|
+
/** Object metadata key holding the sha256 of a company-vault offload upload. */
|
|
73
|
+
export const VAULT_SHA256_METADATA_KEY = "hq-sha256";
|
|
56
74
|
const DAY_MS = 86_400_000;
|
|
57
75
|
const PERSONAL_BUCKET_PREFIX = "hq-vault-prs-";
|
|
58
76
|
/** Top-level HQ-root folder that holds offloaded blobs in the personal vault. */
|
|
@@ -193,96 +211,268 @@ function isNotFound(err) {
|
|
|
193
211
|
const e = err;
|
|
194
212
|
return e?.name === "NotFound" || e?.name === "NoSuchKey" || e?.$metadata?.httpStatusCode === 404;
|
|
195
213
|
}
|
|
214
|
+
// ── Vault request errors and credential refresh ─────────────────────────────
|
|
215
|
+
/** Error codes S3 and STS return when the request's credentials are refused. */
|
|
216
|
+
const CREDENTIAL_ERROR_HINTS = {
|
|
217
|
+
ExpiredToken: "credentials expired",
|
|
218
|
+
ExpiredTokenException: "credentials expired",
|
|
219
|
+
TokenRefreshRequired: "credentials expired",
|
|
220
|
+
RequestExpired: "credentials expired",
|
|
221
|
+
InvalidToken: "credentials rejected",
|
|
222
|
+
InvalidAccessKeyId: "credentials rejected",
|
|
223
|
+
InvalidSecurityToken: "credentials rejected",
|
|
224
|
+
InvalidClientTokenId: "credentials rejected",
|
|
225
|
+
AccessDenied: "access denied",
|
|
226
|
+
};
|
|
227
|
+
/** Placeholder names the SDK uses when the response carried no error code. */
|
|
228
|
+
const NO_CODE = new Set(["", "Unknown", "UnknownError", "Error"]);
|
|
229
|
+
function awsErrorCode(err) {
|
|
230
|
+
const candidates = [err.Code, err.code, err.name, err.$response?.headers?.["x-amz-error-code"]];
|
|
231
|
+
return candidates.find((c) => typeof c === "string" && !NO_CODE.has(c));
|
|
232
|
+
}
|
|
233
|
+
function isAwsError(err) {
|
|
234
|
+
if (!err || typeof err !== "object")
|
|
235
|
+
return false;
|
|
236
|
+
const e = err;
|
|
237
|
+
return typeof e.$metadata === "object" || typeof e.$fault === "string" || e.name === "Unknown";
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* True when S3 refused the request's credentials: an expired, invalid or
|
|
241
|
+
* denied token, or a 403 with no error body (what S3 sends for HEAD).
|
|
242
|
+
*/
|
|
243
|
+
export function isCredentialRejection(err) {
|
|
244
|
+
if (!isAwsError(err))
|
|
245
|
+
return false;
|
|
246
|
+
const status = err.$metadata?.httpStatusCode;
|
|
247
|
+
const code = awsErrorCode(err);
|
|
248
|
+
if (code)
|
|
249
|
+
return code in CREDENTIAL_ERROR_HINTS && (status === undefined || status === 400 || status === 403);
|
|
250
|
+
return status === 403;
|
|
251
|
+
}
|
|
252
|
+
/** A failed vault request, described with its HTTP status, error code and vault. */
|
|
253
|
+
export class VaultRequestError extends Error {
|
|
254
|
+
status;
|
|
255
|
+
code;
|
|
256
|
+
constructor(message, opts) {
|
|
257
|
+
super(message, { cause: opts.cause });
|
|
258
|
+
this.name = "VaultRequestError";
|
|
259
|
+
this.status = opts.status;
|
|
260
|
+
this.code = opts.code;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Describes an AWS error from `op` against `where` so the user sees the HTTP
|
|
265
|
+
* status, the error code and the vault, never the SDK's bare "UnknownError".
|
|
266
|
+
* Errors that did not come from AWS pass through unchanged.
|
|
267
|
+
*/
|
|
268
|
+
export function describeVaultError(err, op, where, retried = false) {
|
|
269
|
+
if (err instanceof VaultRequestError || !isAwsError(err))
|
|
270
|
+
return err;
|
|
271
|
+
const status = err.$metadata?.httpStatusCode;
|
|
272
|
+
const code = awsErrorCode(err);
|
|
273
|
+
let what;
|
|
274
|
+
if (code) {
|
|
275
|
+
const hint = CREDENTIAL_ERROR_HINTS[code];
|
|
276
|
+
what = hint ? `${code} (${hint})` : code;
|
|
277
|
+
}
|
|
278
|
+
else if (status === 403) {
|
|
279
|
+
what = "Forbidden (no error body: credentials expired or access denied)";
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
what = "error with no error code";
|
|
283
|
+
}
|
|
284
|
+
let message = `${op} failed in ${where}: ${status ? `HTTP ${status} ` : ""}${what}`;
|
|
285
|
+
if (retried)
|
|
286
|
+
message += ", after re-vending credentials once";
|
|
287
|
+
const detail = err.message?.trim();
|
|
288
|
+
if (detail && !NO_CODE.has(detail) && detail !== code)
|
|
289
|
+
message += `: ${detail}`;
|
|
290
|
+
return new VaultRequestError(message, { status, code, cause: err });
|
|
291
|
+
}
|
|
292
|
+
/** Turns any per-file failure into one line for the run's error list. */
|
|
293
|
+
export function failureMessage(err) {
|
|
294
|
+
const described = describeVaultError(err, "Request", "the vault");
|
|
295
|
+
return described instanceof Error ? described.message : String(described);
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Runs vault requests on `initial` and, when S3 refuses the credentials,
|
|
299
|
+
* re-vends once and retries the request once on the new client. Requests
|
|
300
|
+
* refused on the same client share one re-vend. A request refused again
|
|
301
|
+
* after the re-vend fails with a described error; it is never retried twice.
|
|
302
|
+
*/
|
|
303
|
+
export function credentialRetry(initial, where, revend) {
|
|
304
|
+
let client = initial;
|
|
305
|
+
let generation = 0;
|
|
306
|
+
let pending;
|
|
307
|
+
return async function call(op, fn) {
|
|
308
|
+
const used = generation;
|
|
309
|
+
try {
|
|
310
|
+
return await fn(client);
|
|
311
|
+
}
|
|
312
|
+
catch (err) {
|
|
313
|
+
if (!revend || !isCredentialRejection(err))
|
|
314
|
+
throw describeVaultError(err, op, where);
|
|
315
|
+
if (generation === used) {
|
|
316
|
+
pending ??= revend()
|
|
317
|
+
.then((fresh) => {
|
|
318
|
+
client = fresh;
|
|
319
|
+
generation += 1;
|
|
320
|
+
})
|
|
321
|
+
.finally(() => {
|
|
322
|
+
pending = undefined;
|
|
323
|
+
});
|
|
324
|
+
try {
|
|
325
|
+
await pending;
|
|
326
|
+
}
|
|
327
|
+
catch (vendErr) {
|
|
328
|
+
const refused = describeVaultError(err, op, where);
|
|
329
|
+
throw new Error(`${refused.message}; re-vending credentials for ${where} failed: ${vendErr?.message ?? String(vendErr)}`, { cause: vendErr });
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
try {
|
|
333
|
+
return await fn(client);
|
|
334
|
+
}
|
|
335
|
+
catch (again) {
|
|
336
|
+
throw describeVaultError(again, op, where, true);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
const CLIENT_METHOD_OPS = {
|
|
342
|
+
create: "CreateMultipartUpload",
|
|
343
|
+
uploadPart: "UploadPart",
|
|
344
|
+
listParts: "ListParts",
|
|
345
|
+
complete: "CompleteMultipartUpload",
|
|
346
|
+
abort: "AbortMultipartUpload",
|
|
347
|
+
head: "HeadObject",
|
|
348
|
+
};
|
|
349
|
+
/**
|
|
350
|
+
* Wraps every method of `initial` (a multipart client) so each call gets
|
|
351
|
+
* {@link credentialRetry}: one re-vend and one retry when S3 refuses the
|
|
352
|
+
* credentials, and a described error otherwise.
|
|
353
|
+
*/
|
|
354
|
+
export function credentialRetryingClient(initial, where, revend) {
|
|
355
|
+
const call = credentialRetry(initial, where, revend);
|
|
356
|
+
const wrapped = {};
|
|
357
|
+
for (const name of Object.keys(initial)) {
|
|
358
|
+
if (typeof initial[name] !== "function")
|
|
359
|
+
continue;
|
|
360
|
+
wrapped[name] = (...args) => call(CLIENT_METHOD_OPS[name] ?? name, (c) => c[name].apply(c, args));
|
|
361
|
+
}
|
|
362
|
+
return { ...initial, ...wrapped };
|
|
363
|
+
}
|
|
364
|
+
function pipeToNewFile(body, dest) {
|
|
365
|
+
return new Promise((resolve, reject) => {
|
|
366
|
+
const out = fs.createWriteStream(dest, { flags: "wx" });
|
|
367
|
+
body.on("error", reject);
|
|
368
|
+
out.on("error", reject);
|
|
369
|
+
out.on("finish", () => resolve());
|
|
370
|
+
body.pipe(out);
|
|
371
|
+
});
|
|
372
|
+
}
|
|
196
373
|
/**
|
|
197
374
|
* S3-backed store over the caller's personal vault bucket. Accepts only
|
|
198
375
|
* personal vault buckets (`hq-vault-prs-*`) so a mis-resolved entity can never
|
|
199
376
|
* receive offloaded bytes.
|
|
200
377
|
*/
|
|
201
|
-
export function s3BlobStore(bucket, s3) {
|
|
378
|
+
export function s3BlobStore(bucket, s3, opts = {}) {
|
|
202
379
|
if (!bucket || !bucket.startsWith(PERSONAL_BUCKET_PREFIX)) {
|
|
203
380
|
throw new Error(`Refusing to offload into '${bucket}': offload only writes to your personal vault.`);
|
|
204
381
|
}
|
|
382
|
+
const call = credentialRetry(s3, opts.where ?? `vault '${bucket}'`, opts.revend);
|
|
205
383
|
return {
|
|
206
384
|
vault: "personal",
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
385
|
+
head(key) {
|
|
386
|
+
return call("HEAD", async (client) => {
|
|
387
|
+
try {
|
|
388
|
+
const res = (await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })));
|
|
389
|
+
const fromChecksum = res.ChecksumSHA256 && !res.ChecksumSHA256.includes("-")
|
|
390
|
+
? Buffer.from(res.ChecksumSHA256, "base64").toString("hex")
|
|
391
|
+
: undefined;
|
|
392
|
+
// The server-computed checksum wins; client metadata is a fallback for
|
|
393
|
+
// objects stored without one.
|
|
394
|
+
return { size: res.ContentLength ?? -1, sha256: fromChecksum ?? res.Metadata?.sha256 };
|
|
395
|
+
}
|
|
396
|
+
catch (err) {
|
|
397
|
+
if (isNotFound(err))
|
|
398
|
+
return null;
|
|
399
|
+
throw err;
|
|
400
|
+
}
|
|
401
|
+
});
|
|
222
402
|
},
|
|
223
403
|
async put(key, file, digest) {
|
|
224
|
-
await
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
404
|
+
await call("PUT", async (client) => {
|
|
405
|
+
// A fresh stream per attempt: a refused attempt may have read from the last one.
|
|
406
|
+
const body = fs.createReadStream(file);
|
|
407
|
+
try {
|
|
408
|
+
await client.send(new PutObjectCommand({
|
|
409
|
+
Bucket: bucket,
|
|
410
|
+
Key: key,
|
|
411
|
+
Body: body,
|
|
412
|
+
ContentLength: digest.size,
|
|
413
|
+
ChecksumSHA256: Buffer.from(digest.sha256, "hex").toString("base64"),
|
|
414
|
+
StorageClass: "STANDARD",
|
|
415
|
+
Metadata: { sha256: digest.sha256, "git-blob": digest.gitBlob },
|
|
416
|
+
}));
|
|
417
|
+
}
|
|
418
|
+
finally {
|
|
419
|
+
body.destroy();
|
|
420
|
+
}
|
|
421
|
+
});
|
|
233
422
|
},
|
|
234
423
|
async download(key, dest) {
|
|
235
|
-
|
|
424
|
+
// Only the request is retried: the destination is created by the pipe below.
|
|
425
|
+
const res = await call("GET", async (client) => (await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }))));
|
|
236
426
|
const body = res.Body;
|
|
237
427
|
if (!body)
|
|
238
428
|
throw new Error(`Empty download for ${key}.`);
|
|
239
|
-
await
|
|
240
|
-
const out = fs.createWriteStream(dest, { flags: "wx" });
|
|
241
|
-
body.on("error", reject);
|
|
242
|
-
out.on("error", reject);
|
|
243
|
-
out.on("finish", () => resolve());
|
|
244
|
-
body.pipe(out);
|
|
245
|
-
});
|
|
429
|
+
await pipeToNewFile(body, dest);
|
|
246
430
|
},
|
|
247
431
|
};
|
|
248
432
|
}
|
|
249
|
-
export function s3VaultObjectStore(bucket, s3) {
|
|
433
|
+
export function s3VaultObjectStore(bucket, s3, opts = {}) {
|
|
434
|
+
const call = credentialRetry(s3, opts.where ?? `vault '${bucket}'`, opts.revend);
|
|
250
435
|
return {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
436
|
+
head(key) {
|
|
437
|
+
return call("HEAD", async (client) => {
|
|
438
|
+
try {
|
|
439
|
+
const res = (await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })));
|
|
440
|
+
const head = {
|
|
441
|
+
size: res.ContentLength ?? -1,
|
|
442
|
+
etag: normalizeEtag(res.ETag),
|
|
443
|
+
};
|
|
444
|
+
const sha = res.Metadata?.[VAULT_SHA256_METADATA_KEY];
|
|
445
|
+
if (sha && HEX64.test(sha))
|
|
446
|
+
head.sha256 = sha;
|
|
447
|
+
return head;
|
|
448
|
+
}
|
|
449
|
+
catch (err) {
|
|
450
|
+
if (isNotFound(err))
|
|
451
|
+
return null;
|
|
452
|
+
throw err;
|
|
453
|
+
}
|
|
454
|
+
});
|
|
261
455
|
},
|
|
262
456
|
async download(key, dest, etag) {
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
catch (err) {
|
|
268
|
-
const e = err;
|
|
269
|
-
if (e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412) {
|
|
270
|
-
throw new Error("The cloud copy of this file changed since HQ sync last saw it. Run `hq sync`, then try again.");
|
|
457
|
+
// Only the request is retried: the destination is created by the pipe below.
|
|
458
|
+
const res = await call("GET", async (client) => {
|
|
459
|
+
try {
|
|
460
|
+
return (await client.send(new GetObjectCommand({ Bucket: bucket, Key: key, IfMatch: `"${normalizeEtag(etag)}"` })));
|
|
271
461
|
}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
462
|
+
catch (err) {
|
|
463
|
+
const e = err;
|
|
464
|
+
if (e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412) {
|
|
465
|
+
throw new Error("The cloud copy of this file changed since HQ sync last saw it. Run `hq sync`, then try again.");
|
|
466
|
+
}
|
|
467
|
+
if (isNotFound(err))
|
|
468
|
+
throw new Error(`This file is no longer in the cloud (${key}).`);
|
|
469
|
+
throw err;
|
|
470
|
+
}
|
|
471
|
+
});
|
|
276
472
|
const body = res.Body;
|
|
277
473
|
if (!body)
|
|
278
474
|
throw new Error(`Empty download for ${key}.`);
|
|
279
|
-
await
|
|
280
|
-
const out = fs.createWriteStream(dest, { flags: "wx" });
|
|
281
|
-
body.on("error", reject);
|
|
282
|
-
out.on("error", reject);
|
|
283
|
-
out.on("finish", () => resolve());
|
|
284
|
-
body.pipe(out);
|
|
285
|
-
});
|
|
475
|
+
await pipeToNewFile(body, dest);
|
|
286
476
|
return normalizeEtag(res.ETag);
|
|
287
477
|
},
|
|
288
478
|
};
|
|
@@ -516,6 +706,122 @@ async function pointCurrentFileAtVault(file, vault, deps, fsOps) {
|
|
|
516
706
|
}
|
|
517
707
|
await fsOps.removeOriginal(abs);
|
|
518
708
|
}
|
|
709
|
+
/** Resumable upload state for one company vault key. */
|
|
710
|
+
function uploadStatePath(deps, target) {
|
|
711
|
+
const dir = deps.uploadStateDir ? deps.uploadStateDir() : path.join(os.homedir(), ".hq", "offload-uploads");
|
|
712
|
+
const id = crypto.createHash("sha256").update(`${target.slug}\0${target.key}`).digest("hex");
|
|
713
|
+
return path.join(dir, `${id}.json`);
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* ETag of a verified upload this machine made to the key but did not record
|
|
717
|
+
* (the file changed before it was recorded), so the next run may replace it.
|
|
718
|
+
*/
|
|
719
|
+
function readOwnUpload(stateFile) {
|
|
720
|
+
try {
|
|
721
|
+
const etag = JSON.parse(fs.readFileSync(`${stateFile}.uploaded`, "utf8")).etag;
|
|
722
|
+
return typeof etag === "string" ? etag : undefined;
|
|
723
|
+
}
|
|
724
|
+
catch (err) {
|
|
725
|
+
if (err.code === "ENOENT")
|
|
726
|
+
return undefined;
|
|
727
|
+
throw new Error(`Cannot read offload upload state ${stateFile}.uploaded: ${err.message}`);
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
/**
|
|
731
|
+
* What the company vault key allows for a local file over the sync limit:
|
|
732
|
+
* upload when the key is empty, still holds the version this machine last
|
|
733
|
+
* recorded, or holds this machine's own unrecorded upload; adopt when it
|
|
734
|
+
* already holds these exact bytes; otherwise a different copy is there.
|
|
735
|
+
*/
|
|
736
|
+
function companyKeyState(head, digest, record, ownEtag) {
|
|
737
|
+
if (!head)
|
|
738
|
+
return "upload";
|
|
739
|
+
if (head.sha256 === digest.sha256 && head.size === digest.size)
|
|
740
|
+
return "adopt";
|
|
741
|
+
if (record?.remoteEtag && head.etag === normalizeEtag(record.remoteEtag))
|
|
742
|
+
return "upload";
|
|
743
|
+
if (ownEtag && head.etag === normalizeEtag(ownEtag))
|
|
744
|
+
return "upload";
|
|
745
|
+
return "conflict";
|
|
746
|
+
}
|
|
747
|
+
const DIFFERENT_COPY = "A different copy is already in the company vault. Run hq sync to get it, then try again.";
|
|
748
|
+
/**
|
|
749
|
+
* Upload one company file over the sync limit to its company vault key, then
|
|
750
|
+
* offload it. Order: verified upload, journal record (synced, cloud-only),
|
|
751
|
+
* fsynced `x.hqcloud`, and only then remove `x`. An interrupted upload leaves
|
|
752
|
+
* `x` untouched, records nothing, and keeps the state file so the next run
|
|
753
|
+
* resumes.
|
|
754
|
+
*/
|
|
755
|
+
async function uploadCompanyFile(file, uploader, vault, deps, fsOps) {
|
|
756
|
+
const { plan, target, stat: before, digest, record } = file;
|
|
757
|
+
if (target.vault !== "company" || !before || !digest)
|
|
758
|
+
throw new Error(`'${plan.path}' is not a company file.`);
|
|
759
|
+
if (uploader.slug !== target.slug || uploader.bucket.startsWith(PERSONAL_BUCKET_PREFIX)) {
|
|
760
|
+
throw new Error(`The upload target '${uploader.bucket}' is not the company vault for '${target.slug}'. Nothing was uploaded.`);
|
|
761
|
+
}
|
|
762
|
+
if (!deps.recordCloudOnly)
|
|
763
|
+
throw new Error(`This HQ build cannot record cloud-only files; ${UPDATE_SYNC}.`);
|
|
764
|
+
const abs = containedPath(deps.hqRoot, plan.path);
|
|
765
|
+
const unchanged = async () => {
|
|
766
|
+
const st = await fs.promises.stat(abs);
|
|
767
|
+
return st.size === before.size && st.mtimeMs === before.mtimeMs && st.size === digest.size;
|
|
768
|
+
};
|
|
769
|
+
const changed = new Error(`'${plan.path}' changed while uploading. Nothing was replaced.`);
|
|
770
|
+
if (!(await unchanged()))
|
|
771
|
+
throw changed;
|
|
772
|
+
const stateFile = uploadStatePath(deps, target);
|
|
773
|
+
const head = await vault.head(target.key);
|
|
774
|
+
const state = companyKeyState(head, digest, record, readOwnUpload(stateFile));
|
|
775
|
+
if (state === "conflict")
|
|
776
|
+
throw new Error(`${DIFFERENT_COPY} Nothing was uploaded.`);
|
|
777
|
+
let object;
|
|
778
|
+
if (state === "adopt" && head) {
|
|
779
|
+
object = { etag: head.etag, size: head.size, sha256: digest.sha256 };
|
|
780
|
+
}
|
|
781
|
+
else {
|
|
782
|
+
object = await uploader.upload({ key: target.key, file: abs, stateFile });
|
|
783
|
+
if (object.sha256 !== digest.sha256 || object.size !== digest.size) {
|
|
784
|
+
throw new Error(`The uploaded copy of '${plan.path}' does not match the local file. Nothing was replaced.`);
|
|
785
|
+
}
|
|
786
|
+
if (!(await unchanged())) {
|
|
787
|
+
await fs.promises.mkdir(path.dirname(stateFile), { recursive: true });
|
|
788
|
+
await fs.promises.writeFile(`${stateFile}.uploaded`, JSON.stringify({ etag: object.etag }));
|
|
789
|
+
throw changed;
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
await deps.recordCloudOnly(target, object);
|
|
793
|
+
await fs.promises.rm(`${stateFile}.uploaded`, { force: true });
|
|
794
|
+
const placeholder = `${abs}${PLACEHOLDER_SUFFIX}`;
|
|
795
|
+
await fsOps.writePlaceholder(placeholder, renderPointer({
|
|
796
|
+
path: plan.path,
|
|
797
|
+
size: digest.size,
|
|
798
|
+
sha256: digest.sha256,
|
|
799
|
+
vault: "company",
|
|
800
|
+
key: target.key,
|
|
801
|
+
offloadedAt: deps.now().toISOString().replace(/\.\d{3}Z$/, "Z"),
|
|
802
|
+
remoteEtag: object.etag,
|
|
803
|
+
}));
|
|
804
|
+
if (!(await unchanged())) {
|
|
805
|
+
await fs.promises.rm(placeholder, { force: true });
|
|
806
|
+
throw changed;
|
|
807
|
+
}
|
|
808
|
+
await fsOps.removeOriginal(abs);
|
|
809
|
+
}
|
|
810
|
+
/** Plain name of an HQ sync ignore layer, for skip reasons. */
|
|
811
|
+
function exclusionReason(ex) {
|
|
812
|
+
if (ex.layer === ".hqinclude allowlist") {
|
|
813
|
+
const add = ex.includeLine ? ` To sync and offload it, add this line to .hqinclude: ${ex.includeLine}` : "";
|
|
814
|
+
return `HQ sync skips this path because the .hqinclude allowlist does not list it.${add}`;
|
|
815
|
+
}
|
|
816
|
+
const by = ex.layer === "built-in"
|
|
817
|
+
? "HQ sync's built-in exclusions"
|
|
818
|
+
: ex.layer === "defaults"
|
|
819
|
+
? "HQ sync's default exclusions"
|
|
820
|
+
: ex.layer
|
|
821
|
+
? `a rule in ${ex.layer}`
|
|
822
|
+
: "an HQ sync ignore rule";
|
|
823
|
+
return `HQ sync skips this path because of ${by}, so HQ will not offload it.`;
|
|
824
|
+
}
|
|
519
825
|
/** Whether a working-tree path is never a current-offload candidate. */
|
|
520
826
|
function skippedByName(rel) {
|
|
521
827
|
const parts = rel.split("/");
|
|
@@ -615,10 +921,34 @@ async function planCurrentFiles(rels, opts, deps, openVault) {
|
|
|
615
921
|
plan.synced = head !== null && head.etag === normalizeEtag(record.remoteEtag) && head.size === st.size;
|
|
616
922
|
}
|
|
617
923
|
}
|
|
618
|
-
if (plan.synced)
|
|
924
|
+
if (plan.synced) {
|
|
619
925
|
plan.action = "point-at-vault";
|
|
620
|
-
|
|
621
|
-
|
|
926
|
+
}
|
|
927
|
+
else if (target.vault === "company") {
|
|
928
|
+
const ex = deps.syncExclusion?.(rel);
|
|
929
|
+
if (ex?.excluded) {
|
|
930
|
+
plan.excluded_by = ex.layer ?? "ignore rules";
|
|
931
|
+
plan.reason = exclusionReason(ex);
|
|
932
|
+
}
|
|
933
|
+
else if (st.size <= (deps.syncMaxFileBytes ?? SYNC_MAX_FILE_BYTES)) {
|
|
934
|
+
plan.reason = RUN_SYNC_FIRST;
|
|
935
|
+
}
|
|
936
|
+
else if (!deps.openCompanyUploader || !deps.recordCloudOnly || !deps.openVaultObjects) {
|
|
937
|
+
plan.reason = UPDATE_SYNC;
|
|
938
|
+
}
|
|
939
|
+
else {
|
|
940
|
+
digest ??= await digestFile(abs);
|
|
941
|
+
const head = await (await openVault(target)).head(target.key);
|
|
942
|
+
const state = companyKeyState(head, digest, record, readOwnUpload(uploadStatePath(deps, target)));
|
|
943
|
+
if (state === "conflict") {
|
|
944
|
+
plan.reason = DIFFERENT_COPY;
|
|
945
|
+
}
|
|
946
|
+
else {
|
|
947
|
+
plan.action = "upload+offload";
|
|
948
|
+
plan.vault_key = target.key;
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
}
|
|
622
952
|
planned.push({ plan, target, stat: st, digest, record: record ?? undefined });
|
|
623
953
|
}
|
|
624
954
|
return planned;
|
|
@@ -670,7 +1000,11 @@ async function runCurrent(opts, deps, result) {
|
|
|
670
1000
|
if (opts.dryRun || todo.length === 0)
|
|
671
1001
|
return "ok";
|
|
672
1002
|
if (!opts.yes) {
|
|
673
|
-
const
|
|
1003
|
+
const uploads = todo.filter((p) => p.plan.action === "upload+offload");
|
|
1004
|
+
const uploadNote = uploads.length > 0
|
|
1005
|
+
? ` ${uploads.length} of them (${uploads.reduce((n, p) => n + p.plan.bytes, 0)} bytes) are uploaded to their company vault first.`
|
|
1006
|
+
: "";
|
|
1007
|
+
const ok = await deps.confirm(`Replace ${todo.length} current file(s) with links to their HQ cloud copies and remove the local copies?${uploadNote}`);
|
|
674
1008
|
if (!ok) {
|
|
675
1009
|
deps.log("Aborted - nothing was uploaded or changed.");
|
|
676
1010
|
return "aborted";
|
|
@@ -678,29 +1012,39 @@ async function runCurrent(opts, deps, result) {
|
|
|
678
1012
|
}
|
|
679
1013
|
const fsOps = deps.placeholderFs ?? durablePlaceholderFs;
|
|
680
1014
|
let personal;
|
|
1015
|
+
const uploaders = new Map();
|
|
681
1016
|
for (const file of todo) {
|
|
682
|
-
const { plan } = file;
|
|
1017
|
+
const { plan, target } = file;
|
|
683
1018
|
try {
|
|
1019
|
+
let action;
|
|
684
1020
|
if (plan.action === "point-at-vault") {
|
|
685
|
-
await pointCurrentFileAtVault(file, await openVault(
|
|
1021
|
+
await pointCurrentFileAtVault(file, await openVault(target), deps, fsOps);
|
|
1022
|
+
action = "point-at-vault";
|
|
1023
|
+
}
|
|
1024
|
+
else if (plan.action === "upload+offload") {
|
|
1025
|
+
if (target.vault !== "company" || !deps.openCompanyUploader)
|
|
1026
|
+
throw new Error(UPDATE_SYNC);
|
|
1027
|
+
let uploader = uploaders.get(target.slug);
|
|
1028
|
+
if (!uploader) {
|
|
1029
|
+
uploader = deps.openCompanyUploader(target.slug);
|
|
1030
|
+
uploaders.set(target.slug, uploader);
|
|
1031
|
+
}
|
|
1032
|
+
await uploadCompanyFile(file, await uploader, await openVault(target), deps, fsOps);
|
|
1033
|
+
action = "upload+offload";
|
|
686
1034
|
}
|
|
687
1035
|
else {
|
|
688
1036
|
personal ??= await deps.openPersonal();
|
|
689
1037
|
await offloadCurrentFile(plan.path, personal, deps, fsOps);
|
|
1038
|
+
action = "upload";
|
|
690
1039
|
}
|
|
691
|
-
const done = {
|
|
692
|
-
path: plan.path,
|
|
693
|
-
bytes: plan.bytes,
|
|
694
|
-
vault: plan.vault,
|
|
695
|
-
action: plan.action === "point-at-vault" ? "point-at-vault" : "upload",
|
|
696
|
-
};
|
|
1040
|
+
const done = { path: plan.path, bytes: plan.bytes, vault: plan.vault, action };
|
|
697
1041
|
if (plan.company)
|
|
698
1042
|
done.company = plan.company;
|
|
699
1043
|
current.offloaded.push(done);
|
|
700
1044
|
current.freed_bytes += plan.bytes;
|
|
701
1045
|
}
|
|
702
1046
|
catch (err) {
|
|
703
|
-
result.errors.push(`${plan.path}: ${err
|
|
1047
|
+
result.errors.push(`${plan.path}: ${failureMessage(err)}`);
|
|
704
1048
|
}
|
|
705
1049
|
}
|
|
706
1050
|
return "ok";
|
|
@@ -844,7 +1188,7 @@ export async function runOffload(opts, deps) {
|
|
|
844
1188
|
});
|
|
845
1189
|
}
|
|
846
1190
|
catch (err) {
|
|
847
|
-
result.errors.push(`${blob.paths[0] ?? blob.sha}: ${err
|
|
1191
|
+
result.errors.push(`${blob.paths[0] ?? blob.sha}: ${failureMessage(err)}`);
|
|
848
1192
|
}
|
|
849
1193
|
}
|
|
850
1194
|
if (manifest.length === 0)
|
|
@@ -1057,6 +1401,7 @@ async function fetchVaultObject(pointer, target, deps) {
|
|
|
1057
1401
|
if (!deps.openVaultObjects)
|
|
1058
1402
|
throw new Error("This HQ build cannot fetch files recorded by HQ sync.");
|
|
1059
1403
|
let store;
|
|
1404
|
+
let owner;
|
|
1060
1405
|
if (pointer.vault === "company") {
|
|
1061
1406
|
const slug = companyForPointer(pointer);
|
|
1062
1407
|
const active = deps.activeCompanySlugs ? await deps.activeCompanySlugs() : [];
|
|
@@ -1064,19 +1409,35 @@ async function fetchVaultObject(pointer, target, deps) {
|
|
|
1064
1409
|
throw new Error(`You are not an active member of '${slug}', so HQ cannot download this file for you.`);
|
|
1065
1410
|
}
|
|
1066
1411
|
store = await deps.openVaultObjects({ vault: "company", slug });
|
|
1412
|
+
owner = { vault: "company", slug, key: pointer.key };
|
|
1067
1413
|
}
|
|
1068
1414
|
else {
|
|
1069
1415
|
if (pointer.key !== pointer.path) {
|
|
1070
1416
|
throw new Error(`Placeholder for '${pointer.path}' names vault key '${pointer.key}', which does not match its location. Nothing was changed.`);
|
|
1071
1417
|
}
|
|
1072
1418
|
store = await deps.openVaultObjects({ vault: "personal" });
|
|
1419
|
+
owner = { vault: "personal", key: pointer.key };
|
|
1073
1420
|
}
|
|
1074
1421
|
const etag = pointer.remoteEtag;
|
|
1075
|
-
|
|
1076
|
-
|
|
1422
|
+
// A teammate's placeholder has no sha256; an offload upload carries it as
|
|
1423
|
+
// object metadata, so check against that when the pinned object has it.
|
|
1424
|
+
let sha256 = pointer.sha256;
|
|
1425
|
+
if (sha256 == null) {
|
|
1426
|
+
const head = await store.head(pointer.key);
|
|
1427
|
+
if (head?.sha256 && head.etag === normalizeEtag(etag))
|
|
1428
|
+
sha256 = head.sha256;
|
|
1429
|
+
}
|
|
1430
|
+
const digest = await downloadVerified((partial) => store.download(pointer.key, partial, etag), pointer.key, target, {
|
|
1431
|
+
sha256,
|
|
1077
1432
|
size: pointer.size,
|
|
1078
1433
|
etag,
|
|
1079
1434
|
});
|
|
1435
|
+
// Sync never transfers a file over its limit, so record the restored file
|
|
1436
|
+
// as the vault's version; otherwise sync would treat it as an unknown local file.
|
|
1437
|
+
if (digest.size > (deps.syncMaxFileBytes ?? SYNC_MAX_FILE_BYTES) && deps.recordCloudOnly) {
|
|
1438
|
+
await deps.recordCloudOnly(owner, { etag: normalizeEtag(etag), size: digest.size, sha256: digest.sha256 });
|
|
1439
|
+
}
|
|
1440
|
+
return digest;
|
|
1080
1441
|
}
|
|
1081
1442
|
export async function runFetchGitBlob(sha, out, deps) {
|
|
1082
1443
|
return fetchGitBlob(sha, out, deps);
|
|
@@ -1197,7 +1558,4 @@ export function gitListPlaceholders(hqRoot) {
|
|
|
1197
1558
|
export function defaultTmpDir() {
|
|
1198
1559
|
return os.tmpdir();
|
|
1199
1560
|
}
|
|
1200
|
-
export function personalS3Store(input) {
|
|
1201
|
-
return s3BlobStore(input.bucket, new S3Client({ region: input.region, credentials: input.credentials }));
|
|
1202
|
-
}
|
|
1203
1561
|
//# sourceMappingURL=storage-offload.js.map
|