@indigoai-us/hq-cli 5.345.137 → 5.345.139

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,45 +1,56 @@
1
1
  /**
2
- * `hq storage offload` / `hq storage fetch` — move big files to the caller's
3
- * PERSONAL vault (S3 Standard, content-addressed) and leave small linked
4
- * pointers behind. Shared contract: owner-approved offload design, 2026-10-09.
2
+ * `hq storage offload` / `hq storage fetch` — move big files out of the HQ
3
+ * folder into HQ cloud storage and leave small linked placeholders behind.
4
+ * Shared contract: owner-approved offload design, 2026-10-09; company upload
5
+ * for files over the sync limit approved 2026-10-15.
5
6
  *
6
7
  * Two kinds of big file:
7
8
  * - History copies (old versions inside the HQ root git history). The CLI
8
- * uploads each blob, verifies size + sha256 + git blob id, then hands only
9
- * verified blobs to core `hq-storage.sh strip-large`, which rewrites history
10
- * to the pointer text. Working-tree files are never touched.
9
+ * uploads each blob to the caller's PERSONAL vault, verifies size + sha256
10
+ * + git blob id, then hands only verified blobs to core `hq-storage.sh
11
+ * strip-large`, which rewrites history to the pointer text. Working-tree
12
+ * files are never touched.
11
13
  * - Current files (`--current [paths...]`, idle big files in the working
12
- * tree). Each file goes to the vault that owns its path. A file under
13
- * `companies/<slug>/` is never uploaded: when HQ sync already stored this
14
- * exact version in that company's vault (journal hash + size, and the vault
15
- * object still has the journal's ETag), the file is replaced by a
16
- * placeholder pointing at that object. Otherwise it is skipped with "run
17
- * hq sync first", so sync stays the only writer of company vault keys. Any
18
- * other file points at its personal-vault sync object the same way, or is
19
- * uploaded to the personal vault's `.hq-offload/blobs/`. Current-file
20
- * offload runs only when the installed HQ sync holds every `.hqcloud`
21
- * sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and skips
22
- * `.hq-offload/`; otherwise it reports "update HQ sync".
14
+ * tree). Each file goes to the vault that owns its path:
15
+ * - A file in a company folder belongs to that company's vault, and only
16
+ * an active member can offload it. When HQ sync already stored this
17
+ * exact version (journal hash + size, and the vault object still has
18
+ * the journal's ETag), the file is replaced by a placeholder pointing
19
+ * at that object. A file over the sync limit (SYNC_MAX_FILE_BYTES),
20
+ * which sync never uploads, is uploaded to the company vault at its
21
+ * sync key by a resumable, checksum-verified multipart upload
22
+ * (`upload+offload`). Once the vault copy is verified the file is
23
+ * recorded in the sync journal as synced and cloud-only, then the
24
+ * placeholder is written, then the local file is removed. A path the
25
+ * sync ignore rules exclude is refused and the excluding rule is named
26
+ * (for `.hqinclude`, with the exact line to add). Other unsynced
27
+ * company files are skipped with "run hq sync first".
28
+ * - Any other file points at its personal-vault sync object the same
29
+ * way, or is uploaded to the personal vault's `.hq-offload/blobs/`.
30
+ * Current-file offload runs only when the installed HQ sync holds every
31
+ * `.hqcloud` sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and
32
+ * skips `.hq-offload/`; otherwise it reports "update HQ sync".
23
33
  *
24
- * Fetch also reads placeholders that HQ sync rewrote after a teammate changed
25
- * the file (`vault: company`, the company vault key, `sha256: unknown`,
26
- * `remote-etag`). Those download from the company vault, only for an active
27
- * member, and verify by size and ETag. A placeholder marked `orphaned: true`
28
- * (a teammate deleted the file) is refused unless `--force-orphan` is given,
29
- * which writes the caller's own personal-vault copy to a file outside HQ so
30
- * it can never sync back.
34
+ * Fetch also reads placeholders that HQ sync wrote on a teammate's machine
35
+ * (`vault: company`, the company vault key, `sha256: unknown`, `remote-etag`).
36
+ * Those download from the company vault, only for an active member, and
37
+ * verify by size, ETag and, when the object carries it, its `hq-sha256`
38
+ * metadata. A placeholder marked `orphaned: true` (a teammate deleted the
39
+ * file) is refused unless `--force-orphan` is given, which writes the caller's
40
+ * own personal-vault copy to a file outside HQ so it can never sync back.
31
41
  *
32
- * Tenant isolation: uploads only ever go to the caller's personal vault, and
33
- * the personal store refuses a company vault bucket. A company vault is opened
34
- * read-only (HEAD and ETag-pinned GET), and only for a company the caller is
35
- * an active member of.
42
+ * Tenant isolation: history and personal uploads only ever go to the caller's
43
+ * personal vault, and the personal store refuses a company vault bucket. A
44
+ * company vault is opened only for a company the caller is an active member
45
+ * of, and only for a file in that company's folder; the company uploader must
46
+ * name that company and refuses a personal bucket.
36
47
  */
37
48
  import * as crypto from "node:crypto";
38
49
  import * as fs from "node:fs";
39
50
  import * as os from "node:os";
40
51
  import * as path from "node:path";
41
52
  import { spawn, execFile } from "node:child_process";
42
- import { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand, } from "@aws-sdk/client-s3";
53
+ import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, } from "@aws-sdk/client-s3";
43
54
  import { checkPreconditions, coreHasFlag, coreSupports, dropExpiredBackups, PRECONDITION_FALLBACK, coreKeepsUndo, reclaimFields, reclaimWindowHours, runCoreScript, } from "./storage-core.js";
44
55
  // ── Constants ───────────────────────────────────────────────────────────────
45
56
  export const PLACEHOLDER_SUFFIX = ".hqcloud";
@@ -53,6 +64,13 @@ const UPDATE_HQ = "update HQ";
53
64
  export const UPDATE_SYNC = "update HQ sync";
54
65
  /** Reason on company files HQ sync has not stored in the company vault yet. */
55
66
  export const RUN_SYNC_FIRST = "run hq sync first";
67
+ /**
68
+ * HQ sync's per-file limit (hq-cloud SYNC_MAX_FILE_BYTES; sync skips files
69
+ * strictly larger). storage.ts passes hq-cloud's value when it exports one.
70
+ */
71
+ export const SYNC_MAX_FILE_BYTES = 50 * 1024 * 1024;
72
+ /** Object metadata key holding the sha256 of a company-vault offload upload. */
73
+ export const VAULT_SHA256_METADATA_KEY = "hq-sha256";
56
74
  const DAY_MS = 86_400_000;
57
75
  const PERSONAL_BUCKET_PREFIX = "hq-vault-prs-";
58
76
  /** Top-level HQ-root folder that holds offloaded blobs in the personal vault. */
@@ -193,96 +211,268 @@ function isNotFound(err) {
193
211
  const e = err;
194
212
  return e?.name === "NotFound" || e?.name === "NoSuchKey" || e?.$metadata?.httpStatusCode === 404;
195
213
  }
214
+ // ── Vault request errors and credential refresh ─────────────────────────────
215
+ /** Error codes S3 and STS return when the request's credentials are refused. */
216
+ const CREDENTIAL_ERROR_HINTS = {
217
+ ExpiredToken: "credentials expired",
218
+ ExpiredTokenException: "credentials expired",
219
+ TokenRefreshRequired: "credentials expired",
220
+ RequestExpired: "credentials expired",
221
+ InvalidToken: "credentials rejected",
222
+ InvalidAccessKeyId: "credentials rejected",
223
+ InvalidSecurityToken: "credentials rejected",
224
+ InvalidClientTokenId: "credentials rejected",
225
+ AccessDenied: "access denied",
226
+ };
227
+ /** Placeholder names the SDK uses when the response carried no error code. */
228
+ const NO_CODE = new Set(["", "Unknown", "UnknownError", "Error"]);
229
+ function awsErrorCode(err) {
230
+ const candidates = [err.Code, err.code, err.name, err.$response?.headers?.["x-amz-error-code"]];
231
+ return candidates.find((c) => typeof c === "string" && !NO_CODE.has(c));
232
+ }
233
+ function isAwsError(err) {
234
+ if (!err || typeof err !== "object")
235
+ return false;
236
+ const e = err;
237
+ return typeof e.$metadata === "object" || typeof e.$fault === "string" || e.name === "Unknown";
238
+ }
239
+ /**
240
+ * True when S3 refused the request's credentials: an expired, invalid or
241
+ * denied token, or a 403 with no error body (what S3 sends for HEAD).
242
+ */
243
+ export function isCredentialRejection(err) {
244
+ if (!isAwsError(err))
245
+ return false;
246
+ const status = err.$metadata?.httpStatusCode;
247
+ const code = awsErrorCode(err);
248
+ if (code)
249
+ return code in CREDENTIAL_ERROR_HINTS && (status === undefined || status === 400 || status === 403);
250
+ return status === 403;
251
+ }
252
+ /** A failed vault request, described with its HTTP status, error code and vault. */
253
+ export class VaultRequestError extends Error {
254
+ status;
255
+ code;
256
+ constructor(message, opts) {
257
+ super(message, { cause: opts.cause });
258
+ this.name = "VaultRequestError";
259
+ this.status = opts.status;
260
+ this.code = opts.code;
261
+ }
262
+ }
263
+ /**
264
+ * Describes an AWS error from `op` against `where` so the user sees the HTTP
265
+ * status, the error code and the vault, never the SDK's bare "UnknownError".
266
+ * Errors that did not come from AWS pass through unchanged.
267
+ */
268
+ export function describeVaultError(err, op, where, retried = false) {
269
+ if (err instanceof VaultRequestError || !isAwsError(err))
270
+ return err;
271
+ const status = err.$metadata?.httpStatusCode;
272
+ const code = awsErrorCode(err);
273
+ let what;
274
+ if (code) {
275
+ const hint = CREDENTIAL_ERROR_HINTS[code];
276
+ what = hint ? `${code} (${hint})` : code;
277
+ }
278
+ else if (status === 403) {
279
+ what = "Forbidden (no error body: credentials expired or access denied)";
280
+ }
281
+ else {
282
+ what = "error with no error code";
283
+ }
284
+ let message = `${op} failed in ${where}: ${status ? `HTTP ${status} ` : ""}${what}`;
285
+ if (retried)
286
+ message += ", after re-vending credentials once";
287
+ const detail = err.message?.trim();
288
+ if (detail && !NO_CODE.has(detail) && detail !== code)
289
+ message += `: ${detail}`;
290
+ return new VaultRequestError(message, { status, code, cause: err });
291
+ }
292
+ /** Turns any per-file failure into one line for the run's error list. */
293
+ export function failureMessage(err) {
294
+ const described = describeVaultError(err, "Request", "the vault");
295
+ return described instanceof Error ? described.message : String(described);
296
+ }
297
+ /**
298
+ * Runs vault requests on `initial` and, when S3 refuses the credentials,
299
+ * re-vends once and retries the request once on the new client. Requests
300
+ * refused on the same client share one re-vend. A request refused again
301
+ * after the re-vend fails with a described error; it is never retried twice.
302
+ */
303
+ export function credentialRetry(initial, where, revend) {
304
+ let client = initial;
305
+ let generation = 0;
306
+ let pending;
307
+ return async function call(op, fn) {
308
+ const used = generation;
309
+ try {
310
+ return await fn(client);
311
+ }
312
+ catch (err) {
313
+ if (!revend || !isCredentialRejection(err))
314
+ throw describeVaultError(err, op, where);
315
+ if (generation === used) {
316
+ pending ??= revend()
317
+ .then((fresh) => {
318
+ client = fresh;
319
+ generation += 1;
320
+ })
321
+ .finally(() => {
322
+ pending = undefined;
323
+ });
324
+ try {
325
+ await pending;
326
+ }
327
+ catch (vendErr) {
328
+ const refused = describeVaultError(err, op, where);
329
+ throw new Error(`${refused.message}; re-vending credentials for ${where} failed: ${vendErr?.message ?? String(vendErr)}`, { cause: vendErr });
330
+ }
331
+ }
332
+ try {
333
+ return await fn(client);
334
+ }
335
+ catch (again) {
336
+ throw describeVaultError(again, op, where, true);
337
+ }
338
+ }
339
+ };
340
+ }
341
+ const CLIENT_METHOD_OPS = {
342
+ create: "CreateMultipartUpload",
343
+ uploadPart: "UploadPart",
344
+ listParts: "ListParts",
345
+ complete: "CompleteMultipartUpload",
346
+ abort: "AbortMultipartUpload",
347
+ head: "HeadObject",
348
+ };
349
+ /**
350
+ * Wraps every method of `initial` (a multipart client) so each call gets
351
+ * {@link credentialRetry}: one re-vend and one retry when S3 refuses the
352
+ * credentials, and a described error otherwise.
353
+ */
354
+ export function credentialRetryingClient(initial, where, revend) {
355
+ const call = credentialRetry(initial, where, revend);
356
+ const wrapped = {};
357
+ for (const name of Object.keys(initial)) {
358
+ if (typeof initial[name] !== "function")
359
+ continue;
360
+ wrapped[name] = (...args) => call(CLIENT_METHOD_OPS[name] ?? name, (c) => c[name].apply(c, args));
361
+ }
362
+ return { ...initial, ...wrapped };
363
+ }
364
+ function pipeToNewFile(body, dest) {
365
+ return new Promise((resolve, reject) => {
366
+ const out = fs.createWriteStream(dest, { flags: "wx" });
367
+ body.on("error", reject);
368
+ out.on("error", reject);
369
+ out.on("finish", () => resolve());
370
+ body.pipe(out);
371
+ });
372
+ }
196
373
  /**
197
374
  * S3-backed store over the caller's personal vault bucket. Accepts only
198
375
  * personal vault buckets (`hq-vault-prs-*`) so a mis-resolved entity can never
199
376
  * receive offloaded bytes.
200
377
  */
201
- export function s3BlobStore(bucket, s3) {
378
+ export function s3BlobStore(bucket, s3, opts = {}) {
202
379
  if (!bucket || !bucket.startsWith(PERSONAL_BUCKET_PREFIX)) {
203
380
  throw new Error(`Refusing to offload into '${bucket}': offload only writes to your personal vault.`);
204
381
  }
382
+ const call = credentialRetry(s3, opts.where ?? `vault '${bucket}'`, opts.revend);
205
383
  return {
206
384
  vault: "personal",
207
- async head(key) {
208
- try {
209
- const res = (await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })));
210
- const fromChecksum = res.ChecksumSHA256 && !res.ChecksumSHA256.includes("-")
211
- ? Buffer.from(res.ChecksumSHA256, "base64").toString("hex")
212
- : undefined;
213
- // The server-computed checksum wins; client metadata is a fallback for
214
- // objects stored without one.
215
- return { size: res.ContentLength ?? -1, sha256: fromChecksum ?? res.Metadata?.sha256 };
216
- }
217
- catch (err) {
218
- if (isNotFound(err))
219
- return null;
220
- throw err;
221
- }
385
+ head(key) {
386
+ return call("HEAD", async (client) => {
387
+ try {
388
+ const res = (await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })));
389
+ const fromChecksum = res.ChecksumSHA256 && !res.ChecksumSHA256.includes("-")
390
+ ? Buffer.from(res.ChecksumSHA256, "base64").toString("hex")
391
+ : undefined;
392
+ // The server-computed checksum wins; client metadata is a fallback for
393
+ // objects stored without one.
394
+ return { size: res.ContentLength ?? -1, sha256: fromChecksum ?? res.Metadata?.sha256 };
395
+ }
396
+ catch (err) {
397
+ if (isNotFound(err))
398
+ return null;
399
+ throw err;
400
+ }
401
+ });
222
402
  },
223
403
  async put(key, file, digest) {
224
- await s3.send(new PutObjectCommand({
225
- Bucket: bucket,
226
- Key: key,
227
- Body: fs.createReadStream(file),
228
- ContentLength: digest.size,
229
- ChecksumSHA256: Buffer.from(digest.sha256, "hex").toString("base64"),
230
- StorageClass: "STANDARD",
231
- Metadata: { sha256: digest.sha256, "git-blob": digest.gitBlob },
232
- }));
404
+ await call("PUT", async (client) => {
405
+ // A fresh stream per attempt: a refused attempt may have read from the last one.
406
+ const body = fs.createReadStream(file);
407
+ try {
408
+ await client.send(new PutObjectCommand({
409
+ Bucket: bucket,
410
+ Key: key,
411
+ Body: body,
412
+ ContentLength: digest.size,
413
+ ChecksumSHA256: Buffer.from(digest.sha256, "hex").toString("base64"),
414
+ StorageClass: "STANDARD",
415
+ Metadata: { sha256: digest.sha256, "git-blob": digest.gitBlob },
416
+ }));
417
+ }
418
+ finally {
419
+ body.destroy();
420
+ }
421
+ });
233
422
  },
234
423
  async download(key, dest) {
235
- const res = (await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })));
424
+ // Only the request is retried: the destination is created by the pipe below.
425
+ const res = await call("GET", async (client) => (await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }))));
236
426
  const body = res.Body;
237
427
  if (!body)
238
428
  throw new Error(`Empty download for ${key}.`);
239
- await new Promise((resolve, reject) => {
240
- const out = fs.createWriteStream(dest, { flags: "wx" });
241
- body.on("error", reject);
242
- out.on("error", reject);
243
- out.on("finish", () => resolve());
244
- body.pipe(out);
245
- });
429
+ await pipeToNewFile(body, dest);
246
430
  },
247
431
  };
248
432
  }
249
- export function s3VaultObjectStore(bucket, s3) {
433
+ export function s3VaultObjectStore(bucket, s3, opts = {}) {
434
+ const call = credentialRetry(s3, opts.where ?? `vault '${bucket}'`, opts.revend);
250
435
  return {
251
- async head(key) {
252
- try {
253
- const res = (await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })));
254
- return { size: res.ContentLength ?? -1, etag: normalizeEtag(res.ETag) };
255
- }
256
- catch (err) {
257
- if (isNotFound(err))
258
- return null;
259
- throw err;
260
- }
436
+ head(key) {
437
+ return call("HEAD", async (client) => {
438
+ try {
439
+ const res = (await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })));
440
+ const head = {
441
+ size: res.ContentLength ?? -1,
442
+ etag: normalizeEtag(res.ETag),
443
+ };
444
+ const sha = res.Metadata?.[VAULT_SHA256_METADATA_KEY];
445
+ if (sha && HEX64.test(sha))
446
+ head.sha256 = sha;
447
+ return head;
448
+ }
449
+ catch (err) {
450
+ if (isNotFound(err))
451
+ return null;
452
+ throw err;
453
+ }
454
+ });
261
455
  },
262
456
  async download(key, dest, etag) {
263
- let res;
264
- try {
265
- res = (await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key, IfMatch: `"${normalizeEtag(etag)}"` })));
266
- }
267
- catch (err) {
268
- const e = err;
269
- if (e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412) {
270
- throw new Error("The cloud copy of this file changed since HQ sync last saw it. Run `hq sync`, then try again.");
457
+ // Only the request is retried: the destination is created by the pipe below.
458
+ const res = await call("GET", async (client) => {
459
+ try {
460
+ return (await client.send(new GetObjectCommand({ Bucket: bucket, Key: key, IfMatch: `"${normalizeEtag(etag)}"` })));
271
461
  }
272
- if (isNotFound(err))
273
- throw new Error(`This file is no longer in the cloud (${key}).`);
274
- throw err;
275
- }
462
+ catch (err) {
463
+ const e = err;
464
+ if (e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412) {
465
+ throw new Error("The cloud copy of this file changed since HQ sync last saw it. Run `hq sync`, then try again.");
466
+ }
467
+ if (isNotFound(err))
468
+ throw new Error(`This file is no longer in the cloud (${key}).`);
469
+ throw err;
470
+ }
471
+ });
276
472
  const body = res.Body;
277
473
  if (!body)
278
474
  throw new Error(`Empty download for ${key}.`);
279
- await new Promise((resolve, reject) => {
280
- const out = fs.createWriteStream(dest, { flags: "wx" });
281
- body.on("error", reject);
282
- out.on("error", reject);
283
- out.on("finish", () => resolve());
284
- body.pipe(out);
285
- });
475
+ await pipeToNewFile(body, dest);
286
476
  return normalizeEtag(res.ETag);
287
477
  },
288
478
  };
@@ -516,6 +706,122 @@ async function pointCurrentFileAtVault(file, vault, deps, fsOps) {
516
706
  }
517
707
  await fsOps.removeOriginal(abs);
518
708
  }
709
+ /** Resumable upload state for one company vault key. */
710
+ function uploadStatePath(deps, target) {
711
+ const dir = deps.uploadStateDir ? deps.uploadStateDir() : path.join(os.homedir(), ".hq", "offload-uploads");
712
+ const id = crypto.createHash("sha256").update(`${target.slug}\0${target.key}`).digest("hex");
713
+ return path.join(dir, `${id}.json`);
714
+ }
715
+ /**
716
+ * ETag of a verified upload this machine made to the key but did not record
717
+ * (the file changed before it was recorded), so the next run may replace it.
718
+ */
719
+ function readOwnUpload(stateFile) {
720
+ try {
721
+ const etag = JSON.parse(fs.readFileSync(`${stateFile}.uploaded`, "utf8")).etag;
722
+ return typeof etag === "string" ? etag : undefined;
723
+ }
724
+ catch (err) {
725
+ if (err.code === "ENOENT")
726
+ return undefined;
727
+ throw new Error(`Cannot read offload upload state ${stateFile}.uploaded: ${err.message}`);
728
+ }
729
+ }
730
+ /**
731
+ * What the company vault key allows for a local file over the sync limit:
732
+ * upload when the key is empty, still holds the version this machine last
733
+ * recorded, or holds this machine's own unrecorded upload; adopt when it
734
+ * already holds these exact bytes; otherwise a different copy is there.
735
+ */
736
+ function companyKeyState(head, digest, record, ownEtag) {
737
+ if (!head)
738
+ return "upload";
739
+ if (head.sha256 === digest.sha256 && head.size === digest.size)
740
+ return "adopt";
741
+ if (record?.remoteEtag && head.etag === normalizeEtag(record.remoteEtag))
742
+ return "upload";
743
+ if (ownEtag && head.etag === normalizeEtag(ownEtag))
744
+ return "upload";
745
+ return "conflict";
746
+ }
747
+ const DIFFERENT_COPY = "A different copy is already in the company vault. Run hq sync to get it, then try again.";
748
+ /**
749
+ * Upload one company file over the sync limit to its company vault key, then
750
+ * offload it. Order: verified upload, journal record (synced, cloud-only),
751
+ * fsynced `x.hqcloud`, and only then remove `x`. An interrupted upload leaves
752
+ * `x` untouched, records nothing, and keeps the state file so the next run
753
+ * resumes.
754
+ */
755
+ async function uploadCompanyFile(file, uploader, vault, deps, fsOps) {
756
+ const { plan, target, stat: before, digest, record } = file;
757
+ if (target.vault !== "company" || !before || !digest)
758
+ throw new Error(`'${plan.path}' is not a company file.`);
759
+ if (uploader.slug !== target.slug || uploader.bucket.startsWith(PERSONAL_BUCKET_PREFIX)) {
760
+ throw new Error(`The upload target '${uploader.bucket}' is not the company vault for '${target.slug}'. Nothing was uploaded.`);
761
+ }
762
+ if (!deps.recordCloudOnly)
763
+ throw new Error(`This HQ build cannot record cloud-only files; ${UPDATE_SYNC}.`);
764
+ const abs = containedPath(deps.hqRoot, plan.path);
765
+ const unchanged = async () => {
766
+ const st = await fs.promises.stat(abs);
767
+ return st.size === before.size && st.mtimeMs === before.mtimeMs && st.size === digest.size;
768
+ };
769
+ const changed = new Error(`'${plan.path}' changed while uploading. Nothing was replaced.`);
770
+ if (!(await unchanged()))
771
+ throw changed;
772
+ const stateFile = uploadStatePath(deps, target);
773
+ const head = await vault.head(target.key);
774
+ const state = companyKeyState(head, digest, record, readOwnUpload(stateFile));
775
+ if (state === "conflict")
776
+ throw new Error(`${DIFFERENT_COPY} Nothing was uploaded.`);
777
+ let object;
778
+ if (state === "adopt" && head) {
779
+ object = { etag: head.etag, size: head.size, sha256: digest.sha256 };
780
+ }
781
+ else {
782
+ object = await uploader.upload({ key: target.key, file: abs, stateFile });
783
+ if (object.sha256 !== digest.sha256 || object.size !== digest.size) {
784
+ throw new Error(`The uploaded copy of '${plan.path}' does not match the local file. Nothing was replaced.`);
785
+ }
786
+ if (!(await unchanged())) {
787
+ await fs.promises.mkdir(path.dirname(stateFile), { recursive: true });
788
+ await fs.promises.writeFile(`${stateFile}.uploaded`, JSON.stringify({ etag: object.etag }));
789
+ throw changed;
790
+ }
791
+ }
792
+ await deps.recordCloudOnly(target, object);
793
+ await fs.promises.rm(`${stateFile}.uploaded`, { force: true });
794
+ const placeholder = `${abs}${PLACEHOLDER_SUFFIX}`;
795
+ await fsOps.writePlaceholder(placeholder, renderPointer({
796
+ path: plan.path,
797
+ size: digest.size,
798
+ sha256: digest.sha256,
799
+ vault: "company",
800
+ key: target.key,
801
+ offloadedAt: deps.now().toISOString().replace(/\.\d{3}Z$/, "Z"),
802
+ remoteEtag: object.etag,
803
+ }));
804
+ if (!(await unchanged())) {
805
+ await fs.promises.rm(placeholder, { force: true });
806
+ throw changed;
807
+ }
808
+ await fsOps.removeOriginal(abs);
809
+ }
810
+ /** Plain name of an HQ sync ignore layer, for skip reasons. */
811
+ function exclusionReason(ex) {
812
+ if (ex.layer === ".hqinclude allowlist") {
813
+ const add = ex.includeLine ? ` To sync and offload it, add this line to .hqinclude: ${ex.includeLine}` : "";
814
+ return `HQ sync skips this path because the .hqinclude allowlist does not list it.${add}`;
815
+ }
816
+ const by = ex.layer === "built-in"
817
+ ? "HQ sync's built-in exclusions"
818
+ : ex.layer === "defaults"
819
+ ? "HQ sync's default exclusions"
820
+ : ex.layer
821
+ ? `a rule in ${ex.layer}`
822
+ : "an HQ sync ignore rule";
823
+ return `HQ sync skips this path because of ${by}, so HQ will not offload it.`;
824
+ }
519
825
  /** Whether a working-tree path is never a current-offload candidate. */
520
826
  function skippedByName(rel) {
521
827
  const parts = rel.split("/");
@@ -615,10 +921,34 @@ async function planCurrentFiles(rels, opts, deps, openVault) {
615
921
  plan.synced = head !== null && head.etag === normalizeEtag(record.remoteEtag) && head.size === st.size;
616
922
  }
617
923
  }
618
- if (plan.synced)
924
+ if (plan.synced) {
619
925
  plan.action = "point-at-vault";
620
- else if (target.vault === "company")
621
- plan.reason = RUN_SYNC_FIRST;
926
+ }
927
+ else if (target.vault === "company") {
928
+ const ex = deps.syncExclusion?.(rel);
929
+ if (ex?.excluded) {
930
+ plan.excluded_by = ex.layer ?? "ignore rules";
931
+ plan.reason = exclusionReason(ex);
932
+ }
933
+ else if (st.size <= (deps.syncMaxFileBytes ?? SYNC_MAX_FILE_BYTES)) {
934
+ plan.reason = RUN_SYNC_FIRST;
935
+ }
936
+ else if (!deps.openCompanyUploader || !deps.recordCloudOnly || !deps.openVaultObjects) {
937
+ plan.reason = UPDATE_SYNC;
938
+ }
939
+ else {
940
+ digest ??= await digestFile(abs);
941
+ const head = await (await openVault(target)).head(target.key);
942
+ const state = companyKeyState(head, digest, record, readOwnUpload(uploadStatePath(deps, target)));
943
+ if (state === "conflict") {
944
+ plan.reason = DIFFERENT_COPY;
945
+ }
946
+ else {
947
+ plan.action = "upload+offload";
948
+ plan.vault_key = target.key;
949
+ }
950
+ }
951
+ }
622
952
  planned.push({ plan, target, stat: st, digest, record: record ?? undefined });
623
953
  }
624
954
  return planned;
@@ -670,7 +1000,11 @@ async function runCurrent(opts, deps, result) {
670
1000
  if (opts.dryRun || todo.length === 0)
671
1001
  return "ok";
672
1002
  if (!opts.yes) {
673
- const ok = await deps.confirm(`Replace ${todo.length} current file(s) with links to their HQ cloud copies and remove the local copies?`);
1003
+ const uploads = todo.filter((p) => p.plan.action === "upload+offload");
1004
+ const uploadNote = uploads.length > 0
1005
+ ? ` ${uploads.length} of them (${uploads.reduce((n, p) => n + p.plan.bytes, 0)} bytes) are uploaded to their company vault first.`
1006
+ : "";
1007
+ const ok = await deps.confirm(`Replace ${todo.length} current file(s) with links to their HQ cloud copies and remove the local copies?${uploadNote}`);
674
1008
  if (!ok) {
675
1009
  deps.log("Aborted - nothing was uploaded or changed.");
676
1010
  return "aborted";
@@ -678,29 +1012,39 @@ async function runCurrent(opts, deps, result) {
678
1012
  }
679
1013
  const fsOps = deps.placeholderFs ?? durablePlaceholderFs;
680
1014
  let personal;
1015
+ const uploaders = new Map();
681
1016
  for (const file of todo) {
682
- const { plan } = file;
1017
+ const { plan, target } = file;
683
1018
  try {
1019
+ let action;
684
1020
  if (plan.action === "point-at-vault") {
685
- await pointCurrentFileAtVault(file, await openVault(file.target), deps, fsOps);
1021
+ await pointCurrentFileAtVault(file, await openVault(target), deps, fsOps);
1022
+ action = "point-at-vault";
1023
+ }
1024
+ else if (plan.action === "upload+offload") {
1025
+ if (target.vault !== "company" || !deps.openCompanyUploader)
1026
+ throw new Error(UPDATE_SYNC);
1027
+ let uploader = uploaders.get(target.slug);
1028
+ if (!uploader) {
1029
+ uploader = deps.openCompanyUploader(target.slug);
1030
+ uploaders.set(target.slug, uploader);
1031
+ }
1032
+ await uploadCompanyFile(file, await uploader, await openVault(target), deps, fsOps);
1033
+ action = "upload+offload";
686
1034
  }
687
1035
  else {
688
1036
  personal ??= await deps.openPersonal();
689
1037
  await offloadCurrentFile(plan.path, personal, deps, fsOps);
1038
+ action = "upload";
690
1039
  }
691
- const done = {
692
- path: plan.path,
693
- bytes: plan.bytes,
694
- vault: plan.vault,
695
- action: plan.action === "point-at-vault" ? "point-at-vault" : "upload",
696
- };
1040
+ const done = { path: plan.path, bytes: plan.bytes, vault: plan.vault, action };
697
1041
  if (plan.company)
698
1042
  done.company = plan.company;
699
1043
  current.offloaded.push(done);
700
1044
  current.freed_bytes += plan.bytes;
701
1045
  }
702
1046
  catch (err) {
703
- result.errors.push(`${plan.path}: ${err.message}`);
1047
+ result.errors.push(`${plan.path}: ${failureMessage(err)}`);
704
1048
  }
705
1049
  }
706
1050
  return "ok";
@@ -844,7 +1188,7 @@ export async function runOffload(opts, deps) {
844
1188
  });
845
1189
  }
846
1190
  catch (err) {
847
- result.errors.push(`${blob.paths[0] ?? blob.sha}: ${err.message}`);
1191
+ result.errors.push(`${blob.paths[0] ?? blob.sha}: ${failureMessage(err)}`);
848
1192
  }
849
1193
  }
850
1194
  if (manifest.length === 0)
@@ -1057,6 +1401,7 @@ async function fetchVaultObject(pointer, target, deps) {
1057
1401
  if (!deps.openVaultObjects)
1058
1402
  throw new Error("This HQ build cannot fetch files recorded by HQ sync.");
1059
1403
  let store;
1404
+ let owner;
1060
1405
  if (pointer.vault === "company") {
1061
1406
  const slug = companyForPointer(pointer);
1062
1407
  const active = deps.activeCompanySlugs ? await deps.activeCompanySlugs() : [];
@@ -1064,19 +1409,35 @@ async function fetchVaultObject(pointer, target, deps) {
1064
1409
  throw new Error(`You are not an active member of '${slug}', so HQ cannot download this file for you.`);
1065
1410
  }
1066
1411
  store = await deps.openVaultObjects({ vault: "company", slug });
1412
+ owner = { vault: "company", slug, key: pointer.key };
1067
1413
  }
1068
1414
  else {
1069
1415
  if (pointer.key !== pointer.path) {
1070
1416
  throw new Error(`Placeholder for '${pointer.path}' names vault key '${pointer.key}', which does not match its location. Nothing was changed.`);
1071
1417
  }
1072
1418
  store = await deps.openVaultObjects({ vault: "personal" });
1419
+ owner = { vault: "personal", key: pointer.key };
1073
1420
  }
1074
1421
  const etag = pointer.remoteEtag;
1075
- return downloadVerified((partial) => store.download(pointer.key, partial, etag), pointer.key, target, {
1076
- sha256: pointer.sha256,
1422
+ // A teammate's placeholder has no sha256; an offload upload carries it as
1423
+ // object metadata, so check against that when the pinned object has it.
1424
+ let sha256 = pointer.sha256;
1425
+ if (sha256 == null) {
1426
+ const head = await store.head(pointer.key);
1427
+ if (head?.sha256 && head.etag === normalizeEtag(etag))
1428
+ sha256 = head.sha256;
1429
+ }
1430
+ const digest = await downloadVerified((partial) => store.download(pointer.key, partial, etag), pointer.key, target, {
1431
+ sha256,
1077
1432
  size: pointer.size,
1078
1433
  etag,
1079
1434
  });
1435
+ // Sync never transfers a file over its limit, so record the restored file
1436
+ // as the vault's version; otherwise sync would treat it as an unknown local file.
1437
+ if (digest.size > (deps.syncMaxFileBytes ?? SYNC_MAX_FILE_BYTES) && deps.recordCloudOnly) {
1438
+ await deps.recordCloudOnly(owner, { etag: normalizeEtag(etag), size: digest.size, sha256: digest.sha256 });
1439
+ }
1440
+ return digest;
1080
1441
  }
1081
1442
  export async function runFetchGitBlob(sha, out, deps) {
1082
1443
  return fetchGitBlob(sha, out, deps);
@@ -1197,7 +1558,4 @@ export function gitListPlaceholders(hqRoot) {
1197
1558
  export function defaultTmpDir() {
1198
1559
  return os.tmpdir();
1199
1560
  }
1200
- export function personalS3Store(input) {
1201
- return s3BlobStore(input.bucket, new S3Client({ region: input.region, credentials: input.credentials }));
1202
- }
1203
1561
  //# sourceMappingURL=storage-offload.js.map