@indigoai-us/hq-cli 5.345.137 → 5.345.138
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -0
- package/dist/commands/storage-offload.d.ts +110 -31
- package/dist/commands/storage-offload.js +239 -43
- package/dist/commands/storage.js +62 -7
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +577 -441
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.345.138] — 2026-10-15
|
|
6
|
+
- `hq storage offload --current` can now free company files larger than the 50 MB sync limit. It uploads each one to its company vault, checks the vault copy against the file's sha256 and size, records it as cloud-only, writes the `.hqcloud` placeholder, and only then removes the local copy. An interrupted upload leaves the file in place and resumes on the next run. The dry run shows `upload+offload` with the byte count and the vault key. A path the `.hqinclude` allowlist excludes is refused with the line to add. `hq storage fetch` checks the vault copy's sha256 before restoring it. The upload path turns on once the bundled hq-cloud includes the new upload exports; until then these files report that HQ sync needs an update.
|
|
7
|
+
|
|
5
8
|
## [5.345.137] — 2026-10-15
|
|
6
9
|
- The bundled sync engine now uses hq-cloud 6.18.76. On Windows, the hq-core update and rescue lay down native links after the overlay, turn git symlink stubs from Windows clones into real links, and upload symlinks with the correct target. Sync merges `board.json` by entry id instead of replacing it, merges append-only `.jsonl` conflicts instead of parking them, keeps any `.hqcloud` placeholder on pull, and no longer syncs live-meeting runtime state or server sync manifests.
|
|
7
10
|
|
|
@@ -1,38 +1,49 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `hq storage offload` / `hq storage fetch` — move big files
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* `hq storage offload` / `hq storage fetch` — move big files out of the HQ
|
|
3
|
+
* folder into HQ cloud storage and leave small linked placeholders behind.
|
|
4
|
+
* Shared contract: owner-approved offload design, 2026-10-09; company upload
|
|
5
|
+
* for files over the sync limit approved 2026-10-15.
|
|
5
6
|
*
|
|
6
7
|
* Two kinds of big file:
|
|
7
8
|
* - History copies (old versions inside the HQ root git history). The CLI
|
|
8
|
-
* uploads each blob, verifies size + sha256
|
|
9
|
-
* verified blobs to core `hq-storage.sh
|
|
10
|
-
* to the pointer text. Working-tree
|
|
9
|
+
* uploads each blob to the caller's PERSONAL vault, verifies size + sha256
|
|
10
|
+
* + git blob id, then hands only verified blobs to core `hq-storage.sh
|
|
11
|
+
* strip-large`, which rewrites history to the pointer text. Working-tree
|
|
12
|
+
* files are never touched.
|
|
11
13
|
* - Current files (`--current [paths...]`, idle big files in the working
|
|
12
|
-
* tree). Each file goes to the vault that owns its path
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
14
|
+
* tree). Each file goes to the vault that owns its path:
|
|
15
|
+
* - A file in a company folder belongs to that company's vault, and only
|
|
16
|
+
* an active member can offload it. When HQ sync already stored this
|
|
17
|
+
* exact version (journal hash + size, and the vault object still has
|
|
18
|
+
* the journal's ETag), the file is replaced by a placeholder pointing
|
|
19
|
+
* at that object. A file over the sync limit (SYNC_MAX_FILE_BYTES),
|
|
20
|
+
* which sync never uploads, is uploaded to the company vault at its
|
|
21
|
+
* sync key by a resumable, checksum-verified multipart upload
|
|
22
|
+
* (`upload+offload`). Once the vault copy is verified the file is
|
|
23
|
+
* recorded in the sync journal as synced and cloud-only, then the
|
|
24
|
+
* placeholder is written, then the local file is removed. A path the
|
|
25
|
+
* sync ignore rules exclude is refused and the excluding rule is named
|
|
26
|
+
* (for `.hqinclude`, with the exact line to add). Other unsynced
|
|
27
|
+
* company files are skipped with "run hq sync first".
|
|
28
|
+
* - Any other file points at its personal-vault sync object the same
|
|
29
|
+
* way, or is uploaded to the personal vault's `.hq-offload/blobs/`.
|
|
30
|
+
* Current-file offload runs only when the installed HQ sync holds every
|
|
31
|
+
* `.hqcloud` sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and
|
|
32
|
+
* skips `.hq-offload/`; otherwise it reports "update HQ sync".
|
|
23
33
|
*
|
|
24
|
-
* Fetch also reads placeholders that HQ sync
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* it can never sync back.
|
|
34
|
+
* Fetch also reads placeholders that HQ sync wrote on a teammate's machine
|
|
35
|
+
* (`vault: company`, the company vault key, `sha256: unknown`, `remote-etag`).
|
|
36
|
+
* Those download from the company vault, only for an active member, and
|
|
37
|
+
* verify by size, ETag and, when the object carries it, its `hq-sha256`
|
|
38
|
+
* metadata. A placeholder marked `orphaned: true` (a teammate deleted the
|
|
39
|
+
* file) is refused unless `--force-orphan` is given, which writes the caller's
|
|
40
|
+
* own personal-vault copy to a file outside HQ so it can never sync back.
|
|
31
41
|
*
|
|
32
|
-
* Tenant isolation: uploads only ever go to the caller's
|
|
33
|
-
* the personal store refuses a company vault bucket. A
|
|
34
|
-
*
|
|
35
|
-
*
|
|
42
|
+
* Tenant isolation: history and personal uploads only ever go to the caller's
|
|
43
|
+
* personal vault, and the personal store refuses a company vault bucket. A
|
|
44
|
+
* company vault is opened only for a company the caller is an active member
|
|
45
|
+
* of, and only for a file in that company's folder; the company uploader must
|
|
46
|
+
* name that company and refuses a personal bucket.
|
|
36
47
|
*/
|
|
37
48
|
import { GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
38
49
|
import type { ScriptRunner } from "./storage.js";
|
|
@@ -47,6 +58,13 @@ export declare const ORPHANED_MESSAGE = "This file was deleted by a teammate. It
|
|
|
47
58
|
export declare const UPDATE_SYNC = "update HQ sync";
|
|
48
59
|
/** Reason on company files HQ sync has not stored in the company vault yet. */
|
|
49
60
|
export declare const RUN_SYNC_FIRST = "run hq sync first";
|
|
61
|
+
/**
|
|
62
|
+
* HQ sync's per-file limit (hq-cloud SYNC_MAX_FILE_BYTES; sync skips files
|
|
63
|
+
* strictly larger). storage.ts passes hq-cloud's value when it exports one.
|
|
64
|
+
*/
|
|
65
|
+
export declare const SYNC_MAX_FILE_BYTES: number;
|
|
66
|
+
/** Object metadata key holding the sha256 of a company-vault offload upload. */
|
|
67
|
+
export declare const VAULT_SHA256_METADATA_KEY = "hq-sha256";
|
|
50
68
|
/** Top-level HQ-root folder that holds offloaded blobs in the personal vault. */
|
|
51
69
|
export declare const OFFLOAD_TOP_LEVEL = ".hq-offload";
|
|
52
70
|
/**
|
|
@@ -120,10 +138,14 @@ export interface OffloadS3Client {
|
|
|
120
138
|
export declare function s3BlobStore(bucket: string, s3: OffloadS3Client): BlobStore;
|
|
121
139
|
/** Read-only access to vault objects at their sync keys, pinned to an ETag. */
|
|
122
140
|
export interface VaultObjectStore {
|
|
123
|
-
/**
|
|
141
|
+
/**
|
|
142
|
+
* Size and unquoted ETag of `key`, or null when it is not there. `sha256` is
|
|
143
|
+
* the object's `hq-sha256` metadata, set on company-vault offload uploads.
|
|
144
|
+
*/
|
|
124
145
|
head(key: string): Promise<{
|
|
125
146
|
size: number;
|
|
126
147
|
etag: string;
|
|
148
|
+
sha256?: string;
|
|
127
149
|
} | null>;
|
|
128
150
|
/** Download `key` to `dest` only if its ETag is `etag`; returns the served ETag. */
|
|
129
151
|
download(key: string, dest: string, etag: string): Promise<string>;
|
|
@@ -169,6 +191,55 @@ export interface OffloadDeps {
|
|
|
169
191
|
syncRecord?: (target: OffloadTarget) => Promise<SyncRecord | null>;
|
|
170
192
|
/** File operations for current-file offload (tests inject to see the order). */
|
|
171
193
|
placeholderFs?: PlaceholderFs;
|
|
194
|
+
/** HQ sync's per-file limit in bytes. Defaults to SYNC_MAX_FILE_BYTES. */
|
|
195
|
+
syncMaxFileBytes?: number;
|
|
196
|
+
/**
|
|
197
|
+
* Whether HQ sync's ignore rules exclude an HQ-root-relative path, and which
|
|
198
|
+
* rule does (hq-cloud createIgnoreExplainer). Null or absent: not excluded.
|
|
199
|
+
*/
|
|
200
|
+
syncExclusion?: (rel: string) => SyncExclusion | null;
|
|
201
|
+
/**
|
|
202
|
+
* Open a multipart uploader for a company vault. Called only for a company
|
|
203
|
+
* the caller is an active member of. Absent: company files over the sync
|
|
204
|
+
* limit report "update HQ sync".
|
|
205
|
+
*/
|
|
206
|
+
openCompanyUploader?: (slug: string) => Promise<CompanyUploader>;
|
|
207
|
+
/**
|
|
208
|
+
* Record `target` in the sync journal as synced and cloud-only (hq-cloud
|
|
209
|
+
* recordOffloadedObject), so sync neither re-uploads it nor pulls it back.
|
|
210
|
+
*/
|
|
211
|
+
recordCloudOnly?: (target: OffloadTarget, object: VerifiedUpload) => Promise<void>;
|
|
212
|
+
/** Folder for resumable multipart upload state. Defaults to ~/.hq/offload-uploads. */
|
|
213
|
+
uploadStateDir?: () => string;
|
|
214
|
+
}
|
|
215
|
+
/** Why HQ sync skips a path: the ignore layer that excludes it. */
|
|
216
|
+
export interface SyncExclusion {
|
|
217
|
+
excluded: boolean;
|
|
218
|
+
/** "built-in", "defaults", ".hqignore", ".hqsyncignore" or ".hqinclude allowlist". */
|
|
219
|
+
layer: string | null;
|
|
220
|
+
/** For the .hqinclude allowlist: the line that would include the path. */
|
|
221
|
+
includeLine?: string;
|
|
222
|
+
}
|
|
223
|
+
/** A vault object whose size and sha256 were verified after upload. */
|
|
224
|
+
export interface VerifiedUpload {
|
|
225
|
+
etag: string;
|
|
226
|
+
size: number;
|
|
227
|
+
sha256: string;
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Resumable, checksum-verified multipart upload into one company vault
|
|
231
|
+
* (hq-cloud uploadLargeObjectVerified). `upload` resumes from `stateFile`
|
|
232
|
+
* when it holds an upload for the same file, and resolves only after the
|
|
233
|
+
* object's size, part count and sha256 metadata are verified.
|
|
234
|
+
*/
|
|
235
|
+
export interface CompanyUploader {
|
|
236
|
+
slug: string;
|
|
237
|
+
bucket: string;
|
|
238
|
+
upload(input: {
|
|
239
|
+
key: string;
|
|
240
|
+
file: string;
|
|
241
|
+
stateFile: string;
|
|
242
|
+
}): Promise<VerifiedUpload>;
|
|
172
243
|
}
|
|
173
244
|
/** The vault that owns a working-tree path, and the object key there. */
|
|
174
245
|
export type OffloadTarget = {
|
|
@@ -273,7 +344,15 @@ export interface CurrentFilePlan {
|
|
|
273
344
|
company?: string;
|
|
274
345
|
/** HQ sync already stored this exact version in the owning vault. */
|
|
275
346
|
synced: boolean;
|
|
276
|
-
|
|
347
|
+
/**
|
|
348
|
+
* upload+offload: a company file over the sync limit, uploaded to the
|
|
349
|
+
* company vault at `vault_key`, then replaced by a placeholder.
|
|
350
|
+
*/
|
|
351
|
+
action: "point-at-vault" | "upload" | "upload+offload" | "skip";
|
|
352
|
+
/** Company vault key for upload+offload. */
|
|
353
|
+
vault_key?: string;
|
|
354
|
+
/** The HQ sync ignore layer that excludes this path. */
|
|
355
|
+
excluded_by?: string;
|
|
277
356
|
reason?: string;
|
|
278
357
|
}
|
|
279
358
|
export interface CurrentOffloaded {
|
|
@@ -281,7 +360,7 @@ export interface CurrentOffloaded {
|
|
|
281
360
|
bytes: number;
|
|
282
361
|
vault: "personal" | "company";
|
|
283
362
|
company?: string;
|
|
284
|
-
action: "point-at-vault" | "upload";
|
|
363
|
+
action: "point-at-vault" | "upload" | "upload+offload";
|
|
285
364
|
}
|
|
286
365
|
export interface OffloadOptions {
|
|
287
366
|
minBytes: number;
|
|
@@ -1,38 +1,49 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `hq storage offload` / `hq storage fetch` — move big files
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* `hq storage offload` / `hq storage fetch` — move big files out of the HQ
|
|
3
|
+
* folder into HQ cloud storage and leave small linked placeholders behind.
|
|
4
|
+
* Shared contract: owner-approved offload design, 2026-10-09; company upload
|
|
5
|
+
* for files over the sync limit approved 2026-10-15.
|
|
5
6
|
*
|
|
6
7
|
* Two kinds of big file:
|
|
7
8
|
* - History copies (old versions inside the HQ root git history). The CLI
|
|
8
|
-
* uploads each blob, verifies size + sha256
|
|
9
|
-
* verified blobs to core `hq-storage.sh
|
|
10
|
-
* to the pointer text. Working-tree
|
|
9
|
+
* uploads each blob to the caller's PERSONAL vault, verifies size + sha256
|
|
10
|
+
* + git blob id, then hands only verified blobs to core `hq-storage.sh
|
|
11
|
+
* strip-large`, which rewrites history to the pointer text. Working-tree
|
|
12
|
+
* files are never touched.
|
|
11
13
|
* - Current files (`--current [paths...]`, idle big files in the working
|
|
12
|
-
* tree). Each file goes to the vault that owns its path
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
14
|
+
* tree). Each file goes to the vault that owns its path:
|
|
15
|
+
* - A file in a company folder belongs to that company's vault, and only
|
|
16
|
+
* an active member can offload it. When HQ sync already stored this
|
|
17
|
+
* exact version (journal hash + size, and the vault object still has
|
|
18
|
+
* the journal's ETag), the file is replaced by a placeholder pointing
|
|
19
|
+
* at that object. A file over the sync limit (SYNC_MAX_FILE_BYTES),
|
|
20
|
+
* which sync never uploads, is uploaded to the company vault at its
|
|
21
|
+
* sync key by a resumable, checksum-verified multipart upload
|
|
22
|
+
* (`upload+offload`). Once the vault copy is verified the file is
|
|
23
|
+
* recorded in the sync journal as synced and cloud-only, then the
|
|
24
|
+
* placeholder is written, then the local file is removed. A path the
|
|
25
|
+
* sync ignore rules exclude is refused and the excluding rule is named
|
|
26
|
+
* (for `.hqinclude`, with the exact line to add). Other unsynced
|
|
27
|
+
* company files are skipped with "run hq sync first".
|
|
28
|
+
* - Any other file points at its personal-vault sync object the same
|
|
29
|
+
* way, or is uploaded to the personal vault's `.hq-offload/blobs/`.
|
|
30
|
+
* Current-file offload runs only when the installed HQ sync holds every
|
|
31
|
+
* `.hqcloud` sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and
|
|
32
|
+
* skips `.hq-offload/`; otherwise it reports "update HQ sync".
|
|
23
33
|
*
|
|
24
|
-
* Fetch also reads placeholders that HQ sync
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* it can never sync back.
|
|
34
|
+
* Fetch also reads placeholders that HQ sync wrote on a teammate's machine
|
|
35
|
+
* (`vault: company`, the company vault key, `sha256: unknown`, `remote-etag`).
|
|
36
|
+
* Those download from the company vault, only for an active member, and
|
|
37
|
+
* verify by size, ETag and, when the object carries it, its `hq-sha256`
|
|
38
|
+
* metadata. A placeholder marked `orphaned: true` (a teammate deleted the
|
|
39
|
+
* file) is refused unless `--force-orphan` is given, which writes the caller's
|
|
40
|
+
* own personal-vault copy to a file outside HQ so it can never sync back.
|
|
31
41
|
*
|
|
32
|
-
* Tenant isolation: uploads only ever go to the caller's
|
|
33
|
-
* the personal store refuses a company vault bucket. A
|
|
34
|
-
*
|
|
35
|
-
*
|
|
42
|
+
* Tenant isolation: history and personal uploads only ever go to the caller's
|
|
43
|
+
* personal vault, and the personal store refuses a company vault bucket. A
|
|
44
|
+
* company vault is opened only for a company the caller is an active member
|
|
45
|
+
* of, and only for a file in that company's folder; the company uploader must
|
|
46
|
+
* name that company and refuses a personal bucket.
|
|
36
47
|
*/
|
|
37
48
|
import * as crypto from "node:crypto";
|
|
38
49
|
import * as fs from "node:fs";
|
|
@@ -53,6 +64,13 @@ const UPDATE_HQ = "update HQ";
|
|
|
53
64
|
export const UPDATE_SYNC = "update HQ sync";
|
|
54
65
|
/** Reason on company files HQ sync has not stored in the company vault yet. */
|
|
55
66
|
export const RUN_SYNC_FIRST = "run hq sync first";
|
|
67
|
+
/**
|
|
68
|
+
* HQ sync's per-file limit (hq-cloud SYNC_MAX_FILE_BYTES; sync skips files
|
|
69
|
+
* strictly larger). storage.ts passes hq-cloud's value when it exports one.
|
|
70
|
+
*/
|
|
71
|
+
export const SYNC_MAX_FILE_BYTES = 50 * 1024 * 1024;
|
|
72
|
+
/** Object metadata key holding the sha256 of a company-vault offload upload. */
|
|
73
|
+
export const VAULT_SHA256_METADATA_KEY = "hq-sha256";
|
|
56
74
|
const DAY_MS = 86_400_000;
|
|
57
75
|
const PERSONAL_BUCKET_PREFIX = "hq-vault-prs-";
|
|
58
76
|
/** Top-level HQ-root folder that holds offloaded blobs in the personal vault. */
|
|
@@ -251,7 +269,14 @@ export function s3VaultObjectStore(bucket, s3) {
|
|
|
251
269
|
async head(key) {
|
|
252
270
|
try {
|
|
253
271
|
const res = (await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })));
|
|
254
|
-
|
|
272
|
+
const head = {
|
|
273
|
+
size: res.ContentLength ?? -1,
|
|
274
|
+
etag: normalizeEtag(res.ETag),
|
|
275
|
+
};
|
|
276
|
+
const sha = res.Metadata?.[VAULT_SHA256_METADATA_KEY];
|
|
277
|
+
if (sha && HEX64.test(sha))
|
|
278
|
+
head.sha256 = sha;
|
|
279
|
+
return head;
|
|
255
280
|
}
|
|
256
281
|
catch (err) {
|
|
257
282
|
if (isNotFound(err))
|
|
@@ -516,6 +541,122 @@ async function pointCurrentFileAtVault(file, vault, deps, fsOps) {
|
|
|
516
541
|
}
|
|
517
542
|
await fsOps.removeOriginal(abs);
|
|
518
543
|
}
|
|
544
|
+
/** Resumable upload state for one company vault key. */
|
|
545
|
+
function uploadStatePath(deps, target) {
|
|
546
|
+
const dir = deps.uploadStateDir ? deps.uploadStateDir() : path.join(os.homedir(), ".hq", "offload-uploads");
|
|
547
|
+
const id = crypto.createHash("sha256").update(`${target.slug}\0${target.key}`).digest("hex");
|
|
548
|
+
return path.join(dir, `${id}.json`);
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* ETag of a verified upload this machine made to the key but did not record
|
|
552
|
+
* (the file changed before it was recorded), so the next run may replace it.
|
|
553
|
+
*/
|
|
554
|
+
function readOwnUpload(stateFile) {
|
|
555
|
+
try {
|
|
556
|
+
const etag = JSON.parse(fs.readFileSync(`${stateFile}.uploaded`, "utf8")).etag;
|
|
557
|
+
return typeof etag === "string" ? etag : undefined;
|
|
558
|
+
}
|
|
559
|
+
catch (err) {
|
|
560
|
+
if (err.code === "ENOENT")
|
|
561
|
+
return undefined;
|
|
562
|
+
throw new Error(`Cannot read offload upload state ${stateFile}.uploaded: ${err.message}`);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* What the company vault key allows for a local file over the sync limit:
|
|
567
|
+
* upload when the key is empty, still holds the version this machine last
|
|
568
|
+
* recorded, or holds this machine's own unrecorded upload; adopt when it
|
|
569
|
+
* already holds these exact bytes; otherwise a different copy is there.
|
|
570
|
+
*/
|
|
571
|
+
function companyKeyState(head, digest, record, ownEtag) {
|
|
572
|
+
if (!head)
|
|
573
|
+
return "upload";
|
|
574
|
+
if (head.sha256 === digest.sha256 && head.size === digest.size)
|
|
575
|
+
return "adopt";
|
|
576
|
+
if (record?.remoteEtag && head.etag === normalizeEtag(record.remoteEtag))
|
|
577
|
+
return "upload";
|
|
578
|
+
if (ownEtag && head.etag === normalizeEtag(ownEtag))
|
|
579
|
+
return "upload";
|
|
580
|
+
return "conflict";
|
|
581
|
+
}
|
|
582
|
+
const DIFFERENT_COPY = "A different copy is already in the company vault. Run hq sync to get it, then try again.";
|
|
583
|
+
/**
|
|
584
|
+
* Upload one company file over the sync limit to its company vault key, then
|
|
585
|
+
* offload it. Order: verified upload, journal record (synced, cloud-only),
|
|
586
|
+
* fsynced `x.hqcloud`, and only then remove `x`. An interrupted upload leaves
|
|
587
|
+
* `x` untouched, records nothing, and keeps the state file so the next run
|
|
588
|
+
* resumes.
|
|
589
|
+
*/
|
|
590
|
+
async function uploadCompanyFile(file, uploader, vault, deps, fsOps) {
|
|
591
|
+
const { plan, target, stat: before, digest, record } = file;
|
|
592
|
+
if (target.vault !== "company" || !before || !digest)
|
|
593
|
+
throw new Error(`'${plan.path}' is not a company file.`);
|
|
594
|
+
if (uploader.slug !== target.slug || uploader.bucket.startsWith(PERSONAL_BUCKET_PREFIX)) {
|
|
595
|
+
throw new Error(`The upload target '${uploader.bucket}' is not the company vault for '${target.slug}'. Nothing was uploaded.`);
|
|
596
|
+
}
|
|
597
|
+
if (!deps.recordCloudOnly)
|
|
598
|
+
throw new Error(`This HQ build cannot record cloud-only files; ${UPDATE_SYNC}.`);
|
|
599
|
+
const abs = containedPath(deps.hqRoot, plan.path);
|
|
600
|
+
const unchanged = async () => {
|
|
601
|
+
const st = await fs.promises.stat(abs);
|
|
602
|
+
return st.size === before.size && st.mtimeMs === before.mtimeMs && st.size === digest.size;
|
|
603
|
+
};
|
|
604
|
+
const changed = new Error(`'${plan.path}' changed while uploading. Nothing was replaced.`);
|
|
605
|
+
if (!(await unchanged()))
|
|
606
|
+
throw changed;
|
|
607
|
+
const stateFile = uploadStatePath(deps, target);
|
|
608
|
+
const head = await vault.head(target.key);
|
|
609
|
+
const state = companyKeyState(head, digest, record, readOwnUpload(stateFile));
|
|
610
|
+
if (state === "conflict")
|
|
611
|
+
throw new Error(`${DIFFERENT_COPY} Nothing was uploaded.`);
|
|
612
|
+
let object;
|
|
613
|
+
if (state === "adopt" && head) {
|
|
614
|
+
object = { etag: head.etag, size: head.size, sha256: digest.sha256 };
|
|
615
|
+
}
|
|
616
|
+
else {
|
|
617
|
+
object = await uploader.upload({ key: target.key, file: abs, stateFile });
|
|
618
|
+
if (object.sha256 !== digest.sha256 || object.size !== digest.size) {
|
|
619
|
+
throw new Error(`The uploaded copy of '${plan.path}' does not match the local file. Nothing was replaced.`);
|
|
620
|
+
}
|
|
621
|
+
if (!(await unchanged())) {
|
|
622
|
+
await fs.promises.mkdir(path.dirname(stateFile), { recursive: true });
|
|
623
|
+
await fs.promises.writeFile(`${stateFile}.uploaded`, JSON.stringify({ etag: object.etag }));
|
|
624
|
+
throw changed;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
await deps.recordCloudOnly(target, object);
|
|
628
|
+
await fs.promises.rm(`${stateFile}.uploaded`, { force: true });
|
|
629
|
+
const placeholder = `${abs}${PLACEHOLDER_SUFFIX}`;
|
|
630
|
+
await fsOps.writePlaceholder(placeholder, renderPointer({
|
|
631
|
+
path: plan.path,
|
|
632
|
+
size: digest.size,
|
|
633
|
+
sha256: digest.sha256,
|
|
634
|
+
vault: "company",
|
|
635
|
+
key: target.key,
|
|
636
|
+
offloadedAt: deps.now().toISOString().replace(/\.\d{3}Z$/, "Z"),
|
|
637
|
+
remoteEtag: object.etag,
|
|
638
|
+
}));
|
|
639
|
+
if (!(await unchanged())) {
|
|
640
|
+
await fs.promises.rm(placeholder, { force: true });
|
|
641
|
+
throw changed;
|
|
642
|
+
}
|
|
643
|
+
await fsOps.removeOriginal(abs);
|
|
644
|
+
}
|
|
645
|
+
/** Plain name of an HQ sync ignore layer, for skip reasons. */
|
|
646
|
+
function exclusionReason(ex) {
|
|
647
|
+
if (ex.layer === ".hqinclude allowlist") {
|
|
648
|
+
const add = ex.includeLine ? ` To sync and offload it, add this line to .hqinclude: ${ex.includeLine}` : "";
|
|
649
|
+
return `HQ sync skips this path because the .hqinclude allowlist does not list it.${add}`;
|
|
650
|
+
}
|
|
651
|
+
const by = ex.layer === "built-in"
|
|
652
|
+
? "HQ sync's built-in exclusions"
|
|
653
|
+
: ex.layer === "defaults"
|
|
654
|
+
? "HQ sync's default exclusions"
|
|
655
|
+
: ex.layer
|
|
656
|
+
? `a rule in ${ex.layer}`
|
|
657
|
+
: "an HQ sync ignore rule";
|
|
658
|
+
return `HQ sync skips this path because of ${by}, so HQ will not offload it.`;
|
|
659
|
+
}
|
|
519
660
|
/** Whether a working-tree path is never a current-offload candidate. */
|
|
520
661
|
function skippedByName(rel) {
|
|
521
662
|
const parts = rel.split("/");
|
|
@@ -615,10 +756,34 @@ async function planCurrentFiles(rels, opts, deps, openVault) {
|
|
|
615
756
|
plan.synced = head !== null && head.etag === normalizeEtag(record.remoteEtag) && head.size === st.size;
|
|
616
757
|
}
|
|
617
758
|
}
|
|
618
|
-
if (plan.synced)
|
|
759
|
+
if (plan.synced) {
|
|
619
760
|
plan.action = "point-at-vault";
|
|
620
|
-
|
|
621
|
-
|
|
761
|
+
}
|
|
762
|
+
else if (target.vault === "company") {
|
|
763
|
+
const ex = deps.syncExclusion?.(rel);
|
|
764
|
+
if (ex?.excluded) {
|
|
765
|
+
plan.excluded_by = ex.layer ?? "ignore rules";
|
|
766
|
+
plan.reason = exclusionReason(ex);
|
|
767
|
+
}
|
|
768
|
+
else if (st.size <= (deps.syncMaxFileBytes ?? SYNC_MAX_FILE_BYTES)) {
|
|
769
|
+
plan.reason = RUN_SYNC_FIRST;
|
|
770
|
+
}
|
|
771
|
+
else if (!deps.openCompanyUploader || !deps.recordCloudOnly || !deps.openVaultObjects) {
|
|
772
|
+
plan.reason = UPDATE_SYNC;
|
|
773
|
+
}
|
|
774
|
+
else {
|
|
775
|
+
digest ??= await digestFile(abs);
|
|
776
|
+
const head = await (await openVault(target)).head(target.key);
|
|
777
|
+
const state = companyKeyState(head, digest, record, readOwnUpload(uploadStatePath(deps, target)));
|
|
778
|
+
if (state === "conflict") {
|
|
779
|
+
plan.reason = DIFFERENT_COPY;
|
|
780
|
+
}
|
|
781
|
+
else {
|
|
782
|
+
plan.action = "upload+offload";
|
|
783
|
+
plan.vault_key = target.key;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
}
|
|
622
787
|
planned.push({ plan, target, stat: st, digest, record: record ?? undefined });
|
|
623
788
|
}
|
|
624
789
|
return planned;
|
|
@@ -670,7 +835,11 @@ async function runCurrent(opts, deps, result) {
|
|
|
670
835
|
if (opts.dryRun || todo.length === 0)
|
|
671
836
|
return "ok";
|
|
672
837
|
if (!opts.yes) {
|
|
673
|
-
const
|
|
838
|
+
const uploads = todo.filter((p) => p.plan.action === "upload+offload");
|
|
839
|
+
const uploadNote = uploads.length > 0
|
|
840
|
+
? ` ${uploads.length} of them (${uploads.reduce((n, p) => n + p.plan.bytes, 0)} bytes) are uploaded to their company vault first.`
|
|
841
|
+
: "";
|
|
842
|
+
const ok = await deps.confirm(`Replace ${todo.length} current file(s) with links to their HQ cloud copies and remove the local copies?${uploadNote}`);
|
|
674
843
|
if (!ok) {
|
|
675
844
|
deps.log("Aborted - nothing was uploaded or changed.");
|
|
676
845
|
return "aborted";
|
|
@@ -678,22 +847,32 @@ async function runCurrent(opts, deps, result) {
|
|
|
678
847
|
}
|
|
679
848
|
const fsOps = deps.placeholderFs ?? durablePlaceholderFs;
|
|
680
849
|
let personal;
|
|
850
|
+
const uploaders = new Map();
|
|
681
851
|
for (const file of todo) {
|
|
682
|
-
const { plan } = file;
|
|
852
|
+
const { plan, target } = file;
|
|
683
853
|
try {
|
|
854
|
+
let action;
|
|
684
855
|
if (plan.action === "point-at-vault") {
|
|
685
|
-
await pointCurrentFileAtVault(file, await openVault(
|
|
856
|
+
await pointCurrentFileAtVault(file, await openVault(target), deps, fsOps);
|
|
857
|
+
action = "point-at-vault";
|
|
858
|
+
}
|
|
859
|
+
else if (plan.action === "upload+offload") {
|
|
860
|
+
if (target.vault !== "company" || !deps.openCompanyUploader)
|
|
861
|
+
throw new Error(UPDATE_SYNC);
|
|
862
|
+
let uploader = uploaders.get(target.slug);
|
|
863
|
+
if (!uploader) {
|
|
864
|
+
uploader = deps.openCompanyUploader(target.slug);
|
|
865
|
+
uploaders.set(target.slug, uploader);
|
|
866
|
+
}
|
|
867
|
+
await uploadCompanyFile(file, await uploader, await openVault(target), deps, fsOps);
|
|
868
|
+
action = "upload+offload";
|
|
686
869
|
}
|
|
687
870
|
else {
|
|
688
871
|
personal ??= await deps.openPersonal();
|
|
689
872
|
await offloadCurrentFile(plan.path, personal, deps, fsOps);
|
|
873
|
+
action = "upload";
|
|
690
874
|
}
|
|
691
|
-
const done = {
|
|
692
|
-
path: plan.path,
|
|
693
|
-
bytes: plan.bytes,
|
|
694
|
-
vault: plan.vault,
|
|
695
|
-
action: plan.action === "point-at-vault" ? "point-at-vault" : "upload",
|
|
696
|
-
};
|
|
875
|
+
const done = { path: plan.path, bytes: plan.bytes, vault: plan.vault, action };
|
|
697
876
|
if (plan.company)
|
|
698
877
|
done.company = plan.company;
|
|
699
878
|
current.offloaded.push(done);
|
|
@@ -1057,6 +1236,7 @@ async function fetchVaultObject(pointer, target, deps) {
|
|
|
1057
1236
|
if (!deps.openVaultObjects)
|
|
1058
1237
|
throw new Error("This HQ build cannot fetch files recorded by HQ sync.");
|
|
1059
1238
|
let store;
|
|
1239
|
+
let owner;
|
|
1060
1240
|
if (pointer.vault === "company") {
|
|
1061
1241
|
const slug = companyForPointer(pointer);
|
|
1062
1242
|
const active = deps.activeCompanySlugs ? await deps.activeCompanySlugs() : [];
|
|
@@ -1064,19 +1244,35 @@ async function fetchVaultObject(pointer, target, deps) {
|
|
|
1064
1244
|
throw new Error(`You are not an active member of '${slug}', so HQ cannot download this file for you.`);
|
|
1065
1245
|
}
|
|
1066
1246
|
store = await deps.openVaultObjects({ vault: "company", slug });
|
|
1247
|
+
owner = { vault: "company", slug, key: pointer.key };
|
|
1067
1248
|
}
|
|
1068
1249
|
else {
|
|
1069
1250
|
if (pointer.key !== pointer.path) {
|
|
1070
1251
|
throw new Error(`Placeholder for '${pointer.path}' names vault key '${pointer.key}', which does not match its location. Nothing was changed.`);
|
|
1071
1252
|
}
|
|
1072
1253
|
store = await deps.openVaultObjects({ vault: "personal" });
|
|
1254
|
+
owner = { vault: "personal", key: pointer.key };
|
|
1073
1255
|
}
|
|
1074
1256
|
const etag = pointer.remoteEtag;
|
|
1075
|
-
|
|
1076
|
-
|
|
1257
|
+
// A teammate's placeholder has no sha256; an offload upload carries it as
|
|
1258
|
+
// object metadata, so check against that when the pinned object has it.
|
|
1259
|
+
let sha256 = pointer.sha256;
|
|
1260
|
+
if (sha256 == null) {
|
|
1261
|
+
const head = await store.head(pointer.key);
|
|
1262
|
+
if (head?.sha256 && head.etag === normalizeEtag(etag))
|
|
1263
|
+
sha256 = head.sha256;
|
|
1264
|
+
}
|
|
1265
|
+
const digest = await downloadVerified((partial) => store.download(pointer.key, partial, etag), pointer.key, target, {
|
|
1266
|
+
sha256,
|
|
1077
1267
|
size: pointer.size,
|
|
1078
1268
|
etag,
|
|
1079
1269
|
});
|
|
1270
|
+
// Sync never transfers a file over its limit, so record the restored file
|
|
1271
|
+
// as the vault's version; otherwise sync would treat it as an unknown local file.
|
|
1272
|
+
if (digest.size > (deps.syncMaxFileBytes ?? SYNC_MAX_FILE_BYTES) && deps.recordCloudOnly) {
|
|
1273
|
+
await deps.recordCloudOnly(owner, { etag: normalizeEtag(etag), size: digest.size, sha256: digest.sha256 });
|
|
1274
|
+
}
|
|
1275
|
+
return digest;
|
|
1080
1276
|
}
|
|
1081
1277
|
export async function runFetchGitBlob(sha, out, deps) {
|
|
1082
1278
|
return fetchGitBlob(sha, out, deps);
|