@indigoai-us/hq-cli 5.345.134 → 5.345.136
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/assets/scaffold/core/scripts/worktree-gc.sh +98 -8
- package/dist/command-catalog.generated.d.ts +62 -6
- package/dist/command-catalog.generated.js +79 -7
- package/dist/command-registration-plan.d.ts +6 -0
- package/dist/command-registration-plan.js +1 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.d.ts +3 -1
- package/dist/commands/core-bucket-c-handoff-knowledge-native.js +8 -4
- package/dist/commands/core-checkpoint-native.d.ts +10 -0
- package/dist/commands/core-checkpoint-native.js +60 -15
- package/dist/commands/core-handoff.d.ts +2 -0
- package/dist/commands/core-handoff.js +3 -1
- package/dist/commands/core-pipeline-workflow-runner-native.js +7 -5
- package/dist/commands/daemon.js +3 -1
- package/dist/commands/fingerprint.d.ts +10 -0
- package/dist/commands/fingerprint.js +191 -0
- package/dist/commands/integrations-core.d.ts +22 -0
- package/dist/commands/integrations-core.js +25 -0
- package/dist/commands/integrations-manage.d.ts +24 -1
- package/dist/commands/integrations-manage.js +92 -17
- package/dist/commands/integrations.d.ts +11 -0
- package/dist/commands/integrations.js +100 -19
- package/dist/commands/lanes-loop.d.ts +9 -0
- package/dist/commands/lanes-loop.js +9 -1
- package/dist/commands/lanes.js +78 -55
- package/dist/commands/meetings.d.ts +2 -0
- package/dist/commands/meetings.js +57 -14
- package/dist/commands/mesh.d.ts +19 -1
- package/dist/commands/mesh.js +158 -70
- package/dist/commands/reindex.js +80 -5
- package/dist/commands/storage-offload.d.ts +93 -28
- package/dist/commands/storage-offload.js +290 -34
- package/dist/commands/storage.js +60 -7
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +2550 -1748
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/index.js +4 -0
- package/dist/lib/capacity/route.js +24 -0
- package/dist/lib/core-utils/session-company.d.ts +11 -0
- package/dist/lib/core-utils/session-company.js +34 -0
- package/dist/lib/core-utils/worktree-resume.js +44 -4
- package/dist/lib/core-utils/worktree-session-owner.d.ts +27 -0
- package/dist/lib/core-utils/worktree-session-owner.js +157 -0
- package/dist/lib/daemon/catalog.d.ts +2 -0
- package/dist/lib/daemon/catalog.js +3 -3
- package/dist/lib/daemon/hq-child-bootstrap-stage.d.ts +5 -0
- package/dist/lib/daemon/hq-child-bootstrap-stage.js +3 -11
- package/dist/lib/daemon/server.js +10 -1
- package/dist/lib/daemon/supervisor.js +6 -0
- package/dist/lib/daemon/sync-runner-bundle.d.ts +5 -0
- package/dist/lib/daemon/sync-runner-bundle.js +4 -12
- package/dist/lib/doctor/checks/claude-wiring.js +2 -1
- package/dist/lib/doctor/fix/apply.d.ts +20 -0
- package/dist/lib/doctor/fix/apply.js +63 -3
- package/dist/lib/doctor/fix/remediation.d.ts +1 -1
- package/dist/lib/doctor/fix/remediation.js +16 -0
- package/dist/lib/doctor/shared.d.ts +11 -1
- package/dist/lib/doctor/shared.js +16 -2
- package/dist/lib/fingerprint/fixtures.d.ts +19 -0
- package/dist/lib/fingerprint/fixtures.js +61 -0
- package/dist/lib/fingerprint/format.d.ts +50 -0
- package/dist/lib/fingerprint/format.js +105 -0
- package/dist/lib/fingerprint/offline.d.ts +16 -0
- package/dist/lib/fingerprint/offline.js +44 -0
- package/dist/lib/fingerprint/registry-client.d.ts +74 -0
- package/dist/lib/fingerprint/registry-client.js +141 -0
- package/dist/lib/fingerprint/resolve-local.d.ts +14 -0
- package/dist/lib/fingerprint/resolve-local.js +67 -0
- package/dist/lib/fingerprint/self-core.d.ts +139 -0
- package/dist/lib/fingerprint/self-core.js +371 -0
- package/dist/lib/fingerprint/self.d.ts +44 -0
- package/dist/lib/fingerprint/self.js +191 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/lanes/monitor.js +14 -2
- package/dist/lib/lanes/spawn.d.ts +19 -1
- package/dist/lib/lanes/spawn.js +50 -6
- package/dist/lib/lanes/story-ledger.d.ts +1 -0
- package/dist/lib/lanes/story-ledger.js +3 -1
- package/dist/lib/lanes/types.d.ts +12 -0
- package/dist/lib/lanes/types.js +12 -1
- package/dist/lib/mesh/live/daemon/install.d.ts +12 -0
- package/dist/lib/mesh/live/daemon/install.js +22 -0
- package/dist/lib/mesh/live/flush.js +106 -3
- package/dist/lib/mesh/live/session-events-client.d.ts +9 -0
- package/dist/lib/mesh/live/session-events-client.js +15 -0
- package/dist/lib/work-context/state.d.ts +10 -0
- package/dist/lib/work-context/state.js +5 -0
- package/dist/utils/cli-network-lifecycle.d.ts +2 -0
- package/dist/utils/cli-network-lifecycle.js +5 -1
- package/dist/utils/cognito-session.d.ts +2 -21
- package/dist/utils/cognito-session.js +3 -84
- package/dist/utils/fsync-directory.d.ts +17 -0
- package/dist/utils/fsync-directory.js +27 -0
- package/dist/utils/person-token-cache.d.ts +32 -0
- package/dist/utils/person-token-cache.js +95 -0
- package/dist/utils/vault-api.d.ts +21 -1
- package/dist/utils/vault-api.js +54 -5
- package/dist/utils/windows-hide-default.d.ts +42 -0
- package/dist/utils/windows-hide-default.js +98 -0
- package/package.json +2 -2
|
@@ -8,13 +8,18 @@
|
|
|
8
8
|
* uploads each blob, verifies size + sha256 + git blob id, then hands only
|
|
9
9
|
* verified blobs to core `hq-storage.sh strip-large`, which rewrites history
|
|
10
10
|
* to the pointer text. Working-tree files are never touched.
|
|
11
|
-
* - Current files (idle big files in the working
|
|
12
|
-
* the
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
11
|
+
* - Current files (`--current [paths...]`, idle big files in the working
|
|
12
|
+
* tree). Each file goes to the vault that owns its path. A file under
|
|
13
|
+
* `companies/<slug>/` is never uploaded: when HQ sync already stored this
|
|
14
|
+
* exact version in that company's vault (journal hash + size, and the vault
|
|
15
|
+
* object still has the journal's ETag), the file is replaced by a
|
|
16
|
+
* placeholder pointing at that object. Otherwise it is skipped with "run
|
|
17
|
+
* hq sync first", so sync stays the only writer of company vault keys. Any
|
|
18
|
+
* other file points at its personal-vault sync object the same way, or is
|
|
19
|
+
* uploaded to the personal vault's `.hq-offload/blobs/`. Current-file
|
|
20
|
+
* offload runs only when the installed HQ sync holds every `.hqcloud`
|
|
21
|
+
* sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and skips
|
|
22
|
+
* `.hq-offload/`; otherwise it reports "update HQ sync".
|
|
18
23
|
*
|
|
19
24
|
* Fetch also reads placeholders that HQ sync rewrote after a teammate changed
|
|
20
25
|
* the file (`vault: company`, the company vault key, `sha256: unknown`,
|
|
@@ -24,9 +29,10 @@
|
|
|
24
29
|
* which writes the caller's own personal-vault copy to a file outside HQ so
|
|
25
30
|
* it can never sync back.
|
|
26
31
|
*
|
|
27
|
-
* Tenant isolation:
|
|
28
|
-
* personal store refuses a company vault bucket
|
|
29
|
-
*
|
|
32
|
+
* Tenant isolation: uploads only ever go to the caller's personal vault, and
|
|
33
|
+
* the personal store refuses a company vault bucket. A company vault is opened
|
|
34
|
+
* read-only (HEAD and ETag-pinned GET), and only for a company the caller is
|
|
35
|
+
* an active member of.
|
|
30
36
|
*/
|
|
31
37
|
import * as crypto from "node:crypto";
|
|
32
38
|
import * as fs from "node:fs";
|
|
@@ -42,15 +48,12 @@ export const POINTER_HEADER = "HQ-CLOUD-FILE v1";
|
|
|
42
48
|
export const POINTER_FOOTER = "This file is stored in HQ cloud storage. Open it in HQ, or run the command above to download it.";
|
|
43
49
|
export const DEFAULT_MIN_BYTES = 50 * 1024 * 1024;
|
|
44
50
|
export const DEFAULT_IDLE_DAYS = 14;
|
|
45
|
-
export const CURRENT_BLOCKED_REASON = "Offloading current files is not available yet: HQ sync would treat the replaced file as deleted.";
|
|
46
|
-
/**
|
|
47
|
-
* Gate for `--current`. Stays false until the owner enables current-file
|
|
48
|
-
* offload on top of hq-cloud's cloud-only placeholders (PR #919, 6.18.61).
|
|
49
|
-
*/
|
|
50
|
-
export const CURRENT_OFFLOAD_ENABLED = false;
|
|
51
51
|
export const ORPHANED_MESSAGE = "This file was deleted by a teammate. It's no longer in the cloud for your team.";
|
|
52
52
|
const UPDATE_HQ = "update HQ";
|
|
53
53
|
export const UPDATE_SYNC = "update HQ sync";
|
|
54
|
+
/** Reason on company files HQ sync has not stored in the company vault yet. */
|
|
55
|
+
export const RUN_SYNC_FIRST = "run hq sync first";
|
|
56
|
+
const DAY_MS = 86_400_000;
|
|
54
57
|
const PERSONAL_BUCKET_PREFIX = "hq-vault-prs-";
|
|
55
58
|
/** Top-level HQ-root folder that holds offloaded blobs in the personal vault. */
|
|
56
59
|
export const OFFLOAD_TOP_LEVEL = ".hq-offload";
|
|
@@ -63,7 +66,7 @@ export const OFFLOAD_TOP_LEVEL = ".hq-offload";
|
|
|
63
66
|
* @indigoai-us/hq-cloud 6.18.61 (PR #919) lists ".hq-offload" in
|
|
64
67
|
* PERSONAL_VAULT_EXCLUDED_TOP_LEVEL; storage.ts passes that export here. The
|
|
65
68
|
* check stays at runtime so an older bundled HQ sync still reports
|
|
66
|
-
* "update HQ sync".
|
|
69
|
+
* "update HQ sync". Current-file offload also needs syncHoldsPlaceholders.
|
|
67
70
|
*/
|
|
68
71
|
export function syncExcludesOffload(excludedTopLevel) {
|
|
69
72
|
return Array.isArray(excludedTopLevel) && excludedTopLevel.includes(OFFLOAD_TOP_LEVEL);
|
|
@@ -245,6 +248,17 @@ export function s3BlobStore(bucket, s3) {
|
|
|
245
248
|
}
|
|
246
249
|
export function s3VaultObjectStore(bucket, s3) {
|
|
247
250
|
return {
|
|
251
|
+
async head(key) {
|
|
252
|
+
try {
|
|
253
|
+
const res = (await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })));
|
|
254
|
+
return { size: res.ContentLength ?? -1, etag: normalizeEtag(res.ETag) };
|
|
255
|
+
}
|
|
256
|
+
catch (err) {
|
|
257
|
+
if (isNotFound(err))
|
|
258
|
+
return null;
|
|
259
|
+
throw err;
|
|
260
|
+
}
|
|
261
|
+
},
|
|
248
262
|
async download(key, dest, etag) {
|
|
249
263
|
let res;
|
|
250
264
|
try {
|
|
@@ -273,6 +287,21 @@ export function s3VaultObjectStore(bucket, s3) {
|
|
|
273
287
|
},
|
|
274
288
|
};
|
|
275
289
|
}
|
|
290
|
+
/**
|
|
291
|
+
* `companies/<slug>/<rest>` belongs to that company's vault at key `<rest>`.
|
|
292
|
+
* Every other path belongs to the personal vault at its HQ-root-relative path.
|
|
293
|
+
*/
|
|
294
|
+
export function resolveOffloadTarget(rel) {
|
|
295
|
+
const posix = toPosix(rel);
|
|
296
|
+
const m = /^companies\/([^/]+)\/(.+)$/.exec(posix);
|
|
297
|
+
if (m)
|
|
298
|
+
return { vault: "company", slug: m[1], key: m[2] };
|
|
299
|
+
return { vault: "personal", key: posix };
|
|
300
|
+
}
|
|
301
|
+
/** Whether current-file offload can run with the installed HQ sync. */
|
|
302
|
+
export function currentSyncReady(deps) {
|
|
303
|
+
return deps.syncExcludesOffload() && deps.syncHoldsPlaceholders?.() === true;
|
|
304
|
+
}
|
|
276
305
|
// ── Core script capability ──────────────────────────────────────────────────
|
|
277
306
|
export function storageScriptPath(hqRoot) {
|
|
278
307
|
return path.join(hqRoot, "core", "scripts", "hq-storage.sh");
|
|
@@ -408,16 +437,17 @@ export const durablePlaceholderFs = {
|
|
|
408
437
|
},
|
|
409
438
|
};
|
|
410
439
|
/**
|
|
411
|
-
* Offload one current file: upload and verify,
|
|
412
|
-
* and only then remove `x`. With the placeholder
|
|
413
|
-
* watcher never sees `x` missing without its
|
|
414
|
-
*
|
|
415
|
-
*
|
|
416
|
-
* Not called while CURRENT_OFFLOAD_ENABLED is false.
|
|
440
|
+
* Offload one personal-scope current file by upload: upload and verify,
|
|
441
|
+
* write `x.hqcloud` (fsynced), and only then remove `x`. With the placeholder
|
|
442
|
+
* written first, a sync watcher never sees `x` missing without its
|
|
443
|
+
* placeholder. Company files never take this path (see pointCurrentFileAtVault).
|
|
417
444
|
*/
|
|
418
445
|
export async function offloadCurrentFile(rel, store, deps, fsOps = durablePlaceholderFs) {
|
|
419
446
|
if (store.vault !== "personal")
|
|
420
447
|
throw new Error("Offload only writes to your personal vault.");
|
|
448
|
+
if (resolveOffloadTarget(rel).vault === "company") {
|
|
449
|
+
throw new Error(`'${rel}' is a company file. Company files are never uploaded to your personal vault.`);
|
|
450
|
+
}
|
|
421
451
|
assertRestorable(rel);
|
|
422
452
|
const abs = containedPath(deps.hqRoot, rel);
|
|
423
453
|
const before = await fs.promises.stat(abs);
|
|
@@ -449,23 +479,251 @@ export async function offloadCurrentFile(rel, store, deps, fsOps = durablePlaceh
|
|
|
449
479
|
await fsOps.removeOriginal(abs);
|
|
450
480
|
return { path: toPosix(rel), bytes: digest.size };
|
|
451
481
|
}
|
|
482
|
+
/**
|
|
483
|
+
* Point one current file at the vault object HQ sync stored for it: confirm
|
|
484
|
+
* the object still has the recorded ETag and size, write `x.hqcloud`
|
|
485
|
+
* (fsynced), and only then remove `x`. Nothing is uploaded.
|
|
486
|
+
*/
|
|
487
|
+
async function pointCurrentFileAtVault(file, vault, deps, fsOps) {
|
|
488
|
+
const { plan, target, stat: before, digest, record } = file;
|
|
489
|
+
if (!before || !digest || !record?.remoteEtag)
|
|
490
|
+
throw new Error(`'${plan.path}' has no synced version.`);
|
|
491
|
+
const abs = containedPath(deps.hqRoot, plan.path);
|
|
492
|
+
const head = await vault.head(target.key);
|
|
493
|
+
if (!head || head.etag !== normalizeEtag(record.remoteEtag) || head.size !== digest.size) {
|
|
494
|
+
throw new Error(`The cloud copy of '${plan.path}' changed. Nothing was replaced; ${RUN_SYNC_FIRST}.`);
|
|
495
|
+
}
|
|
496
|
+
const unchanged = async () => {
|
|
497
|
+
const st = await fs.promises.stat(abs);
|
|
498
|
+
return st.size === before.size && st.mtimeMs === before.mtimeMs;
|
|
499
|
+
};
|
|
500
|
+
const changed = new Error(`'${plan.path}' changed while offloading. Nothing was replaced.`);
|
|
501
|
+
if (!(await unchanged()))
|
|
502
|
+
throw changed;
|
|
503
|
+
const placeholder = `${abs}${PLACEHOLDER_SUFFIX}`;
|
|
504
|
+
await fsOps.writePlaceholder(placeholder, renderPointer({
|
|
505
|
+
path: plan.path,
|
|
506
|
+
size: digest.size,
|
|
507
|
+
sha256: digest.sha256,
|
|
508
|
+
vault: target.vault,
|
|
509
|
+
key: target.key,
|
|
510
|
+
offloadedAt: deps.now().toISOString().replace(/\.\d{3}Z$/, "Z"),
|
|
511
|
+
remoteEtag: head.etag,
|
|
512
|
+
}));
|
|
513
|
+
if (!(await unchanged())) {
|
|
514
|
+
await fs.promises.rm(placeholder, { force: true });
|
|
515
|
+
throw changed;
|
|
516
|
+
}
|
|
517
|
+
await fsOps.removeOriginal(abs);
|
|
518
|
+
}
|
|
519
|
+
/** Whether a working-tree path is never a current-offload candidate. */
|
|
520
|
+
function skippedByName(rel) {
|
|
521
|
+
const parts = rel.split("/");
|
|
522
|
+
const name = parts[parts.length - 1];
|
|
523
|
+
return (parts[0] === OFFLOAD_TOP_LEVEL ||
|
|
524
|
+
parts.includes(".git") ||
|
|
525
|
+
(parts[0] === ".claude" && parts[1] === "hooks") ||
|
|
526
|
+
name.endsWith(PLACEHOLDER_SUFFIX) ||
|
|
527
|
+
name.endsWith(".hqcloud-download"));
|
|
528
|
+
}
|
|
529
|
+
/**
|
|
530
|
+
* Regular files at or under each path argument, HQ-root-relative and sorted.
|
|
531
|
+
* Symlinks are not followed, and a path outside HQ throws.
|
|
532
|
+
*/
|
|
533
|
+
function walkCurrentPaths(hqRoot, args) {
|
|
534
|
+
const realRoot = fs.realpathSync(path.resolve(hqRoot));
|
|
535
|
+
const out = [];
|
|
536
|
+
const seen = new Set();
|
|
537
|
+
const visit = (abs) => {
|
|
538
|
+
const rel = toPosix(path.relative(realRoot, abs));
|
|
539
|
+
if (rel === "" ? false : skippedByName(rel))
|
|
540
|
+
return;
|
|
541
|
+
const st = fs.lstatSync(abs);
|
|
542
|
+
if (st.isSymbolicLink())
|
|
543
|
+
return;
|
|
544
|
+
if (st.isDirectory()) {
|
|
545
|
+
for (const name of fs.readdirSync(abs).sort())
|
|
546
|
+
visit(path.join(abs, name));
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
if (st.isFile() && !seen.has(rel)) {
|
|
550
|
+
seen.add(rel);
|
|
551
|
+
out.push(rel);
|
|
552
|
+
}
|
|
553
|
+
};
|
|
554
|
+
for (const arg of args) {
|
|
555
|
+
const lexical = path.resolve(realRoot, arg);
|
|
556
|
+
const abs = containedPath(hqRoot, arg);
|
|
557
|
+
if (!fs.existsSync(abs))
|
|
558
|
+
throw new Error(`'${arg}' does not exist in HQ.`);
|
|
559
|
+
if (fs.lstatSync(lexical).isSymbolicLink())
|
|
560
|
+
continue;
|
|
561
|
+
visit(abs);
|
|
562
|
+
}
|
|
563
|
+
return out;
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* Decide what `--current` does with each candidate. Read-only: stats and
|
|
567
|
+
* hashes local files, reads sync records and HEADs vault objects.
|
|
568
|
+
*/
|
|
569
|
+
async function planCurrentFiles(rels, opts, deps, openVault) {
|
|
570
|
+
let active;
|
|
571
|
+
const planned = [];
|
|
572
|
+
const now = deps.now().getTime();
|
|
573
|
+
for (const rel of rels) {
|
|
574
|
+
if (skippedByName(rel))
|
|
575
|
+
continue;
|
|
576
|
+
const abs = containedPath(deps.hqRoot, rel);
|
|
577
|
+
let st;
|
|
578
|
+
try {
|
|
579
|
+
st = fs.lstatSync(abs);
|
|
580
|
+
}
|
|
581
|
+
catch (err) {
|
|
582
|
+
if (err.code === "ENOENT")
|
|
583
|
+
continue;
|
|
584
|
+
throw err;
|
|
585
|
+
}
|
|
586
|
+
if (!st.isFile() || fs.existsSync(`${abs}${PLACEHOLDER_SUFFIX}`))
|
|
587
|
+
continue;
|
|
588
|
+
const idle = Math.max(0, Math.floor((now - st.mtimeMs) / DAY_MS));
|
|
589
|
+
if (st.size < opts.minBytes || idle < opts.idleDays)
|
|
590
|
+
continue;
|
|
591
|
+
const target = resolveOffloadTarget(rel);
|
|
592
|
+
const plan = {
|
|
593
|
+
path: rel,
|
|
594
|
+
bytes: st.size,
|
|
595
|
+
idle_days: idle,
|
|
596
|
+
vault: target.vault,
|
|
597
|
+
synced: false,
|
|
598
|
+
action: target.vault === "company" ? "skip" : "upload",
|
|
599
|
+
};
|
|
600
|
+
if (target.vault === "company") {
|
|
601
|
+
plan.company = target.slug;
|
|
602
|
+
active ??= deps.activeCompanySlugs ? await deps.activeCompanySlugs() : [];
|
|
603
|
+
if (!active.includes(target.slug)) {
|
|
604
|
+
plan.reason = `You are not an active member of '${target.slug}'.`;
|
|
605
|
+
planned.push({ plan, target });
|
|
606
|
+
continue;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
const record = deps.syncRecord ? await deps.syncRecord(target) : null;
|
|
610
|
+
let digest;
|
|
611
|
+
if (record?.remoteEtag && record.hash && record.size === st.size) {
|
|
612
|
+
digest = await digestFile(abs);
|
|
613
|
+
if (digest.sha256 === record.hash.toLowerCase()) {
|
|
614
|
+
const head = await (await openVault(target)).head(target.key);
|
|
615
|
+
plan.synced = head !== null && head.etag === normalizeEtag(record.remoteEtag) && head.size === st.size;
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
if (plan.synced)
|
|
619
|
+
plan.action = "point-at-vault";
|
|
620
|
+
else if (target.vault === "company")
|
|
621
|
+
plan.reason = RUN_SYNC_FIRST;
|
|
622
|
+
planned.push({ plan, target, stat: st, digest, record: record ?? undefined });
|
|
623
|
+
}
|
|
624
|
+
return planned;
|
|
625
|
+
}
|
|
626
|
+
/** `--current`: plan, confirm, then offload each file to the vault that owns it. */
|
|
627
|
+
async function runCurrent(opts, deps, result) {
|
|
628
|
+
const current = result.current;
|
|
629
|
+
if (!currentSyncReady(deps)) {
|
|
630
|
+
current.available = false;
|
|
631
|
+
current.reason = UPDATE_SYNC;
|
|
632
|
+
return "ok";
|
|
633
|
+
}
|
|
634
|
+
let rels;
|
|
635
|
+
if (opts.paths && opts.paths.length > 0) {
|
|
636
|
+
rels = walkCurrentPaths(deps.hqRoot, opts.paths);
|
|
637
|
+
}
|
|
638
|
+
else {
|
|
639
|
+
const large = await getLarge(deps, opts.minBytes);
|
|
640
|
+
if (!large.available) {
|
|
641
|
+
current.available = false;
|
|
642
|
+
current.reason = large.reason;
|
|
643
|
+
return "ok";
|
|
644
|
+
}
|
|
645
|
+
rels = large.current.map((f) => toPosix(f.path));
|
|
646
|
+
}
|
|
647
|
+
current.available = true;
|
|
648
|
+
const vaults = new Map();
|
|
649
|
+
const openVault = (t) => {
|
|
650
|
+
if (!deps.openVaultObjects)
|
|
651
|
+
throw new Error("This HQ build cannot read files stored by HQ sync.");
|
|
652
|
+
const id = t.vault === "company" ? `company:${t.slug}` : "personal";
|
|
653
|
+
let v = vaults.get(id);
|
|
654
|
+
if (!v) {
|
|
655
|
+
v = deps.openVaultObjects(t.vault === "company" ? { vault: "company", slug: t.slug } : { vault: "personal" });
|
|
656
|
+
vaults.set(id, v);
|
|
657
|
+
}
|
|
658
|
+
return v;
|
|
659
|
+
};
|
|
660
|
+
const planned = await planCurrentFiles(rels, opts, deps, openVault);
|
|
661
|
+
current.files = planned.map((p) => p.plan);
|
|
662
|
+
const unsynced = planned.filter((p) => p.plan.reason === RUN_SYNC_FIRST).map((p) => p.plan.path);
|
|
663
|
+
if (unsynced.length > 0) {
|
|
664
|
+
current.unsynced = unsynced;
|
|
665
|
+
current.message =
|
|
666
|
+
`${unsynced.length} company file(s) are not in the company vault yet, so they were left as they are. ` +
|
|
667
|
+
`To offload them, ${RUN_SYNC_FIRST}, then run offload again: ${unsynced.join(", ")}`;
|
|
668
|
+
}
|
|
669
|
+
const todo = planned.filter((p) => p.plan.action !== "skip");
|
|
670
|
+
if (opts.dryRun || todo.length === 0)
|
|
671
|
+
return "ok";
|
|
672
|
+
if (!opts.yes) {
|
|
673
|
+
const ok = await deps.confirm(`Replace ${todo.length} current file(s) with links to their HQ cloud copies and remove the local copies?`);
|
|
674
|
+
if (!ok) {
|
|
675
|
+
deps.log("Aborted - nothing was uploaded or changed.");
|
|
676
|
+
return "aborted";
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
const fsOps = deps.placeholderFs ?? durablePlaceholderFs;
|
|
680
|
+
let personal;
|
|
681
|
+
for (const file of todo) {
|
|
682
|
+
const { plan } = file;
|
|
683
|
+
try {
|
|
684
|
+
if (plan.action === "point-at-vault") {
|
|
685
|
+
await pointCurrentFileAtVault(file, await openVault(file.target), deps, fsOps);
|
|
686
|
+
}
|
|
687
|
+
else {
|
|
688
|
+
personal ??= await deps.openPersonal();
|
|
689
|
+
await offloadCurrentFile(plan.path, personal, deps, fsOps);
|
|
690
|
+
}
|
|
691
|
+
const done = {
|
|
692
|
+
path: plan.path,
|
|
693
|
+
bytes: plan.bytes,
|
|
694
|
+
vault: plan.vault,
|
|
695
|
+
action: plan.action === "point-at-vault" ? "point-at-vault" : "upload",
|
|
696
|
+
};
|
|
697
|
+
if (plan.company)
|
|
698
|
+
done.company = plan.company;
|
|
699
|
+
current.offloaded.push(done);
|
|
700
|
+
current.freed_bytes += plan.bytes;
|
|
701
|
+
}
|
|
702
|
+
catch (err) {
|
|
703
|
+
result.errors.push(`${plan.path}: ${err.message}`);
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
return "ok";
|
|
707
|
+
}
|
|
452
708
|
export async function runOffload(opts, deps) {
|
|
453
|
-
|
|
454
|
-
const doHistory =
|
|
455
|
-
const doCurrent =
|
|
709
|
+
// Current files are offloaded only on request; a bare run touches history only.
|
|
710
|
+
const doHistory = opts.history || !opts.current;
|
|
711
|
+
const doCurrent = opts.current;
|
|
456
712
|
const result = {
|
|
457
713
|
history: { uploaded: 0, bytes: 0, freed_bytes: 0 },
|
|
458
714
|
current: { offloaded: [], freed_bytes: 0 },
|
|
459
715
|
errors: [],
|
|
460
716
|
dry_run: opts.dryRun,
|
|
461
717
|
};
|
|
462
|
-
if (doCurrent)
|
|
463
|
-
result.current.
|
|
464
|
-
|
|
465
|
-
|
|
718
|
+
if (!doCurrent)
|
|
719
|
+
result.current.skipped = true;
|
|
720
|
+
if (doCurrent && (await runCurrent(opts, deps, result)) === "aborted")
|
|
721
|
+
return undefined;
|
|
466
722
|
const script = storageScriptPath(deps.hqRoot);
|
|
467
|
-
if (!doHistory)
|
|
723
|
+
if (!doHistory) {
|
|
724
|
+
result.history.skipped = true;
|
|
468
725
|
return result;
|
|
726
|
+
}
|
|
469
727
|
if (!deps.syncExcludesOffload()) {
|
|
470
728
|
result.history.available = false;
|
|
471
729
|
result.history.reason = UPDATE_SYNC;
|
|
@@ -843,8 +1101,6 @@ async function fetchGitBlob(sha, out, deps) {
|
|
|
843
1101
|
}
|
|
844
1102
|
/** Current-file offload availability, shared by status and its error fallback. */
|
|
845
1103
|
export function currentAvailability(syncReady) {
|
|
846
|
-
if (!CURRENT_OFFLOAD_ENABLED)
|
|
847
|
-
return { current_available: false, current_reason: CURRENT_BLOCKED_REASON };
|
|
848
1104
|
if (!syncReady)
|
|
849
1105
|
return { current_available: false, current_reason: UPDATE_SYNC };
|
|
850
1106
|
return { current_available: true };
|
|
@@ -854,7 +1110,7 @@ export async function getOffloadStatus(deps, opts) {
|
|
|
854
1110
|
history_candidates: { count: 0, bytes: 0 },
|
|
855
1111
|
current_candidates: { count: 0, bytes: 0 },
|
|
856
1112
|
placeholders: { count: 0, bytes: 0 },
|
|
857
|
-
...currentAvailability(deps
|
|
1113
|
+
...currentAvailability(currentSyncReady(deps)),
|
|
858
1114
|
};
|
|
859
1115
|
for (const rel of await deps.listPlaceholders()) {
|
|
860
1116
|
const pointer = readPointerFile(containedPath(deps.hqRoot, rel));
|
package/dist/commands/storage.js
CHANGED
|
@@ -41,6 +41,7 @@ import { execFile } from "node:child_process";
|
|
|
41
41
|
import { dropExpiredBackups, reclaimFields, reclaimWindowHours, runCoreScript, } from "./storage-core.js";
|
|
42
42
|
import { S3Client, ListObjectVersionsCommand, DeleteObjectsCommand, } from "@aws-sdk/client-s3";
|
|
43
43
|
import { PERSONAL_VAULT_EXCLUDED_TOP_LEVEL, VaultClient } from "@indigoai-us/hq-cloud";
|
|
44
|
+
import * as hqCloud from "@indigoai-us/hq-cloud";
|
|
44
45
|
import { buildVaultConfig, DEFAULT_COGNITO } from "../utils/cognito-session.js";
|
|
45
46
|
import { resolveVaultCredential } from "../utils/resolve-vault-credential.js";
|
|
46
47
|
import { vaultApiFetch } from "../utils/vault-api.js";
|
|
@@ -775,7 +776,10 @@ export function formatOffloadStatus(o) {
|
|
|
775
776
|
export function formatOffload(r) {
|
|
776
777
|
const lines = [];
|
|
777
778
|
const h = r.history;
|
|
778
|
-
if (h.
|
|
779
|
+
if (h.skipped) {
|
|
780
|
+
// History was not part of this run.
|
|
781
|
+
}
|
|
782
|
+
else if (h.refused) {
|
|
779
783
|
// Core's and the CLI's refusal messages may already say nothing changed;
|
|
780
784
|
// add that sentence only when the message lacks it.
|
|
781
785
|
const msg = (h.message ?? "").trim();
|
|
@@ -797,7 +801,7 @@ export function formatOffload(r) {
|
|
|
797
801
|
else {
|
|
798
802
|
lines.push(`History: uploaded ${h.uploaded} old version(s), ${formatBytes(h.bytes)}; freed ${formatBytes(h.freed_bytes)}`);
|
|
799
803
|
}
|
|
800
|
-
if (h.available !== false) {
|
|
804
|
+
if (!h.skipped && h.available !== false) {
|
|
801
805
|
const pending = reclaimLine(h);
|
|
802
806
|
if (pending)
|
|
803
807
|
lines.push(` ${pending}`);
|
|
@@ -806,16 +810,44 @@ export function formatOffload(r) {
|
|
|
806
810
|
lines.push(` ${h.held_by_worktrees.count} more old version(s) (${formatBytes(h.held_by_worktrees.bytes)}) stay: ${names} still uses this history.`);
|
|
807
811
|
}
|
|
808
812
|
}
|
|
809
|
-
|
|
810
|
-
|
|
813
|
+
const c = r.current;
|
|
814
|
+
if (c.skipped) {
|
|
815
|
+
// Current files were not part of this run.
|
|
816
|
+
}
|
|
817
|
+
else if (c.available === false) {
|
|
818
|
+
lines.push(`Current files: not available - ${c.reason ?? "unknown"}`);
|
|
811
819
|
}
|
|
812
820
|
else {
|
|
813
|
-
|
|
821
|
+
const files = c.files ?? [];
|
|
822
|
+
if (r.dry_run) {
|
|
823
|
+
const todo = files.filter((f) => f.action !== "skip");
|
|
824
|
+
lines.push(`Current files: would offload ${todo.length} of ${files.length}, ${formatBytes(todo.reduce((n, f) => n + f.bytes, 0))}`);
|
|
825
|
+
}
|
|
826
|
+
else {
|
|
827
|
+
lines.push(`Current files: moved ${c.offloaded.length}, freed ${formatBytes(c.freed_bytes)}`);
|
|
828
|
+
}
|
|
829
|
+
for (const f of files)
|
|
830
|
+
lines.push(` ${formatCurrentFile(f)}`);
|
|
831
|
+
if (c.message)
|
|
832
|
+
lines.push(` ${c.message}`);
|
|
814
833
|
}
|
|
815
834
|
for (const e of r.errors)
|
|
816
835
|
lines.push(chalk.red(` ${e}`));
|
|
817
836
|
return lines.join("\n");
|
|
818
837
|
}
|
|
838
|
+
const CURRENT_ACTION_LABEL = {
|
|
839
|
+
"point-at-vault": "point at vault",
|
|
840
|
+
upload: "upload",
|
|
841
|
+
skip: "skip",
|
|
842
|
+
};
|
|
843
|
+
/** One `--current` plan line: path, size, owning vault, sync state, action. */
|
|
844
|
+
function formatCurrentFile(f) {
|
|
845
|
+
const vault = f.vault === "company" ? `company ${f.company ?? "?"}` : "personal vault";
|
|
846
|
+
const parts = [f.path, formatBytes(f.bytes), vault, f.synced ? "synced" : "not synced", CURRENT_ACTION_LABEL[f.action]];
|
|
847
|
+
if (f.action === "skip" && f.reason)
|
|
848
|
+
parts.push(`(${f.reason})`);
|
|
849
|
+
return parts.join(" ");
|
|
850
|
+
}
|
|
819
851
|
/** One line on when backed-up space comes back, or nothing. */
|
|
820
852
|
export function reclaimLine(r) {
|
|
821
853
|
const pending = num(r.pending_reclaim_bytes);
|
|
@@ -993,6 +1025,7 @@ async function buildOffloadDeps() {
|
|
|
993
1025
|
}
|
|
994
1026
|
return companiesFromMemberships(await res.json());
|
|
995
1027
|
};
|
|
1028
|
+
const journals = new Map();
|
|
996
1029
|
return {
|
|
997
1030
|
hqRoot,
|
|
998
1031
|
now: () => new Date(),
|
|
@@ -1008,6 +1041,25 @@ async function buildOffloadDeps() {
|
|
|
1008
1041
|
return personalS3Store({ bucket, region: DEFAULT_COGNITO.region, credentials });
|
|
1009
1042
|
},
|
|
1010
1043
|
activeCompanySlugs: async () => (await activeCompanies()).map((c) => c.slug),
|
|
1044
|
+
// Current-file offload replaces a file with a placeholder only when the
|
|
1045
|
+
// installed sync engine holds every `.hqcloud` sibling (no delete, no
|
|
1046
|
+
// download over it). Older engines lack the flag, so this stays false.
|
|
1047
|
+
syncHoldsPlaceholders: () => hqCloud.CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING === true,
|
|
1048
|
+
// What HQ sync last stored for a file, read from its local journal.
|
|
1049
|
+
// Company journal keys are relative to the company folder; personal keys
|
|
1050
|
+
// are relative to the HQ root. Read-only: offload never writes a journal.
|
|
1051
|
+
syncRecord: async (target) => {
|
|
1052
|
+
const slug = target.vault === "company" ? target.slug : hqCloud.PERSONAL_VAULT_JOURNAL_SLUG;
|
|
1053
|
+
let journal = journals.get(slug);
|
|
1054
|
+
if (!journal) {
|
|
1055
|
+
journal = hqCloud.readJournal(slug);
|
|
1056
|
+
journals.set(slug, journal);
|
|
1057
|
+
}
|
|
1058
|
+
const entry = hqCloud.getEntry(journal, target.key);
|
|
1059
|
+
if (!entry || hqCloud.isTombstone(entry))
|
|
1060
|
+
return null;
|
|
1061
|
+
return { hash: entry.hash, size: entry.size, remoteEtag: entry.remoteEtag };
|
|
1062
|
+
},
|
|
1011
1063
|
// Read-only, for placeholders that HQ sync rewrote. Company credentials
|
|
1012
1064
|
// come from the existing membership-scoped /sts/vend path.
|
|
1013
1065
|
openVaultObjects: async (target) => {
|
|
@@ -1169,11 +1221,11 @@ export function registerStorageCommand(program) {
|
|
|
1169
1221
|
});
|
|
1170
1222
|
storage
|
|
1171
1223
|
.command("offload")
|
|
1172
|
-
.description("Move big files to
|
|
1224
|
+
.description("Move big files to HQ cloud storage and leave links behind (old versions in history; with --current, idle working files to the vault that owns them)")
|
|
1173
1225
|
.option("--min-size <size>", "Only files at least this big, e.g. 50MB (env HQ_OFFLOAD_MIN_BYTES)")
|
|
1174
1226
|
.option("--idle-days <days>", "Current files must be untouched this many days", String(DEFAULT_IDLE_DAYS))
|
|
1175
1227
|
.option("--history", "Only old versions in local autosave history")
|
|
1176
|
-
.option("--current", "
|
|
1228
|
+
.option("--current [paths...]", "Idle working files instead of history; optional HQ paths limit it to those files or folders")
|
|
1177
1229
|
.option("--dry-run", "Show what would move without uploading or changing anything")
|
|
1178
1230
|
.option("--yes", "Skip the confirmation prompt")
|
|
1179
1231
|
.option("--json", "Print JSON")
|
|
@@ -1188,6 +1240,7 @@ export function registerStorageCommand(program) {
|
|
|
1188
1240
|
idleDays,
|
|
1189
1241
|
history: Boolean(opts.history),
|
|
1190
1242
|
current: Boolean(opts.current),
|
|
1243
|
+
paths: Array.isArray(opts.current) ? opts.current : undefined,
|
|
1191
1244
|
dryRun: Boolean(opts.dryRun),
|
|
1192
1245
|
yes: Boolean(opts.yes),
|
|
1193
1246
|
}, await buildOffloadDeps());
|