@indigoai-us/hq-cli 5.345.80 → 5.345.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/command-catalog.generated.d.ts +142 -0
- package/dist/command-catalog.generated.js +173 -0
- package/dist/command-help.js +2 -0
- package/dist/command-registration-plan.d.ts +6 -0
- package/dist/command-registration-plan.js +1 -0
- package/dist/commands/storage-core.d.ts +98 -0
- package/dist/commands/storage-core.js +223 -0
- package/dist/commands/storage-gate.d.ts +35 -0
- package/dist/commands/storage-gate.js +85 -0
- package/dist/commands/storage-offload.d.ts +342 -0
- package/dist/commands/storage-offload.js +943 -0
- package/dist/commands/storage.d.ts +199 -0
- package/dist/commands/storage.js +931 -0
- package/dist/fast-scaffold.js +5 -0
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +5140 -3412
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/cli-flag-keys.d.ts +6 -0
- package/dist/lib/cli-flag-keys.js +6 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/storage-feature-flag.d.ts +56 -0
- package/dist/lib/storage-feature-flag.js +104 -0
- package/dist/main.js +12 -1
- package/dist/utils/resident-package-preload.js +3 -1
- package/package.json +2 -2
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared calls into core `hq-storage.sh` for `hq storage` status, prune and
|
|
3
|
+
* offload.
|
|
4
|
+
*
|
|
5
|
+
* - `runCoreScript` turns one script run into a JSON report. Exit 4 from
|
|
6
|
+
* `prune` ("would free nothing") becomes a structured refusal with the
|
|
7
|
+
* branches, tags or stash that hold the old history, never raw stderr.
|
|
8
|
+
* - `dropExpiredBackups` runs core `drop-backups --expired` right before a
|
|
9
|
+
* real prune or offload (never for status, dry runs or fetch). With no
|
|
10
|
+
* expired backup it takes no lock and runs no gc; it never expires reflogs.
|
|
11
|
+
* A failure is reported but never blocks the command.
|
|
12
|
+
* - `checkPreconditions` runs core `check` so offload can refuse before it
|
|
13
|
+
* uploads anything. Exit 5 from a core rewrite (a failed precondition)
|
|
14
|
+
* becomes a structured refusal with plain text, never raw stderr.
|
|
15
|
+
* - Backups: by default core keeps a backup ref for 24 h after prune or
|
|
16
|
+
* strip-large, so space comes back after `reclaim_after`, not at once.
|
|
17
|
+
* `withReclaim` adds `backup_ref`, `reclaim_after` and
|
|
18
|
+
* `pending_reclaim_bytes` to a result so a UI can say when space is freed.
|
|
19
|
+
*/
|
|
20
|
+
import * as fs from "node:fs";
|
|
21
|
+
/** Core's default backup grace period (`HQ_STORAGE_BACKUP_GRACE_HOURS`). */
|
|
22
|
+
export const BACKUP_GRACE_HOURS = 24;
|
|
23
|
+
/**
|
|
24
|
+
* Git's default expiry for reflog entries that point at rewritten history
|
|
25
|
+
* (gc.reflogExpireUnreachable). Core that keeps undo history frees the space
|
|
26
|
+
* a prune or offload rewrote only after these entries age out.
|
|
27
|
+
*/
|
|
28
|
+
export const UNDO_WINDOW_DAYS = 30;
|
|
29
|
+
/** Whether the installed core keeps git's undo history (never expires reflogs early). */
|
|
30
|
+
export function coreKeepsUndo(script) {
|
|
31
|
+
return coreHasFlag(script, "hq-storage-undo-pending");
|
|
32
|
+
}
|
|
33
|
+
/** Hours after a rewrite until its space is freed, for this core. */
|
|
34
|
+
export function reclaimWindowHours(script) {
|
|
35
|
+
return coreKeepsUndo(script) ? UNDO_WINDOW_DAYS * 24 : BACKUP_GRACE_HOURS;
|
|
36
|
+
}
|
|
37
|
+
/** Whether the installed core script has a `case` arm for `sub`. */
|
|
38
|
+
export function coreSupports(script, sub) {
|
|
39
|
+
if (!fs.existsSync(script))
|
|
40
|
+
return false;
|
|
41
|
+
const text = fs.readFileSync(script, "utf8");
|
|
42
|
+
return new RegExp(`(^|[\\s|(])${sub}\\)`, "m").test(text);
|
|
43
|
+
}
|
|
44
|
+
/** Whether the installed core script mentions a flag, e.g. `--no-backup`. */
|
|
45
|
+
export function coreHasFlag(script, flag) {
|
|
46
|
+
return fs.existsSync(script) && fs.readFileSync(script, "utf8").includes(flag);
|
|
47
|
+
}
|
|
48
|
+
function normalizeHoldingRefs(raw) {
|
|
49
|
+
if (!Array.isArray(raw))
|
|
50
|
+
return [];
|
|
51
|
+
const refs = [];
|
|
52
|
+
for (const r of raw) {
|
|
53
|
+
if (typeof r === "string")
|
|
54
|
+
refs.push({ name: r, commit_date: null });
|
|
55
|
+
else if (r && typeof r === "object") {
|
|
56
|
+
const o = r;
|
|
57
|
+
const name = typeof o.name === "string" ? o.name : typeof o.ref === "string" ? o.ref : undefined;
|
|
58
|
+
if (!name)
|
|
59
|
+
continue;
|
|
60
|
+
const date = typeof o.commit_date === "string" ? o.commit_date : null;
|
|
61
|
+
refs.push({ name, commit_date: date });
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return refs;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Structured exit-4 refusal. Newer core prints
|
|
68
|
+
* `{"refused":"nothing_to_free","holding_refs":[...],"holding_refs_count":N}`
|
|
69
|
+
* on stdout. Older core printed only a stderr line naming `(ref, ref)`; the
|
|
70
|
+
* ref names are taken from it and the line itself (which tells the user to
|
|
71
|
+
* delete them) is dropped.
|
|
72
|
+
*/
|
|
73
|
+
export function parseRefusal(stdout, stderr) {
|
|
74
|
+
let refs = [];
|
|
75
|
+
let count;
|
|
76
|
+
try {
|
|
77
|
+
const parsed = JSON.parse(stdout);
|
|
78
|
+
refs = normalizeHoldingRefs(parsed.holding_refs);
|
|
79
|
+
if (typeof parsed.holding_refs_count === "number")
|
|
80
|
+
count = parsed.holding_refs_count;
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
const m = /\(([^()]*refs\/[^()]*)\)/.exec(stderr);
|
|
84
|
+
if (m) {
|
|
85
|
+
refs = m[1]
|
|
86
|
+
.split(/[,\s]+/)
|
|
87
|
+
.filter((s) => s.startsWith("refs/"))
|
|
88
|
+
.map((name) => ({ name, commit_date: null }));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return {
|
|
92
|
+
available: true,
|
|
93
|
+
refused: "nothing_to_free",
|
|
94
|
+
holding_refs: refs,
|
|
95
|
+
holding_refs_count: count ?? refs.length,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
/** Core exit code for a failed precondition (remote-tracking branches, ...). */
|
|
99
|
+
export const CORE_PRECONDITION_EXIT = 5;
|
|
100
|
+
/** Plain text for a precondition refusal from core that printed no message. */
|
|
101
|
+
export const PRECONDITION_FALLBACK = "HQ is not in a state where its history can be rewritten. Nothing was changed.";
|
|
102
|
+
/** `{"refused":"<code>","message":".."}` on stdout, or a plain fallback. */
|
|
103
|
+
export function parsePreconditionRefusal(stdout) {
|
|
104
|
+
try {
|
|
105
|
+
const o = JSON.parse(stdout);
|
|
106
|
+
if (typeof o.refused === "string") {
|
|
107
|
+
return { refused: o.refused, message: typeof o.message === "string" ? o.message : PRECONDITION_FALLBACK };
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
// fall through
|
|
112
|
+
}
|
|
113
|
+
return { refused: "precondition", message: PRECONDITION_FALLBACK };
|
|
114
|
+
}
|
|
115
|
+
/** Run one core subcommand and return its JSON as a report. */
|
|
116
|
+
export async function runCoreScript(runScript, script, args) {
|
|
117
|
+
const res = await runScript(script, args);
|
|
118
|
+
if (res.code === 4 && args[0] === "prune")
|
|
119
|
+
return parseRefusal(res.stdout, res.stderr);
|
|
120
|
+
if (res.code === CORE_PRECONDITION_EXIT) {
|
|
121
|
+
return { available: true, ...parsePreconditionRefusal(res.stdout) };
|
|
122
|
+
}
|
|
123
|
+
// strip-large exit 4: nothing to free (linked worktrees still use every
|
|
124
|
+
// listed file). Structured, never raw stderr.
|
|
125
|
+
if (res.code === 4 && args[0] === "strip-large") {
|
|
126
|
+
let extra = {};
|
|
127
|
+
try {
|
|
128
|
+
extra = JSON.parse(res.stdout);
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
// fall through
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
...extra,
|
|
135
|
+
available: true,
|
|
136
|
+
refused: "nothing_to_free",
|
|
137
|
+
message: typeof extra.message === "string" ? extra.message : "Offloading would free nothing. Nothing was changed.",
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
if (res.code !== 0) {
|
|
141
|
+
return {
|
|
142
|
+
available: false,
|
|
143
|
+
reason: (res.stderr.trim() || `hq-storage.sh exited ${res.code}`).slice(0, 500),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
return { available: true, ...JSON.parse(res.stdout) };
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
return { available: false, reason: `hq-storage.sh returned invalid JSON: ${err.message}` };
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Run core `drop-backups --expired` when the script has it; undefined when it
|
|
155
|
+
* does not. Call only right before a real prune or offload.
|
|
156
|
+
*/
|
|
157
|
+
export async function dropExpiredBackups(runScript, script) {
|
|
158
|
+
if (!coreSupports(script, "drop-backups"))
|
|
159
|
+
return undefined;
|
|
160
|
+
const r = await runCoreScript(runScript, script, ["drop-backups", "--expired"]);
|
|
161
|
+
if (r.available === false)
|
|
162
|
+
return { available: false, reason: r.reason };
|
|
163
|
+
return {
|
|
164
|
+
available: true,
|
|
165
|
+
dropped: typeof r.dropped === "number" ? r.dropped : 0,
|
|
166
|
+
freed_bytes: typeof r.freed_bytes === "number" ? r.freed_bytes : 0,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Core `check`: undefined when core lacks it (or it fails to run), else null
|
|
171
|
+
* when every precondition holds, or the refusal.
|
|
172
|
+
*/
|
|
173
|
+
export async function checkPreconditions(runScript, script, forCmd) {
|
|
174
|
+
if (!coreSupports(script, "check"))
|
|
175
|
+
return undefined;
|
|
176
|
+
const res = await runScript(script, ["check", "--for", forCmd]);
|
|
177
|
+
if (res.code !== 0)
|
|
178
|
+
return undefined;
|
|
179
|
+
try {
|
|
180
|
+
const o = JSON.parse(res.stdout);
|
|
181
|
+
if (o.ok === true)
|
|
182
|
+
return null;
|
|
183
|
+
if (typeof o.refused === "string") {
|
|
184
|
+
return { refused: o.refused, message: typeof o.message === "string" ? o.message : PRECONDITION_FALLBACK };
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
// fall through
|
|
189
|
+
}
|
|
190
|
+
return undefined;
|
|
191
|
+
}
|
|
192
|
+
export function reclaimEstimate(now, hours = BACKUP_GRACE_HOURS) {
|
|
193
|
+
return new Date(now.getTime() + hours * 3600_000).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Backup fields for a prune or strip-large result.
|
|
197
|
+
* - dry run: `reclaim_after` is an estimate (now + `windowHours`: 24 h for
|
|
198
|
+
* the backup, or 30 days when core keeps undo history) and all of
|
|
199
|
+
* `expectedBytes` is pending.
|
|
200
|
+
* - real run: core's `backup_ref` / `reclaim_after`; `pending_reclaim_bytes`
|
|
201
|
+
* is core's value when it reports one, else `expectedBytes - freed` while a
|
|
202
|
+
* backup is held, else 0.
|
|
203
|
+
*/
|
|
204
|
+
export function reclaimFields(core, opts) {
|
|
205
|
+
if (opts.dryRun) {
|
|
206
|
+
// Nothing to free means no date to promise.
|
|
207
|
+
const pending = Math.max(0, opts.expectedBytes);
|
|
208
|
+
return {
|
|
209
|
+
backup_ref: null,
|
|
210
|
+
reclaim_after: pending > 0 ? reclaimEstimate(opts.now, opts.windowHours) : null,
|
|
211
|
+
pending_reclaim_bytes: pending,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
const backupRef = typeof core.backup_ref === "string" ? core.backup_ref : null;
|
|
215
|
+
const reclaimAfter = typeof core.reclaim_after === "string" ? core.reclaim_after : null;
|
|
216
|
+
const pending = typeof core.pending_reclaim_bytes === "number"
|
|
217
|
+
? core.pending_reclaim_bytes
|
|
218
|
+
: backupRef
|
|
219
|
+
? Math.max(0, opts.expectedBytes - opts.freedBytes)
|
|
220
|
+
: 0;
|
|
221
|
+
return { backup_ref: backupRef, reclaim_after: reclaimAfter, pending_reclaim_bytes: pending };
|
|
222
|
+
}
|
|
223
|
+
//# sourceMappingURL=storage-core.js.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live HQ Storage gate and refusal surface for `hq storage`.
|
|
3
|
+
*
|
|
4
|
+
* Allowlist gate, fails closed (owner decision 2026-10-11, Indigo-only until
|
|
5
|
+
* the owner widens it; see the storage-feature-flag module in src/lib). This
|
|
6
|
+
* is the deliberate exception to the default-on kill-switch rule: any error,
|
|
7
|
+
* a signed-out caller, an API-key caller, a membership read failure, or an
|
|
8
|
+
* unreadable registry all mean Storage is OFF.
|
|
9
|
+
*
|
|
10
|
+
* Resolved before the command tree is registered, because that path is
|
|
11
|
+
* already async; the registrar and every subcommand stay synchronous.
|
|
12
|
+
*/
|
|
13
|
+
import { Command } from "commander";
|
|
14
|
+
import { hideStorageInCatalog } from "../lib/storage-feature-flag.js";
|
|
15
|
+
export interface StorageGateDeps {
|
|
16
|
+
/** Synchronous cached answer; true short-circuits the network. */
|
|
17
|
+
cached?: () => boolean;
|
|
18
|
+
/** Signed-in Cognito token, or null when signed out / API-key auth. */
|
|
19
|
+
token?: () => Promise<string | null>;
|
|
20
|
+
/** Active company uids for the caller (from the membership API). */
|
|
21
|
+
companyUids?: (token: string) => Promise<string[]>;
|
|
22
|
+
/** Resolve the flag for one company the caller belongs to. */
|
|
23
|
+
resolveForCompany?: (companyUid: string, token: string) => Promise<boolean>;
|
|
24
|
+
}
|
|
25
|
+
/** True when `hq-storage` resolves on for the caller. Never throws. */
|
|
26
|
+
export declare function storageFeatureEnabled(deps?: StorageGateDeps): Promise<boolean>;
|
|
27
|
+
/** Print the refusal: JSON on stdout for `--json`, plain text on stderr otherwise. */
|
|
28
|
+
export declare function printStorageRefusal(argv: readonly string[]): void;
|
|
29
|
+
/**
|
|
30
|
+
* Register a hidden `storage` root that refuses every invocation, including
|
|
31
|
+
* `--help` and every subcommand, without describing what Storage does.
|
|
32
|
+
*/
|
|
33
|
+
export declare function registerStorageRefusal(program: Command, argv?: readonly string[]): Command;
|
|
34
|
+
export { hideStorageInCatalog };
|
|
35
|
+
//# sourceMappingURL=storage-gate.d.ts.map
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live HQ Storage gate and refusal surface for `hq storage`.
|
|
3
|
+
*
|
|
4
|
+
* Allowlist gate, fails closed (owner decision 2026-10-11, Indigo-only until
|
|
5
|
+
* the owner widens it; see the storage-feature-flag module in src/lib). This
|
|
6
|
+
* is the deliberate exception to the default-on kill-switch rule: any error,
|
|
7
|
+
* a signed-out caller, an API-key caller, a membership read failure, or an
|
|
8
|
+
* unreadable registry all mean Storage is OFF.
|
|
9
|
+
*
|
|
10
|
+
* Resolved before the command tree is registered, because that path is
|
|
11
|
+
* already async; the registrar and every subcommand stay synchronous.
|
|
12
|
+
*/
|
|
13
|
+
import { Command } from "commander";
|
|
14
|
+
import { resolveCompanyFlagGate } from "../lib/flag-registry.js";
|
|
15
|
+
import { HQ_STORAGE_FLAG, STORAGE_FEATURE_DISABLED_MESSAGE, storageFeatureDisabledJson, storageFeatureEnabledFromCache, hideStorageInCatalog, } from "../lib/storage-feature-flag.js";
|
|
16
|
+
const MEMBERSHIP_TIMEOUT_MS = 3_000;
|
|
17
|
+
async function defaultToken() {
|
|
18
|
+
const { resolveVaultCredential } = await import("../utils/resolve-vault-credential.js");
|
|
19
|
+
const cred = await resolveVaultCredential({ interactive: false });
|
|
20
|
+
return cred.kind === "api-key" ? null : cred.token;
|
|
21
|
+
}
|
|
22
|
+
async function defaultCompanyUids(token) {
|
|
23
|
+
const [{ vaultApiFetch }, { companiesFromMemberships }] = await Promise.all([
|
|
24
|
+
import("../utils/vault-api.js"),
|
|
25
|
+
import("./storage.js"),
|
|
26
|
+
]);
|
|
27
|
+
const res = await vaultApiFetch({
|
|
28
|
+
token,
|
|
29
|
+
path: "/membership/me",
|
|
30
|
+
signal: AbortSignal.timeout(MEMBERSHIP_TIMEOUT_MS),
|
|
31
|
+
});
|
|
32
|
+
if (!res.ok)
|
|
33
|
+
return [];
|
|
34
|
+
return companiesFromMemberships(await res.json()).map((company) => company.uid);
|
|
35
|
+
}
|
|
36
|
+
function defaultResolveForCompany(companyUid, token) {
|
|
37
|
+
// Shared evaluator, bounded client (1.2 s transport timeout). Fallback is
|
|
38
|
+
// false: an unreadable or absent row keeps Storage off.
|
|
39
|
+
return resolveCompanyFlagGate(companyUid, token, HQ_STORAGE_FLAG, { fallback: false }, () => false);
|
|
40
|
+
}
|
|
41
|
+
/** True when `hq-storage` resolves on for the caller. Never throws. */
|
|
42
|
+
export async function storageFeatureEnabled(deps = {}) {
|
|
43
|
+
try {
|
|
44
|
+
if ((deps.cached ?? storageFeatureEnabledFromCache)())
|
|
45
|
+
return true;
|
|
46
|
+
const token = await (deps.token ?? defaultToken)();
|
|
47
|
+
if (!token)
|
|
48
|
+
return false;
|
|
49
|
+
const uids = await (deps.companyUids ?? defaultCompanyUids)(token);
|
|
50
|
+
if (uids.length === 0)
|
|
51
|
+
return false;
|
|
52
|
+
const resolve = deps.resolveForCompany ?? defaultResolveForCompany;
|
|
53
|
+
const answers = await Promise.all(uids.map((uid) => resolve(uid, token).then((value) => value === true, () => false)));
|
|
54
|
+
return answers.some(Boolean);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** Print the refusal: JSON on stdout for `--json`, plain text on stderr otherwise. */
|
|
61
|
+
export function printStorageRefusal(argv) {
|
|
62
|
+
if (argv.includes("--json"))
|
|
63
|
+
console.log(storageFeatureDisabledJson());
|
|
64
|
+
else
|
|
65
|
+
console.error(STORAGE_FEATURE_DISABLED_MESSAGE);
|
|
66
|
+
process.exitCode = 1;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Register a hidden `storage` root that refuses every invocation, including
|
|
70
|
+
* `--help` and every subcommand, without describing what Storage does.
|
|
71
|
+
*/
|
|
72
|
+
export function registerStorageRefusal(program, argv = process.argv) {
|
|
73
|
+
const storage = new Command("storage")
|
|
74
|
+
.helpOption(false)
|
|
75
|
+
.allowUnknownOption()
|
|
76
|
+
.allowExcessArguments()
|
|
77
|
+
.argument("[args...]")
|
|
78
|
+
.action(() => printStorageRefusal(argv));
|
|
79
|
+
// `hq help storage` prints help instead of running the action.
|
|
80
|
+
storage.helpInformation = () => `${STORAGE_FEATURE_DISABLED_MESSAGE}\n`;
|
|
81
|
+
program.addCommand(storage, { hidden: true });
|
|
82
|
+
return storage;
|
|
83
|
+
}
|
|
84
|
+
export { hideStorageInCatalog };
|
|
85
|
+
//# sourceMappingURL=storage-gate.js.map
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hq storage offload` / `hq storage fetch` — move big files to the caller's
|
|
3
|
+
* PERSONAL vault (S3 Standard, content-addressed) and leave small linked
|
|
4
|
+
* pointers behind. Shared contract: owner-approved offload design, 2026-10-09.
|
|
5
|
+
*
|
|
6
|
+
* Two kinds of big file:
|
|
7
|
+
* - History copies (old versions inside the HQ root git history). The CLI
|
|
8
|
+
* uploads each blob, verifies size + sha256 + git blob id, then hands only
|
|
9
|
+
* verified blobs to core `hq-storage.sh strip-large`, which rewrites history
|
|
10
|
+
* to the pointer text. Working-tree files are never touched.
|
|
11
|
+
* - Current files (idle big files in the working tree). NOT offloaded yet:
|
|
12
|
+
* the sync delete planner lives in @indigoai-us/hq-cloud (`computePullPlan`
|
|
13
|
+
* and the push delete planner in dist/cli/sync.js), so replacing `x.mp4`
|
|
14
|
+
* with `x.mp4.hqcloud` would still propagate as a remote delete of `x.mp4`.
|
|
15
|
+
* `--current` reports `{ available: false }` until hq-cloud learns about
|
|
16
|
+
* placeholders (CURRENT_OFFLOAD_ENABLED). `offloadCurrentFile` is built
|
|
17
|
+
* and tested but not called.
|
|
18
|
+
*
|
|
19
|
+
* Fetch also reads placeholders that HQ sync rewrote after a teammate changed
|
|
20
|
+
* the file (`vault: company`, the company vault key, `sha256: unknown`,
|
|
21
|
+
* `remote-etag`). Those download from the company vault, only for an active
|
|
22
|
+
* member, and verify by size and ETag. A placeholder marked `orphaned: true`
|
|
23
|
+
* (a teammate deleted the file) is refused unless `--force-orphan` is given,
|
|
24
|
+
* which writes the caller's own personal-vault copy to a file outside HQ so
|
|
25
|
+
* it can never sync back.
|
|
26
|
+
*
|
|
27
|
+
* Tenant isolation: bytes only ever go to the caller's personal vault. The
|
|
28
|
+
* personal store refuses a company vault bucket, and nothing here lists or
|
|
29
|
+
* opens company memberships.
|
|
30
|
+
*/
|
|
31
|
+
import { GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
32
|
+
import type { ScriptRunner } from "./storage.js";
|
|
33
|
+
import { type ReclaimReport } from "./storage-core.js";
|
|
34
|
+
export declare const PLACEHOLDER_SUFFIX = ".hqcloud";
|
|
35
|
+
export declare const OFFLOAD_PREFIX = ".hq-offload/blobs/";
|
|
36
|
+
export declare const POINTER_HEADER = "HQ-CLOUD-FILE v1";
|
|
37
|
+
export declare const POINTER_FOOTER = "This file is stored in HQ cloud storage. Open it in HQ, or run the command above to download it.";
|
|
38
|
+
export declare const DEFAULT_MIN_BYTES: number;
|
|
39
|
+
export declare const DEFAULT_IDLE_DAYS = 14;
|
|
40
|
+
export declare const CURRENT_BLOCKED_REASON = "Offloading current files is not available yet: HQ sync would treat the replaced file as deleted.";
|
|
41
|
+
/**
|
|
42
|
+
* Gate for `--current`. Stays false until the owner enables current-file
|
|
43
|
+
* offload on top of hq-cloud's cloud-only placeholders (PR #919, 6.18.61).
|
|
44
|
+
*/
|
|
45
|
+
export declare const CURRENT_OFFLOAD_ENABLED = false;
|
|
46
|
+
export declare const ORPHANED_MESSAGE = "This file was deleted by a teammate. It's no longer in the cloud for your team.";
|
|
47
|
+
export declare const UPDATE_SYNC = "update HQ sync";
|
|
48
|
+
/** Top-level HQ-root folder that holds offloaded blobs in the personal vault. */
|
|
49
|
+
export declare const OFFLOAD_TOP_LEVEL = ".hq-offload";
|
|
50
|
+
/**
|
|
51
|
+
* Whether personal-vault sync skips `.hq-offload/`. Without that exclusion a
|
|
52
|
+
* later `hq sync` pulls every offloaded blob back to disk (and to every other
|
|
53
|
+
* machine), undoing the savings. Pass hq-cloud's
|
|
54
|
+
* `PERSONAL_VAULT_EXCLUDED_TOP_LEVEL`.
|
|
55
|
+
*
|
|
56
|
+
* @indigoai-us/hq-cloud 6.18.61 (PR #919) lists ".hq-offload" in
|
|
57
|
+
* PERSONAL_VAULT_EXCLUDED_TOP_LEVEL; storage.ts passes that export here. The
|
|
58
|
+
* check stays at runtime so an older bundled HQ sync still reports
|
|
59
|
+
* "update HQ sync". CURRENT_OFFLOAD_ENABLED is a separate owner decision.
|
|
60
|
+
*/
|
|
61
|
+
export declare function syncExcludesOffload(excludedTopLevel: readonly string[] | undefined): boolean;
|
|
62
|
+
export declare function offloadKey(gitBlobSha: string): string;
|
|
63
|
+
export declare function parseSize(value: string): number;
|
|
64
|
+
export declare function resolveMinBytes(flag: string | undefined, env: NodeJS.ProcessEnv): number;
|
|
65
|
+
export interface Pointer {
|
|
66
|
+
path: string;
|
|
67
|
+
size: number;
|
|
68
|
+
/** Hex sha256, or null when recorded as `unknown` (sync-updated placeholders). */
|
|
69
|
+
sha256: string | null;
|
|
70
|
+
gitBlob?: string;
|
|
71
|
+
vault: "personal" | "company";
|
|
72
|
+
key: string;
|
|
73
|
+
offloadedAt?: string;
|
|
74
|
+
/** Vault object ETag, written by sync when it records a teammate's version. */
|
|
75
|
+
remoteEtag?: string;
|
|
76
|
+
/** Set by sync when a teammate deleted the remote file. */
|
|
77
|
+
orphaned?: boolean;
|
|
78
|
+
}
|
|
79
|
+
export declare function renderPointer(p: Pointer): string;
|
|
80
|
+
/** ETags compare without the surrounding quotes S3 returns. */
|
|
81
|
+
export declare function normalizeEtag(etag: string | undefined): string;
|
|
82
|
+
/**
|
|
83
|
+
* Parse pointer text. Returns null for anything that is not a valid v1 pointer.
|
|
84
|
+
*
|
|
85
|
+
* Two shapes are valid:
|
|
86
|
+
* - an offload copy: `vault: personal`, key = offloadKey(git-blob), with a
|
|
87
|
+
* git blob id to verify against;
|
|
88
|
+
* - a vault object recorded by sync: either vault, a non-offload key, and a
|
|
89
|
+
* `remote-etag` to verify against (sha256 may be `unknown`).
|
|
90
|
+
*/
|
|
91
|
+
export declare function parsePointer(text: string): Pointer | null;
|
|
92
|
+
export interface FileDigest {
|
|
93
|
+
size: number;
|
|
94
|
+
sha256: string;
|
|
95
|
+
gitBlob: string;
|
|
96
|
+
}
|
|
97
|
+
/** sha256 and git blob id (sha1 of "blob <size>\0" + bytes) in one pass. */
|
|
98
|
+
export declare function digestFile(file: string): Promise<FileDigest>;
|
|
99
|
+
export interface StoredObject {
|
|
100
|
+
size: number;
|
|
101
|
+
/** Hex sha256, from object metadata or the S3 full-object checksum. */
|
|
102
|
+
sha256?: string;
|
|
103
|
+
}
|
|
104
|
+
export interface BlobStore {
|
|
105
|
+
readonly vault: "personal";
|
|
106
|
+
head(key: string): Promise<StoredObject | null>;
|
|
107
|
+
put(key: string, file: string, digest: FileDigest): Promise<void>;
|
|
108
|
+
download(key: string, dest: string): Promise<void>;
|
|
109
|
+
}
|
|
110
|
+
export interface OffloadS3Client {
|
|
111
|
+
send(cmd: PutObjectCommand | HeadObjectCommand | GetObjectCommand): Promise<unknown>;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* S3-backed store over the caller's personal vault bucket. Accepts only
|
|
115
|
+
* personal vault buckets (`hq-vault-prs-*`) so a mis-resolved entity can never
|
|
116
|
+
* receive offloaded bytes.
|
|
117
|
+
*/
|
|
118
|
+
export declare function s3BlobStore(bucket: string, s3: OffloadS3Client): BlobStore;
|
|
119
|
+
/** Read-only access to vault objects at their sync keys, pinned to an ETag. */
|
|
120
|
+
export interface VaultObjectStore {
|
|
121
|
+
/** Download `key` to `dest` only if its ETag is `etag`; returns the served ETag. */
|
|
122
|
+
download(key: string, dest: string, etag: string): Promise<string>;
|
|
123
|
+
}
|
|
124
|
+
export interface VaultObjectS3Client {
|
|
125
|
+
send(cmd: GetObjectCommand): Promise<unknown>;
|
|
126
|
+
}
|
|
127
|
+
export declare function s3VaultObjectStore(bucket: string, s3: VaultObjectS3Client): VaultObjectStore;
|
|
128
|
+
export interface OffloadDeps {
|
|
129
|
+
hqRoot: string;
|
|
130
|
+
now: () => Date;
|
|
131
|
+
runScript: ScriptRunner;
|
|
132
|
+
confirm: (message: string) => Promise<boolean>;
|
|
133
|
+
log: (line: string) => void;
|
|
134
|
+
/** Open the caller's personal vault store. The only store offload uses. */
|
|
135
|
+
openPersonal: () => Promise<BlobStore>;
|
|
136
|
+
/** Write git blob `sha` from the HQ root repo to `dest`. */
|
|
137
|
+
exportBlob: (sha: string, dest: string) => Promise<void>;
|
|
138
|
+
/** HQ-root-relative paths of `*.hqcloud` placeholders. */
|
|
139
|
+
listPlaceholders: () => Promise<string[]>;
|
|
140
|
+
tmpDir: () => string;
|
|
141
|
+
/** Whether the installed HQ sync skips `.hq-offload/` (see syncExcludesOffload). */
|
|
142
|
+
syncExcludesOffload: () => boolean;
|
|
143
|
+
/** Slugs of companies where the caller is an active member. */
|
|
144
|
+
activeCompanySlugs?: () => Promise<string[]>;
|
|
145
|
+
/**
|
|
146
|
+
* Open a vault for objects recorded by sync. Company vaults are vended
|
|
147
|
+
* through the caller's membership; runFetch checks membership first.
|
|
148
|
+
*/
|
|
149
|
+
openVaultObjects?: (target: {
|
|
150
|
+
vault: "personal";
|
|
151
|
+
} | {
|
|
152
|
+
vault: "company";
|
|
153
|
+
slug: string;
|
|
154
|
+
}) => Promise<VaultObjectStore>;
|
|
155
|
+
}
|
|
156
|
+
export declare function storageScriptPath(hqRoot: string): string;
|
|
157
|
+
/** Whether the installed core script has a `case` arm for `sub`. */
|
|
158
|
+
export declare function scriptSupports(hqRoot: string, sub: "large" | "strip-large"): boolean;
|
|
159
|
+
export interface LargeBlob {
|
|
160
|
+
sha: string;
|
|
161
|
+
bytes: number;
|
|
162
|
+
paths: string[];
|
|
163
|
+
in_head: boolean;
|
|
164
|
+
/** Linked worktrees (paths) whose HEAD still reaches this blob. */
|
|
165
|
+
held_by: string[];
|
|
166
|
+
}
|
|
167
|
+
/** A linked worktree that keeps old history alive. HQ never changes it. */
|
|
168
|
+
export interface WorktreeHolder {
|
|
169
|
+
path: string;
|
|
170
|
+
branch: string | null;
|
|
171
|
+
}
|
|
172
|
+
export interface LargeCurrent {
|
|
173
|
+
path: string;
|
|
174
|
+
bytes: number;
|
|
175
|
+
idle_days: number;
|
|
176
|
+
}
|
|
177
|
+
export interface LargeReport {
|
|
178
|
+
available: boolean;
|
|
179
|
+
reason?: string;
|
|
180
|
+
blobs: LargeBlob[];
|
|
181
|
+
current: LargeCurrent[];
|
|
182
|
+
worktree_holders: WorktreeHolder[];
|
|
183
|
+
}
|
|
184
|
+
export declare function getLarge(deps: OffloadDeps, minBytes: number): Promise<LargeReport>;
|
|
185
|
+
/** Eligible history blobs split into ones offload can free and ones a linked worktree keeps. */
|
|
186
|
+
export declare function splitHeld(blobs: LargeBlob[]): {
|
|
187
|
+
free: LargeBlob[];
|
|
188
|
+
held: LargeBlob[];
|
|
189
|
+
};
|
|
190
|
+
/** Distinct worktree paths that hold any of `held`, in first-seen order. */
|
|
191
|
+
export declare function heldWorktrees(held: LargeBlob[]): string[];
|
|
192
|
+
/** Plain refusal when every eligible old file is still used by a linked worktree. */
|
|
193
|
+
export declare function worktreeHoldsAllCopy(held: LargeBlob[]): string;
|
|
194
|
+
/**
|
|
195
|
+
* History blobs eligible for offload: at least `minBytes`, and not part of the
|
|
196
|
+
* current checkout. Blobs in HEAD are current files; stripping them would
|
|
197
|
+
* leave the working tree out of step with HEAD.
|
|
198
|
+
*/
|
|
199
|
+
export declare function selectHistoryBlobs(blobs: LargeBlob[], minBytes: number): LargeBlob[];
|
|
200
|
+
export declare function matchesDigest(obj: StoredObject | null, digest: FileDigest): boolean;
|
|
201
|
+
/**
|
|
202
|
+
* Export, hash, upload and verify one history blob. Resolves only when the
|
|
203
|
+
* stored object matches size and sha256. Dedupes on the content-addressed key:
|
|
204
|
+
* an already-verified object is not re-uploaded.
|
|
205
|
+
*/
|
|
206
|
+
export declare function uploadBlob(blob: LargeBlob, store: BlobStore, deps: OffloadDeps): Promise<{
|
|
207
|
+
digest: FileDigest;
|
|
208
|
+
uploaded: boolean;
|
|
209
|
+
}>;
|
|
210
|
+
/** File operations for current-file offload, injectable so tests can see the order. */
|
|
211
|
+
export interface PlaceholderFs {
|
|
212
|
+
/** Create the placeholder (never replacing one) and fsync it and its folder. */
|
|
213
|
+
writePlaceholder(file: string, text: string): Promise<void>;
|
|
214
|
+
removeOriginal(file: string): Promise<void>;
|
|
215
|
+
}
|
|
216
|
+
export declare const durablePlaceholderFs: PlaceholderFs;
|
|
217
|
+
/**
|
|
218
|
+
* Offload one current file: upload and verify, write `x.hqcloud` (fsynced),
|
|
219
|
+
* and only then remove `x`. With the placeholder written first, a sync
|
|
220
|
+
* watcher never sees `x` missing without its placeholder, which it would
|
|
221
|
+
* push as a delete.
|
|
222
|
+
*
|
|
223
|
+
* Not called while CURRENT_OFFLOAD_ENABLED is false.
|
|
224
|
+
*/
|
|
225
|
+
export declare function offloadCurrentFile(rel: string, store: BlobStore, deps: Pick<OffloadDeps, "hqRoot" | "now">, fsOps?: PlaceholderFs): Promise<{
|
|
226
|
+
path: string;
|
|
227
|
+
bytes: number;
|
|
228
|
+
}>;
|
|
229
|
+
export interface OffloadOptions {
|
|
230
|
+
minBytes: number;
|
|
231
|
+
idleDays: number;
|
|
232
|
+
history: boolean;
|
|
233
|
+
current: boolean;
|
|
234
|
+
dryRun: boolean;
|
|
235
|
+
yes: boolean;
|
|
236
|
+
}
|
|
237
|
+
export interface OffloadResult {
|
|
238
|
+
history: {
|
|
239
|
+
uploaded: number;
|
|
240
|
+
bytes: number;
|
|
241
|
+
freed_bytes: number;
|
|
242
|
+
available?: boolean;
|
|
243
|
+
reason?: string;
|
|
244
|
+
candidates?: number;
|
|
245
|
+
candidate_bytes?: number;
|
|
246
|
+
/** Backup kept by strip-large; space it holds is freed after reclaim_after. */
|
|
247
|
+
backup_ref?: string | null;
|
|
248
|
+
reclaim_after?: string | null;
|
|
249
|
+
pending_reclaim_bytes?: number;
|
|
250
|
+
/** Old versions a linked worktree still uses: left in place, not counted as freed. */
|
|
251
|
+
held_by_worktrees?: {
|
|
252
|
+
count: number;
|
|
253
|
+
bytes: number;
|
|
254
|
+
};
|
|
255
|
+
/** Worktrees (paths) that still use old history. */
|
|
256
|
+
worktree_holders?: string[];
|
|
257
|
+
/**
|
|
258
|
+
* Set when a precondition stops the history rewrite (e.g.
|
|
259
|
+
* remote_tracking_branches). Checked before any upload, so nothing was
|
|
260
|
+
* uploaded; a dry run sets it when the real run would stop.
|
|
261
|
+
*/
|
|
262
|
+
refused?: string;
|
|
263
|
+
message?: string;
|
|
264
|
+
};
|
|
265
|
+
current: {
|
|
266
|
+
offloaded: {
|
|
267
|
+
path: string;
|
|
268
|
+
bytes: number;
|
|
269
|
+
}[];
|
|
270
|
+
freed_bytes: number;
|
|
271
|
+
available?: boolean;
|
|
272
|
+
reason?: string;
|
|
273
|
+
};
|
|
274
|
+
errors: string[];
|
|
275
|
+
dry_run: boolean;
|
|
276
|
+
/** Expired backups dropped before a real offload; absent otherwise. */
|
|
277
|
+
reclaim?: ReclaimReport;
|
|
278
|
+
}
|
|
279
|
+
export declare function runOffload(opts: OffloadOptions, deps: OffloadDeps): Promise<OffloadResult | undefined>;
|
|
280
|
+
/**
|
|
281
|
+
* Resolve `rel` inside HQ, following symlinks: the deepest existing ancestor
|
|
282
|
+
* is realpath'd and must still be inside the realpath'd HQ root, so a
|
|
283
|
+
* symlinked folder cannot redirect a write outside HQ.
|
|
284
|
+
*/
|
|
285
|
+
export declare function containedPath(hqRoot: string, rel: string): string;
|
|
286
|
+
/** Restore targets that fetch must never write: git internals and agent hooks. */
|
|
287
|
+
export declare function assertRestorable(rel: string): void;
|
|
288
|
+
/** Resolve a fetch argument (original path or placeholder path) to its placeholder file. */
|
|
289
|
+
export declare function resolvePlaceholder(hqRoot: string, arg: string): string;
|
|
290
|
+
export interface FetchResult {
|
|
291
|
+
path: string;
|
|
292
|
+
bytes: number;
|
|
293
|
+
sha256: string;
|
|
294
|
+
}
|
|
295
|
+
export interface FetchOptions {
|
|
296
|
+
/** Restore the caller's own personal-vault copy of a teammate-deleted file. */
|
|
297
|
+
forceOrphan?: boolean;
|
|
298
|
+
/** Destination for --force-orphan; must be outside HQ. */
|
|
299
|
+
out?: string;
|
|
300
|
+
}
|
|
301
|
+
/** Company slug for a company-vault placeholder, checked against its key. */
|
|
302
|
+
export declare function companyForPointer(pointer: Pointer): string;
|
|
303
|
+
export declare function runFetch(arg: string, deps: OffloadDeps, opts?: FetchOptions): Promise<FetchResult>;
|
|
304
|
+
export declare function runFetchGitBlob(sha: string, out: string, deps: OffloadDeps): Promise<FetchResult>;
|
|
305
|
+
export interface CountBytes {
|
|
306
|
+
count: number;
|
|
307
|
+
bytes: number;
|
|
308
|
+
}
|
|
309
|
+
export interface OffloadStatus {
|
|
310
|
+
/** Old versions offload can move and free (not used by another worktree). */
|
|
311
|
+
history_candidates: CountBytes;
|
|
312
|
+
/** Old versions linked worktrees still use: offload leaves them. */
|
|
313
|
+
history_held?: CountBytes & {
|
|
314
|
+
worktrees: string[];
|
|
315
|
+
};
|
|
316
|
+
current_candidates: CountBytes;
|
|
317
|
+
placeholders: CountBytes;
|
|
318
|
+
/** Whether `offload --current` can run. The desktop shows the current-files row only when true. */
|
|
319
|
+
current_available: boolean;
|
|
320
|
+
current_reason?: string;
|
|
321
|
+
available?: boolean;
|
|
322
|
+
reason?: string;
|
|
323
|
+
}
|
|
324
|
+
/** Current-file offload availability, shared by status and its error fallback. */
|
|
325
|
+
export declare function currentAvailability(syncReady: boolean): Pick<OffloadStatus, "current_available" | "current_reason">;
|
|
326
|
+
export declare function getOffloadStatus(deps: OffloadDeps, opts: {
|
|
327
|
+
minBytes: number;
|
|
328
|
+
idleDays: number;
|
|
329
|
+
}): Promise<OffloadStatus>;
|
|
330
|
+
export declare function gitExportBlob(hqRoot: string): (sha: string, dest: string) => Promise<void>;
|
|
331
|
+
export declare function gitListPlaceholders(hqRoot: string): () => Promise<string[]>;
|
|
332
|
+
export declare function defaultTmpDir(): string;
|
|
333
|
+
export declare function personalS3Store(input: {
|
|
334
|
+
bucket: string;
|
|
335
|
+
region: string;
|
|
336
|
+
credentials: {
|
|
337
|
+
accessKeyId: string;
|
|
338
|
+
secretAccessKey: string;
|
|
339
|
+
sessionToken: string;
|
|
340
|
+
};
|
|
341
|
+
}): BlobStore;
|
|
342
|
+
//# sourceMappingURL=storage-offload.d.ts.map
|