@indigoai-us/hq-cli 5.345.137 → 5.345.139

package/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## [Unreleased]
 
+## [5.345.139] — 2026-10-15
+- `hq storage offload` and `hq storage fetch` no longer fail about 15 to 25 minutes into a run. Personal and company
+  vault credentials are now re-vended before they expire, for the whole run and in the middle of a multipart upload.
+  When S3 still refuses a request's credentials, HQ re-vends once and retries that request once. A request refused
+  again fails, and an interrupted company upload resumes from its last finished part on the next run.
+- Vault errors now name the request, the vault, the HTTP status and the error code, for example
+  `HEAD failed in company vault 'acme': HTTP 403 Forbidden (no error body: credentials expired or access denied)`,
+  instead of a bare `UnknownError`.
+
+## [5.345.138] — 2026-10-15
+- `hq storage offload --current` can now free company files larger than the 50 MB sync limit. It uploads each one to its company vault, checks the vault copy against the file's sha256 and size, records it as cloud-only, writes the `.hqcloud` placeholder, and only then removes the local copy. An interrupted upload leaves the file in place and resumes on the next run. The dry run shows `upload+offload` with the byte count and the vault key. A path the `.hqinclude` allowlist excludes is refused with the line to add. `hq storage fetch` checks the vault copy's sha256 before restoring it. The upload path turns on once the bundled hq-cloud includes the new upload exports; until then these files report that HQ sync needs an update.
+
 ## [5.345.137] — 2026-10-15
 - The bundled sync engine now uses hq-cloud 6.18.76. On Windows, the hq-core update and rescue lay down native links after the overlay, turn git symlink stubs from Windows clones into real links, and upload symlinks with the correct target. Sync merges `board.json` by entry id instead of replacing it, merges append-only `.jsonl` conflicts instead of parking them, keeps any `.hqcloud` placeholder on pull, and no longer syncs live-meeting runtime state or server sync manifests.
 

package/dist/commands/storage-offload.d.ts

@@ -1,38 +1,49 @@
 /**
- * `hq storage offload` / `hq storage fetch` — move big files to the caller's
- * PERSONAL vault (S3 Standard, content-addressed) and leave small linked
- * pointers behind. Shared contract: owner-approved offload design, 2026-10-09.
+ * `hq storage offload` / `hq storage fetch` — move big files out of the HQ
+ * folder into HQ cloud storage and leave small linked placeholders behind.
+ * Shared contract: owner-approved offload design, 2026-10-09; company upload
+ * for files over the sync limit approved 2026-10-15.
  *
  * Two kinds of big file:
  *   - History copies (old versions inside the HQ root git history). The CLI
- *     uploads each blob, verifies size + sha256 + git blob id, then hands only
- *     verified blobs to core `hq-storage.sh strip-large`, which rewrites history
- *     to the pointer text. Working-tree files are never touched.
+ *     uploads each blob to the caller's PERSONAL vault, verifies size + sha256
+ *     + git blob id, then hands only verified blobs to core `hq-storage.sh
+ *     strip-large`, which rewrites history to the pointer text. Working-tree
+ *     files are never touched.
  *   - Current files (`--current [paths...]`, idle big files in the working
- *     tree). Each file goes to the vault that owns its path. A file under
- *     `companies/<slug>/` is never uploaded: when HQ sync already stored this
- *     exact version in that company's vault (journal hash + size, and the vault
- *     object still has the journal's ETag), the file is replaced by a
- *     placeholder pointing at that object. Otherwise it is skipped with "run
- *     hq sync first", so sync stays the only writer of company vault keys. Any
- *     other file points at its personal-vault sync object the same way, or is
- *     uploaded to the personal vault's `.hq-offload/blobs/`. Current-file
- *     offload runs only when the installed HQ sync holds every `.hqcloud`
- *     sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and skips
- *     `.hq-offload/`; otherwise it reports "update HQ sync".
+ *     tree). Each file goes to the vault that owns its path:
+ *       - A file in a company folder belongs to that company's vault, and only
+ *         an active member can offload it. When HQ sync already stored this
+ *         exact version (journal hash + size, and the vault object still has
+ *         the journal's ETag), the file is replaced by a placeholder pointing
+ *         at that object. A file over the sync limit (SYNC_MAX_FILE_BYTES),
+ *         which sync never uploads, is uploaded to the company vault at its
+ *         sync key by a resumable, checksum-verified multipart upload
+ *         (`upload+offload`). Once the vault copy is verified the file is
+ *         recorded in the sync journal as synced and cloud-only, then the
+ *         placeholder is written, then the local file is removed. A path the
+ *         sync ignore rules exclude is refused and the excluding rule is named
+ *         (for `.hqinclude`, with the exact line to add). Other unsynced
+ *         company files are skipped with "run hq sync first".
+ *       - Any other file points at its personal-vault sync object the same
+ *         way, or is uploaded to the personal vault's `.hq-offload/blobs/`.
+ *     Current-file offload runs only when the installed HQ sync holds every
+ *     `.hqcloud` sibling (hq-cloud CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING) and
+ *     skips `.hq-offload/`; otherwise it reports "update HQ sync".
  *
- * Fetch also reads placeholders that HQ sync rewrote after a teammate changed
- * the file (`vault: company`, the company vault key, `sha256: unknown`,
- * `remote-etag`). Those download from the company vault, only for an active
- * member, and verify by size and ETag. A placeholder marked `orphaned: true`
- * (a teammate deleted the file) is refused unless `--force-orphan` is given,
- * which writes the caller's own personal-vault copy to a file outside HQ so
- * it can never sync back.
+ * Fetch also reads placeholders that HQ sync wrote on a teammate's machine
+ * (`vault: company`, the company vault key, `sha256: unknown`, `remote-etag`).
+ * Those download from the company vault, only for an active member, and
+ * verify by size, ETag and, when the object carries it, its `hq-sha256`
+ * metadata. A placeholder marked `orphaned: true` (a teammate deleted the
+ * file) is refused unless `--force-orphan` is given, which writes the caller's
+ * own personal-vault copy to a file outside HQ so it can never sync back.
  *
- * Tenant isolation: uploads only ever go to the caller's personal vault, and
- * the personal store refuses a company vault bucket. A company vault is opened
- * read-only (HEAD and ETag-pinned GET), and only for a company the caller is
- * an active member of.
+ * Tenant isolation: history and personal uploads only ever go to the caller's
+ * personal vault, and the personal store refuses a company vault bucket. A
+ * company vault is opened only for a company the caller is an active member
+ * of, and only for a file in that company's folder; the company uploader must
+ * name that company and refuses a personal bucket.
  */
 import { GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
 import type { ScriptRunner } from "./storage.js";
@@ -47,6 +58,13 @@ export declare const ORPHANED_MESSAGE = "This file was deleted by a teammate. It
 export declare const UPDATE_SYNC = "update HQ sync";
 /** Reason on company files HQ sync has not stored in the company vault yet. */
 export declare const RUN_SYNC_FIRST = "run hq sync first";
+/**
+ * HQ sync's per-file limit (hq-cloud SYNC_MAX_FILE_BYTES; sync skips files
+ * strictly larger). storage.ts passes hq-cloud's value when it exports one.
+ */
+export declare const SYNC_MAX_FILE_BYTES: number;
+/** Object metadata key holding the sha256 of a company-vault offload upload. */
+export declare const VAULT_SHA256_METADATA_KEY = "hq-sha256";
 /** Top-level HQ-root folder that holds offloaded blobs in the personal vault. */
 export declare const OFFLOAD_TOP_LEVEL = ".hq-offload";
 /**
@@ -112,18 +130,64 @@ export interface BlobStore {
 export interface OffloadS3Client {
     send(cmd: PutObjectCommand | HeadObjectCommand | GetObjectCommand): Promise<unknown>;
 }
+/**
+ * True when S3 refused the request's credentials: an expired, invalid or
+ * denied token, or a 403 with no error body (what S3 sends for HEAD).
+ */
+export declare function isCredentialRejection(err: unknown): boolean;
+/** A failed vault request, described with its HTTP status, error code and vault. */
+export declare class VaultRequestError extends Error {
+    readonly status?: number;
+    readonly code?: string;
+    constructor(message: string, opts: {
+        status?: number;
+        code?: string;
+        cause: unknown;
+    });
+}
+/**
+ * Describes an AWS error from `op` against `where` so the user sees the HTTP
+ * status, the error code and the vault, never the SDK's bare "UnknownError".
+ * Errors that did not come from AWS pass through unchanged.
+ */
+export declare function describeVaultError(err: unknown, op: string, where: string, retried?: boolean): unknown;
+/** Turns any per-file failure into one line for the run's error list. */
+export declare function failureMessage(err: unknown): string;
+/**
+ * Runs vault requests on `initial` and, when S3 refuses the credentials,
+ * re-vends once and retries the request once on the new client. Requests
+ * refused on the same client share one re-vend. A request refused again
+ * after the re-vend fails with a described error; it is never retried twice.
+ */
+export declare function credentialRetry<C>(initial: C, where: string, revend?: () => Promise<C>): <T>(op: string, fn: (client: C) => Promise<T>) => Promise<T>;
+/**
+ * Wraps every method of `initial` (a multipart client) so each call gets
+ * {@link credentialRetry}: one re-vend and one retry when S3 refuses the
+ * credentials, and a described error otherwise.
+ */
+export declare function credentialRetryingClient<C extends object>(initial: C, where: string, revend: () => Promise<C>): C;
+export interface S3StoreOptions<C> {
+    /** How errors name the vault, e.g. "personal vault" or "company vault 'acme'". */
+    where?: string;
+    /** Vends fresh credentials and returns a client that uses them. */
+    revend?: () => Promise<C>;
+}
 /**
  * S3-backed store over the caller's personal vault bucket. Accepts only
  * personal vault buckets (`hq-vault-prs-*`) so a mis-resolved entity can never
  * receive offloaded bytes.
  */
-export declare function s3BlobStore(bucket: string, s3: OffloadS3Client): BlobStore;
+export declare function s3BlobStore(bucket: string, s3: OffloadS3Client, opts?: S3StoreOptions<OffloadS3Client>): BlobStore;
 /** Read-only access to vault objects at their sync keys, pinned to an ETag. */
 export interface VaultObjectStore {
-    /** Size and unquoted ETag of `key`, or null when it is not there. */
+    /**
+     * Size and unquoted ETag of `key`, or null when it is not there. `sha256` is
+     * the object's `hq-sha256` metadata, set on company-vault offload uploads.
+     */
     head(key: string): Promise<{
         size: number;
         etag: string;
+        sha256?: string;
     } | null>;
     /** Download `key` to `dest` only if its ETag is `etag`; returns the served ETag. */
     download(key: string, dest: string, etag: string): Promise<string>;
@@ -131,7 +195,7 @@ export interface VaultObjectStore {
 export interface VaultObjectS3Client {
     send(cmd: GetObjectCommand | HeadObjectCommand): Promise<unknown>;
 }
-export declare function s3VaultObjectStore(bucket: string, s3: VaultObjectS3Client): VaultObjectStore;
+export declare function s3VaultObjectStore(bucket: string, s3: VaultObjectS3Client, opts?: S3StoreOptions<VaultObjectS3Client>): VaultObjectStore;
 export interface OffloadDeps {
     hqRoot: string;
     now: () => Date;
@@ -169,6 +233,55 @@ export interface OffloadDeps {
     syncRecord?: (target: OffloadTarget) => Promise<SyncRecord | null>;
     /** File operations for current-file offload (tests inject to see the order). */
     placeholderFs?: PlaceholderFs;
+    /** HQ sync's per-file limit in bytes. Defaults to SYNC_MAX_FILE_BYTES. */
+    syncMaxFileBytes?: number;
+    /**
+     * Whether HQ sync's ignore rules exclude an HQ-root-relative path, and which
+     * rule does (hq-cloud createIgnoreExplainer). Null or absent: not excluded.
+     */
+    syncExclusion?: (rel: string) => SyncExclusion | null;
+    /**
+     * Open a multipart uploader for a company vault. Called only for a company
+     * the caller is an active member of. Absent: company files over the sync
+     * limit report "update HQ sync".
+     */
+    openCompanyUploader?: (slug: string) => Promise<CompanyUploader>;
+    /**
+     * Record `target` in the sync journal as synced and cloud-only (hq-cloud
+     * recordOffloadedObject), so sync neither re-uploads it nor pulls it back.
+     */
+    recordCloudOnly?: (target: OffloadTarget, object: VerifiedUpload) => Promise<void>;
+    /** Folder for resumable multipart upload state. Defaults to ~/.hq/offload-uploads. */
+    uploadStateDir?: () => string;
+}
+/** Why HQ sync skips a path: the ignore layer that excludes it. */
+export interface SyncExclusion {
+    excluded: boolean;
+    /** "built-in", "defaults", ".hqignore", ".hqsyncignore" or ".hqinclude allowlist". */
+    layer: string | null;
+    /** For the .hqinclude allowlist: the line that would include the path. */
+    includeLine?: string;
+}
+/** A vault object whose size and sha256 were verified after upload. */
+export interface VerifiedUpload {
+    etag: string;
+    size: number;
+    sha256: string;
+}
+/**
+ * Resumable, checksum-verified multipart upload into one company vault
+ * (hq-cloud uploadLargeObjectVerified). `upload` resumes from `stateFile`
+ * when it holds an upload for the same file, and resolves only after the
+ * object's size, part count and sha256 metadata are verified.
+ */
+export interface CompanyUploader {
+    slug: string;
+    bucket: string;
+    upload(input: {
+        key: string;
+        file: string;
+        stateFile: string;
+    }): Promise<VerifiedUpload>;
 }
 /** The vault that owns a working-tree path, and the object key there. */
 export type OffloadTarget = {
@@ -273,7 +386,15 @@ export interface CurrentFilePlan {
     company?: string;
     /** HQ sync already stored this exact version in the owning vault. */
     synced: boolean;
-    action: "point-at-vault" | "upload" | "skip";
+    /**
+     * upload+offload: a company file over the sync limit, uploaded to the
+     * company vault at `vault_key`, then replaced by a placeholder.
+     */
+    action: "point-at-vault" | "upload" | "upload+offload" | "skip";
+    /** Company vault key for upload+offload. */
+    vault_key?: string;
+    /** The HQ sync ignore layer that excludes this path. */
+    excluded_by?: string;
     reason?: string;
 }
 export interface CurrentOffloaded {
@@ -281,7 +402,7 @@ export interface CurrentOffloaded {
     bytes: number;
     vault: "personal" | "company";
     company?: string;
-    action: "point-at-vault" | "upload";
+    action: "point-at-vault" | "upload" | "upload+offload";
 }
 export interface OffloadOptions {
     minBytes: number;
@@ -405,13 +526,4 @@ export declare function getOffloadStatus(deps: OffloadDeps, opts: {
 export declare function gitExportBlob(hqRoot: string): (sha: string, dest: string) => Promise<void>;
 export declare function gitListPlaceholders(hqRoot: string): () => Promise<string[]>;
 export declare function defaultTmpDir(): string;
-export declare function personalS3Store(input: {
-    bucket: string;
-    region: string;
-    credentials: {
-        accessKeyId: string;
-        secretAccessKey: string;
-        sessionToken: string;
-    };
-}): BlobStore;
 //# sourceMappingURL=storage-offload.d.ts.map