@indigoai-us/hq-cli 5.345.137 → 5.345.139

package/dist/commands/storage.d.ts

@@ -35,7 +35,7 @@
  */
 import { Command } from "commander";
 import { type ReclaimReport } from "./storage-core.js";
-import { ListObjectVersionsCommand, DeleteObjectsCommand, type ListObjectVersionsCommandOutput, type DeleteObjectsCommandOutput } from "@aws-sdk/client-s3";
+import { S3Client, ListObjectVersionsCommand, DeleteObjectsCommand, type ListObjectVersionsCommandOutput, type DeleteObjectsCommandOutput } from "@aws-sdk/client-s3";
 import { type OffloadResult, type OffloadStatus } from "./storage-offload.js";
 export type TrancheId = "7d" | "30d" | "90d" | "365d" | "older";
 export declare const TRANCHES: readonly {
@@ -265,6 +265,17 @@ export declare const NEEDS_YES = "No terminal to confirm in. Re-run with --yes t
 export declare function confirmOnTty(message: string, input?: NodeJS.ReadableStream & {
     isTTY?: boolean;
 }, output?: NodeJS.WritableStream): Promise<boolean>;
+/**
+ * An S3 client for one vault whose credentials are re-vended before they
+ * expire, plus `revend`, which vends now and returns a client on the new
+ * credentials (the SDK caches credentials per client, so a forced refresh
+ * needs a new client).
+ */
+export declare function vendedS3(vend: () => Promise<VendedCredentials>, bucket: string, makeClient?: (credentials: ReturnType<typeof refreshingVendedCredentials>) => S3Client): {
+    bucket: string;
+    client: S3Client;
+    revend: () => Promise<S3Client>;
+};
 /** STS credentials from one vend, with the time they stop working. */
 export interface VendedCredentials {
     accessKeyId: string;

package/dist/commands/storage.js

@@ -48,7 +48,7 @@ import { vaultApiFetch } from "../utils/vault-api.js";
 import { findHqRoot } from "../utils/manifest.js";
 import { maxLength } from "../utils/table-width.js";
 import { resolveCanonicalPersonUid } from "./cloud.js";
-import { DEFAULT_IDLE_DAYS, currentAvailability, defaultTmpDir, getOffloadStatus, gitExportBlob, gitListPlaceholders, personalS3Store, resolveMinBytes, runFetch, runFetchGitBlob, runOffload, s3VaultObjectStore, syncExcludesOffload, } from "./storage-offload.js";
+import { DEFAULT_IDLE_DAYS, currentAvailability, defaultTmpDir, getOffloadStatus, gitExportBlob, gitListPlaceholders, credentialRetryingClient, resolveMinBytes, runFetch, runFetchGitBlob, runOffload, s3BlobStore, s3VaultObjectStore, syncExcludesOffload, } from "./storage-offload.js";
 export const TRANCHES = [
     { id: "7d", label: "Last 7 days", maxAgeDays: 7 },
     { id: "30d", label: "8-30 days", maxAgeDays: 30 },
@@ -838,12 +838,16 @@ export function formatOffload(r) {
 const CURRENT_ACTION_LABEL = {
     "point-at-vault": "point at vault",
     upload: "upload",
+    "upload+offload": "upload+offload",
     skip: "skip",
 };
 /** One `--current` plan line: path, size, owning vault, sync state, action. */
 function formatCurrentFile(f) {
     const vault = f.vault === "company" ? `company ${f.company ?? "?"}` : "personal vault";
     const parts = [f.path, formatBytes(f.bytes), vault, f.synced ? "synced" : "not synced", CURRENT_ACTION_LABEL[f.action]];
+    if (f.action === "upload+offload") {
+        parts.push(`(${f.bytes} bytes to the company ${f.company ?? "?"} vault at ${f.vault_key ?? f.path})`);
+    }
     if (f.action === "skip" && f.reason)
         parts.push(`(${f.reason})`);
     return parts.join("  ");
@@ -1014,8 +1018,14 @@ async function buildOffloadDeps() {
         }
         if (!entity.bucketName)
             throw new Error("Your personal vault has no storage yet. Run `hq onboard`.");
-        const vend = await vault.sts.vendSelf({ personUid: entity.uid });
-        return { bucket: entity.bucketName, credentials: vend.credentials };
+        const uid = entity.uid;
+        // A run can outlast one STS session, so every client re-vends before the
+        // credentials expire, and once more when S3 refuses them anyway.
+        return vendedS3(async () => {
+            const { vault } = await signedInVault();
+            const vend = await vault.sts.vendSelf({ personUid: uid });
+            return { ...vend.credentials, expiresAt: vend.expiresAt };
+        }, entity.bucketName);
     };
     const activeCompanies = async () => {
         const { token } = await signedInVault();
@@ -1026,6 +1036,30 @@ async function buildOffloadDeps() {
         return companiesFromMemberships(await res.json());
     };
     const journals = new Map();
+    const journalSlug = (target) => target.vault === "company" ? target.slug : hqCloud.PERSONAL_VAULT_JOURNAL_SLUG;
+    const companyVault = async (slug) => {
+        const company = (await activeCompanies()).find((c) => c.slug === slug);
+        if (!company)
+            throw new Error(`You are not an active member of '${slug}'.`);
+        const { vault } = await signedInVault();
+        const entity = await vault.entity.get(company.uid);
+        const bucket = entity.bucketName ?? `hq-vault-cmp-${company.slug}`;
+        const s3 = vendedS3(async () => {
+            const { vault } = await signedInVault();
+            const vend = await vault.sts.vend({ companyUid: company.uid });
+            return { ...vend.credentials, expiresAt: vend.expiresAt };
+        }, bucket);
+        return { company, ...s3 };
+    };
+    // Company upload, the ignore explainer and the cloud-only journal row ship
+    // in a newer @indigoai-us/hq-cloud. Until the bundled one exports them,
+    // company files over the sync limit report "update HQ sync".
+    const cloud = hqCloud;
+    const explainIgnore = cloud.createIgnoreExplainer;
+    const uploadLarge = cloud.uploadLargeObjectVerified;
+    const multipartClient = cloud.s3MultipartClient;
+    const recordOffloaded = cloud.recordOffloadedObject;
+    let explainer;
     return {
         hqRoot,
         now: () => new Date(),
@@ -1037,8 +1071,8 @@ async function buildOffloadDeps() {
         tmpDir: defaultTmpDir,
         syncExcludesOffload: () => syncExcludesOffload(PERSONAL_VAULT_EXCLUDED_TOP_LEVEL),
         openPersonal: async () => {
-            const { bucket, credentials } = await personalVault();
-            return personalS3Store({ bucket, region: DEFAULT_COGNITO.region, credentials });
+            const { bucket, client, revend } = await personalVault();
+            return s3BlobStore(bucket, client, { where: "personal vault", revend });
         },
         activeCompanySlugs: async () => (await activeCompanies()).map((c) => c.slug),
         // Current-file offload replaces a file with a placeholder only when the
@@ -1047,9 +1081,9 @@ async function buildOffloadDeps() {
         syncHoldsPlaceholders: () => hqCloud.CLOUD_PLACEHOLDER_HOLDS_ANY_SIBLING === true,
         // What HQ sync last stored for a file, read from its local journal.
         // Company journal keys are relative to the company folder; personal keys
-        // are relative to the HQ root. Read-only: offload never writes a journal.
+        // are relative to the HQ root. The only journal write is recordCloudOnly.
         syncRecord: async (target) => {
-            const slug = target.vault === "company" ? target.slug : hqCloud.PERSONAL_VAULT_JOURNAL_SLUG;
+            const slug = journalSlug(target);
             let journal = journals.get(slug);
             if (!journal) {
                 journal = hqCloud.readJournal(slug);
@@ -1064,17 +1098,63 @@ async function buildOffloadDeps() {
         // come from the existing membership-scoped /sts/vend path.
         openVaultObjects: async (target) => {
             if (target.vault === "personal") {
-                const { bucket, credentials } = await personalVault();
-                return s3VaultObjectStore(bucket, new S3Client({ region: DEFAULT_COGNITO.region, credentials }));
+                const { bucket, client, revend } = await personalVault();
+                return s3VaultObjectStore(bucket, client, { where: "personal vault", revend });
             }
-            const company = (await activeCompanies()).find((c) => c.slug === target.slug);
-            if (!company)
-                throw new Error(`You are not an active member of '${target.slug}'.`);
-            const { vault } = await signedInVault();
-            const vend = await vault.sts.vend({ companyUid: company.uid });
-            const entity = await vault.entity.get(company.uid);
-            const bucket = entity.bucketName ?? `hq-vault-cmp-${company.slug}`;
-            return s3VaultObjectStore(bucket, new S3Client({ region: DEFAULT_COGNITO.region, credentials: vend.credentials }));
+            const { company, bucket, client, revend } = await companyVault(target.slug);
+            return s3VaultObjectStore(bucket, client, { where: `company vault '${company.slug}'`, revend });
+        },
+        syncMaxFileBytes: typeof cloud.SYNC_MAX_FILE_BYTES === "number" ? cloud.SYNC_MAX_FILE_BYTES : undefined,
+        syncExclusion: explainIgnore
+            ? (rel) => {
+                explainer ??= explainIgnore(hqRoot);
+                return explainer(rel);
+            }
+            : undefined,
+        // Multipart upload into the company vault, through the caller's
+        // membership-scoped /sts/vend credentials (re-vended before they expire,
+        // since a multi-GB upload can outlast one STS session).
+        openCompanyUploader: uploadLarge && multipartClient && recordOffloaded
+            ? async (slug) => {
+                const { company, bucket, client: s3, revend } = await companyVault(slug);
+                if (bucket.startsWith("hq-vault-prs-")) {
+                    throw new Error(`'${bucket}' is a personal vault, not the company vault for '${slug}'.`);
+                }
+                // Each multipart request re-vends once if S3 still refuses the
+                // credentials; a part refused twice leaves the upload resumable.
+                const client = credentialRetryingClient(multipartClient(s3, bucket), `company vault '${company.slug}'`, async () => multipartClient(await revend(), bucket));
+                return {
+                    slug: company.slug,
+                    bucket,
+                    upload: ({ key, file, stateFile }) => uploadLarge({ client, key, filePath: file, stateFile }),
+                };
+            }
+            : undefined,
+        recordCloudOnly: recordOffloaded
+            ? async (target, object) => {
+                const slug = journalSlug(target);
+                recordOffloaded({ journalSlug: slug, key: target.key, ...object });
+                // The cached journal is stale now; reread it on the next lookup.
+                journals.delete(slug);
+            }
+            : undefined,
+    };
+}
+/**
+ * An S3 client for one vault whose credentials are re-vended before they
+ * expire, plus `revend`, which vends now and returns a client on the new
+ * credentials (the SDK caches credentials per client, so a forced refresh
+ * needs a new client).
+ */
+export function vendedS3(vend, bucket, makeClient = (credentials) => new S3Client({ region: DEFAULT_COGNITO.region, credentials })) {
+    return {
+        bucket,
+        client: makeClient(refreshingVendedCredentials(vend)),
+        revend: async () => {
+            const credentials = refreshingVendedCredentials(vend);
+            // Vend now, so a failed vend is reported as one rather than as a refused request.
+            await credentials();
+            return makeClient(credentials);
         },
     };
 }