@indigoai-us/hq-cloud 6.18.16 → 6.18.17

package/dist/cli/share.js

@@ -13,7 +13,7 @@ import { createSyncProgressRecorder } from "../sync-progress.js";
 import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
 import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
 import * as crypto from "crypto";
-import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, S3SdkObjectIO, } from "../object-io.js";
+import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, } from "../object-io.js";
 import { primePrivateFolderUpload } from "../private-folder-transport.js";
 import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
 import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
@@ -32,7 +32,9 @@ import { isOntologyEntityKey, ONTOLOGY_MULTI_WRITER_FLAG, resolveOntologyEntityP
 import { entityIsFreshForCreate, outboxAllowsEntityPush, readOntologyOutbox, } from "../lib/ontology-outbox.js";
 import { resolveSkillRegisterLandedBytes } from "../lib/skill-register-landed-bytes.js";
 import { resolvePersonalConnectorSettingsSync } from "../lib/personal-connector-settings-sync.js";
-import { VaultAuthError, VaultClient, VaultClientError, VaultConflictError, VaultPermissionDeniedError, } from "../vault-client.js";
+import { planLimitRefusalFromError, VaultAuthError, VaultClient, VaultClientError, VaultConflictError, VaultPermissionDeniedError, } from "../vault-client.js";
+import { formatPlanLimitNotice } from "../plan-limit.js";
+import { PlanLimitPauseStore, planLimitPauseNow, PLAN_LIMIT_PAUSE_RETRY_MS, } from "../plan-limit-pause.js";
 import { describeError } from "../lib/describe-error.js";
 import { isTransientHttpStatusError } from "../lib/net-errors.js";
 import { isRetryableIdentityRefusal, isTerminalIdentityRefusal, } from "../cognito-auth.js";
@@ -1243,6 +1245,8 @@ function createShareCounters() {
         filesRefusedStale: 0,
         filesSuppressedByTombstone: 0,
         uploadedPaths: [],
+        filesPlanLimited: 0,
+        planLimitedPaths: [],
     };
 }
 const CANONICAL_COMPANY_SKILL_RE = /^skills\/([a-z0-9][a-z0-9-]*)\/SKILL\.md$/;
@@ -2051,15 +2055,48 @@ async function refreshPushJournalAfterWalk(run, collectedKeys = [], deleteScopeR
         await yieldToCooperativeCheckpoint(checkpointWork, run.options.cooperativeCheckpoint, true);
     }
 }
-function planLimitUpgradeUrlForError(error) {
-    if (!(error instanceof VaultClientError) || !error.upgradeUrl)
-        return undefined;
-    if (error.statusCode === 402 && error.code === "plan_limit_reached") {
-        return error.upgradeUrl;
+/** Refused paths carried on a `plan-limit` event and in `ShareResult`. */
+const PLAN_LIMITED_PATHS_CAP = 50;
+/**
+ * One files admission check for a vault with paused uploads (see
+ * `ObjectIO.checkFilesAdmission`). `allowed` clears the vault's pauses so the
+ * files upload in this pass; `refused` keeps them and refreshes the stored
+ * refusal; anything else keeps the hourly retry. Never throws: an unknown
+ * answer is logged at debug level only.
+ */
+async function checkPlanLimitAdmission(run, pauses) {
+    if (!run.ctx.uid.startsWith("cmp_"))
+        return;
+    let outcome;
+    try {
+        const io = resolveObjectIO(run.ctx);
+        if (typeof io.checkFilesAdmission !== "function")
+            return;
+        outcome = await io.checkFilesAdmission();
+    }
+    catch (err) {
+        outcome = { state: "unknown", reason: describeError(err) };
     }
-    if (error.code === "PLAN_LIMIT_REACHED")
-        return error.upgradeUrl;
-    return undefined;
+    if (outcome.state === "allowed") {
+        pauses.clear();
+    }
+    else if (outcome.state === "refused") {
+        pauses.refreshRefusal(outcome.refusal);
+    }
+    else {
+        debugPlanLimit({
+            event: "plan-limit.admission.unknown",
+            companyUid: run.ctx.uid,
+            reason: outcome.reason,
+        });
+    }
+}
+/** Debug-level diagnostic on stderr, only when `LOG_LEVEL` is debug/trace. */
+function debugPlanLimit(fields) {
+    const level = process.env.LOG_LEVEL;
+    if (level !== "debug" && level !== "trace")
+        return;
+    process.stderr.write(`${JSON.stringify({ level: "debug", component: "plan-limit", ...fields })}\n`);
 }
 async function executeUploads(run, pushPlan, counters, conflictPaths) {
     counters.filesSkipped += pushPlan.preplannedFilesSkipped;
@@ -2085,21 +2122,51 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
         const entry = journalRows(run.journal).get(item.relativePath);
         return entry !== undefined && entry.hash === item.localHash;
     };
+    // Plan-limit pauses (plan-limit-pause.ts). Read once per pass; the pass's
+    // refusals and successes are applied in finishPlanLimitPass below.
+    const planLimitPauses = PlanLimitPauseStore.open(run.ctx.uid);
+    const planLimitPassStartedAt = planLimitPauseNow();
+    const honorPlanLimitPauses = run.options.planLimitPause === true;
+    const uploadedBeforePass = counters.filesUploaded;
+    // Refused by the server in this pass.
+    const planLimitRefusedPaths = new Set();
+    // Skipped without a presign because an earlier refusal paused them, with
+    // the end of each pause window.
+    const planLimitPausedSkips = new Map();
+    let planLimitRefusal;
+    const notePlanLimitedPath = (relativePath) => {
+        counters.filesPlanLimited++;
+        if (counters.planLimitedPaths.length < PLAN_LIMITED_PATHS_CAP) {
+            counters.planLimitedPaths.push(relativePath);
+        }
+    };
+    const planLimitPausedUntil = (item) => honorPlanLimitPauses
+        ? planLimitPauses.pausedUntil(item.relativePath, planLimitPassStartedAt)
+        : undefined;
+    const isPlanLimitPaused = (item) => planLimitPausedUntil(item) !== undefined;
     // Prime from the preview's likely uploads for transport efficiency. The
     // candidate is refreshed after the plan event and tombstone suppression is
     // applied BEFORE any presigns are minted. The queue below still re-plans
     // every preview item before admission, so a later edit cannot disappear from
     // the pass or carry a stale hash into its journal entry.
-    const initiallyPlannedUploads = [];
+    const plannedUploadCandidates = [];
     for (const previewItem of pushPlan.items) {
         if (previewItem.action !== "upload")
             continue;
         const item = replanPushItemAtExecution(previewItem, run.journal, run.skipUnchanged === true, run.options.onHeartbeat, run.ontologyEntityPush);
-        if (item?.action === "upload" &&
-            !isUploadSuppressedByTombstone(item)) {
-            initiallyPlannedUploads.push(item);
+        if (item?.action === "upload" && !isUploadSuppressedByTombstone(item)) {
+            plannedUploadCandidates.push(item);
         }
     }
+    // Before skipping paused files, ask the company vault once whether it
+    // accepts uploads again (an upgrade, or files deleted to get under the
+    // limit). Allowed: drop the pauses and upload them in this pass. Refused:
+    // keep them. Anything else: keep the hourly retry.
+    if (plannedUploadCandidates.some(isPlanLimitPaused)) {
+        await checkPlanLimitAdmission(run, planLimitPauses);
+    }
+    // A paused file is not uploaded this pass, so do not presign it either.
+    const initiallyPlannedUploads = plannedUploadCandidates.filter((item) => !isPlanLimitPaused(item));
     if (run.options.cooperativeCheckpoint?.shouldAbort?.() === true) {
         return {
             aborted: true,
@@ -2122,18 +2189,59 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
     let aborted = false;
     let abortFlightConflictPaths = [];
     let hadUploadFailure = false;
-    let emittedPlanLimitNotice = false;
-    const emitPlanLimitSkip = (error) => {
-        const upgradeUrl = planLimitUpgradeUrlForError(error);
-        if (!upgradeUrl)
+    // A plan-limit refusal is a deliberate per-file skip, never an error: the
+    // file is counted, remembered for the end-of-pass notice, and paused.
+    const emitPlanLimitSkip = (error, relativePath) => {
+        const refusal = planLimitRefusalFromError(error);
+        if (!refusal)
             return false;
         counters.filesSkipped++;
-        if (!emittedPlanLimitNotice) {
-            emittedPlanLimitNotice = true;
-            run.emit({ type: "plan-limit", upgradeUrl });
+        planLimitRefusal ??= refusal;
+        if (!planLimitRefusedPaths.has(relativePath)) {
+            planLimitRefusedPaths.add(relativePath);
+            notePlanLimitedPath(relativePath);
         }
         return true;
     };
+    // Apply the pass's outcome to the pause record and report every file the
+    // plan limit kept back in this pass, in ONE `plan-limit` event: files the
+    // server refused now, and files skipped because an earlier refusal paused
+    // them (`fromPause: true` when only the latter). A successful upload means
+    // the server is accepting writes for this vault again, so earlier pauses
+    // are dropped before this pass's refusals are recorded.
+    const finishPlanLimitPass = () => {
+        const now = planLimitPauseNow();
+        // Read before clear(): the stored refusal describes the paused files.
+        const storedRefusal = planLimitPauses.refusal;
+        const pausesCleared = counters.filesUploaded > uploadedBeforePass;
+        if (pausesCleared)
+            planLimitPauses.clear();
+        const refused = [...planLimitRefusedPaths];
+        if (planLimitRefusal && refused.length > 0) {
+            planLimitPauses.pause(refused, planLimitRefusal, now);
+        }
+        const pausedSkips = [...planLimitPausedSkips.keys()].filter((p) => !planLimitRefusedPaths.has(p));
+        const refusal = planLimitRefusal ?? storedRefusal;
+        if (refusal && refused.length + pausedSkips.length > 0) {
+            counters.planLimit ??= refusal;
+            // The earliest moment any of these files is retried. Paused files whose
+            // pause was just cleared by a successful upload retry on the next pass.
+            let retryAt = refused.length > 0 ? now + PLAN_LIMIT_PAUSE_RETRY_MS : Infinity;
+            for (const p of pausedSkips) {
+                retryAt = Math.min(retryAt, pausesCleared ? now : planLimitPausedSkips.get(p));
+            }
+            run.emit({
+                type: "plan-limit",
+                ...refusal,
+                upgradeUrl: refusal.upgradeUrl,
+                count: refused.length + pausedSkips.length,
+                paths: [...refused, ...pausedSkips].slice(0, PLAN_LIMITED_PATHS_CAP),
+                pausedUntil: new Date(retryAt).toISOString(),
+                fromPause: refused.length === 0,
+            });
+        }
+        planLimitPauses.save(now);
+    };
     const settleFreshSkip = (item) => {
         if (item.action === "refuse-server-owned") {
             if (run.options.serverOwnedPathPolicy !== "skip") {
@@ -2782,7 +2890,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
             // full-journal write after every later PUT.
             if (err instanceof JournalCheckpointError)
                 throw err;
-            if (emitPlanLimitSkip(err))
+            if (emitPlanLimitSkip(err, relativePath))
                 return;
             if (isPreconditionFailed(err)) {
                 conflictPaths.push(relativePath);
@@ -2837,7 +2945,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                             }
                             if (retryErr instanceof JournalCheckpointError)
                                 throw retryErr;
-                            if (emitPlanLimitSkip(retryErr))
+                            if (emitPlanLimitSkip(retryErr, relativePath))
                                 return;
                             if (isPreconditionFailed(retryErr)) {
                                 counters.filesSkipped++;
@@ -2925,7 +3033,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                             isRetryableIdentityRefusal(retryErr)) {
                             throw retryErr;
                         }
-                        if (emitPlanLimitSkip(retryErr))
+                        if (emitPlanLimitSkip(retryErr, relativePath))
                             return;
                         hadUploadFailure = true;
                         run.emit({
@@ -3056,6 +3164,20 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                     skippedSinceCooperativeCheckpoint++;
                     continue;
                 }
+                const pausedUntil = planLimitPausedUntil(item);
+                if (pausedUntil !== undefined) {
+                    // Refused by the plan limit within the last hour: skip it without a
+                    // presign until the pause lapses, the admission check clears it, or
+                    // an upload to this vault succeeds. Reported at the end of the pass.
+                    queue.shift();
+                    counters.filesSkipped++;
+                    if (!planLimitPausedSkips.has(item.relativePath)) {
+                        planLimitPausedSkips.set(item.relativePath, pausedUntil);
+                        notePlanLimitedPath(item.relativePath);
+                    }
+                    skippedSinceCooperativeCheckpoint++;
+                    continue;
+                }
                 // Reserve the file's CURRENT on-disk size, not the planned size: an
                 // active log/session file can grow between planning and upload, and
                 // uploadFile buffers the whole current body. A stale planned size would
@@ -3135,6 +3257,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
             }
         }
     }
+    finishPlanLimitPass();
     return { aborted, abortFlightConflictPaths, workerErrors, hadUploadFailure };
 }
 /**
@@ -3946,6 +4069,9 @@ function buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths,
         linkedSubtreesNotShipped: [...run.linkedSubtreeSet],
         conflictPaths,
         pathResults,
+        filesPlanLimited: counters.filesPlanLimited,
+        planLimitedPaths: counters.planLimitedPaths,
+        ...(counters.planLimit ? { planLimit: counters.planLimit } : {}),
         aborted,
     };
 }
@@ -3960,7 +4086,7 @@ function defaultConsoleLogger(event) {
         }
     }
     else if (event.type === "plan-limit") {
-        console.warn(`  ! New files are paused by your plan. Upgrade: ${event.upgradeUrl}`);
+        console.warn(formatPlanLimitNotice(event));
     }
     else if (event.type === "progress") {
         if (event.deleted) {