@indigoai-us/hq-cloud 6.18.94 → 6.18.95

package/dist/cli/share.js

@@ -23,10 +23,10 @@ import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemo
 import * as crypto from "crypto";
 import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, } from "../object-io.js";
 import { primePrivateFolderUpload } from "../private-folder-transport.js";
-import { getJournalPath, isUnderAnyRoot, readJournalForPass, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
-import { createIgnoreExplainer, createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isMachineLocalRuntimePath, isPathExcludedFromSync, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
+import { getJournalPath, isUnderAnyRoot, readJournalForPass, readJournalReadOnly, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, configureRejectedDeltaRecovery, getJournalRecoveryMarker, } from "../journal.js";
+import { createIgnoreExplainer, createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isHqGeneratedIndexFile, isMachineLocalRuntimePath, isPathExcludedFromSync, isSecretFolderPath, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
 import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
-import { claimNeverSyncedLocalNotice, formatSecretsNotSyncedLine, isNeverSyncedVaultPath, isSecretBearingVaultPath, isSkipSecretPathsEnabled, } from "../sync/secret-path-policy.js";
+import { claimNeverSyncedLocalNotices, formatSecretsNotSyncedLine, isNeverSyncedVaultPath, isSecretBearingVaultPath, isSkipSecretPathsEnabled, isWalRejectedDeltaRecoveryEnabled, } from "../sync/secret-path-policy.js";
 import { getActiveFullPassDecodeWindow, getActiveFullPassUnitSize, } from "../sync/full-pass-caps.js";
 import { isGeneratedCoreMirrorKey, isPersonalVaultDeletionExcluded, isSessionLogDeletionKey, purgeExcludedPersonalVaultJournalEntries, } from "../personal-vault.js";
 import { pickWinner, resolveConflict, usesVersionAwareWinner } from "./conflict.js";
@@ -885,10 +885,13 @@ export class UnreachablePushPathsError extends Error {
     paths;
     /** Per-path reason, keyed by the same original spelling. */
     reasons;
-    constructor(unreachable, syncRoot) {
+    constructor(unreachable, syncRoot, ignoreRules = new Map()) {
         const paths = [...unreachable.keys()];
         const lines = paths.map((p) => {
             const reason = unreachable.get(p);
+            if (reason === "ignored") {
+                return `  · ${p} — ${ignoreRules.get(p) ?? "excluded by a sync ignore rule"}; it was not uploaded`;
+            }
             if (reason === "outside-company") {
                 return `  · ${p} — resolves outside the company folder (${syncRoot})`;
             }
@@ -1367,6 +1370,10 @@ async function createPushRunContext(options) {
             ? await isSkipSecretPathsEnabled(secretPathFlagConfig, ctx.uid, true)
             : await isSkipSecretPathsEnabled(secretPathFlagConfig, ctx.uid)
         : preVendedCompanyPush;
+    const walRecoveryEnabled = secretPathFlagConfig
+        ? await isWalRejectedDeltaRecoveryEnabled(secretPathFlagConfig, ctx.uid)
+        : true;
+    configureRejectedDeltaRecovery(walRecoveryEnabled);
     const ignoreFilter = createIgnoreFilter(hqRoot, {
         allowRootMcpConfig: personalConnectorSettingsSync,
     });
@@ -1429,8 +1436,7 @@ async function createPushRunContext(options) {
             return false;
         return baseFilter(filePath, isDir);
     };
-    const neverSyncedLocalPath = claimNeverSyncedLocalNotice(hqRoot, syncRoot);
-    if (neverSyncedLocalPath) {
+    for (const neverSyncedLocalPath of claimNeverSyncedLocalNotices(hqRoot, syncRoot)) {
         emit({ type: "secrets-not-synced", path: neverSyncedLocalPath });
     }
     const shouldSync = options.prefixSet !== undefined
@@ -1453,6 +1459,7 @@ async function createPushRunContext(options) {
     // known. Full/cadence callers retain the complete snapshot here.
     const scopedJournal = options.scopedJournal === true;
     const journal = scopedJournal ? emptyJournalForScopedPass() : readJournalForPass(journalSlug, ctx.uid);
+    const holdRemoteDeletes = getJournalRecoveryMarker(journalSlug) !== undefined;
     return {
         options,
         ontologyEntityPush: false,
@@ -1481,7 +1488,15 @@ async function createPushRunContext(options) {
         syncRoot,
         shouldSync,
         durableShouldSync: baseFilter,
+        ignoreFilter,
+        explainIgnore: (filePath, isDir) => {
+            explainIgnored ??= createIgnoreExplainer(hqRoot, {
+                allowRootMcpConfig: personalConnectorSettingsSync,
+            });
+            return explainIgnored(filePath, isDir);
+        },
         skipSecretPaths,
+        holdRemoteDeletes,
         journalSlug,
         quarantinedPaths,
         journal,
@@ -1489,12 +1504,14 @@ async function createPushRunContext(options) {
         excludedSet,
         excludedById,
         scopeExcludedSet,
+        uploadFailedSet: new Set(),
         ignoreExcludedSet,
         ignoreExcludedTotal,
         ignoreExcludedLayers,
         ignoreIncludeLines,
         secretExcludedSet,
         unreachablePaths,
+        ignoredNamedPathRules: new Map(),
         linkedSubtreeSet,
         skillRegistrationConflicts,
         hostLoadAtStart,
@@ -2184,6 +2201,7 @@ async function buildSharePlans(run) {
             if (rememberLinkWarning(run.linkWarnings, warning))
                 run.emit(brokenLinkSkipEvent(warning));
         },
+        onFilteredNamedPath: (namedPath, absolutePath, isDir) => recordIgnoredNamedPath(run, namedPath, absolutePath, isDir),
         onLinkedSubtree: (rel) => run.linkedSubtreeSet.add(rel),
         onHeartbeat: run.options.onHeartbeat,
         cooperativeCheckpoint: run.options.cooperativeCheckpoint,
@@ -2200,7 +2218,7 @@ async function buildSharePlans(run) {
     const fatal = collectFatalUnreachablePaths(run);
     if (fatal.size > 0) {
         emitUnreachablePathEvent(run);
-        throw new UnreachablePushPathsError(fatal, run.syncRoot);
+        throw new UnreachablePushPathsError(fatal, run.syncRoot, run.ignoredNamedPathRules);
     }
     const deleteScopeRoots = run.propagateDeletes === true
         ? resolveDeleteScopeRoots(run.paths, run.hqRoot, run.syncRoot, run.options.deleteScopeRoots)
@@ -2291,8 +2309,9 @@ async function buildSharePlans(run) {
         companySlug: run.ctx.slug,
         reportedUnsafeSymlinkPaths: run.reportedUnsafeSymlinkPaths,
     });
+    const recoveryHoldActive = run.holdRemoteDeletes || getJournalRecoveryMarker(run.journalSlug) !== undefined;
     const deletePlan = run.propagateDeletes === true
-        ? await computeDeletePlan(run.journal, run.syncRoot, deleteScopeRoots, run.shouldSync, run.propagateDeletePolicy, run.ctx, run.options.personalMode !== true, run.skipSecretPaths, run.quarantinedPaths)
+        ? await computeDeletePlan(run.journal, run.syncRoot, deleteScopeRoots, run.shouldSync, run.propagateDeletePolicy, run.ctx, run.options.personalMode !== true, run.skipSecretPaths, run.quarantinedPaths, recoveryHoldActive, run.scopedJournal ? () => readJournalReadOnly(run.journalSlug) : undefined)
         : { toDelete: [], toTombstone: [], refusedStale: [] };
     for (const warning of deletePlan.brokenLinkSkipped ?? []) {
         if (rememberLinkWarning(run.linkWarnings, warning))
@@ -3174,6 +3193,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                     if (outcome === "write-failed") {
                         // The merge already emitted the error. A second skip line repeats it.
                         hadUploadFailure = true;
+                        run.uploadFailedSet.add(relativePath);
                     }
                     counters.filesSkipped++;
                     return;
@@ -3232,6 +3252,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                             if (outcome === "write-failed") {
                                 // The merge already emitted the error. A second skip line repeats it.
                                 hadUploadFailure = true;
+                                run.uploadFailedSet.add(relativePath);
                             }
                             counters.filesSkipped++;
                             return;
@@ -3710,6 +3731,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                             if (outcome === "write-failed") {
                                 // The merge already emitted the error. A second skip line repeats it.
                                 hadUploadFailure = true;
+                                run.uploadFailedSet.add(relativePath);
                             }
                             else if (outcome === "publish-local") {
                                 counters.filesSkippedConflict++;
@@ -3788,6 +3810,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                                 return;
                             }
                             hadUploadFailure = true;
+                            run.uploadFailedSet.add(relativePath);
                             run.emit({
                                 type: "error",
                                 path: relativePath,
@@ -3890,6 +3913,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                         if (emitPlanLimitSkip(retryErr, relativePath))
                             return;
                         hadUploadFailure = true;
+                        run.uploadFailedSet.add(relativePath);
                         run.emit({
                             type: "error",
                             path: relativePath,
@@ -3939,6 +3963,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                 });
             }
             else {
+                run.uploadFailedSet.add(relativePath);
                 run.emit({
                     type: "error",
                     path: relativePath,
@@ -5210,11 +5235,59 @@ function collectFatalUnreachablePaths(run) {
     if (run.options.unreachablePathPolicy === "warn")
         return fatal;
     for (const [namedPath, reason] of run.unreachablePaths) {
-        if (reason === "outside-company")
+        if (reason === "outside-company" || reason === "ignored")
             fatal.set(namedPath, reason);
     }
     return fatal;
 }
+/**
+ * A caller-named path that the ignore stack rejects was dropped with no event:
+ * `.hq-*` names and generated INDEX.md files count as expected noise, so the
+ * ignore warning skips them, and the push printed "✓ Pushed 0 file(s)" with
+ * exit 0 (feedback_63bfc302, feedback_8c294805). Record it as `"ignored"` so
+ * the default policy fails the push and names the rule.
+ *
+ * Not recorded:
+ * - Runs without `failOnIgnoredNamedPaths`. Their paths are enumerated roots,
+ *   not user input, and an ignored root there is routine.
+ * - `"warn"` runs. Their paths are watcher and walk roots, where an ignored
+ *   file is routine and already filtered.
+ * - Paths the raw ignore stack admits. Scope, personal-vault, and secret-path
+ *   wrappers rejected those and report them through their own events.
+ * - Secret-pattern files (dotenv files, credentials) and secret folders
+ *   (company `secrets/` and `settings/`, `.secrets`). They are left out by
+ *   design (US-178), and watcher runs name them routinely.
+ */
+function recordIgnoredNamedPath(run, namedPath, absolutePath, isDir) {
+    if (run.options.failOnIgnoredNamedPaths !== true)
+        return;
+    if (run.options.unreachablePathPolicy === "warn")
+        return;
+    if (run.unreachablePaths.has(namedPath))
+        return;
+    if (run.ignoreFilter(absolutePath, isDir))
+        return;
+    const rel = vaultKeyForLocalPath(run.hqRoot, absolutePath);
+    if (isSecretFolderPath(rel))
+        return;
+    if (!isDir && isSecretPatternIgnore(rel))
+        return;
+    run.unreachablePaths.set(namedPath, "ignored");
+    run.ignoredNamedPathRules.set(namedPath, describeIgnoreRule(run, absolutePath, isDir));
+}
+function describeIgnoreRule(run, absolutePath, isDir) {
+    if (!isDir && isHqGeneratedIndexFile(absolutePath)) {
+        return "an HQ-generated INDEX.md (it starts with \"> Auto-generated.\"), which each machine rebuilds and sync never uploads";
+    }
+    const { layer } = run.explainIgnore(absolutePath, isDir);
+    if (layer === ".hqignore" || layer === ".hqsyncignore") {
+        return `excluded by a rule in ${layer}`;
+    }
+    if (layer === ".hqinclude allowlist") {
+        return "not listed in .hqinclude";
+    }
+    return "excluded by a built-in sync ignore rule";
+}
 /**
  * Emit the `not-shipped` / `unreachable-path` event for a run, if any named
  * path went unshipped. Shared by BOTH policies so the operator sees the same
@@ -5311,6 +5384,8 @@ function buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths,
         filesTombstoned: counters.filesTombstoned,
         filesRefusedStale: counters.filesRefusedStale,
         filesRefusedStalePaths,
+        filesUploadFailed: run.uploadFailedSet.size,
+        uploadFailedPaths: [...run.uploadFailedSet].slice(0, REFUSED_STALE_PATH_CAP),
         filesSuppressedByTombstone: counters.filesSuppressedByTombstone,
         ...(run.skillRegisterLandedBytes
             ? { skillRegistrationFallbacks: run.skillRegistrationFallbacks }
@@ -5448,6 +5523,9 @@ function defaultConsoleLogger(event, humanSkipLines) {
             console.error(`  ⚠ cloud-placeholder, kept on remote: ${event.path} ` +
                 `(a .hqcloud placeholder stands in for this file, so it was not deleted)`);
         }
+        else if (event.reason === "state-rebuilt-deletes-held") {
+            console.error(`  ⚠ state-rebuilt-deletes-held, kept on remote: ${event.path} (complete a full authoritative sync pass to release the hold)`);
+        }
         else if (event.reason === "marker-mismatch") {
             console.error(`  ⚠ marker-mismatch, kept on remote: ${event.path} ` +
                 `(the remote object is not the symlink revision reindex replaced)`);
@@ -5895,8 +5973,10 @@ async function collectFiles(paths, hqRoot, syncRoot, filter, hooks = {}) {
             // symlinks under allowlist mode (the motivating case for this
             // whole branch). The filter is pure path lookup with no I/O,
             // so two calls are free.
-            if (!filter(absolutePath, false) && !filter(absolutePath, true))
+            if (!filter(absolutePath, false) && !filter(absolutePath, true)) {
+                hooks.onFilteredNamedPath?.(p, absolutePath, false);
                 continue;
+            }
             const target = readlinkOrNull(absolutePath);
             if (target === null) {
                 console.error(`  Warning: ${p} is an unreadable symbolic link, skipping.`);
@@ -5924,17 +6004,22 @@ async function collectFiles(paths, hqRoot, syncRoot, filter, hooks = {}) {
             continue;
         }
         if (lstat.isDirectory()) {
-            if (!filter(absolutePath, true))
+            if (!filter(absolutePath, true)) {
+                hooks.onFilteredNamedPath?.(p, absolutePath, true);
                 continue;
+            }
             await walkDir(absolutePath, syncRoot, filter, results, hooks, cooperativeWork);
         }
         else if (lstat.isFile()) {
             const relativePath = vaultKeyForLocalPath(syncRoot, absolutePath);
+            if (!filter(absolutePath)) {
+                hooks.onFilteredNamedPath?.(p, absolutePath, false);
+                continue;
+            }
             // Cloud-only placeholders are local stand-ins; never upload them,
             // except library asset placeholders, which sync as small text files.
-            if (filter(absolutePath) &&
-                (!isCloudPlaceholderName(path.basename(absolutePath)) ||
-                    isSyncedCloudPlaceholderFile(absolutePath))) {
+            if (!isCloudPlaceholderName(path.basename(absolutePath)) ||
+                isSyncedCloudPlaceholderFile(absolutePath)) {
                 noteCooperativeRowVisit("collect", relativePath);
                 results.push({ kind: "file", absolutePath, relativePath });
             }
@@ -6232,6 +6317,13 @@ function resolveDeleteScopeRoots(paths, hqRoot, syncRoot, explicitRoots = []) {
  *     already opts out of safety gates (currency check, owned-only).
  *     This guard honors that explicit opt-out.
  *
+ * `inScope` in a watcher (scoped) push is NOT the delete roots alone. The
+ * watcher names each deleted key as its own delete root, so counting only
+ * rows under those roots makes the ratio ~100% and refuses every batch of
+ * MIN_ABS or more deletes, which the next pull then restores. When the local
+ * ratio trips, a scoped push re-counts the denominator over the whole journal
+ * (`bulkBaselineJournal`), the same rows a full pass's root would cover.
+ *
  * `"owned-only"` and `"currency-gated"` both go through the guard. The
  * `direction === "up"` filter in `owned-only` is no defense once a user
  * has ever run full bidirectional sync (every uploaded file gets
@@ -6388,7 +6480,10 @@ export async function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSyn
 // True for a COMPANY-scoped push (personalMode !== true). This distinguishes
 // the canonical knowledge-marker layout from the personal-vault layout; it
 // does not reinterpret ordinary vault-relative path segments.
-companyScoped = true, skipSecretPaths = false, excludedPaths = new Set()) {
+companyScoped = true, skipSecretPaths = false, excludedPaths = new Set(), holdRemoteDeletes = false, 
+// Scoped (watcher) pushes only: reads the whole journal so the breaker's
+// denominator matches a full pass. Called only when the scoped ratio trips.
+bulkBaselineJournal) {
     const plan = { toDelete: [], toTombstone: [], refusedStale: [] };
     if (scopeRoots.length === 0)
         return plan;
@@ -6517,6 +6612,15 @@ companyScoped = true, skipSecretPaths = false, excludedPaths = new Set()) {
                 continue;
             }
         }
+        if (holdRemoteDeletes && !presentLocally) {
+            plan.refusedStale.push({
+                key: relativeKey,
+                journalEtag: entry.remoteEtag ?? "<unknown>",
+                remoteEtag: "<not-checked>",
+                reason: "state-rebuilt-deletes-held",
+            });
+            continue;
+        }
         const materializedKnowledgeDir = presentLocally &&
             isMaterializedKnowledgeDirectory(localPath, relativeKey, entry, companyScoped);
         if (presentLocally && !materializedKnowledgeDir)
@@ -6770,11 +6874,30 @@ companyScoped = true, skipSecretPaths = false, excludedPaths = new Set()) {
     // Bulk-asymmetry circuit-breaker. See `BULK_ASYMMETRY_*` constants for
     // rationale + bypass paths. Skipped under policy:"all" (caller asserted
     // intent) and under `HQ_SYNC_DELETE_BULK_OVERRIDE` (operator rollback knob).
-    if (policy !== "all" &&
+    const bulkRatioTrips = () => inScopeJournalEntries > 0 &&
+        bulkCandidatePicks / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO;
+    const bulkGuardApplies = policy !== "all" &&
         !isBulkAsymmetryOverride() &&
-        bulkCandidatePicks >= BULK_ASYMMETRY_MIN_ABS &&
-        inScopeJournalEntries > 0 &&
-        bulkCandidatePicks / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO) {
+        bulkCandidatePicks >= BULK_ASYMMETRY_MIN_ABS;
+    if (bulkGuardApplies && bulkRatioTrips() && bulkBaselineJournal) {
+        // The rows a full pass would count, with the same hard filters as the
+        // Stage-1 loop. Rows it could not lstat (broken links) are not removed,
+        // so this can only exceed a full pass's denominator by those rows.
+        let baseline = 0;
+        for (const [key, entry] of journalRows(bulkBaselineJournal()).entries()) {
+            if (excludedPaths.has(key))
+                continue;
+            if (isNeverSyncedVaultPath(key))
+                continue;
+            if (skipSecretPaths && isSecretBearingVaultPath(key))
+                continue;
+            if (isTombstone(entry) && entry.removedReason !== "local-delete")
+                continue;
+            baseline++;
+        }
+        inScopeJournalEntries = Math.max(inScopeJournalEntries, baseline);
+    }
+    if (bulkGuardApplies && bulkRatioTrips()) {
         // Move every staged candidate (both already-bucketed toDelete from
         // owned-only and queued headCandidates from currency-gated) into
         // refusedStale with reason "bulk-asymmetry", and re-label the intent-less