@indigoai-us/hq-cloud 6.18.60 → 6.18.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli/share.js CHANGED
@@ -7,6 +7,7 @@
7
7
  import * as fs from "fs";
8
8
  import * as os from "os";
9
9
  import * as path from "path";
10
+ import { hasCloudPlaceholderSibling, isCloudOnlyLocally, isCloudPlaceholderName, } from "../cloud-placeholder.js";
10
11
  import { resolveSymlinkTargetRefusalFlag } from "../sync/symlink-target-refusal-flag.js";
11
12
  import { inspectLocalSymlinkScope, inspectSymlinkTarget, materializedTargetMatchesWire, personalSymlinkTargetLinkPath, } from "../symlink-target-policy.js";
12
13
  import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
@@ -21,7 +22,7 @@ import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemo
21
22
  import * as crypto from "crypto";
22
23
  import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, } from "../object-io.js";
23
24
  import { primePrivateFolderUpload } from "../private-folder-transport.js";
24
- import { isUnderAnyRoot, readJournalForPass, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
25
+ import { getJournalPath, isUnderAnyRoot, readJournalForPass, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
25
26
  import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isPathExcludedFromSync, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
26
27
  import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
27
28
  import { isSecretBearingVaultPath, isSkipSecretPathsEnabled, } from "../sync/secret-path-policy.js";
@@ -33,6 +34,7 @@ import { compileScopeMatcher, } from "../prefix-coalesce.js";
33
34
  import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, resolveUploadByteBudget, readHostLoad, } from "../sync-core.js";
34
35
  import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
35
36
  import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
37
+ import { findLandedUpload, followLandedUploads, SamePathUploadHold, samePathUploadKey, SamePathUploadSupersededError, } from "../lib/same-path-upload-lock.js";
36
38
  import { isLocallyIneligibleForPush } from "../lib/cloud-authoritative.js";
37
39
  import { isOntologyEntityKey, ONTOLOGY_MULTI_WRITER_FLAG, resolveOntologyEntityPush, } from "../lib/ontology-entity-push.js";
38
40
  import { entityIsFreshForCreate, outboxAllowsEntityPush, readOntologyOutbox, } from "../lib/ontology-outbox.js";
@@ -2533,7 +2535,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2533
2535
  });
2534
2536
  return true;
2535
2537
  };
2536
- const processUploadItem = async (item, replanAttempts = 0) => {
2538
+ const processUploadItem = async (item, replanAttempts = 0, samePathHold) => {
2537
2539
  if (aborted)
2538
2540
  return;
2539
2541
  const { absolutePath, relativePath, localHash } = item;
@@ -2566,8 +2568,35 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2566
2568
  counters.filesSkipped++;
2567
2569
  return;
2568
2570
  }
2569
- await processUploadItem(refreshed, replanAttempts + 1);
2571
+ await processUploadItem(refreshed, replanAttempts + 1, samePathHold);
2570
2572
  };
2573
+ // Another pusher in this process landed this key after the decision above
2574
+ // was made. Decide again from the etag it wrote; the restart budget in
2575
+ // restartForChangedBody bounds how often a hot key can do this.
2576
+ const restartAfterSupersede = async () => {
2577
+ const refreshed = refreshUploadItem();
2578
+ if (refreshed === null)
2579
+ return;
2580
+ await restartForChangedBody(refreshed);
2581
+ };
2582
+ // Same-path serialization: a watcher push and this pass may both push
2583
+ // this key in one process. Hold the per-path lock across the decision
2584
+ // and the PUT, and first adopt anything another pusher in this process
2585
+ // landed for the key, so this push fences on the etag that pusher wrote
2586
+ // instead of hitting a 412 or a stale two-sided conflict.
2587
+ if (samePathHold) {
2588
+ await samePathHold.acquire();
2589
+ if (samePathHold.takeWaited()) {
2590
+ // Another pusher held this key; the body may have moved meanwhile.
2591
+ const afterLock = refreshUploadItem();
2592
+ if (afterLock === null)
2593
+ return;
2594
+ if (!uploadBodyMatchesPlan(item, afterLock)) {
2595
+ await restartForChangedBody(afterLock);
2596
+ return;
2597
+ }
2598
+ }
2599
+ }
2571
2600
  // Set when `publish-local` was chosen for a FRESH two-sided conflict rather
2572
2601
  // than for the `localDiverges` recovery. The two cases need different write
2573
2602
  // fences — see the precondition block below.
@@ -2611,6 +2640,21 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2611
2640
  }
2612
2641
  throw headErr;
2613
2642
  }
2643
+ // The remote now sits at an etag a pusher in this process wrote. That is
2644
+ // proof of what the object holds, so take it as this run's baseline: the
2645
+ // decision below then sees an ordinary one-sided local change (or no
2646
+ // change) instead of a stale fence or a two-sided conflict.
2647
+ if (samePathHold &&
2648
+ remoteMeta &&
2649
+ adoptLandedSamePathUpload(run, item, findLandedUpload(samePathHold.key, relativePath, normalizeEtag(remoteMeta.etag)))) {
2650
+ const afterAdopt = refreshUploadItem();
2651
+ if (afterAdopt === null)
2652
+ return;
2653
+ if (!uploadBodyMatchesPlan(item, afterAdopt)) {
2654
+ await restartForChangedBody(afterAdopt);
2655
+ return;
2656
+ }
2657
+ }
2614
2658
  // A fresh archived session log has no journal row, so it always needs the
2615
2659
  // remote body to decide a collision. HEAD already proves that body cannot
2616
2660
  // be read; preserve that cheap skip before the post-HEAD refresh hashes the
@@ -3053,7 +3097,16 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3053
3097
  : undefined;
3054
3098
  if (!isSymlinkUpload)
3055
3099
  setUploadContentHash(absolutePath, localHash);
3056
- const journalSnapshot = JSON.stringify(journalRows(run.journal).get(relativePath) ?? null);
3100
+ const rowBeforePut = journalRows(run.journal).get(relativePath);
3101
+ const journalSnapshot = JSON.stringify(rowBeforePut ?? null);
3102
+ const landedFromEtags = [
3103
+ rowBeforePut?.remoteEtag ? normalizeEtag(rowBeforePut.remoteEtag) : null,
3104
+ ];
3105
+ if (pc && "ifMatch" in pc && pc.ifMatch) {
3106
+ const fenced = normalizeEtag(pc.ifMatch);
3107
+ if (!landedFromEtags.includes(fenced))
3108
+ landedFromEtags.push(fenced);
3109
+ }
3057
3110
  const isPrivateFolderKey = run.privateFolderMatcher.coversPath(relativePath);
3058
3111
  const selectedTransport = isPrivateFolderKey
3059
3112
  ? run.privateFolderTransport
@@ -3110,10 +3163,51 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3110
3163
  throw err;
3111
3164
  }
3112
3165
  };
3166
+ const sendAndRecordLanded = async () => {
3167
+ const result = await sendPayload();
3168
+ samePathHold?.recordLanded({
3169
+ relativePath,
3170
+ fromEtags: landedFromEtags,
3171
+ etag: normalizeEtag(result.etag),
3172
+ hash: localHash,
3173
+ size,
3174
+ mtimeMs,
3175
+ ctimeMs,
3176
+ kind: item.kind,
3177
+ ...(item.kind === "symlink" ? { wireSymlinkTarget: item.target } : {}),
3178
+ });
3179
+ return result;
3180
+ };
3113
3181
  const runNetworkTransfer = run.options.cooperativeCheckpoint?.runChoreNetworkTransfer;
3114
- const transfer = runNetworkTransfer
3115
- ? await runNetworkTransfer(sendPayload)
3116
- : { value: await sendPayload(), realtimeRan: false, complete: () => undefined };
3182
+ let transfer;
3183
+ if (runNetworkTransfer) {
3184
+ // The hand-off lets realtime passes run until every in-flight chore
3185
+ // transfer settles and realtime drains. Holding the path lock across
3186
+ // that wait would deadlock a watcher push of this key, so release it
3187
+ // now, take it back for the PUT itself, and restart the decision if
3188
+ // another pusher landed this key in between.
3189
+ const decidedAtSeq = samePathHold?.snapshot();
3190
+ samePathHold?.release();
3191
+ transfer = await runNetworkTransfer(async () => {
3192
+ if (samePathHold) {
3193
+ await samePathHold.acquire();
3194
+ samePathHold.takeWaited();
3195
+ if (decidedAtSeq !== undefined && samePathHold.landedSince(decidedAtSeq)) {
3196
+ samePathHold.release();
3197
+ throw new SamePathUploadSupersededError(relativePath);
3198
+ }
3199
+ }
3200
+ try {
3201
+ return await sendAndRecordLanded();
3202
+ }
3203
+ finally {
3204
+ samePathHold?.release();
3205
+ }
3206
+ });
3207
+ }
3208
+ else {
3209
+ transfer = { value: await sendAndRecordLanded(), realtimeRan: false, complete: () => undefined };
3210
+ }
3117
3211
  const { etag, alreadyCurrent } = transfer.value;
3118
3212
  uploadCompleted = true;
3119
3213
  // The PUT has completed at this point. A realtime pass may have advanced
@@ -3136,7 +3230,22 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3136
3230
  if (transfer.realtimeRan) {
3137
3231
  const latestJournal = readJournalScoped(run.journalSlug, [relativePath]);
3138
3232
  const latestEntry = journalRows(latestJournal).get(relativePath);
3139
- if (JSON.stringify(latestEntry ?? null) !== journalSnapshot) {
3233
+ // A watcher push admitted during the hand-off may have built on the
3234
+ // object just written. Its row is the newer baseline for this key.
3235
+ const builtOnThisPut = samePathHold
3236
+ ? followLandedUploads(samePathHold.key, relativePath, normalizeEtag(etag))
3237
+ : undefined;
3238
+ if (builtOnThisPut) {
3239
+ adoptLandedSamePathUpload(run, item, builtOnThisPut);
3240
+ return;
3241
+ }
3242
+ // A row on disk at an etag this process has since replaced came from
3243
+ // a pass that never saw those uploads. Adopting it would rewind the
3244
+ // journal behind the object just written.
3245
+ const diskRowSuperseded = samePathHold && latestEntry
3246
+ ? followLandedUploads(samePathHold.key, relativePath, latestEntry.remoteEtag ? normalizeEtag(latestEntry.remoteEtag) : null) !== undefined
3247
+ : false;
3248
+ if (!diskRowSuperseded && JSON.stringify(latestEntry ?? null) !== journalSnapshot) {
3140
3249
  if (latestEntry)
3141
3250
  journalRows(run.journal).set(relativePath, latestEntry);
3142
3251
  else
@@ -3203,6 +3312,10 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3203
3312
  await performUpload(precondition);
3204
3313
  }
3205
3314
  catch (err) {
3315
+ if (err instanceof SamePathUploadSupersededError) {
3316
+ await restartAfterSupersede();
3317
+ return;
3318
+ }
3206
3319
  // A session-level auth refusal (vault 401/403, or a mid-pass Cognito
3207
3320
  // identity refusal from the bearer getter) is a run-level fact, not a
3208
3321
  // per-file upload failure — propagate it so one refusal aborts the pass
@@ -3319,6 +3432,10 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3319
3432
  await performUpload({ ifMatch: normalizeEtag(latest.etag) });
3320
3433
  }
3321
3434
  catch (retryErr) {
3435
+ if (retryErr instanceof SamePathUploadSupersededError) {
3436
+ await restartAfterSupersede();
3437
+ return;
3438
+ }
3322
3439
  if (retryErr instanceof VaultAuthError ||
3323
3440
  isTerminalIdentityRefusal(retryErr) ||
3324
3441
  isRetryableIdentityRefusal(retryErr)) {
@@ -3418,6 +3535,10 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3418
3535
  await performUpload(undefined);
3419
3536
  }
3420
3537
  catch (retryErr) {
3538
+ if (retryErr instanceof SamePathUploadSupersededError) {
3539
+ await restartAfterSupersede();
3540
+ return;
3541
+ }
3421
3542
  if (retryErr instanceof VaultAuthError ||
3422
3543
  isTerminalIdentityRefusal(retryErr) ||
3423
3544
  isRetryableIdentityRefusal(retryErr)) {
@@ -3589,7 +3710,16 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3589
3710
  admittedSinceCooperativeCheckpoint++;
3590
3711
  inFlightBytes += nextBytes;
3591
3712
  const upload = async () => {
3592
- await processUploadItem(item);
3713
+ const samePathHold = new SamePathUploadHold(
3714
+ // One journal file is one lock namespace; the same relative
3715
+ // path in another vault or state dir never shares a lock.
3716
+ samePathUploadKey(getJournalPath(run.journalSlug), item.relativePath), item.relativePath);
3717
+ try {
3718
+ await processUploadItem(item, 0, samePathHold);
3719
+ }
3720
+ finally {
3721
+ samePathHold.release();
3722
+ }
3593
3723
  if (realtimeIsWaiting())
3594
3724
  realtimeYieldRequested = true;
3595
3725
  };
@@ -3667,6 +3797,41 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
3667
3797
  finishPlanLimitPass();
3668
3798
  return { aborted, abortFlightConflictPaths, workerErrors, hadUploadFailure };
3669
3799
  }
3800
+ /**
3801
+ * Overlay an upload another pusher in this process landed for the key onto
3802
+ * this run's journal. Each pass keeps its own journal, so without this a
3803
+ * second pusher fences on an etag the first one already replaced. The overlay
3804
+ * states a fact this process observed: the object at that etag holds that
3805
+ * hash. Returns false when there is nothing to adopt or the row already says so.
3806
+ */
3807
+ function adoptLandedSamePathUpload(run, item, landed) {
3808
+ if (!landed)
3809
+ return false;
3810
+ const row = journalRows(run.journal).get(item.relativePath);
3811
+ if (row &&
3812
+ row.hash === landed.hash &&
3813
+ row.remoteEtag !== undefined &&
3814
+ normalizeEtag(row.remoteEtag) === landed.etag &&
3815
+ !row.localDiverges) {
3816
+ return false;
3817
+ }
3818
+ try {
3819
+ updateEntry(run.journal, item.relativePath, landed.hash, landed.size, "up", item.absolutePath, {
3820
+ remoteEtag: landed.etag,
3821
+ mtimeMs: landed.mtimeMs,
3822
+ ...(landed.ctimeMs !== undefined ? { ctimeMs: landed.ctimeMs } : {}),
3823
+ kind: landed.kind,
3824
+ ...(landed.wireSymlinkTarget !== undefined
3825
+ ? { wireSymlinkTarget: landed.wireSymlinkTarget }
3826
+ : {}),
3827
+ });
3828
+ }
3829
+ catch {
3830
+ // The local path vanished; the normal replan settles it.
3831
+ return false;
3832
+ }
3833
+ return true;
3834
+ }
3670
3835
  /**
3671
3836
  * Exact-prefix and storage-class-specific, matching the pull planner: only
3672
3837
  * session transcripts in DEEP_ARCHIVE are frozen. Other archived objects keep
@@ -4180,6 +4345,14 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
4180
4345
  materializedKnowledgeDir: false,
4181
4346
  })),
4182
4347
  ];
4348
+ for (const relativePath of deletePlan.placeholderRefused ?? []) {
4349
+ pathResults.push({
4350
+ path: relativePath,
4351
+ status: "refused",
4352
+ operation: "delete",
4353
+ reason: "cloud-placeholder",
4354
+ });
4355
+ }
4183
4356
  const deleteKeys = deleteItems.map((item) => item.key);
4184
4357
  await primeObjectTransport(run.ctx, "delete", deleteKeys);
4185
4358
  for (const item of deleteItems) {
@@ -4205,8 +4378,11 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
4205
4378
  const localPath = localPathForVaultKey(run.syncRoot, relativePath);
4206
4379
  const localStateStillSafe = item.materializedKnowledgeDir
4207
4380
  ? isMaterializedKnowledgeDirectory(localPath, relativePath, entry, run.options.personalMode !== true)
4208
- : isLocallyAbsent(localPath);
4381
+ : isLocallyAbsent(localPath) && !hasCloudPlaceholderSibling(localPath);
4209
4382
  const recreatedLocally = !item.decommission && !localStateStillSafe;
4383
+ // `hq storage offload` wrote `<name>.hqcloud` after planning: the file
4384
+ // is cloud-only now, not deleted.
4385
+ const placeholderAppeared = recreatedLocally && !item.materializedKnowledgeDir && hasCloudPlaceholderSibling(localPath);
4210
4386
  const intentChanged = !item.decommission &&
4211
4387
  item.intentVersion !== null &&
4212
4388
  (!hasCurrentLocalDeleteIntent(entry) ||
@@ -4216,9 +4392,11 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
4216
4392
  const { localDeleteIntent: _localDeleteIntent, ...cleared } = entry;
4217
4393
  journalRows(run.journal).set(relativePath, cleared);
4218
4394
  }
4219
- const refusalReason = recreatedLocally
4220
- ? "recreated-locally"
4221
- : "intent-changed";
4395
+ const refusalReason = placeholderAppeared
4396
+ ? "cloud-placeholder"
4397
+ : recreatedLocally
4398
+ ? "recreated-locally"
4399
+ : "intent-changed";
4222
4400
  counters.filesRefusedStale++;
4223
4401
  if (filesRefusedStalePaths.length < REFUSED_STALE_PATH_CAP) {
4224
4402
  filesRefusedStalePaths.push(relativePath);
@@ -4818,6 +4996,10 @@ function defaultConsoleLogger(event) {
4818
4996
  console.error(` ⚠ recreated-locally, kept on remote: ${event.path} ` +
4819
4997
  `(the file exists locally again, so the delete was abandoned)`);
4820
4998
  }
4999
+ else if (event.reason === "cloud-placeholder") {
5000
+ console.error(` ⚠ cloud-placeholder, kept on remote: ${event.path} ` +
5001
+ `(a .hqcloud placeholder stands in for this file, so it was not deleted)`);
5002
+ }
4821
5003
  else if (event.reason === "marker-mismatch") {
4822
5004
  console.error(` ⚠ marker-mismatch, kept on remote: ${event.path} ` +
4823
5005
  `(the remote object is not the symlink revision reindex replaced)`);
@@ -5265,7 +5447,8 @@ async function collectFiles(paths, hqRoot, syncRoot, filter, hooks = {}) {
5265
5447
  }
5266
5448
  else if (lstat.isFile()) {
5267
5449
  const relativePath = vaultKeyForLocalPath(syncRoot, absolutePath);
5268
- if (filter(absolutePath)) {
5450
+ // Cloud-only placeholders are local stand-ins; never upload them.
5451
+ if (filter(absolutePath) && !isCloudPlaceholderName(path.basename(absolutePath))) {
5269
5452
  noteCooperativeRowVisit("collect", relativePath);
5270
5453
  results.push({ kind: "file", absolutePath, relativePath });
5271
5454
  }
@@ -5357,6 +5540,10 @@ async function walkDir(dir, syncRoot, filter, results, hooks = {}, cooperativeWo
5357
5540
  enterDirectory(absolutePath);
5358
5541
  }
5359
5542
  else if (entry.isFile()) {
5543
+ // `.hqcloud` placeholders are local stand-ins; never upload one, valid
5544
+ // or not (an invalid one may still name a teammate-visible file).
5545
+ if (isCloudPlaceholderName(entry.name))
5546
+ continue;
5360
5547
  const relativePath = vaultKeyForLocalPath(syncRoot, absolutePath);
5361
5548
  noteCooperativeRowVisit("collect", relativePath);
5362
5549
  results.push({
@@ -5816,6 +6003,17 @@ companyScoped = true, skipSecretPaths = false, excludedPaths = new Set()) {
5816
6003
  isMaterializedKnowledgeDirectory(localPath, relativeKey, entry, companyScoped);
5817
6004
  if (presentLocally && !materializedKnowledgeDir)
5818
6005
  continue;
6006
+ // `hq storage offload` replaced this file with a `<name>.hqcloud`
6007
+ // placeholder: the bytes live in the cloud, the user did not delete it.
6008
+ // ANY sibling placeholder blocks the delete. A valid one is simply not a
6009
+ // candidate; an invalid one (malformed, wrong path, future format) is
6010
+ // reported as a refused delete so the run does not look converged.
6011
+ if (!presentLocally && hasCloudPlaceholderSibling(localPath)) {
6012
+ if (!isCloudOnlyLocally(localPath)) {
6013
+ (plan.placeholderRefused ??= []).push(relativeKey);
6014
+ }
6015
+ continue;
6016
+ }
5819
6017
  // Reindex's legacy Knowledge migration is a representation transition,
5820
6018
  // not a user-content deletion: the exact key changed from a symlink object
5821
6019
  // to a real directory whose children are uploaded separately. The normal