@indigoai-us/hq-cloud 6.18.103 → 6.18.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/legacy-sync-wake.d.ts +25 -0
- package/dist/bin/legacy-sync-wake.d.ts.map +1 -0
- package/dist/bin/legacy-sync-wake.js +159 -0
- package/dist/bin/legacy-sync-wake.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +6 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +76 -1
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +3 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +9 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/conflict.d.ts +8 -2
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +15 -10
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/share.d.ts +8 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +245 -34
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +10 -2
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +265 -5
- package/dist/cli/sync.js.map +1 -1
- package/dist/lib/same-path-upload-lock.d.ts +39 -1
- package/dist/lib/same-path-upload-lock.d.ts.map +1 -1
- package/dist/lib/same-path-upload-lock.js +177 -1
- package/dist/lib/same-path-upload-lock.js.map +1 -1
- package/dist/manifest/upload-manifest.d.ts +1 -0
- package/dist/manifest/upload-manifest.d.ts.map +1 -1
- package/dist/manifest/upload-manifest.js +2 -0
- package/dist/manifest/upload-manifest.js.map +1 -1
- package/dist/object-io.d.ts +12 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +28 -1
- package/dist/object-io.js.map +1 -1
- package/dist/outposts/command.d.ts.map +1 -1
- package/dist/outposts/command.js +3 -1
- package/dist/outposts/command.js.map +1 -1
- package/dist/outposts/company.d.ts +10 -1
- package/dist/outposts/company.d.ts.map +1 -1
- package/dist/outposts/company.js +17 -0
- package/dist/outposts/company.js.map +1 -1
- package/dist/push-identity-fence.d.ts +30 -0
- package/dist/push-identity-fence.d.ts.map +1 -1
- package/dist/push-identity-fence.js +95 -0
- package/dist/push-identity-fence.js.map +1 -1
- package/dist/sync-core.d.ts +13 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +22 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/types.d.ts +6 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/cli/share.js
CHANGED
|
@@ -24,10 +24,10 @@ import { readPendingConflictNotices, recordConflictParked } from "../conflict-no
|
|
|
24
24
|
import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
|
|
25
25
|
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
26
26
|
import * as crypto from "crypto";
|
|
27
|
-
import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, } from "../object-io.js";
|
|
27
|
+
import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, isPresignDeletedNewer, } from "../object-io.js";
|
|
28
28
|
import { primePrivateFolderUpload } from "../private-folder-transport.js";
|
|
29
29
|
import { getJournalPath, isUnderAnyRoot, readJournalForPass, bindPersonalVaultJournalToIdentity, readJournalReadOnly, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, configureRejectedDeltaRecovery, getJournalRecoveryMarker, getStateDir, personalVaultHasOtherIdentityState, readPersonalVaultOwnerUid, } from "../journal.js";
|
|
30
|
-
import { admitPushIdentity, machineIdentitySwitchRecorded, notePushIdentityHeld, } from "../push-identity-fence.js";
|
|
30
|
+
import { admitPushIdentity, machineIdentitySwitchRecorded, notePushIdentityHeld, noteCompanyRebind, } from "../push-identity-fence.js";
|
|
31
31
|
import { createIgnoreExplainer, createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isHqGeneratedIndexFile, isMachineLocalRuntimePath, isPathExcludedFromSync, isSecretFolderPath, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
32
32
|
import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
33
33
|
import { claimNeverSyncedLocalNotices, formatSecretsNotSyncedLine, isNeverSyncedVaultPath, isSecretBearingVaultPath, isSkipSecretPathsEnabled, isWalRejectedDeltaRecoveryEnabled, } from "../sync/secret-path-policy.js";
|
|
@@ -38,11 +38,11 @@ import { createHumanSkipLineLog, formatReconcileSkipLine, isUpToDateSkipReason,
|
|
|
38
38
|
import { fetchFileTombstones, } from "./tombstones.js";
|
|
39
39
|
import { compileScopeMatcher, } from "../prefix-coalesce.js";
|
|
40
40
|
import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, resolveUploadByteBudget, readHostLoad, } from "../sync-core.js";
|
|
41
|
-
import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
41
|
+
import { buildConflictBackupPath, CONFLICT_BACKUPS_DIR, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
42
42
|
import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
|
|
43
43
|
import { appendIfUnchanged, isAppendOnlyPath, mergeAppendOnly } from "../lib/append-only-merge.js";
|
|
44
44
|
import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/board-merge.js";
|
|
45
|
-
import { findLandedUpload, followLandedUploads, SamePathUploadHold, samePathUploadKey, SamePathUploadSupersededError, } from "../lib/same-path-upload-lock.js";
|
|
45
|
+
import { findLandedUpload, findUploadIntent, followLandedUploads, recordUploadIntent, SamePathUploadHold, samePathUploadKey, SamePathUploadSupersededError, settleUploadIntent, } from "../lib/same-path-upload-lock.js";
|
|
46
46
|
import { isCloudAuthoritative, isLocallyIneligibleForPush } from "../lib/cloud-authoritative.js";
|
|
47
47
|
import { groupServerOwnedSkips } from "../lib/server-owned-skips.js";
|
|
48
48
|
import { isOntologyEntityKey, ONTOLOGY_MULTI_WRITER_FLAG, resolveOntologyEntityPush, } from "../lib/ontology-entity-push.js";
|
|
@@ -114,6 +114,60 @@ async function remoteContentDiffers(ctx, relativePath, localHash, hqRoot, symlin
|
|
|
114
114
|
}
|
|
115
115
|
}
|
|
116
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* A push refused as `deleted-newer`: keep the live local file, copy its bytes
|
|
119
|
+
* once into `.hq/conflict-backups/` (skipped when a backup with the same bytes
|
|
120
|
+
* already exists for this key, so repeated passes do not pile up copies), and
|
|
121
|
+
* drop the journal row so the pull tombstone sweep cannot delete the file.
|
|
122
|
+
*/
|
|
123
|
+
function keepLocalOverDeletedNewer(run, absolutePath, relativePath) {
|
|
124
|
+
let backedUp = false;
|
|
125
|
+
try {
|
|
126
|
+
const localHash = hashFile(absolutePath);
|
|
127
|
+
const vaultKey = vaultKeyForLocalPath(run.hqRoot, absolutePath);
|
|
128
|
+
const probe = localPathForVaultKey(run.hqRoot, buildConflictBackupPath(vaultKey, new Date().toISOString(), readShortMachineId(run.hqRoot)));
|
|
129
|
+
// The backup must stay inside the sync-ignored backups directory. A key
|
|
130
|
+
// that resolves outside the HQ root skips the backup; the live file and
|
|
131
|
+
// the journal drop below still protect the bytes.
|
|
132
|
+
const backupsRoot = path.resolve(run.hqRoot, CONFLICT_BACKUPS_DIR);
|
|
133
|
+
const relToBackups = path.relative(backupsRoot, path.resolve(probe));
|
|
134
|
+
if (relToBackups.startsWith("..") || path.isAbsolute(relToBackups)) {
|
|
135
|
+
throw new Error("backup path escapes the conflict-backups directory");
|
|
136
|
+
}
|
|
137
|
+
const dir = path.dirname(probe);
|
|
138
|
+
const base = path.basename(vaultKey);
|
|
139
|
+
const existing = fs.existsSync(dir)
|
|
140
|
+
? fs.readdirSync(dir).filter((name) => name.startsWith(`${base}.`) && name !== base)
|
|
141
|
+
: [];
|
|
142
|
+
const already = existing.some((name) => {
|
|
143
|
+
try {
|
|
144
|
+
return hashFile(path.join(dir, name)) === localHash;
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
if (!already) {
|
|
151
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
152
|
+
fs.copyFileSync(absolutePath, probe, fs.constants.COPYFILE_EXCL);
|
|
153
|
+
}
|
|
154
|
+
backedUp = true;
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
// A failed backup still leaves the live file in place; the journal drop
|
|
158
|
+
// below is what keeps the pull from deleting it.
|
|
159
|
+
}
|
|
160
|
+
if (journalRows(run.journal).delete(relativePath))
|
|
161
|
+
flushShareJournal(run);
|
|
162
|
+
run.emit({
|
|
163
|
+
type: "error",
|
|
164
|
+
diagnostic: true,
|
|
165
|
+
path: relativePath,
|
|
166
|
+
message: `server has a newer deleted version of '${relativePath}', kept your copy` +
|
|
167
|
+
(backedUp ? " (backup in .hq/conflict-backups/)" : "") +
|
|
168
|
+
`. Restore the newer version with 'hq files restore --version-id', or delete your copy.`,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
117
171
|
/**
|
|
118
172
|
* Local-only ephemeral artifacts: conflict-side files written whenever a
|
|
119
173
|
* 3-way merge preserves the displaced side for inspection. Depending on the
|
|
@@ -1180,6 +1234,16 @@ async function runShareOnce(options, run) {
|
|
|
1180
1234
|
return buildShareResult(run, counters, filesRefusedStalePaths, uploadResult.abortFlightConflictPaths, pathResults, true);
|
|
1181
1235
|
}
|
|
1182
1236
|
await executeDeletes(run, plans.deletePlan, plans.decommissionPlan, counters, filesRefusedStalePaths, pathResults, !uploadResult.hadUploadFailure && uploadResult.workerErrors.length === 0);
|
|
1237
|
+
if (run.writeForbiddenSet.size > REFUSED_STALE_PATH_CAP) {
|
|
1238
|
+
const more = run.writeForbiddenSet.size - REFUSED_STALE_PATH_CAP;
|
|
1239
|
+
run.emit({
|
|
1240
|
+
type: "error",
|
|
1241
|
+
diagnostic: true,
|
|
1242
|
+
path: "(write-forbidden)",
|
|
1243
|
+
message: `${more} more files were refused by the vault (no write access, server put denied: FILES_PRESIGN_FORBIDDEN); ` +
|
|
1244
|
+
`${run.writeForbiddenSet.size} refused in total, kept local-only`,
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
1183
1247
|
finalizeShareJournal(run);
|
|
1184
1248
|
throwUploadWorkerErrors(uploadResult.workerErrors, run.emit);
|
|
1185
1249
|
// A reconcile cursor is an all-or-nothing boundary for this bounded apply
|
|
@@ -1508,7 +1572,12 @@ async function createPushRunContext(options) {
|
|
|
1508
1572
|
// known. Full/cadence callers retain the complete snapshot here.
|
|
1509
1573
|
const scopedJournal = options.scopedJournal === true;
|
|
1510
1574
|
const journal = scopedJournal ? emptyJournalForScopedPass() : readJournalForPass(journalSlug, ctx.uid);
|
|
1511
|
-
|
|
1575
|
+
// feedback_d1296824: the same person switched this folder to another
|
|
1576
|
+
// company. Files and journal rows on disk may be the previous company's, so
|
|
1577
|
+
// the push uploads and deletes nothing until a clean full pull settles it.
|
|
1578
|
+
const companyRebindHold = !personalScope && noteCompanyRebind(getStateDir(), journalSlug, journal.pulls, ctx.uid);
|
|
1579
|
+
const effectivePushFence = pushIdentityFence ?? (companyRebindHold ? { uid: "", sinceMs: Date.now() } : undefined);
|
|
1580
|
+
const holdRemoteDeletes = getJournalRecoveryMarker(journalSlug) !== undefined || effectivePushFence !== undefined;
|
|
1512
1581
|
return {
|
|
1513
1582
|
options,
|
|
1514
1583
|
ontologyEntityPush: false,
|
|
@@ -1555,7 +1624,8 @@ async function createPushRunContext(options) {
|
|
|
1555
1624
|
},
|
|
1556
1625
|
skipSecretPaths,
|
|
1557
1626
|
holdRemoteDeletes,
|
|
1558
|
-
...(
|
|
1627
|
+
...(effectivePushFence ? { pushIdentityFence: effectivePushFence } : {}),
|
|
1628
|
+
...(pushIdentityFence === undefined && companyRebindHold ? { companyRebindHold: true } : {}),
|
|
1559
1629
|
journalSlug,
|
|
1560
1630
|
quarantinedPaths,
|
|
1561
1631
|
journal,
|
|
@@ -1581,6 +1651,7 @@ async function createPushRunContext(options) {
|
|
|
1581
1651
|
threshold: checkpointThresholdFor(journal, options.durableFullPassProgress === true),
|
|
1582
1652
|
baselineLastSync: journal.lastSync,
|
|
1583
1653
|
error: null,
|
|
1654
|
+
unjournaledIntents: [],
|
|
1584
1655
|
},
|
|
1585
1656
|
};
|
|
1586
1657
|
}
|
|
@@ -2503,7 +2574,11 @@ function holdFilesFromBeforeIdentitySwitch(run, entries) {
|
|
|
2503
2574
|
held.push(entry.relativePath);
|
|
2504
2575
|
}
|
|
2505
2576
|
notePushIdentityHeld(getStateDir(), run.journalSlug, held.length, held);
|
|
2506
|
-
run.emit({
|
|
2577
|
+
run.emit({
|
|
2578
|
+
type: "identity-switch-push-held",
|
|
2579
|
+
count: held.length,
|
|
2580
|
+
...(run.companyRebindHold ? { reason: "company-rebind" } : {}),
|
|
2581
|
+
});
|
|
2507
2582
|
return [];
|
|
2508
2583
|
}
|
|
2509
2584
|
/**
|
|
@@ -3046,6 +3121,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3046
3121
|
await restartForChangedBody(afterHead);
|
|
3047
3122
|
return;
|
|
3048
3123
|
}
|
|
3124
|
+
// The remote may be an upload this machine started whose journal row
|
|
3125
|
+
// never reached disk (the pass was killed after its PUT). Take it as the
|
|
3126
|
+
// baseline before classifying, so the next local edit is an ordinary
|
|
3127
|
+
// one-sided change instead of a conflict against our own write.
|
|
3128
|
+
if (remoteMeta && item.kind === "file")
|
|
3129
|
+
adoptOwnUploadIntent(run, item, remoteMeta);
|
|
3049
3130
|
// Journal entry for this key (if any). Used both for 3-way conflict
|
|
3050
3131
|
// classification and for the write fence below: Prefer last-synced
|
|
3051
3132
|
// `remoteEtag` over live HEAD so a peer-advanced object cannot be
|
|
@@ -3154,8 +3235,10 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3154
3235
|
// Replaying that unchanged baseline is a stale revert, not an edit. A
|
|
3155
3236
|
// user who deliberately restores some other historic content has a hash
|
|
3156
3237
|
// different from the journal baseline and still reaches normal conflict
|
|
3157
|
-
// resolution below.
|
|
3158
|
-
|
|
3238
|
+
// resolution below. A remote still at the journaled revision has not
|
|
3239
|
+
// superseded anything, whatever its published lineage says.
|
|
3240
|
+
if (remoteChanged &&
|
|
3241
|
+
journalEntry !== undefined &&
|
|
3159
3242
|
journalEntry.localDiverges !== true &&
|
|
3160
3243
|
remoteContentHash !== undefined &&
|
|
3161
3244
|
journalEntry.hash === localHash &&
|
|
@@ -3530,7 +3613,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3530
3613
|
? { ifMatch: journalEntry.remoteEtag }
|
|
3531
3614
|
: { ifMatch: remoteMeta.etag }
|
|
3532
3615
|
: { ifNoneMatch: "*" };
|
|
3533
|
-
const
|
|
3616
|
+
const performUploadOnce = async (pc, observedRemote) => {
|
|
3534
3617
|
const isSymlinkUpload = item.kind === "symlink";
|
|
3535
3618
|
const lstat = fs.lstatSync(absolutePath);
|
|
3536
3619
|
const size = isSymlinkUpload ? 0 : lstat.size;
|
|
@@ -3757,6 +3840,9 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3757
3840
|
else {
|
|
3758
3841
|
// Strictly after the PUT returned and the entry was applied, so an
|
|
3759
3842
|
// interrupted pass can only ever re-upload a file, never skip one.
|
|
3843
|
+
// Until that checkpoint the upload's intent stays on disk, so a pass
|
|
3844
|
+
// killed before it cannot make the next pass take this object for a
|
|
3845
|
+
// peer's edit (feedback_a1696094).
|
|
3760
3846
|
checkpointShareJournal(run);
|
|
3761
3847
|
}
|
|
3762
3848
|
if (skillContentAtPut !== undefined &&
|
|
@@ -3787,6 +3873,36 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3787
3873
|
transfer.complete();
|
|
3788
3874
|
}
|
|
3789
3875
|
};
|
|
3876
|
+
// The intent is on disk before the PUT leaves. A landed PUT keeps it until
|
|
3877
|
+
// the next journal write puts its row on disk (see writeShareJournal),
|
|
3878
|
+
// including when this call then fails on that very write. Only a PUT the
|
|
3879
|
+
// server refused (412), or one superseded before it was sent, is known not
|
|
3880
|
+
// to have landed and forgets it at once.
|
|
3881
|
+
// Any other failure (timeout, reset) may have landed, so the intent stays
|
|
3882
|
+
// for this process's life and ages out on disk. A process that dies with
|
|
3883
|
+
// intents outstanding leaves them for the next pass to recognise those
|
|
3884
|
+
// objects as this machine's own writes instead of a peer's edits.
|
|
3885
|
+
const performUpload = async (pc, observedRemote = remoteMeta) => {
|
|
3886
|
+
const intent = item.kind === "file"
|
|
3887
|
+
? recordUploadIntent(run.journalSlug, relativePath, localHash)
|
|
3888
|
+
: undefined;
|
|
3889
|
+
const landedBefore = uploadCompleted;
|
|
3890
|
+
try {
|
|
3891
|
+
await performUploadOnce(pc, observedRemote);
|
|
3892
|
+
}
|
|
3893
|
+
catch (err) {
|
|
3894
|
+
if (intent) {
|
|
3895
|
+
if (uploadCompleted && !landedBefore)
|
|
3896
|
+
run.checkpoint.unjournaledIntents.push(intent);
|
|
3897
|
+
else if (isPreconditionFailed(err) || err instanceof SamePathUploadSupersededError) {
|
|
3898
|
+
settleUploadIntent(intent);
|
|
3899
|
+
}
|
|
3900
|
+
}
|
|
3901
|
+
throw err;
|
|
3902
|
+
}
|
|
3903
|
+
if (intent)
|
|
3904
|
+
run.checkpoint.unjournaledIntents.push(intent);
|
|
3905
|
+
};
|
|
3790
3906
|
try {
|
|
3791
3907
|
await performUpload(precondition);
|
|
3792
3908
|
}
|
|
@@ -3952,6 +4068,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3952
4068
|
throw retryErr;
|
|
3953
4069
|
if (emitPlanLimitSkip(retryErr, relativePath))
|
|
3954
4070
|
return;
|
|
4071
|
+
if (isPresignDeletedNewer(retryErr)) {
|
|
4072
|
+
keepLocalOverDeletedNewer(run, item.absolutePath, relativePath);
|
|
4073
|
+
counters.filesSkipped++;
|
|
4074
|
+
counters.filesSkippedConflict++;
|
|
4075
|
+
return;
|
|
4076
|
+
}
|
|
3955
4077
|
if (isPreconditionFailed(retryErr)) {
|
|
3956
4078
|
counters.filesSkipped++;
|
|
3957
4079
|
counters.filesSkippedConflict++;
|
|
@@ -4063,6 +4185,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
4063
4185
|
}
|
|
4064
4186
|
if (emitPlanLimitSkip(retryErr, relativePath))
|
|
4065
4187
|
return;
|
|
4188
|
+
if (isPresignDeletedNewer(retryErr)) {
|
|
4189
|
+
keepLocalOverDeletedNewer(run, item.absolutePath, relativePath);
|
|
4190
|
+
counters.filesSkipped++;
|
|
4191
|
+
counters.filesSkippedConflict++;
|
|
4192
|
+
return;
|
|
4193
|
+
}
|
|
4066
4194
|
hadUploadFailure = true;
|
|
4067
4195
|
run.uploadFailedSet.add(relativePath);
|
|
4068
4196
|
run.emit({
|
|
@@ -4083,18 +4211,35 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
4083
4211
|
counters.filesSkippedConflict++;
|
|
4084
4212
|
return;
|
|
4085
4213
|
}
|
|
4214
|
+
if (isPresignDeletedNewer(err)) {
|
|
4215
|
+
// The server holds a NEWER version of this key behind a delete marker
|
|
4216
|
+
// (hq-pro refusalReason "deleted-newer", feedback_7b30e522). Keep the
|
|
4217
|
+
// local file, park one backup copy out of tree, and drop the journal
|
|
4218
|
+
// row so the pull leg sees an untracked local file instead of a
|
|
4219
|
+
// remote delete to apply. Not an upload failure: nothing was lost.
|
|
4220
|
+
keepLocalOverDeletedNewer(run, item.absolutePath, relativePath);
|
|
4221
|
+
counters.filesSkipped++;
|
|
4222
|
+
counters.filesSkippedConflict++;
|
|
4223
|
+
return;
|
|
4224
|
+
}
|
|
4086
4225
|
if (isAccessDenied(err)) {
|
|
4087
4226
|
hadUploadFailure = true;
|
|
4088
4227
|
if (isPresignWriteForbidden(err)) {
|
|
4089
4228
|
// A write denial on a key in this push: report it instead of
|
|
4090
4229
|
// folding it into the silent scope-excluded count (feedback_682e9187).
|
|
4230
|
+
// Name the first REFUSED_STALE_PATH_CAP paths; the rest are folded
|
|
4231
|
+
// into one summary event after the uploads finish, so a member
|
|
4232
|
+
// without write access to a large folder does not get one line per
|
|
4233
|
+
// file on every pass (feedback_f84671f4).
|
|
4091
4234
|
run.writeForbiddenSet.add(relativePath);
|
|
4092
|
-
run.
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4235
|
+
if (run.writeForbiddenSet.size <= REFUSED_STALE_PATH_CAP) {
|
|
4236
|
+
run.emit({
|
|
4237
|
+
type: "error",
|
|
4238
|
+
diagnostic: true,
|
|
4239
|
+
path: relativePath,
|
|
4240
|
+
message: describeError(err),
|
|
4241
|
+
});
|
|
4242
|
+
}
|
|
4098
4243
|
return;
|
|
4099
4244
|
}
|
|
4100
4245
|
run.scopeExcludedSet.add(relativePath);
|
|
@@ -4365,6 +4510,43 @@ function adoptLandedSamePathUpload(run, item, landed) {
|
|
|
4365
4510
|
}
|
|
4366
4511
|
return true;
|
|
4367
4512
|
}
|
|
4513
|
+
/**
|
|
4514
|
+
* Take a remote revision as this run's baseline when its published content
|
|
4515
|
+
* hash matches an upload this machine recorded before a PUT whose journal row
|
|
4516
|
+
* never reached disk (the pass was killed in between). The object then holds
|
|
4517
|
+
* our own bytes, so the decision below sees an ordinary local edit, fenced on
|
|
4518
|
+
* the remote ETag, instead of a two-sided conflict that lets the cloud copy
|
|
4519
|
+
* replace the newer local file (feedback_a1696094).
|
|
4520
|
+
*
|
|
4521
|
+
* The row carries no local stat stamps: they would claim the local body still
|
|
4522
|
+
* hashes to the adopted value. Returns true when the row was replaced.
|
|
4523
|
+
*/
|
|
4524
|
+
function adoptOwnUploadIntent(run, item, remoteMeta) {
|
|
4525
|
+
const remoteHash = publishedContentHash(remoteMeta.metadata);
|
|
4526
|
+
if (remoteHash === undefined || remoteHash === item.localHash)
|
|
4527
|
+
return false;
|
|
4528
|
+
const row = journalRows(run.journal).get(item.relativePath);
|
|
4529
|
+
if (row?.localDiverges === true || row?.hash === remoteHash)
|
|
4530
|
+
return false;
|
|
4531
|
+
const remoteEtag = normalizeEtag(remoteMeta.etag);
|
|
4532
|
+
if (row?.remoteEtag !== undefined && normalizeEtag(row.remoteEtag) === remoteEtag)
|
|
4533
|
+
return false;
|
|
4534
|
+
const rowWrittenAt = row ? Date.parse(row.syncedAt) : Number.NaN;
|
|
4535
|
+
const intent = findUploadIntent(run.journalSlug, item.relativePath, remoteHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined);
|
|
4536
|
+
if (!intent)
|
|
4537
|
+
return false;
|
|
4538
|
+
try {
|
|
4539
|
+
updateEntry(run.journal, item.relativePath, remoteHash, remoteMeta.size, "up", item.absolutePath, {
|
|
4540
|
+
remoteEtag,
|
|
4541
|
+
kind: "file",
|
|
4542
|
+
});
|
|
4543
|
+
}
|
|
4544
|
+
catch {
|
|
4545
|
+
// The local path vanished; the normal replan settles it.
|
|
4546
|
+
return false;
|
|
4547
|
+
}
|
|
4548
|
+
return true;
|
|
4549
|
+
}
|
|
4368
4550
|
/**
|
|
4369
4551
|
* Exact-prefix and storage-class-specific, matching the pull planner: only
|
|
4370
4552
|
* session transcripts in DEEP_ARCHIVE are frozen. Other archived objects keep
|
|
@@ -4684,7 +4866,11 @@ function recordPushConflictNotice(run, item, backupPath, sideKept, decision, rem
|
|
|
4684
4866
|
companySlug: run.options.personalMode === true ? null : run.ctx.slug,
|
|
4685
4867
|
relativePath: vaultKeyForLocalPath(run.hqRoot, item.absolutePath),
|
|
4686
4868
|
backupPath,
|
|
4687
|
-
|
|
4869
|
+
// A remote kept over a NEWER local body is not "remote-newer"; the
|
|
4870
|
+
// decision names why it was kept.
|
|
4871
|
+
winnerReason: remoteChangedSinceJournal ||
|
|
4872
|
+
decision?.reason === "remote-changed" ||
|
|
4873
|
+
decision?.reason === "no-baseline"
|
|
4688
4874
|
? "remote-changed-since-journal"
|
|
4689
4875
|
: sideKept === "local" ? "local-newer" : "remote-newer",
|
|
4690
4876
|
sideKept,
|
|
@@ -4886,7 +5072,7 @@ async function adoptRemoteOnPush(run, item, remoteHash, decision, probe) {
|
|
|
4886
5072
|
// Persist the converged remote-backed entry immediately. A conflict
|
|
4887
5073
|
// resolution is a completed transfer even though the push counter is
|
|
4888
5074
|
// recorded as skipped, and an interrupted pass must not revive it.
|
|
4889
|
-
checkpointShareJournal(run);
|
|
5075
|
+
checkpointShareJournal(run, { persistRow: true });
|
|
4890
5076
|
}
|
|
4891
5077
|
catch (adoptErr) {
|
|
4892
5078
|
if (parkAttempted && (!backupHash || backupHash !== item.localHash)) {
|
|
@@ -5300,25 +5486,34 @@ function checkpointThresholdFor(journal, durableFullPassProgress = false) {
|
|
|
5300
5486
|
* Called after each result is applied to the in-memory journal. The entry is
|
|
5301
5487
|
* written only AFTER its object PUT has returned, so a checkpoint can at worst
|
|
5302
5488
|
* cause a file to be re-uploaded — never mark an unshipped file as shipped.
|
|
5489
|
+
*
|
|
5490
|
+
* `persistRow` writes the journal even between checkpoints, for a result that
|
|
5491
|
+
* changed the vault (an upload, an adopted conflict). The write appends only
|
|
5492
|
+
* the changed rows; the object-event flush and the checkpoint counters keep
|
|
5493
|
+
* their cadence.
|
|
5303
5494
|
*/
|
|
5304
|
-
function checkpointShareJournal(run) {
|
|
5495
|
+
function checkpointShareJournal(run, { persistRow = false } = {}) {
|
|
5305
5496
|
run.checkpoint.appliedSinceFlush += 1;
|
|
5306
5497
|
const override = checkpointCadenceOverride();
|
|
5307
5498
|
if (override !== null) {
|
|
5308
|
-
if (run.checkpoint.appliedSinceFlush >= override)
|
|
5499
|
+
if (run.checkpoint.appliedSinceFlush >= override) {
|
|
5309
5500
|
flushShareJournal(run);
|
|
5310
|
-
|
|
5501
|
+
return;
|
|
5502
|
+
}
|
|
5311
5503
|
}
|
|
5312
|
-
if (run.checkpoint.appliedSinceFlush
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
5316
|
-
|
|
5504
|
+
else if (run.checkpoint.appliedSinceFlush >= run.checkpoint.threshold) {
|
|
5505
|
+
if (run.options.durableFullPassProgress === true) {
|
|
5506
|
+
flushShareJournal(run);
|
|
5507
|
+
return;
|
|
5508
|
+
}
|
|
5509
|
+
const now = Date.now();
|
|
5510
|
+
if (now - run.checkpoint.lastFlushAt >= CHECKPOINT_MIN_INTERVAL_MS) {
|
|
5511
|
+
flushShareJournal(run, now);
|
|
5512
|
+
return;
|
|
5513
|
+
}
|
|
5317
5514
|
}
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
return;
|
|
5321
|
-
flushShareJournal(run, now);
|
|
5515
|
+
if (persistRow)
|
|
5516
|
+
writeShareJournal(run);
|
|
5322
5517
|
}
|
|
5323
5518
|
/**
|
|
5324
5519
|
* Publish the journal as it currently stands, WITHOUT advancing `lastSync`.
|
|
@@ -5326,6 +5521,14 @@ function checkpointShareJournal(run) {
|
|
|
5326
5521
|
* "these objects are shipped", not "this sync finished".
|
|
5327
5522
|
*/
|
|
5328
5523
|
function flushShareJournal(run, now = Date.now()) {
|
|
5524
|
+
writeShareJournal(run);
|
|
5525
|
+
run.checkpoint.appliedSinceFlush = 0;
|
|
5526
|
+
run.checkpoint.lastFlushAt = now;
|
|
5527
|
+
void run.objectEvents.flush();
|
|
5528
|
+
run.checkpoint.threshold = checkpointThresholdFor(run.journal, run.options.durableFullPassProgress === true);
|
|
5529
|
+
}
|
|
5530
|
+
/** The journal write of {@link flushShareJournal}, without its bookkeeping. */
|
|
5531
|
+
function writeShareJournal(run) {
|
|
5329
5532
|
const liveLastSync = run.journal.lastSync;
|
|
5330
5533
|
run.journal.lastSync = run.checkpoint.baselineLastSync;
|
|
5331
5534
|
const release = setJournalSnapshotQuarantineListener(run.journalSlug, (diagnostic) => run.emit({ type: "journal-quarantine", ...diagnostic }));
|
|
@@ -5340,10 +5543,16 @@ function flushShareJournal(run, now = Date.now()) {
|
|
|
5340
5543
|
release();
|
|
5341
5544
|
run.journal.lastSync = liveLastSync;
|
|
5342
5545
|
}
|
|
5343
|
-
run
|
|
5344
|
-
|
|
5345
|
-
|
|
5346
|
-
|
|
5546
|
+
settleJournaledIntents(run);
|
|
5547
|
+
}
|
|
5548
|
+
/** The rows of every landed upload are on disk now; their intents can go. */
|
|
5549
|
+
function settleJournaledIntents(run) {
|
|
5550
|
+
const settled = run.checkpoint.unjournaledIntents;
|
|
5551
|
+
if (settled.length === 0)
|
|
5552
|
+
return;
|
|
5553
|
+
run.checkpoint.unjournaledIntents = [];
|
|
5554
|
+
for (const intent of settled)
|
|
5555
|
+
settleUploadIntent(intent);
|
|
5347
5556
|
}
|
|
5348
5557
|
function finalizeShareJournal(run) {
|
|
5349
5558
|
run.journal.lastSync = new Date().toISOString();
|
|
@@ -5354,6 +5563,7 @@ function finalizeShareJournal(run) {
|
|
|
5354
5563
|
finally {
|
|
5355
5564
|
release();
|
|
5356
5565
|
}
|
|
5566
|
+
settleJournaledIntents(run);
|
|
5357
5567
|
if (run.excludedSet.size > 0) {
|
|
5358
5568
|
const samplePaths = [];
|
|
5359
5569
|
for (const p of run.excludedSet) {
|
|
@@ -5598,6 +5808,7 @@ function buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths,
|
|
|
5598
5808
|
uploadFailedPaths: [...run.uploadFailedSet].slice(0, REFUSED_STALE_PATH_CAP),
|
|
5599
5809
|
filesWriteForbidden: run.writeForbiddenSet.size,
|
|
5600
5810
|
writeForbiddenPaths: [...run.writeForbiddenSet].slice(0, REFUSED_STALE_PATH_CAP),
|
|
5811
|
+
writeForbiddenHoldPaths: [...run.writeForbiddenSet],
|
|
5601
5812
|
filesSuppressedByTombstone: counters.filesSuppressedByTombstone,
|
|
5602
5813
|
...(run.skillRegisterLandedBytes
|
|
5603
5814
|
? { skillRegistrationFallbacks: run.skillRegistrationFallbacks }
|