@indigoai-us/hq-cloud 6.18.109 → 6.18.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/area-ledger-migration.d.ts.map +1 -1
- package/dist/bin/area-ledger-migration.js +5 -3
- package/dist/bin/area-ledger-migration.js.map +1 -1
- package/dist/bin/backup-prune-runner.js +2 -2
- package/dist/bin/backup-prune-runner.js.map +1 -1
- package/dist/bin/runner-output-guard.d.ts +39 -0
- package/dist/bin/runner-output-guard.d.ts.map +1 -0
- package/dist/bin/runner-output-guard.js +104 -0
- package/dist/bin/runner-output-guard.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +9 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +76 -6
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +37 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +37 -3
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/conflict.d.ts +30 -1
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +69 -3
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/reconcile-skip-reason.d.ts +1 -1
- package/dist/cli/reconcile-skip-reason.d.ts.map +1 -1
- package/dist/cli/reconcile-skip-reason.js +2 -0
- package/dist/cli/reconcile-skip-reason.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +228 -43
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +98 -2
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +487 -74
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +37 -9
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/share.d.ts +35 -8
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +437 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +70 -5
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +439 -77
- package/dist/cli/sync.js.map +1 -1
- package/dist/cognito-auth.d.ts +10 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +162 -1
- package/dist/cognito-auth.js.map +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +43 -21
- package/dist/ignore.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +3 -0
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -0
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-decoder.js +2 -0
- package/dist/journal-snapshot-decoder.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -1
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal.d.ts +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +20 -20
- package/dist/journal.js.map +1 -1
- package/dist/lib/append-log-merge.d.ts +43 -0
- package/dist/lib/append-log-merge.d.ts.map +1 -0
- package/dist/lib/append-log-merge.js +424 -0
- package/dist/lib/append-log-merge.js.map +1 -0
- package/dist/lib/same-path-upload-lock.d.ts +27 -1
- package/dist/lib/same-path-upload-lock.d.ts.map +1 -1
- package/dist/lib/same-path-upload-lock.js +32 -1
- package/dist/lib/same-path-upload-lock.js.map +1 -1
- package/dist/object-io.d.ts +36 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +93 -12
- package/dist/object-io.js.map +1 -1
- package/dist/operation-lock.d.ts +27 -1
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +94 -8
- package/dist/operation-lock.js.map +1 -1
- package/dist/outposts/client.d.ts +19 -6
- package/dist/outposts/client.d.ts.map +1 -1
- package/dist/outposts/client.js +16 -5
- package/dist/outposts/client.js.map +1 -1
- package/dist/outposts/command.d.ts +2 -1
- package/dist/outposts/command.d.ts.map +1 -1
- package/dist/outposts/command.js +81 -13
- package/dist/outposts/command.js.map +1 -1
- package/dist/outposts/self-deploy.d.ts.map +1 -1
- package/dist/outposts/self-deploy.js +3 -0
- package/dist/outposts/self-deploy.js.map +1 -1
- package/dist/private-folder-transport.d.ts +0 -2
- package/dist/private-folder-transport.d.ts.map +1 -1
- package/dist/private-folder-transport.js +17 -16
- package/dist/private-folder-transport.js.map +1 -1
- package/dist/push-identity-fence.d.ts +116 -3
- package/dist/push-identity-fence.d.ts.map +1 -1
- package/dist/push-identity-fence.js +625 -62
- package/dist/push-identity-fence.js.map +1 -1
- package/dist/qmd-reindex.d.ts.map +1 -1
- package/dist/qmd-reindex.js +3 -2
- package/dist/qmd-reindex.js.map +1 -1
- package/dist/s3.d.ts +30 -9
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +110 -82
- package/dist/s3.js.map +1 -1
- package/dist/sync/area-ledger-migration.d.ts +17 -0
- package/dist/sync/area-ledger-migration.d.ts.map +1 -1
- package/dist/sync/area-ledger-migration.js +37 -3
- package/dist/sync/area-ledger-migration.js.map +1 -1
- package/dist/sync/scattered-delete-burst.d.ts +61 -0
- package/dist/sync/scattered-delete-burst.d.ts.map +1 -0
- package/dist/sync/scattered-delete-burst.js +111 -0
- package/dist/sync/scattered-delete-burst.js.map +1 -0
- package/dist/sync/unattended-delete-hold.d.ts +102 -0
- package/dist/sync/unattended-delete-hold.d.ts.map +1 -0
- package/dist/sync/unattended-delete-hold.js +164 -0
- package/dist/sync/unattended-delete-hold.js.map +1 -0
- package/dist/sync-core.d.ts +5 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -6
- package/dist/sync-core.js.map +1 -1
- package/dist/types.d.ts +13 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts +6 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +53 -8
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-key-grammar.d.ts +44 -0
- package/dist/vault-key-grammar.d.ts.map +1 -0
- package/dist/vault-key-grammar.js +158 -0
- package/dist/vault-key-grammar.js.map +1 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +4 -1
- package/dist/watcher.js.map +1 -1
- package/dist/windows-extended-path.d.ts +22 -0
- package/dist/windows-extended-path.d.ts.map +1 -0
- package/dist/windows-extended-path.js +55 -0
- package/dist/windows-extended-path.js.map +1 -0
- package/package.json +1 -1
package/dist/cli/share.js
CHANGED
|
@@ -22,18 +22,21 @@ import { noteSwallowedError } from "../best-effort-error.js";
|
|
|
22
22
|
import { appendSyncDecision } from "../sync-decision-log.js";
|
|
23
23
|
import { readPendingConflictNotices, recordConflictParked } from "../conflict-notices.js";
|
|
24
24
|
import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
|
|
25
|
+
import { detectScatteredDeleteBurst } from "../sync/scattered-delete-burst.js";
|
|
26
|
+
import { clearUnattendedHeldRow, describeRemoteCreator, markUnattendedHeldRow, resolveUnattendedDeleteHold, unattendedMayDelete, UNREADABLE_CREATOR, } from "../sync/unattended-delete-hold.js";
|
|
25
27
|
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
26
28
|
import * as crypto from "crypto";
|
|
27
|
-
import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, isPresignDeletedNewer, } from "../object-io.js";
|
|
29
|
+
import { bindObjectIOToContext, PresignKeyRefusedError, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, isPresignDeletedNewer, } from "../object-io.js";
|
|
28
30
|
import { primePrivateFolderUpload } from "../private-folder-transport.js";
|
|
29
31
|
import { getJournalPath, isUnderAnyRoot, readJournalForPass, bindPersonalVaultJournalToIdentity, readJournalReadOnly, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, configureRejectedDeltaRecovery, getJournalRecoveryMarker, getStateDir, personalVaultHasOtherIdentityState, readPersonalVaultOwnerUid, } from "../journal.js";
|
|
30
|
-
import { admitPushIdentity, machineIdentitySwitchRecorded, notePushIdentityHeld, noteCompanyRebind, } from "../push-identity-fence.js";
|
|
32
|
+
import { admitPushIdentity, corruptPushIdentityHold, holdIsProvenDifferentPerson, recordSignedInPerson, machineIdentitySwitchRecorded, notePushIdentityHeld, noteCompanyRebind, pushIdentityHoldMessage, releaseUnprovenPushIdentityHold, signedInPersonalUid, signedInPersonMappingState, } from "../push-identity-fence.js";
|
|
31
33
|
import { createIgnoreExplainer, createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isHqGeneratedIndexFile, isMachineLocalRuntimePath, isPathExcludedFromSync, isSecretFolderPath, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
34
|
+
import { classifyPresignKey } from "../vault-key-grammar.js";
|
|
32
35
|
import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
33
36
|
import { claimNeverSyncedLocalNotices, formatSecretsNotSyncedLine, isNeverSyncedVaultPath, isSecretBearingVaultPath, isSkipSecretPathsEnabled, isWalRejectedDeltaRecoveryEnabled, } from "../sync/secret-path-policy.js";
|
|
34
37
|
import { getActiveFullPassDecodeWindow, getActiveFullPassUnitSize, } from "../sync/full-pass-caps.js";
|
|
35
38
|
import { isGeneratedCoreMirrorKey, isPersonalVaultDeletionExcluded, isSessionLogDeletionKey, purgeExcludedPersonalVaultJournalEntries, } from "../personal-vault.js";
|
|
36
|
-
import { pickWinner, resolveConflict, usesVersionAwareWinner } from "./conflict.js";
|
|
39
|
+
import { COMPANY_MANIFEST_KEY, countManifestCompanies, countManifestCompaniesAt, pickWinner, resolveConflict, usesVersionAwareWinner, } from "./conflict.js";
|
|
37
40
|
import { createHumanSkipLineLog, formatReconcileSkipLine, isUpToDateSkipReason, } from "./reconcile-skip-reason.js";
|
|
38
41
|
import { fetchFileTombstones, } from "./tombstones.js";
|
|
39
42
|
import { compileScopeMatcher, } from "../prefix-coalesce.js";
|
|
@@ -41,6 +44,7 @@ import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransfer
|
|
|
41
44
|
import { buildConflictBackupPath, CONFLICT_BACKUPS_DIR, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
42
45
|
import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
|
|
43
46
|
import { appendIfUnchanged, isAppendOnlyPath, mergeAppendOnly } from "../lib/append-only-merge.js";
|
|
47
|
+
import { isAppendLogPath, mergeAppendLog, writeAppendLogMerge } from "../lib/append-log-merge.js";
|
|
44
48
|
import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/board-merge.js";
|
|
45
49
|
import { findLandedUpload, findUploadIntent, followLandedUploads, recordUploadIntent, SamePathUploadHold, samePathUploadKey, SamePathUploadSupersededError, settleUploadIntent, } from "../lib/same-path-upload-lock.js";
|
|
46
50
|
import { isCloudAuthoritative, isLocallyIneligibleForPush } from "../lib/cloud-authoritative.js";
|
|
@@ -49,7 +53,7 @@ import { isOntologyEntityKey, ONTOLOGY_MULTI_WRITER_FLAG, resolveOntologyEntityP
|
|
|
49
53
|
import { entityIsFreshForCreate, outboxAllowsEntityPush, readOntologyOutbox, } from "../lib/ontology-outbox.js";
|
|
50
54
|
import { resolveSkillRegisterLandedBytes } from "../lib/skill-register-landed-bytes.js";
|
|
51
55
|
import { resolvePersonalConnectorSettingsSync } from "../lib/personal-connector-settings-sync.js";
|
|
52
|
-
import { planLimitRefusalFromError, VaultAuthError, VaultClient, VaultClientError, VaultConflictError, VaultPermissionDeniedError, } from "../vault-client.js";
|
|
56
|
+
import { planLimitRefusalFromError, VaultAuthError, VaultClient, pickCanonicalPersonEntity, VaultClientError, VaultConflictError, VaultPermissionDeniedError, } from "../vault-client.js";
|
|
53
57
|
import { formatPlanLimitNotice } from "../plan-limit.js";
|
|
54
58
|
import { PlanLimitPauseStore, planLimitPauseNow, PLAN_LIMIT_PAUSE_RETRY_MS, } from "../plan-limit-pause.js";
|
|
55
59
|
import { describeError } from "../lib/describe-error.js";
|
|
@@ -1317,6 +1321,38 @@ const REFUSED_STALE_PATH_CAP = 50;
|
|
|
1317
1321
|
* refused to mint a PUT URL with FILES_PRESIGN_FORBIDDEN (see object-io.ts
|
|
1318
1322
|
* isSkipWithLogDenialCode). Its message carries "server put denied: <code>".
|
|
1319
1323
|
*/
|
|
1324
|
+
/**
|
|
1325
|
+
* Does this context's object traffic go through the presign server? Building
|
|
1326
|
+
* the transport can throw (a direct-S3 context with no credentials); that
|
|
1327
|
+
* answers "no", which keeps the push's pre-presign key checks.
|
|
1328
|
+
*/
|
|
1329
|
+
function usesPresignTransport(ctx) {
|
|
1330
|
+
try {
|
|
1331
|
+
return resolveObjectIO(ctx) instanceof PresignObjectIO;
|
|
1332
|
+
}
|
|
1333
|
+
catch {
|
|
1334
|
+
return false;
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
/** Company vaults are `cmp_*`; every other uid is a personal scope. */
|
|
1338
|
+
function presignKeyScope(ctx) {
|
|
1339
|
+
return ctx.uid.startsWith("cmp_") ? "company" : "personal";
|
|
1340
|
+
}
|
|
1341
|
+
/**
|
|
1342
|
+
* The presign server refuses this key's spelling, so a delete can never
|
|
1343
|
+
* succeed. Direct-S3 contexts keep the key: that transport stores it.
|
|
1344
|
+
*/
|
|
1345
|
+
function presignDeleteKeyRefused(ctx, key) {
|
|
1346
|
+
return usesPresignTransport(ctx) && classifyPresignKey(key, presignKeyScope(ctx)) !== null;
|
|
1347
|
+
}
|
|
1348
|
+
function emitInvalidScopedKeySkip(run, path, errorCode) {
|
|
1349
|
+
run.emit({
|
|
1350
|
+
type: "skip-invalid-scoped-key",
|
|
1351
|
+
path,
|
|
1352
|
+
errorCode,
|
|
1353
|
+
...(errorCode === "INVALID_KEY_BACKSLASH" ? { reason: "backslash-key" } : {}),
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
1320
1356
|
export function isPresignWriteForbidden(err) {
|
|
1321
1357
|
const message = err instanceof Error ? err.message : String(err);
|
|
1322
1358
|
return /server put denied: FILES_PRESIGN_FORBIDDEN\b/.test(message);
|
|
@@ -1330,6 +1366,28 @@ function isDirectS3AccessDenied(err) {
|
|
|
1330
1366
|
shaped.Code === "AccessDenied" ||
|
|
1331
1367
|
shaped.$metadata?.httpStatusCode === 403;
|
|
1332
1368
|
}
|
|
1369
|
+
/**
|
|
1370
|
+
* Record `sub` against the caller's own personal vault uid, from the person
|
|
1371
|
+
* list route and the same canonical pick the sync runner uses to choose its
|
|
1372
|
+
* personal target. hq-pro answers that route with the entities owned by the
|
|
1373
|
+
* caller's token (listEntitiesByOwner(callerUid)), or the caller's own agent
|
|
1374
|
+
* entity for a machine identity, so another person's entity is never returned
|
|
1375
|
+
* to an admin. Only a `prs_` person uid is recorded. Any failure records
|
|
1376
|
+
* nothing, which leaves the pusher unproven (fenced). `onError` receives a
|
|
1377
|
+
* short, token-free description of the swallowed failure for diagnostics.
|
|
1378
|
+
*/
|
|
1379
|
+
export async function recordCallerPersonalUid(client, stateDir, sub, onError) {
|
|
1380
|
+
try {
|
|
1381
|
+
const pick = pickCanonicalPersonEntity(await client.entity.listByType("person"));
|
|
1382
|
+
if (pick?.uid.startsWith("prs_"))
|
|
1383
|
+
recordSignedInPerson(stateDir, sub, pick.uid);
|
|
1384
|
+
}
|
|
1385
|
+
catch (err) {
|
|
1386
|
+
// Unproven pusher: stay fenced. Report only the error class, never its
|
|
1387
|
+
// message, which could carry request details.
|
|
1388
|
+
onError?.(`person lookup failed (${err instanceof Error ? err.name : typeof err}); pusher stays held`);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1333
1391
|
async function createPushRunContext(options) {
|
|
1334
1392
|
const { paths, company, message, onConflict, vaultConfig, entityContext, hqRoot, propagateDeletes } = options;
|
|
1335
1393
|
// Personal vault is a bulk tree, not `hq share <file>`. Default skip-unchanged
|
|
@@ -1559,16 +1617,48 @@ async function createPushRunContext(options) {
|
|
|
1559
1617
|
const personalScope = options.personalMode === true || ctx.uid.startsWith("prs_");
|
|
1560
1618
|
const pushIdentityUid = personalScope ? ctx.uid : signedInSub;
|
|
1561
1619
|
const otherIdentityEvidence = personalVaultHasOtherIdentityState();
|
|
1620
|
+
// feedback_a3d39586: a hand-run push of a company scope never ran the sync
|
|
1621
|
+
// runner's sign-in step, so this sub may have no person record. Record it the
|
|
1622
|
+
// way the runner does (the caller's own person entity), before the fence is
|
|
1623
|
+
// checked. A failed lookup records nothing and the scope stays fenced.
|
|
1624
|
+
if (!personalScope && signedInSub && vaultConfig && signedInPersonMappingState(getStateDir(), signedInSub) === "none") {
|
|
1625
|
+
// Only a truly missing mapping is recorded; a corrupt or unreadable file
|
|
1626
|
+
// is left alone and the pusher stays fenced.
|
|
1627
|
+
await recordCallerPersonalUid(new VaultClient(vaultConfig), getStateDir(), signedInSub, (message) => {
|
|
1628
|
+
if (process.env.HQ_DEBUG)
|
|
1629
|
+
console.debug(`[push-identity] ${message}`);
|
|
1630
|
+
});
|
|
1631
|
+
}
|
|
1632
|
+
const personalOwnerUid = readPersonalVaultOwnerUid();
|
|
1633
|
+
const personalUid = personalScope ? ctx.uid : signedInPersonalUid(getStateDir(), signedInSub);
|
|
1634
|
+
if (pushIdentityUid &&
|
|
1635
|
+
!holdIsProvenDifferentPerson(getStateDir(), {
|
|
1636
|
+
otherIdentityEvidence,
|
|
1637
|
+
machineOwnerUid: personalOwnerUid,
|
|
1638
|
+
personalUid,
|
|
1639
|
+
sub: signedInSub,
|
|
1640
|
+
})) {
|
|
1641
|
+
const identities = [personalUid, signedInSub, pushIdentityUid].filter((identity) => typeof identity === "string" && identity.length > 0);
|
|
1642
|
+
const healed = releaseUnprovenPushIdentityHold(getStateDir(), identities);
|
|
1643
|
+
if (healed.released.length > 0) {
|
|
1644
|
+
console.log(`Released held uploads (${healed.released.join(", ")}): the signed-in person matches this computer's history.`);
|
|
1645
|
+
}
|
|
1646
|
+
else if (healed.clearedMarker) {
|
|
1647
|
+
console.log("Cleared a false sign-in switch on this computer: the signed-in person matches this computer's history.");
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1562
1650
|
const pushIdentityFence = pushIdentityUid
|
|
1563
1651
|
? admitPushIdentity(getStateDir(), journalSlug, pushIdentityUid, {
|
|
1564
1652
|
...(signedInSub ? { sub: signedInSub } : {}),
|
|
1565
1653
|
otherIdentityEvidence,
|
|
1566
1654
|
personalScope,
|
|
1567
|
-
personalOwnerUid
|
|
1655
|
+
personalOwnerUid,
|
|
1568
1656
|
})
|
|
1569
|
-
:
|
|
1570
|
-
? { uid: "", sinceMs: Date.now() }
|
|
1571
|
-
:
|
|
1657
|
+
: corruptPushIdentityHold(getStateDir(), journalSlug)
|
|
1658
|
+
? { uid: "", sinceMs: Date.now(), reason: "corrupt-identity" }
|
|
1659
|
+
: machineIdentitySwitchRecorded(getStateDir()) || otherIdentityEvidence || personalOwnerUid !== null
|
|
1660
|
+
? { uid: "", sinceMs: Date.now() }
|
|
1661
|
+
: undefined;
|
|
1572
1662
|
const quarantinedPaths = readQuarantinedJournalPaths(journalSlug);
|
|
1573
1663
|
const flagConfig = vaultConfig ?? privateFolderVaultConfig;
|
|
1574
1664
|
const symlinkTargetRefusal = flagConfig
|
|
@@ -2385,14 +2475,30 @@ async function buildSharePlans(run) {
|
|
|
2385
2475
|
// Finder icon files are normally already gone by here — createIgnoreFilter
|
|
2386
2476
|
// drops them during the walk. This stays as the general guard for any other
|
|
2387
2477
|
// control-char path, and as defense for callers that bypass the walker.
|
|
2478
|
+
//
|
|
2479
|
+
// feedback_65ee9e51: the same rule covers every key the presign server
|
|
2480
|
+
// refuses on spelling (hq-pro toKeyItem: a `*`, a backslash, a double
|
|
2481
|
+
// slash, a `.` or `..` component, or one of those after URI decoding).
|
|
2482
|
+
// That is the presign transport, and a private folder on the direct-S3
|
|
2483
|
+
// path, which is written through presign. Such a key can never be read or
|
|
2484
|
+
// written through presign, so planning it only reaches a refusal. Skip it
|
|
2485
|
+
// here with the same event. Other direct-S3 keys are stored (uploadFile
|
|
2486
|
+
// turns a backslash into a separator), so they keep the control-char check
|
|
2487
|
+
// only.
|
|
2488
|
+
const presignTransport = usesPresignTransport(run.ctx);
|
|
2489
|
+
const keyScope = presignKeyScope(run.ctx);
|
|
2388
2490
|
const controlCharFiltered = collected.filter((entry) => {
|
|
2389
|
-
|
|
2491
|
+
const presignGrammar = presignTransport ||
|
|
2492
|
+
(run.privateFolderTransport !== undefined &&
|
|
2493
|
+
run.privateFolderMatcher.coversPath(entry.relativePath));
|
|
2494
|
+
const errorCode = hasControlCharacters(entry.relativePath)
|
|
2495
|
+
? "INVALID_KEY_CONTROL_CHARS"
|
|
2496
|
+
: presignGrammar
|
|
2497
|
+
? classifyPresignKey(entry.relativePath, keyScope)?.code
|
|
2498
|
+
: undefined;
|
|
2499
|
+
if (errorCode === undefined)
|
|
2390
2500
|
return true;
|
|
2391
|
-
run.
|
|
2392
|
-
type: "skip-invalid-scoped-key",
|
|
2393
|
-
path: entry.relativePath,
|
|
2394
|
-
errorCode: "INVALID_KEY_CONTROL_CHARS",
|
|
2395
|
-
});
|
|
2501
|
+
emitInvalidScopedKeySkip(run, entry.relativePath, errorCode);
|
|
2396
2502
|
return false;
|
|
2397
2503
|
}).filter((entry) => !isLocalCopyOfLibraryAsset(run.journal, entry));
|
|
2398
2504
|
// `companies/` is an ordinary directory name below a company vault root.
|
|
@@ -2454,7 +2560,11 @@ async function buildSharePlans(run) {
|
|
|
2454
2560
|
});
|
|
2455
2561
|
const recoveryHoldActive = run.holdRemoteDeletes || getJournalRecoveryMarker(run.journalSlug) !== undefined;
|
|
2456
2562
|
const deletePlan = run.propagateDeletes === true
|
|
2457
|
-
? await computeDeletePlan(run.journal, run.syncRoot, deleteScopeRoots, run.shouldSync, run.propagateDeletePolicy, run.ctx, run.options.personalMode !== true, run.skipSecretPaths, run.quarantinedPaths, recoveryHoldActive, run.scopedJournal ? () => readJournalReadOnly(run.journalSlug) : undefined)
|
|
2563
|
+
? await computeDeletePlan(run.journal, run.syncRoot, deleteScopeRoots, run.shouldSync, run.propagateDeletePolicy, run.ctx, run.options.personalMode !== true, run.skipSecretPaths, run.quarantinedPaths, recoveryHoldActive, run.scopedJournal ? () => readJournalReadOnly(run.journalSlug) : undefined, new Set(pushPlan.items.flatMap((item) => item.action === "upload" && item.kind === "file" ? [item.localHash] : [])), await resolveUnattendedDeleteHold({
|
|
2564
|
+
personalMode: run.options.personalMode === true,
|
|
2565
|
+
knownSubs: [run.options.author?.userSub, run.options.signedInSub],
|
|
2566
|
+
...(run.vaultConfig ? { authToken: run.vaultConfig.authToken } : {}),
|
|
2567
|
+
}))
|
|
2458
2568
|
: { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
2459
2569
|
for (const warning of deletePlan.brokenLinkSkipped ?? []) {
|
|
2460
2570
|
if (rememberLinkWarning(run.linkWarnings, warning))
|
|
@@ -2530,6 +2640,9 @@ async function buildSharePlans(run) {
|
|
|
2530
2640
|
inScope: deletePlan.bulkAsymmetry.inScope,
|
|
2531
2641
|
ratio: deletePlan.bulkAsymmetry.ratio,
|
|
2532
2642
|
samplePaths: deletePlan.bulkAsymmetry.samplePaths,
|
|
2643
|
+
...(deletePlan.bulkAsymmetry.trigger
|
|
2644
|
+
? { trigger: deletePlan.bulkAsymmetry.trigger, folders: deletePlan.bulkAsymmetry.folders }
|
|
2645
|
+
: {}),
|
|
2533
2646
|
});
|
|
2534
2647
|
}
|
|
2535
2648
|
if (massDeleteGuard) {
|
|
@@ -2585,10 +2698,19 @@ function holdFilesFromBeforeIdentitySwitch(run, entries) {
|
|
|
2585
2698
|
held.push(entry.relativePath);
|
|
2586
2699
|
}
|
|
2587
2700
|
notePushIdentityHeld(getStateDir(), run.journalSlug, held.length, held);
|
|
2701
|
+
const sample = held.slice(0, 20);
|
|
2702
|
+
const corrupt = run.pushIdentityFence?.reason === "corrupt-identity";
|
|
2588
2703
|
run.emit({
|
|
2589
2704
|
type: "identity-switch-push-held",
|
|
2590
2705
|
count: held.length,
|
|
2591
|
-
...(run.companyRebindHold
|
|
2706
|
+
...(run.companyRebindHold
|
|
2707
|
+
? { reason: "company-rebind" }
|
|
2708
|
+
: {
|
|
2709
|
+
samplePaths: sample,
|
|
2710
|
+
message: pushIdentityHoldMessage(held.length, sample, corrupt ? "corrupt-identity" : undefined),
|
|
2711
|
+
action: "hq sync release-held",
|
|
2712
|
+
...(corrupt ? { reason: "corrupt-identity" } : {}),
|
|
2713
|
+
}),
|
|
2592
2714
|
});
|
|
2593
2715
|
return [];
|
|
2594
2716
|
}
|
|
@@ -3378,7 +3500,47 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3378
3500
|
// still at that etag, silently promoting the divergent local to write
|
|
3379
3501
|
// authority. Route through conflict (keep/abort/overwrite) instead.
|
|
3380
3502
|
const divergentLocalHonesty = !!journalEntry?.localDiverges;
|
|
3381
|
-
|
|
3503
|
+
// A blank companies manifest (the hq-core template) never uploads over a
|
|
3504
|
+
// cloud manifest that lists companies. It goes through conflict handling
|
|
3505
|
+
// instead, where the populated cloud copy keeps the name and the blank
|
|
3506
|
+
// local body is parked as the conflict backup (feedback_09af2875). An
|
|
3507
|
+
// explicit `--on-conflict overwrite` or `publish-local` is consent to
|
|
3508
|
+
// publish the local body, so it uploads as before; that is how a person
|
|
3509
|
+
// empties the list on purpose.
|
|
3510
|
+
let blankManifestOverPopulated = false;
|
|
3511
|
+
// The probe that read the cloud manifest, handed to the conflict branch
|
|
3512
|
+
// so it does not fetch the same body a second time.
|
|
3513
|
+
let manifestGateProbe;
|
|
3514
|
+
if (!((localChanged && remoteChanged) || isFreshCollision || divergentLocalHonesty) &&
|
|
3515
|
+
run.options.personalMode === true &&
|
|
3516
|
+
run.onConflict !== "overwrite" &&
|
|
3517
|
+
run.onConflict !== "publish-local" &&
|
|
3518
|
+
item.kind === "file" &&
|
|
3519
|
+
relativePath === COMPANY_MANIFEST_KEY &&
|
|
3520
|
+
countManifestCompaniesAt(absolutePath) === 0) {
|
|
3521
|
+
const manifestProbe = await probeRemoteForWinner(run, item, remoteMeta.lastModified, remoteChangedSinceJournal, remoteMeta.etag, !journalEntry);
|
|
3522
|
+
const remoteBody = manifestProbe.peekBody();
|
|
3523
|
+
if (remoteBody === undefined) {
|
|
3524
|
+
// The cloud manifest could not be read, so the upload cannot be shown
|
|
3525
|
+
// to keep its company list. Leave both sides and report the path as
|
|
3526
|
+
// failed, so a cloud read that keeps failing is not a quiet skip.
|
|
3527
|
+
manifestProbe.discard();
|
|
3528
|
+
hadUploadFailure = true;
|
|
3529
|
+
run.uploadFailedSet.add(relativePath);
|
|
3530
|
+
counters.filesSkipped++;
|
|
3531
|
+
emitPushSkip(run, relativePath, "remote_body_unreadable");
|
|
3532
|
+
return;
|
|
3533
|
+
}
|
|
3534
|
+
blankManifestOverPopulated = (countManifestCompanies(remoteBody) ?? 0) > 0;
|
|
3535
|
+
if (blankManifestOverPopulated) {
|
|
3536
|
+
manifestGateProbe = manifestProbe;
|
|
3537
|
+
process.stderr.write("hq-cloud: not uploading a blank companies/manifest.yaml over the cloud copy that lists companies; resolving as a conflict\n");
|
|
3538
|
+
}
|
|
3539
|
+
else {
|
|
3540
|
+
manifestProbe.discard();
|
|
3541
|
+
}
|
|
3542
|
+
}
|
|
3543
|
+
const twoSidedChange = (localChanged && remoteChanged) || isFreshCollision || divergentLocalHonesty || blankManifestOverPopulated;
|
|
3382
3544
|
// A company board is merged by entry id against the cloud copy before
|
|
3383
3545
|
// every upload, not only on a two-sided change: a local board that lost
|
|
3384
3546
|
// entries (recreated fresh, or written from a stale copy) must never
|
|
@@ -3425,6 +3587,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3425
3587
|
// conflict on a client-owned file, or a flag-gated ontology entity.
|
|
3426
3588
|
conflictPaths.push(relativePath);
|
|
3427
3589
|
if (cloudWinsEntity) {
|
|
3590
|
+
manifestGateProbe?.discard();
|
|
3428
3591
|
// The cloud body wins whatever the conflict strategy. The local body
|
|
3429
3592
|
// is set aside out of the tree, like any adopted conflict.
|
|
3430
3593
|
run.emit({ type: "conflict", path: relativePath, direction: "push", resolution: "keep" });
|
|
@@ -3441,10 +3604,11 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3441
3604
|
let winnerProbe;
|
|
3442
3605
|
let resolution;
|
|
3443
3606
|
if (versionAware) {
|
|
3444
|
-
winnerProbe = await probeRemoteForWinner(run, item, remoteMeta.lastModified, remoteChangedSinceJournal, remoteMeta.etag, !journalEntry);
|
|
3607
|
+
winnerProbe = manifestGateProbe ?? await probeRemoteForWinner(run, item, remoteMeta.lastModified, remoteChangedSinceJournal, remoteMeta.etag, !journalEntry);
|
|
3445
3608
|
resolution = "keep";
|
|
3446
3609
|
}
|
|
3447
3610
|
else {
|
|
3611
|
+
manifestGateProbe?.discard();
|
|
3448
3612
|
resolution = await resolveConflictSerialized({
|
|
3449
3613
|
path: relativePath,
|
|
3450
3614
|
localHash,
|
|
@@ -3460,10 +3624,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3460
3624
|
decision &&
|
|
3461
3625
|
winnerProbe &&
|
|
3462
3626
|
item.kind === "file" &&
|
|
3463
|
-
(isAppendOnlyPath(relativePath) || isBoardJsonPath(relativePath))) {
|
|
3627
|
+
(isAppendOnlyPath(relativePath) || isBoardJsonPath(relativePath) || isAppendLogPath(relativePath))) {
|
|
3464
3628
|
const outcome = isBoardJsonPath(relativePath)
|
|
3465
3629
|
? mergeAppendOnlyPushConflict(run, item, winnerProbe, normalizeEtag(remoteMeta.etag), mergeBoardJson, replaceIfUnchanged)
|
|
3466
|
-
:
|
|
3630
|
+
: isAppendOnlyPath(relativePath)
|
|
3631
|
+
? mergeAppendOnlyPushConflict(run, item, winnerProbe, normalizeEtag(remoteMeta.etag))
|
|
3632
|
+
: mergeAppendLogPushConflict(run, item, winnerProbe, normalizeEtag(remoteMeta.etag));
|
|
3467
3633
|
recordPushMerge(run, item, outcome, {
|
|
3468
3634
|
remoteEtag: normalizeEtag(remoteMeta.etag),
|
|
3469
3635
|
remoteSha256: winnerProbe.remoteContentHash() ?? null,
|
|
@@ -3692,8 +3858,9 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3692
3858
|
};
|
|
3693
3859
|
const primePrivateFolder = (io) => primePrivateFolderUpload({
|
|
3694
3860
|
io,
|
|
3695
|
-
|
|
3696
|
-
|
|
3861
|
+
// Same POSIX key primeUploads and uploadFile cache under. A Windows
|
|
3862
|
+
// separator must not miss the primed PUT.
|
|
3863
|
+
key: toPosixKey(relativePath),
|
|
3697
3864
|
localPath: absolutePath,
|
|
3698
3865
|
contentHash: localHash,
|
|
3699
3866
|
author: run.options.author,
|
|
@@ -3939,6 +4106,14 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3939
4106
|
throw err;
|
|
3940
4107
|
if (emitPlanLimitSkip(err, relativePath))
|
|
3941
4108
|
return;
|
|
4109
|
+
// Spelling refusal from the presign PUT (including the private-folder
|
|
4110
|
+
// fallback). Same quiet skip as the pre-filter: no upload failure, and
|
|
4111
|
+
// the journal row is left as it was. Before isAccessDenied: this error
|
|
4112
|
+
// is named Forbidden.
|
|
4113
|
+
if (err instanceof PresignKeyRefusedError) {
|
|
4114
|
+
emitInvalidScopedKeySkip(run, relativePath, err.presignCode);
|
|
4115
|
+
return;
|
|
4116
|
+
}
|
|
3942
4117
|
if (isPreconditionFailed(err)) {
|
|
3943
4118
|
conflictPaths.push(relativePath);
|
|
3944
4119
|
if (cloudWinsEntity) {
|
|
@@ -3996,8 +4171,13 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3996
4171
|
// pre-flight merge is merged again, never parked. Every outcome
|
|
3997
4172
|
// leaves the journal on the old version, so the next pass merges
|
|
3998
4173
|
// or publishes under a fresh fence.
|
|
3999
|
-
if (decision &&
|
|
4000
|
-
|
|
4174
|
+
if (decision &&
|
|
4175
|
+
latest?.etag &&
|
|
4176
|
+
item.kind === "file" &&
|
|
4177
|
+
(isAppendOnlyPath(relativePath) || isAppendLogPath(relativePath))) {
|
|
4178
|
+
const outcome = isAppendOnlyPath(relativePath)
|
|
4179
|
+
? mergeAppendOnlyPushConflict(run, item, probe, normalizeEtag(latest.etag))
|
|
4180
|
+
: mergeAppendLogPushConflict(run, item, probe, normalizeEtag(latest.etag));
|
|
4001
4181
|
if (outcome !== "not-mergeable") {
|
|
4002
4182
|
const conflictIndex = conflictPaths.lastIndexOf(relativePath);
|
|
4003
4183
|
if (conflictIndex >= 0)
|
|
@@ -4091,6 +4271,10 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
4091
4271
|
emitPushSkip(run, relativePath, "held_for_next_pass");
|
|
4092
4272
|
return;
|
|
4093
4273
|
}
|
|
4274
|
+
if (retryErr instanceof PresignKeyRefusedError) {
|
|
4275
|
+
emitInvalidScopedKeySkip(run, relativePath, retryErr.presignCode);
|
|
4276
|
+
return;
|
|
4277
|
+
}
|
|
4094
4278
|
hadUploadFailure = true;
|
|
4095
4279
|
run.uploadFailedSet.add(relativePath);
|
|
4096
4280
|
run.emit({
|
|
@@ -4202,6 +4386,10 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
4202
4386
|
counters.filesSkippedConflict++;
|
|
4203
4387
|
return;
|
|
4204
4388
|
}
|
|
4389
|
+
if (retryErr instanceof PresignKeyRefusedError) {
|
|
4390
|
+
emitInvalidScopedKeySkip(run, relativePath, retryErr.presignCode);
|
|
4391
|
+
return;
|
|
4392
|
+
}
|
|
4205
4393
|
hadUploadFailure = true;
|
|
4206
4394
|
run.uploadFailedSet.add(relativePath);
|
|
4207
4395
|
run.emit({
|
|
@@ -4543,7 +4731,7 @@ function adoptOwnUploadIntent(run, item, remoteMeta) {
|
|
|
4543
4731
|
if (row?.remoteEtag !== undefined && normalizeEtag(row.remoteEtag) === remoteEtag)
|
|
4544
4732
|
return false;
|
|
4545
4733
|
const rowWrittenAt = row ? Date.parse(row.syncedAt) : Number.NaN;
|
|
4546
|
-
const intent = findUploadIntent(run.journalSlug, item.relativePath, remoteHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined);
|
|
4734
|
+
const intent = findUploadIntent(run.journalSlug, item.relativePath, remoteHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined, remoteMeta.lastModified && !remoteMeta.lastModifiedUnknown ? remoteMeta.lastModified.getTime() : null);
|
|
4547
4735
|
if (!intent)
|
|
4548
4736
|
return false;
|
|
4549
4737
|
try {
|
|
@@ -4737,6 +4925,7 @@ async function probeRemoteForWinner(run, item, remoteMtime, remoteChangedSinceJo
|
|
|
4737
4925
|
remoteChangedSinceJournal,
|
|
4738
4926
|
noJournalBaseline,
|
|
4739
4927
|
path: item.relativePath,
|
|
4928
|
+
personalVault: run.options.personalMode === true,
|
|
4740
4929
|
});
|
|
4741
4930
|
}
|
|
4742
4931
|
}
|
|
@@ -4938,7 +5127,11 @@ function mergeAppendOnlyPushConflict(run, item, probe, remoteEtag, merge = merge
|
|
|
4938
5127
|
run.emit({
|
|
4939
5128
|
type: "error",
|
|
4940
5129
|
path: item.relativePath,
|
|
4941
|
-
message: `${isBoardJsonPath(item.relativePath)
|
|
5130
|
+
message: `${isBoardJsonPath(item.relativePath)
|
|
5131
|
+
? "could not merge the cloud board entries into the local board"
|
|
5132
|
+
: isAppendOnlyPath(item.relativePath)
|
|
5133
|
+
? "could not add the cloud's lines to the local file"
|
|
5134
|
+
: "could not merge the cloud's log entries into the local file"}: ${describeError(error)}`,
|
|
4942
5135
|
});
|
|
4943
5136
|
return "write-failed";
|
|
4944
5137
|
}
|
|
@@ -4959,6 +5152,16 @@ function mergeAppendOnlyPushConflict(run, item, probe, remoteEtag, merge = merge
|
|
|
4959
5152
|
checkpointShareJournal(run);
|
|
4960
5153
|
return "took-remote";
|
|
4961
5154
|
}
|
|
5155
|
+
/**
|
|
5156
|
+
* {@link mergeAppendOnlyPushConflict} for a markdown or text log: entries are
|
|
5157
|
+
* merged only when both sides appended to the body this machine last synced
|
|
5158
|
+
* (the journal hash), so a side that edited or deleted text still takes the
|
|
5159
|
+
* ordinary conflict rule and parks the losing body (feedback_5f5f3ee1).
|
|
5160
|
+
*/
|
|
5161
|
+
function mergeAppendLogPushConflict(run, item, probe, remoteEtag) {
|
|
5162
|
+
const baseHash = journalRows(run.journal).get(item.relativePath)?.hash ?? null;
|
|
5163
|
+
return mergeAppendOnlyPushConflict(run, item, probe, remoteEtag, (localBody, remoteBody) => mergeAppendLog(localBody, remoteBody, baseHash, item.relativePath), writeAppendLogMerge);
|
|
5164
|
+
}
|
|
4962
5165
|
async function adoptRemoteOnPush(run, item, remoteHash, decision, probe) {
|
|
4963
5166
|
let localMovedAside = false;
|
|
4964
5167
|
let remoteAdopted = false;
|
|
@@ -5294,6 +5497,20 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
|
|
|
5294
5497
|
isRetryableIdentityRefusal(err)) {
|
|
5295
5498
|
throw err;
|
|
5296
5499
|
}
|
|
5500
|
+
// The delete presign refused the key's spelling. It can never be
|
|
5501
|
+
// deleted through presign. Scope-exclude it: no error, journal row
|
|
5502
|
+
// stays. Before the transient/error branches, and before Forbidden
|
|
5503
|
+
// handling (this error is named Forbidden).
|
|
5504
|
+
if (err instanceof PresignKeyRefusedError) {
|
|
5505
|
+
run.scopeExcludedSet.add(relativePath);
|
|
5506
|
+
pathResults.push({
|
|
5507
|
+
path: relativePath,
|
|
5508
|
+
status: "refused",
|
|
5509
|
+
operation: "delete",
|
|
5510
|
+
reason: "scope-excluded",
|
|
5511
|
+
});
|
|
5512
|
+
continue;
|
|
5513
|
+
}
|
|
5297
5514
|
// VaultAccessRole explicitly denies s3:DeleteObject on
|
|
5298
5515
|
// `…/workspace/.session-logs/**` (IAM `*` is unanchored). Treat that
|
|
5299
5516
|
// per-key 403 as scope-excluded so it cannot mark the company
|
|
@@ -5416,9 +5633,21 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
|
|
|
5416
5633
|
});
|
|
5417
5634
|
}
|
|
5418
5635
|
const decommissionedSet = decommissionPlan.length > 0 ? new Set(decommissionPlan) : null;
|
|
5636
|
+
for (const key of deletePlan.clearUnattendedHeld ?? [])
|
|
5637
|
+
clearUnattendedHeldRow(run.journal, key);
|
|
5419
5638
|
for (const refused of deletePlan.refusedStale) {
|
|
5420
5639
|
if (decommissionedSet && decommissionedSet.has(refused.key))
|
|
5421
5640
|
continue;
|
|
5641
|
+
if (refused.reason === "unattended-foreign-creator") {
|
|
5642
|
+
markUnattendedHeldRow(run.journal, refused.key, refused.createdBy, refused.createdBySub);
|
|
5643
|
+
}
|
|
5644
|
+
else if (refused.reason === "unattended-creator-unread" && refused.creatorReadDenied) {
|
|
5645
|
+
// A denied creator read will not change by itself. Mark the row like a
|
|
5646
|
+
// foreign-creator hold so the next pull restores the file and a later
|
|
5647
|
+
// push (interactive included) keeps holding the delete. A transient
|
|
5648
|
+
// failure stays unmarked: held for this pass, checked again next sync.
|
|
5649
|
+
markUnattendedHeldRow(run.journal, refused.key, UNREADABLE_CREATOR, undefined);
|
|
5650
|
+
}
|
|
5422
5651
|
counters.filesRefusedStale++;
|
|
5423
5652
|
if (filesRefusedStalePaths.length < REFUSED_STALE_PATH_CAP) {
|
|
5424
5653
|
filesRefusedStalePaths.push(refused.key);
|
|
@@ -5429,6 +5658,8 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
|
|
|
5429
5658
|
journalEtag: refused.journalEtag,
|
|
5430
5659
|
remoteEtag: refused.remoteEtag,
|
|
5431
5660
|
reason: refused.reason,
|
|
5661
|
+
...(refused.createdBy !== undefined ? { createdBy: refused.createdBy } : {}),
|
|
5662
|
+
...(refused.creatorReadDenied ? { creatorReadDenied: true } : {}),
|
|
5432
5663
|
});
|
|
5433
5664
|
pathResults.push({
|
|
5434
5665
|
path: refused.key,
|
|
@@ -5960,6 +6191,18 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5960
6191
|
console.error(` ⚠ cloud-placeholder, kept on remote: ${event.path} ` +
|
|
5961
6192
|
`(a .hqcloud placeholder stands in for this file, so it was not deleted)`);
|
|
5962
6193
|
}
|
|
6194
|
+
else if (event.reason === "unattended-foreign-creator") {
|
|
6195
|
+
console.error(` ⚠ unattended run held a delete, kept on remote: ${event.path} ` +
|
|
6196
|
+
`(created by ${event.createdBy ?? "another member"}; an unattended sync never deletes a file another member created, delete it from an interactive session if that is intended)`);
|
|
6197
|
+
}
|
|
6198
|
+
else if (event.reason === "unattended-creator-unread" && event.creatorReadDenied) {
|
|
6199
|
+
console.error(` ⚠ unattended run held a delete, kept on remote: ${event.path} ` +
|
|
6200
|
+
`(this account cannot read who created it; an unattended sync deletes only a file it can show this account created, delete it from an interactive session if that is intended)`);
|
|
6201
|
+
}
|
|
6202
|
+
else if (event.reason === "unattended-creator-unread") {
|
|
6203
|
+
console.error(` ⚠ unattended run held a delete, kept on remote: ${event.path} ` +
|
|
6204
|
+
`(could not read who created it; the next sync checks again)`);
|
|
6205
|
+
}
|
|
5963
6206
|
else if (event.reason === "state-rebuilt-deletes-held") {
|
|
5964
6207
|
console.error(` ⚠ state-rebuilt-deletes-held, kept on remote: ${event.path} (complete a full authoritative sync pass to release the hold)`);
|
|
5965
6208
|
}
|
|
@@ -5983,6 +6226,12 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5983
6226
|
console.error(` ⚠ stale-etag, kept on remote: ${event.path} (journal=${event.journalEtag}, remote=${event.remoteEtag})`);
|
|
5984
6227
|
}
|
|
5985
6228
|
}
|
|
6229
|
+
else if (event.type === "delete-refused-bulk-asymmetry" && event.trigger === "scattered-deletes") {
|
|
6230
|
+
console.error(` ⚠ scattered-delete circuit-breaker tripped: ${event.candidates} files are missing from folders that still exist, across ${event.folders ?? "several"} top-level folders.\n` +
|
|
6231
|
+
` Refusing to propagate deletes — this looks like the tree was rolled back to an older copy rather than an intentional cleanup. The pull in this sync downloads the files again.\n` +
|
|
6232
|
+
` Sample refused paths: ${event.samplePaths.slice(0, 5).join(", ")}${event.samplePaths.length > 5 ? ", …" : ""}\n` +
|
|
6233
|
+
` To proceed anyway: re-run with HQ_SYNC_DELETE_BULK_OVERRIDE=1 (or propagateDeletePolicy:"all").`);
|
|
6234
|
+
}
|
|
5986
6235
|
else if (event.type === "delete-refused-bulk-asymmetry") {
|
|
5987
6236
|
const pct = (event.ratio * 100).toFixed(1);
|
|
5988
6237
|
console.error(` ⚠ bulk-asymmetry circuit-breaker tripped: ${event.candidates}/${event.inScope} in-scope journal entries (${pct}%) are missing locally.\n` +
|
|
@@ -6934,7 +7183,15 @@ export async function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSyn
|
|
|
6934
7183
|
companyScoped = true, skipSecretPaths = false, excludedPaths = new Set(), holdRemoteDeletes = false,
|
|
6935
7184
|
// Scoped (watcher) pushes only: reads the whole journal so the breaker's
|
|
6936
7185
|
// denominator matches a full pass. Called only when the scoped ratio trips.
|
|
6937
|
-
bulkBaselineJournal
|
|
7186
|
+
bulkBaselineJournal,
|
|
7187
|
+
// Hashes of files this same push uploads. A missing row whose bytes are
|
|
7188
|
+
// among them (or on another live journal row) was moved or renamed, so the
|
|
7189
|
+
// scattered-delete breaker skips it.
|
|
7190
|
+
liveContentHashes,
|
|
7191
|
+
// feedback_65ee9e51: set for unattended runs (see
|
|
7192
|
+
// ../sync/unattended-delete-hold.ts). Every remote delete is then checked
|
|
7193
|
+
// against the HEAD `created-by-sub` and held unless the caller created it.
|
|
7194
|
+
unattendedDeleteHold) {
|
|
6938
7195
|
const plan = { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
6939
7196
|
if (scopeRoots.length === 0)
|
|
6940
7197
|
return plan;
|
|
@@ -7063,6 +7320,41 @@ bulkBaselineJournal) {
|
|
|
7063
7320
|
continue;
|
|
7064
7321
|
}
|
|
7065
7322
|
}
|
|
7323
|
+
// feedback_65ee9e51: a delete an unattended push already held stays held
|
|
7324
|
+
// (any run) until the pull restores the file, and is kept out of both
|
|
7325
|
+
// breakers. A file back on disk loses the marker and syncs normally.
|
|
7326
|
+
if (entry.unattendedHeldBy !== undefined) {
|
|
7327
|
+
// A path the ignore rules now exclude is never deleted or restored, so
|
|
7328
|
+
// drop its marker instead of reporting the hold on every push.
|
|
7329
|
+
const nowIgnored = !presentLocally && isPathExcludedFromSync(shouldSync, localPath);
|
|
7330
|
+
if (presentLocally || nowIgnored) {
|
|
7331
|
+
(plan.clearUnattendedHeld ??= []).push(relativeKey);
|
|
7332
|
+
if (nowIgnored)
|
|
7333
|
+
continue;
|
|
7334
|
+
}
|
|
7335
|
+
else {
|
|
7336
|
+
inScopeJournalEntries--;
|
|
7337
|
+
// A row held because its creator HEAD was denied keeps reporting as
|
|
7338
|
+
// such, so the console line stays accurate; the executor re-marks it.
|
|
7339
|
+
plan.refusedStale.push(entry.unattendedHeldBy === UNREADABLE_CREATOR
|
|
7340
|
+
? {
|
|
7341
|
+
key: relativeKey,
|
|
7342
|
+
journalEtag: entry.remoteEtag ?? "<not-checked>",
|
|
7343
|
+
remoteEtag: "<not-checked>",
|
|
7344
|
+
reason: "unattended-creator-unread",
|
|
7345
|
+
creatorReadDenied: true,
|
|
7346
|
+
}
|
|
7347
|
+
: {
|
|
7348
|
+
key: relativeKey,
|
|
7349
|
+
journalEtag: entry.remoteEtag ?? "<not-checked>",
|
|
7350
|
+
remoteEtag: "<not-checked>",
|
|
7351
|
+
reason: "unattended-foreign-creator",
|
|
7352
|
+
createdBy: entry.unattendedHeldBy,
|
|
7353
|
+
...(entry.createdBySub ? { createdBySub: entry.createdBySub } : {}),
|
|
7354
|
+
});
|
|
7355
|
+
continue;
|
|
7356
|
+
}
|
|
7357
|
+
}
|
|
7066
7358
|
if (holdRemoteDeletes && !presentLocally) {
|
|
7067
7359
|
plan.refusedStale.push({
|
|
7068
7360
|
key: relativeKey,
|
|
@@ -7095,6 +7387,14 @@ bulkBaselineJournal) {
|
|
|
7095
7387
|
if (!presentLocally && hasLocalOriginalForMissingPlaceholder(localPath)) {
|
|
7096
7388
|
continue;
|
|
7097
7389
|
}
|
|
7390
|
+
// feedback_65ee9e51: the presign server refuses this key's spelling on
|
|
7391
|
+
// every op, so a delete can never succeed. Drop it before toDelete,
|
|
7392
|
+
// headCandidates, and the unattended HEAD. Not deleted, not tombstoned,
|
|
7393
|
+
// not a breaker candidate. The journal row stays.
|
|
7394
|
+
if (presignDeleteKeyRefused(ctx, relativeKey)) {
|
|
7395
|
+
inScopeJournalEntries--;
|
|
7396
|
+
continue;
|
|
7397
|
+
}
|
|
7098
7398
|
// Reindex's legacy Knowledge migration is a representation transition,
|
|
7099
7399
|
// not a user-content deletion: the exact key changed from a symlink object
|
|
7100
7400
|
// to a real directory whose children are uploaded separately. The normal
|
|
@@ -7348,7 +7648,17 @@ bulkBaselineJournal) {
|
|
|
7348
7648
|
}
|
|
7349
7649
|
inScopeJournalEntries = Math.max(inScopeJournalEntries, baseline);
|
|
7350
7650
|
}
|
|
7351
|
-
|
|
7651
|
+
// Scattered-delete breaker (feedback_65089095, ../sync/scattered-delete-burst.ts).
|
|
7652
|
+
// Runs only when the ratio breaker did not trip, before any HEAD, under the
|
|
7653
|
+
// same bypasses. The pull leg applies the same rule, so it downloads these
|
|
7654
|
+
// files back instead of tombstoning them.
|
|
7655
|
+
const scattered = policy !== "all" && !isBulkAsymmetryOverride() && !(bulkGuardApplies && bulkRatioTrips())
|
|
7656
|
+
? detectScatteredDeleteBurst([
|
|
7657
|
+
...plan.toDelete.map((item) => item.key),
|
|
7658
|
+
...headCandidates.flatMap((c) => (c.materializedKnowledgeDir ? [] : [c.key])),
|
|
7659
|
+
], journal, syncRoot, liveContentHashes, { countLocalDeleteTombstones: true })
|
|
7660
|
+
: undefined;
|
|
7661
|
+
if ((bulkGuardApplies && bulkRatioTrips()) || scattered) {
|
|
7352
7662
|
// Move every staged candidate (both already-bucketed toDelete from
|
|
7353
7663
|
// owned-only and queued headCandidates from currency-gated) into
|
|
7354
7664
|
// refusedStale with reason "bulk-asymmetry", and re-label the intent-less
|
|
@@ -7384,11 +7694,15 @@ bulkBaselineJournal) {
|
|
|
7384
7694
|
// label. #227 dropped this line, so a tripped breaker stopped nothing it
|
|
7385
7695
|
// had not already refused for another reason.
|
|
7386
7696
|
plan.toDelete = [];
|
|
7697
|
+
// A scattered trip reports only the files that tripped it (not moves or
|
|
7698
|
+
// files in deleted folders it also holds), sampled from those files.
|
|
7699
|
+
const candidates = scattered ? scattered.keys.length : bulkCandidatePicks;
|
|
7387
7700
|
plan.bulkAsymmetry = {
|
|
7388
|
-
candidates
|
|
7701
|
+
candidates,
|
|
7389
7702
|
inScope: inScopeJournalEntries,
|
|
7390
|
-
ratio:
|
|
7391
|
-
samplePaths,
|
|
7703
|
+
ratio: inScopeJournalEntries > 0 ? candidates / inScopeJournalEntries : 1,
|
|
7704
|
+
samplePaths: scattered ? scattered.samplePaths : samplePaths,
|
|
7705
|
+
...(scattered ? { trigger: "scattered-deletes", folders: scattered.areas } : {}),
|
|
7392
7706
|
};
|
|
7393
7707
|
// Litter still drains even when the breaker trips — see `litterToDelete`
|
|
7394
7708
|
// declaration. The breaker protects against corrupt-mirror mass-deletes
|
|
@@ -7408,7 +7722,92 @@ bulkBaselineJournal) {
|
|
|
7408
7722
|
// Warm the GET presigns the HEAD pass reuses so a large candidate set doesn't
|
|
7409
7723
|
// mint one presign per HEAD and trip the presign breaker. Mirrors the
|
|
7410
7724
|
// new-files + tombstone HEAD-pass pre-primes.
|
|
7411
|
-
|
|
7725
|
+
// Unattended hold: `owned-only` and `all` put deletes straight into
|
|
7726
|
+
// `toDelete` without a HEAD. Take them out and send them through a HEAD
|
|
7727
|
+
// too, so the creator check below sees every remote delete.
|
|
7728
|
+
const unheadedDeletes = unattendedDeleteHold ? plan.toDelete : [];
|
|
7729
|
+
if (unattendedDeleteHold)
|
|
7730
|
+
plan.toDelete = [];
|
|
7731
|
+
await primeObjectTransport(ctx, "get", [...headCandidates.map((c) => c.key), ...unheadedDeletes.map((item) => item.key)]);
|
|
7732
|
+
const holdForeignDelete = (key, journalEtag, remote) => {
|
|
7733
|
+
if (!unattendedDeleteHold || unattendedMayDelete(unattendedDeleteHold, remote.metadata)) {
|
|
7734
|
+
return false;
|
|
7735
|
+
}
|
|
7736
|
+
plan.refusedStale.push({
|
|
7737
|
+
key,
|
|
7738
|
+
journalEtag,
|
|
7739
|
+
remoteEtag: normalizeEtag(remote.etag),
|
|
7740
|
+
reason: "unattended-foreign-creator",
|
|
7741
|
+
createdBy: describeRemoteCreator(remote.metadata),
|
|
7742
|
+
...(remote.metadata?.["created-by-sub"]
|
|
7743
|
+
? { createdBySub: remote.metadata["created-by-sub"] }
|
|
7744
|
+
: {}),
|
|
7745
|
+
});
|
|
7746
|
+
return true;
|
|
7747
|
+
};
|
|
7748
|
+
for (let i = 0; i < unheadedDeletes.length; i += DELETE_PLAN_HEAD_CONCURRENCY) {
|
|
7749
|
+
const chunk = unheadedDeletes.slice(i, i + DELETE_PLAN_HEAD_CONCURRENCY);
|
|
7750
|
+
const results = await Promise.all(chunk.map(async (item) => {
|
|
7751
|
+
try {
|
|
7752
|
+
return {
|
|
7753
|
+
item,
|
|
7754
|
+
remote: await headRemoteFile(ctx, item.key),
|
|
7755
|
+
unread: false,
|
|
7756
|
+
denied: false,
|
|
7757
|
+
scopeExcluded: false,
|
|
7758
|
+
};
|
|
7759
|
+
}
|
|
7760
|
+
catch (err) {
|
|
7761
|
+
// A session-level auth refusal is a run-level fact, as at every other
|
|
7762
|
+
// HEAD site: propagate it so the pass aborts once instead of holding
|
|
7763
|
+
// every delete on a dead session.
|
|
7764
|
+
if (isRunLevelRegistrarFailure(err))
|
|
7765
|
+
throw err;
|
|
7766
|
+
// A spelling refusal is permanent and is not "unreadable". Skip this
|
|
7767
|
+
// pass the way the currency-gated HEAD does: no refusedStale entry,
|
|
7768
|
+
// no creatorReadDenied, and the key is not deleted.
|
|
7769
|
+
if (err instanceof PresignKeyRefusedError) {
|
|
7770
|
+
return { item, remote: null, unread: false, denied: false, scopeExcluded: true };
|
|
7771
|
+
}
|
|
7772
|
+
// These policies never HEADed before the hold, so a failed HEAD must
|
|
7773
|
+
// not fail the push. Any failure, a denial (403) included, leaves
|
|
7774
|
+
// the creator unknown, and an unattended run deletes only a file it
|
|
7775
|
+
// can show the caller created: hold this key. A transient failure is
|
|
7776
|
+
// held for this pass only. A denial will not change by itself, so it
|
|
7777
|
+
// is reported as such and the executor marks the row, which makes
|
|
7778
|
+
// the next pull restore the file and later pushes keep the hold.
|
|
7779
|
+
return {
|
|
7780
|
+
item,
|
|
7781
|
+
remote: null,
|
|
7782
|
+
unread: true,
|
|
7783
|
+
denied: isAccessDenied(err),
|
|
7784
|
+
scopeExcluded: false,
|
|
7785
|
+
};
|
|
7786
|
+
}
|
|
7787
|
+
}));
|
|
7788
|
+
for (const { item, remote, unread, denied, scopeExcluded } of results) {
|
|
7789
|
+
if (scopeExcluded)
|
|
7790
|
+
continue;
|
|
7791
|
+
const journalEtag = journalRows(journal).get(item.key)?.remoteEtag ?? "<not-checked>";
|
|
7792
|
+
if (unread) {
|
|
7793
|
+
plan.refusedStale.push({
|
|
7794
|
+
key: item.key,
|
|
7795
|
+
journalEtag,
|
|
7796
|
+
remoteEtag: "<not-checked>",
|
|
7797
|
+
reason: "unattended-creator-unread",
|
|
7798
|
+
...(denied ? { creatorReadDenied: true } : {}),
|
|
7799
|
+
});
|
|
7800
|
+
continue;
|
|
7801
|
+
}
|
|
7802
|
+
if (remote === null) {
|
|
7803
|
+
plan.toTombstone.push(item.key);
|
|
7804
|
+
continue;
|
|
7805
|
+
}
|
|
7806
|
+
if (holdForeignDelete(item.key, journalEtag, remote))
|
|
7807
|
+
continue;
|
|
7808
|
+
plan.toDelete.push(item);
|
|
7809
|
+
}
|
|
7810
|
+
}
|
|
7412
7811
|
for (let i = 0; i < headCandidates.length; i += DELETE_PLAN_HEAD_CONCURRENCY) {
|
|
7413
7812
|
const chunk = headCandidates.slice(i, i + DELETE_PLAN_HEAD_CONCURRENCY);
|
|
7414
7813
|
const results = await Promise.all(chunk.map(async (c) => {
|
|
@@ -7450,6 +7849,12 @@ bulkBaselineJournal) {
|
|
|
7450
7849
|
continue;
|
|
7451
7850
|
}
|
|
7452
7851
|
}
|
|
7852
|
+
// A verified Knowledge marker is a representation change, not a
|
|
7853
|
+
// content delete (its children are uploaded), so it is not held.
|
|
7854
|
+
if (!candidate.materializedKnowledgeDir &&
|
|
7855
|
+
holdForeignDelete(candidate.key, candidate.journalEtag, remote)) {
|
|
7856
|
+
continue;
|
|
7857
|
+
}
|
|
7453
7858
|
plan.toDelete.push({
|
|
7454
7859
|
key: candidate.key,
|
|
7455
7860
|
intentVersion: candidate.intentVersion,
|