@indigoai-us/hq-cloud 6.18.94 → 6.18.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +3 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +12 -3
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +28 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +33 -2
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/conflict.d.ts +12 -1
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +6 -2
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/reindex.d.ts +24 -0
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +26 -1
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/rescue.d.ts +8 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +43 -2
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/share.d.ts +46 -5
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +187 -23
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +48 -2
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +211 -20
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/tombstones.d.ts +8 -0
- package/dist/cli/tombstones.d.ts.map +1 -1
- package/dist/cli/tombstones.js +9 -0
- package/dist/cli/tombstones.js.map +1 -1
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +61 -79
- package/dist/context.js.map +1 -1
- package/dist/hq-root-recovery.d.ts +37 -6
- package/dist/hq-root-recovery.d.ts.map +1 -1
- package/dist/hq-root-recovery.js +266 -22
- package/dist/hq-root-recovery.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts +68 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +265 -2
- package/dist/journal.js.map +1 -1
- package/dist/lib/cloud-authoritative.d.ts +10 -0
- package/dist/lib/cloud-authoritative.d.ts.map +1 -1
- package/dist/lib/cloud-authoritative.js +23 -0
- package/dist/lib/cloud-authoritative.js.map +1 -1
- package/dist/lib/net-errors.d.ts +14 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +49 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/server-owned-skips.d.ts +64 -0
- package/dist/lib/server-owned-skips.d.ts.map +1 -0
- package/dist/lib/server-owned-skips.js +182 -0
- package/dist/lib/server-owned-skips.js.map +1 -0
- package/dist/manifest-reconcile.d.ts +14 -0
- package/dist/manifest-reconcile.d.ts.map +1 -1
- package/dist/manifest-reconcile.js +20 -2
- package/dist/manifest-reconcile.js.map +1 -1
- package/dist/outposts/client.d.ts +6 -1
- package/dist/outposts/client.d.ts.map +1 -1
- package/dist/outposts/client.js +13 -1
- package/dist/outposts/client.js.map +1 -1
- package/dist/outposts/command.d.ts +1 -1
- package/dist/outposts/command.d.ts.map +1 -1
- package/dist/outposts/command.js +58 -2
- package/dist/outposts/command.js.map +1 -1
- package/dist/outposts/company.d.ts +15 -1
- package/dist/outposts/company.d.ts.map +1 -1
- package/dist/outposts/company.js +36 -1
- package/dist/outposts/company.js.map +1 -1
- package/dist/s3.d.ts +22 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +44 -0
- package/dist/s3.js.map +1 -1
- package/dist/session-log-capture.d.ts.map +1 -1
- package/dist/session-log-capture.js +5 -2
- package/dist/session-log-capture.js.map +1 -1
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +3 -3
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/durable-apply.js +2 -2
- package/dist/sync/durable-apply.js.map +1 -1
- package/dist/sync/per-area-maintenance.js +2 -2
- package/dist/sync/per-area-maintenance.js.map +1 -1
- package/dist/sync/reconcile-cursor.js +2 -2
- package/dist/sync/reconcile-cursor.js.map +1 -1
- package/dist/sync/secret-path-policy.d.ts +37 -11
- package/dist/sync/secret-path-policy.d.ts.map +1 -1
- package/dist/sync/secret-path-policy.js +68 -34
- package/dist/sync/secret-path-policy.js.map +1 -1
- package/dist/sync/snapshot-client.js +2 -2
- package/dist/sync/snapshot-client.js.map +1 -1
- package/dist/sync/state-store.d.ts +26 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +275 -8
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/uploader-finalization.d.ts.map +1 -1
- package/dist/sync/uploader-finalization.js +2 -2
- package/dist/sync/uploader-finalization.js.map +1 -1
- package/dist/test-fixtures/unknown-journal-generation.d.ts +9 -0
- package/dist/test-fixtures/unknown-journal-generation.d.ts.map +1 -0
- package/dist/test-fixtures/unknown-journal-generation.js +37 -0
- package/dist/test-fixtures/unknown-journal-generation.js.map +1 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +3 -3
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/share.js
CHANGED
|
@@ -23,10 +23,10 @@ import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemo
|
|
|
23
23
|
import * as crypto from "crypto";
|
|
24
24
|
import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, } from "../object-io.js";
|
|
25
25
|
import { primePrivateFolderUpload } from "../private-folder-transport.js";
|
|
26
|
-
import { getJournalPath, isUnderAnyRoot, readJournalForPass, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
|
|
27
|
-
import { createIgnoreExplainer, createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isMachineLocalRuntimePath, isPathExcludedFromSync, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
26
|
+
import { getJournalPath, isUnderAnyRoot, readJournalForPass, bindPersonalVaultJournalToIdentity, readJournalReadOnly, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, configureRejectedDeltaRecovery, getJournalRecoveryMarker, } from "../journal.js";
|
|
27
|
+
import { createIgnoreExplainer, createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isHqGeneratedIndexFile, isMachineLocalRuntimePath, isPathExcludedFromSync, isSecretFolderPath, isSecretPatternIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
28
28
|
import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
29
|
-
import {
|
|
29
|
+
import { claimNeverSyncedLocalNotices, formatSecretsNotSyncedLine, isNeverSyncedVaultPath, isSecretBearingVaultPath, isSkipSecretPathsEnabled, isWalRejectedDeltaRecoveryEnabled, } from "../sync/secret-path-policy.js";
|
|
30
30
|
import { getActiveFullPassDecodeWindow, getActiveFullPassUnitSize, } from "../sync/full-pass-caps.js";
|
|
31
31
|
import { isGeneratedCoreMirrorKey, isPersonalVaultDeletionExcluded, isSessionLogDeletionKey, purgeExcludedPersonalVaultJournalEntries, } from "../personal-vault.js";
|
|
32
32
|
import { pickWinner, resolveConflict, usesVersionAwareWinner } from "./conflict.js";
|
|
@@ -40,6 +40,7 @@ import { appendIfUnchanged, isAppendOnlyPath, mergeAppendOnly } from "../lib/app
|
|
|
40
40
|
import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/board-merge.js";
|
|
41
41
|
import { findLandedUpload, followLandedUploads, SamePathUploadHold, samePathUploadKey, SamePathUploadSupersededError, } from "../lib/same-path-upload-lock.js";
|
|
42
42
|
import { isCloudAuthoritative, isLocallyIneligibleForPush } from "../lib/cloud-authoritative.js";
|
|
43
|
+
import { groupServerOwnedSkips } from "../lib/server-owned-skips.js";
|
|
43
44
|
import { isOntologyEntityKey, ONTOLOGY_MULTI_WRITER_FLAG, resolveOntologyEntityPush, } from "../lib/ontology-entity-push.js";
|
|
44
45
|
import { entityIsFreshForCreate, outboxAllowsEntityPush, readOntologyOutbox, } from "../lib/ontology-outbox.js";
|
|
45
46
|
import { resolveSkillRegisterLandedBytes } from "../lib/skill-register-landed-bytes.js";
|
|
@@ -885,10 +886,13 @@ export class UnreachablePushPathsError extends Error {
|
|
|
885
886
|
paths;
|
|
886
887
|
/** Per-path reason, keyed by the same original spelling. */
|
|
887
888
|
reasons;
|
|
888
|
-
constructor(unreachable, syncRoot) {
|
|
889
|
+
constructor(unreachable, syncRoot, ignoreRules = new Map()) {
|
|
889
890
|
const paths = [...unreachable.keys()];
|
|
890
891
|
const lines = paths.map((p) => {
|
|
891
892
|
const reason = unreachable.get(p);
|
|
893
|
+
if (reason === "ignored") {
|
|
894
|
+
return ` · ${p} — ${ignoreRules.get(p) ?? "excluded by a sync ignore rule"}; it was not uploaded`;
|
|
895
|
+
}
|
|
892
896
|
if (reason === "outside-company") {
|
|
893
897
|
return ` · ${p} — resolves outside the company folder (${syncRoot})`;
|
|
894
898
|
}
|
|
@@ -941,6 +945,7 @@ function emitServerOwnedRefusalSummary(run, refusals) {
|
|
|
941
945
|
samplePaths: refusals
|
|
942
946
|
.slice(0, SERVER_OWNED_REFUSAL_SAMPLE_CAP)
|
|
943
947
|
.map((item) => item.relativePath),
|
|
948
|
+
folders: groupServerOwnedSkips(refusals.map((item) => item.relativePath), { sampleCap: 0 }).map(({ folder, reason, count }) => ({ folder, reason, count })),
|
|
944
949
|
});
|
|
945
950
|
}
|
|
946
951
|
/** Most uploaded paths a push reports back (the watcher announces these). */
|
|
@@ -1228,6 +1233,15 @@ class JournalCheckpointError extends Error {
|
|
|
1228
1233
|
}
|
|
1229
1234
|
}
|
|
1230
1235
|
const REFUSED_STALE_PATH_CAP = 50;
|
|
1236
|
+
/**
|
|
1237
|
+
* True for the access-denied error the presign layer throws when the server
|
|
1238
|
+
* refused to mint a PUT URL with FILES_PRESIGN_FORBIDDEN (see object-io.ts
|
|
1239
|
+
* isSkipWithLogDenialCode). Its message carries "server put denied: <code>".
|
|
1240
|
+
*/
|
|
1241
|
+
export function isPresignWriteForbidden(err) {
|
|
1242
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1243
|
+
return /server put denied: FILES_PRESIGN_FORBIDDEN\b/.test(message);
|
|
1244
|
+
}
|
|
1231
1245
|
function isDirectS3AccessDenied(err) {
|
|
1232
1246
|
if (!(err instanceof Error))
|
|
1233
1247
|
return false;
|
|
@@ -1367,6 +1381,10 @@ async function createPushRunContext(options) {
|
|
|
1367
1381
|
? await isSkipSecretPathsEnabled(secretPathFlagConfig, ctx.uid, true)
|
|
1368
1382
|
: await isSkipSecretPathsEnabled(secretPathFlagConfig, ctx.uid)
|
|
1369
1383
|
: preVendedCompanyPush;
|
|
1384
|
+
const walRecoveryEnabled = secretPathFlagConfig
|
|
1385
|
+
? await isWalRejectedDeltaRecoveryEnabled(secretPathFlagConfig, ctx.uid)
|
|
1386
|
+
: true;
|
|
1387
|
+
configureRejectedDeltaRecovery(walRecoveryEnabled);
|
|
1370
1388
|
const ignoreFilter = createIgnoreFilter(hqRoot, {
|
|
1371
1389
|
allowRootMcpConfig: personalConnectorSettingsSync,
|
|
1372
1390
|
});
|
|
@@ -1429,14 +1447,26 @@ async function createPushRunContext(options) {
|
|
|
1429
1447
|
return false;
|
|
1430
1448
|
return baseFilter(filePath, isDir);
|
|
1431
1449
|
};
|
|
1432
|
-
const neverSyncedLocalPath
|
|
1433
|
-
if (neverSyncedLocalPath) {
|
|
1450
|
+
for (const neverSyncedLocalPath of claimNeverSyncedLocalNotices(hqRoot, syncRoot)) {
|
|
1434
1451
|
emit({ type: "secrets-not-synced", path: neverSyncedLocalPath });
|
|
1435
1452
|
}
|
|
1436
1453
|
const shouldSync = options.prefixSet !== undefined
|
|
1437
1454
|
? wrapFilterWithScope(transferFilter, syncRoot, options.prefixSet, onScopeExcluded)
|
|
1438
1455
|
: transferFilter;
|
|
1439
1456
|
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
1457
|
+
// hq-cloud#1028: bind the personal journal to this identity before any
|
|
1458
|
+
// journal handle (quarantine, scoped or full read) opens.
|
|
1459
|
+
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) {
|
|
1460
|
+
migratePersonalVaultJournal();
|
|
1461
|
+
const personalBind = bindPersonalVaultJournalToIdentity({
|
|
1462
|
+
uid: ctx.uid,
|
|
1463
|
+
bucket: ctx.bucketName,
|
|
1464
|
+
region: ctx.region,
|
|
1465
|
+
endpoint: vaultConfig?.apiUrl,
|
|
1466
|
+
});
|
|
1467
|
+
if (personalBind.fenced)
|
|
1468
|
+
emit({ type: "personal-journal-identity-fenced", reason: personalBind.fenced });
|
|
1469
|
+
}
|
|
1440
1470
|
const quarantinedPaths = readQuarantinedJournalPaths(journalSlug);
|
|
1441
1471
|
const flagConfig = vaultConfig ?? privateFolderVaultConfig;
|
|
1442
1472
|
const symlinkTargetRefusal = flagConfig
|
|
@@ -1446,13 +1476,12 @@ async function createPushRunContext(options) {
|
|
|
1446
1476
|
companySlug: ctx.slug,
|
|
1447
1477
|
})
|
|
1448
1478
|
: false;
|
|
1449
|
-
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG)
|
|
1450
|
-
migratePersonalVaultJournal();
|
|
1451
1479
|
// The scoped runner has not completed its narrow filesystem walk yet, so
|
|
1452
1480
|
// defer its keyed journal read until the exact K upload/delete keys are
|
|
1453
1481
|
// known. Full/cadence callers retain the complete snapshot here.
|
|
1454
1482
|
const scopedJournal = options.scopedJournal === true;
|
|
1455
1483
|
const journal = scopedJournal ? emptyJournalForScopedPass() : readJournalForPass(journalSlug, ctx.uid);
|
|
1484
|
+
const holdRemoteDeletes = getJournalRecoveryMarker(journalSlug) !== undefined;
|
|
1456
1485
|
return {
|
|
1457
1486
|
options,
|
|
1458
1487
|
ontologyEntityPush: false,
|
|
@@ -1481,7 +1510,15 @@ async function createPushRunContext(options) {
|
|
|
1481
1510
|
syncRoot,
|
|
1482
1511
|
shouldSync,
|
|
1483
1512
|
durableShouldSync: baseFilter,
|
|
1513
|
+
ignoreFilter,
|
|
1514
|
+
explainIgnore: (filePath, isDir) => {
|
|
1515
|
+
explainIgnored ??= createIgnoreExplainer(hqRoot, {
|
|
1516
|
+
allowRootMcpConfig: personalConnectorSettingsSync,
|
|
1517
|
+
});
|
|
1518
|
+
return explainIgnored(filePath, isDir);
|
|
1519
|
+
},
|
|
1484
1520
|
skipSecretPaths,
|
|
1521
|
+
holdRemoteDeletes,
|
|
1485
1522
|
journalSlug,
|
|
1486
1523
|
quarantinedPaths,
|
|
1487
1524
|
journal,
|
|
@@ -1489,12 +1526,15 @@ async function createPushRunContext(options) {
|
|
|
1489
1526
|
excludedSet,
|
|
1490
1527
|
excludedById,
|
|
1491
1528
|
scopeExcludedSet,
|
|
1529
|
+
uploadFailedSet: new Set(),
|
|
1530
|
+
writeForbiddenSet: new Set(),
|
|
1492
1531
|
ignoreExcludedSet,
|
|
1493
1532
|
ignoreExcludedTotal,
|
|
1494
1533
|
ignoreExcludedLayers,
|
|
1495
1534
|
ignoreIncludeLines,
|
|
1496
1535
|
secretExcludedSet,
|
|
1497
1536
|
unreachablePaths,
|
|
1537
|
+
ignoredNamedPathRules: new Map(),
|
|
1498
1538
|
linkedSubtreeSet,
|
|
1499
1539
|
skillRegistrationConflicts,
|
|
1500
1540
|
hostLoadAtStart,
|
|
@@ -2184,6 +2224,7 @@ async function buildSharePlans(run) {
|
|
|
2184
2224
|
if (rememberLinkWarning(run.linkWarnings, warning))
|
|
2185
2225
|
run.emit(brokenLinkSkipEvent(warning));
|
|
2186
2226
|
},
|
|
2227
|
+
onFilteredNamedPath: (namedPath, absolutePath, isDir) => recordIgnoredNamedPath(run, namedPath, absolutePath, isDir),
|
|
2187
2228
|
onLinkedSubtree: (rel) => run.linkedSubtreeSet.add(rel),
|
|
2188
2229
|
onHeartbeat: run.options.onHeartbeat,
|
|
2189
2230
|
cooperativeCheckpoint: run.options.cooperativeCheckpoint,
|
|
@@ -2200,7 +2241,7 @@ async function buildSharePlans(run) {
|
|
|
2200
2241
|
const fatal = collectFatalUnreachablePaths(run);
|
|
2201
2242
|
if (fatal.size > 0) {
|
|
2202
2243
|
emitUnreachablePathEvent(run);
|
|
2203
|
-
throw new UnreachablePushPathsError(fatal, run.syncRoot);
|
|
2244
|
+
throw new UnreachablePushPathsError(fatal, run.syncRoot, run.ignoredNamedPathRules);
|
|
2204
2245
|
}
|
|
2205
2246
|
const deleteScopeRoots = run.propagateDeletes === true
|
|
2206
2247
|
? resolveDeleteScopeRoots(run.paths, run.hqRoot, run.syncRoot, run.options.deleteScopeRoots)
|
|
@@ -2291,8 +2332,9 @@ async function buildSharePlans(run) {
|
|
|
2291
2332
|
companySlug: run.ctx.slug,
|
|
2292
2333
|
reportedUnsafeSymlinkPaths: run.reportedUnsafeSymlinkPaths,
|
|
2293
2334
|
});
|
|
2335
|
+
const recoveryHoldActive = run.holdRemoteDeletes || getJournalRecoveryMarker(run.journalSlug) !== undefined;
|
|
2294
2336
|
const deletePlan = run.propagateDeletes === true
|
|
2295
|
-
? await computeDeletePlan(run.journal, run.syncRoot, deleteScopeRoots, run.shouldSync, run.propagateDeletePolicy, run.ctx, run.options.personalMode !== true, run.skipSecretPaths, run.quarantinedPaths)
|
|
2337
|
+
? await computeDeletePlan(run.journal, run.syncRoot, deleteScopeRoots, run.shouldSync, run.propagateDeletePolicy, run.ctx, run.options.personalMode !== true, run.skipSecretPaths, run.quarantinedPaths, recoveryHoldActive, run.scopedJournal ? () => readJournalReadOnly(run.journalSlug) : undefined)
|
|
2296
2338
|
: { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
2297
2339
|
for (const warning of deletePlan.brokenLinkSkipped ?? []) {
|
|
2298
2340
|
if (rememberLinkWarning(run.linkWarnings, warning))
|
|
@@ -3174,6 +3216,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3174
3216
|
if (outcome === "write-failed") {
|
|
3175
3217
|
// The merge already emitted the error. A second skip line repeats it.
|
|
3176
3218
|
hadUploadFailure = true;
|
|
3219
|
+
run.uploadFailedSet.add(relativePath);
|
|
3177
3220
|
}
|
|
3178
3221
|
counters.filesSkipped++;
|
|
3179
3222
|
return;
|
|
@@ -3201,7 +3244,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3201
3244
|
let winnerProbe;
|
|
3202
3245
|
let resolution;
|
|
3203
3246
|
if (versionAware) {
|
|
3204
|
-
winnerProbe = await probeRemoteForWinner(run, item, remoteMeta.lastModified, remoteChangedSinceJournal, remoteMeta.etag);
|
|
3247
|
+
winnerProbe = await probeRemoteForWinner(run, item, remoteMeta.lastModified, remoteChangedSinceJournal, remoteMeta.etag, !journalEntry);
|
|
3205
3248
|
resolution = "keep";
|
|
3206
3249
|
}
|
|
3207
3250
|
else {
|
|
@@ -3232,6 +3275,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3232
3275
|
if (outcome === "write-failed") {
|
|
3233
3276
|
// The merge already emitted the error. A second skip line repeats it.
|
|
3234
3277
|
hadUploadFailure = true;
|
|
3278
|
+
run.uploadFailedSet.add(relativePath);
|
|
3235
3279
|
}
|
|
3236
3280
|
counters.filesSkipped++;
|
|
3237
3281
|
return;
|
|
@@ -3710,6 +3754,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3710
3754
|
if (outcome === "write-failed") {
|
|
3711
3755
|
// The merge already emitted the error. A second skip line repeats it.
|
|
3712
3756
|
hadUploadFailure = true;
|
|
3757
|
+
run.uploadFailedSet.add(relativePath);
|
|
3713
3758
|
}
|
|
3714
3759
|
else if (outcome === "publish-local") {
|
|
3715
3760
|
counters.filesSkippedConflict++;
|
|
@@ -3788,6 +3833,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3788
3833
|
return;
|
|
3789
3834
|
}
|
|
3790
3835
|
hadUploadFailure = true;
|
|
3836
|
+
run.uploadFailedSet.add(relativePath);
|
|
3791
3837
|
run.emit({
|
|
3792
3838
|
type: "error",
|
|
3793
3839
|
path: relativePath,
|
|
@@ -3890,6 +3936,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3890
3936
|
if (emitPlanLimitSkip(retryErr, relativePath))
|
|
3891
3937
|
return;
|
|
3892
3938
|
hadUploadFailure = true;
|
|
3939
|
+
run.uploadFailedSet.add(relativePath);
|
|
3893
3940
|
run.emit({
|
|
3894
3941
|
type: "error",
|
|
3895
3942
|
path: relativePath,
|
|
@@ -3910,6 +3957,18 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3910
3957
|
}
|
|
3911
3958
|
if (isAccessDenied(err)) {
|
|
3912
3959
|
hadUploadFailure = true;
|
|
3960
|
+
if (isPresignWriteForbidden(err)) {
|
|
3961
|
+
// A write denial on a key in this push: report it instead of
|
|
3962
|
+
// folding it into the silent scope-excluded count (feedback_682e9187).
|
|
3963
|
+
run.writeForbiddenSet.add(relativePath);
|
|
3964
|
+
run.emit({
|
|
3965
|
+
type: "error",
|
|
3966
|
+
diagnostic: true,
|
|
3967
|
+
path: relativePath,
|
|
3968
|
+
message: describeError(err),
|
|
3969
|
+
});
|
|
3970
|
+
return;
|
|
3971
|
+
}
|
|
3913
3972
|
run.scopeExcludedSet.add(relativePath);
|
|
3914
3973
|
return;
|
|
3915
3974
|
}
|
|
@@ -3939,6 +3998,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
3939
3998
|
});
|
|
3940
3999
|
}
|
|
3941
4000
|
else {
|
|
4001
|
+
run.uploadFailedSet.add(relativePath);
|
|
3942
4002
|
run.emit({
|
|
3943
4003
|
type: "error",
|
|
3944
4004
|
path: relativePath,
|
|
@@ -4197,7 +4257,7 @@ function isArchivedSessionLog(relativePath, remoteMeta) {
|
|
|
4197
4257
|
* either side, probe failure); the caller then falls back to the historical
|
|
4198
4258
|
* `keep` outcome (adopt the remote). Session-level auth refusals propagate.
|
|
4199
4259
|
*/
|
|
4200
|
-
async function probeRemoteForWinner(run, item, remoteMtime, remoteChangedSinceJournal = false, observedRemoteEtag) {
|
|
4260
|
+
async function probeRemoteForWinner(run, item, remoteMtime, remoteChangedSinceJournal = false, observedRemoteEtag, noJournalBaseline = false) {
|
|
4201
4261
|
const probeAbs = buildConflictProbePath(run.hqRoot);
|
|
4202
4262
|
let parked = false;
|
|
4203
4263
|
let fetched;
|
|
@@ -4338,6 +4398,7 @@ async function probeRemoteForWinner(run, item, remoteMtime, remoteChangedSinceJo
|
|
|
4338
4398
|
localMtime: localStat.mtime,
|
|
4339
4399
|
remoteMtime,
|
|
4340
4400
|
remoteChangedSinceJournal,
|
|
4401
|
+
noJournalBaseline,
|
|
4341
4402
|
});
|
|
4342
4403
|
}
|
|
4343
4404
|
}
|
|
@@ -5210,11 +5271,59 @@ function collectFatalUnreachablePaths(run) {
|
|
|
5210
5271
|
if (run.options.unreachablePathPolicy === "warn")
|
|
5211
5272
|
return fatal;
|
|
5212
5273
|
for (const [namedPath, reason] of run.unreachablePaths) {
|
|
5213
|
-
if (reason === "outside-company")
|
|
5274
|
+
if (reason === "outside-company" || reason === "ignored")
|
|
5214
5275
|
fatal.set(namedPath, reason);
|
|
5215
5276
|
}
|
|
5216
5277
|
return fatal;
|
|
5217
5278
|
}
|
|
5279
|
+
/**
|
|
5280
|
+
* A caller-named path that the ignore stack rejects was dropped with no event:
|
|
5281
|
+
* `.hq-*` names and generated INDEX.md files count as expected noise, so the
|
|
5282
|
+
* ignore warning skips them, and the push printed "✓ Pushed 0 file(s)" with
|
|
5283
|
+
* exit 0 (feedback_63bfc302, feedback_8c294805). Record it as `"ignored"` so
|
|
5284
|
+
* the default policy fails the push and names the rule.
|
|
5285
|
+
*
|
|
5286
|
+
* Not recorded:
|
|
5287
|
+
* - Runs without `failOnIgnoredNamedPaths`. Their paths are enumerated roots,
|
|
5288
|
+
* not user input, and an ignored root there is routine.
|
|
5289
|
+
* - `"warn"` runs. Their paths are watcher and walk roots, where an ignored
|
|
5290
|
+
* file is routine and already filtered.
|
|
5291
|
+
* - Paths the raw ignore stack admits. Scope, personal-vault, and secret-path
|
|
5292
|
+
* wrappers rejected those and report them through their own events.
|
|
5293
|
+
* - Secret-pattern files (dotenv files, credentials) and secret folders
|
|
5294
|
+
* (company `secrets/` and `settings/`, `.secrets`). They are left out by
|
|
5295
|
+
* design (US-178), and watcher runs name them routinely.
|
|
5296
|
+
*/
|
|
5297
|
+
function recordIgnoredNamedPath(run, namedPath, absolutePath, isDir) {
|
|
5298
|
+
if (run.options.failOnIgnoredNamedPaths !== true)
|
|
5299
|
+
return;
|
|
5300
|
+
if (run.options.unreachablePathPolicy === "warn")
|
|
5301
|
+
return;
|
|
5302
|
+
if (run.unreachablePaths.has(namedPath))
|
|
5303
|
+
return;
|
|
5304
|
+
if (run.ignoreFilter(absolutePath, isDir))
|
|
5305
|
+
return;
|
|
5306
|
+
const rel = vaultKeyForLocalPath(run.hqRoot, absolutePath);
|
|
5307
|
+
if (isSecretFolderPath(rel))
|
|
5308
|
+
return;
|
|
5309
|
+
if (!isDir && isSecretPatternIgnore(rel))
|
|
5310
|
+
return;
|
|
5311
|
+
run.unreachablePaths.set(namedPath, "ignored");
|
|
5312
|
+
run.ignoredNamedPathRules.set(namedPath, describeIgnoreRule(run, absolutePath, isDir));
|
|
5313
|
+
}
|
|
5314
|
+
function describeIgnoreRule(run, absolutePath, isDir) {
|
|
5315
|
+
if (!isDir && isHqGeneratedIndexFile(absolutePath)) {
|
|
5316
|
+
return "an HQ-generated INDEX.md (it starts with \"> Auto-generated.\"), which each machine rebuilds and sync never uploads";
|
|
5317
|
+
}
|
|
5318
|
+
const { layer } = run.explainIgnore(absolutePath, isDir);
|
|
5319
|
+
if (layer === ".hqignore" || layer === ".hqsyncignore") {
|
|
5320
|
+
return `excluded by a rule in ${layer}`;
|
|
5321
|
+
}
|
|
5322
|
+
if (layer === ".hqinclude allowlist") {
|
|
5323
|
+
return "not listed in .hqinclude";
|
|
5324
|
+
}
|
|
5325
|
+
return "excluded by a built-in sync ignore rule";
|
|
5326
|
+
}
|
|
5218
5327
|
/**
|
|
5219
5328
|
* Emit the `not-shipped` / `unreachable-path` event for a run, if any named
|
|
5220
5329
|
* path went unshipped. Shared by BOTH policies so the operator sees the same
|
|
@@ -5311,6 +5420,10 @@ function buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths,
|
|
|
5311
5420
|
filesTombstoned: counters.filesTombstoned,
|
|
5312
5421
|
filesRefusedStale: counters.filesRefusedStale,
|
|
5313
5422
|
filesRefusedStalePaths,
|
|
5423
|
+
filesUploadFailed: run.uploadFailedSet.size,
|
|
5424
|
+
uploadFailedPaths: [...run.uploadFailedSet].slice(0, REFUSED_STALE_PATH_CAP),
|
|
5425
|
+
filesWriteForbidden: run.writeForbiddenSet.size,
|
|
5426
|
+
writeForbiddenPaths: [...run.writeForbiddenSet].slice(0, REFUSED_STALE_PATH_CAP),
|
|
5314
5427
|
filesSuppressedByTombstone: counters.filesSuppressedByTombstone,
|
|
5315
5428
|
...(run.skillRegisterLandedBytes
|
|
5316
5429
|
? { skillRegistrationFallbacks: run.skillRegistrationFallbacks }
|
|
@@ -5388,6 +5501,9 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5388
5501
|
}
|
|
5389
5502
|
else if (event.type === "push-refused-server-owned-summary") {
|
|
5390
5503
|
console.warn(` ! ${event.count} server-owned path${event.count === 1 ? " was" : "s were"} not uploaded; use source ingestion or change the upstream input, then run hq sync pull.`);
|
|
5504
|
+
for (const folder of event.folders ?? []) {
|
|
5505
|
+
console.warn(` ${folder.folder}: ${folder.count} skipped. ${folder.reason}`);
|
|
5506
|
+
}
|
|
5391
5507
|
for (const samplePath of event.samplePaths)
|
|
5392
5508
|
console.warn(` · ${samplePath}`);
|
|
5393
5509
|
if (event.count > event.samplePaths.length) {
|
|
@@ -5448,6 +5564,9 @@ function defaultConsoleLogger(event, humanSkipLines) {
|
|
|
5448
5564
|
console.error(` ⚠ cloud-placeholder, kept on remote: ${event.path} ` +
|
|
5449
5565
|
`(a .hqcloud placeholder stands in for this file, so it was not deleted)`);
|
|
5450
5566
|
}
|
|
5567
|
+
else if (event.reason === "state-rebuilt-deletes-held") {
|
|
5568
|
+
console.error(` ⚠ state-rebuilt-deletes-held, kept on remote: ${event.path} (complete a full authoritative sync pass to release the hold)`);
|
|
5569
|
+
}
|
|
5451
5570
|
else if (event.reason === "marker-mismatch") {
|
|
5452
5571
|
console.error(` ⚠ marker-mismatch, kept on remote: ${event.path} ` +
|
|
5453
5572
|
`(the remote object is not the symlink revision reindex replaced)`);
|
|
@@ -5895,8 +6014,10 @@ async function collectFiles(paths, hqRoot, syncRoot, filter, hooks = {}) {
|
|
|
5895
6014
|
// symlinks under allowlist mode (the motivating case for this
|
|
5896
6015
|
// whole branch). The filter is pure path lookup with no I/O,
|
|
5897
6016
|
// so two calls are free.
|
|
5898
|
-
if (!filter(absolutePath, false) && !filter(absolutePath, true))
|
|
6017
|
+
if (!filter(absolutePath, false) && !filter(absolutePath, true)) {
|
|
6018
|
+
hooks.onFilteredNamedPath?.(p, absolutePath, false);
|
|
5899
6019
|
continue;
|
|
6020
|
+
}
|
|
5900
6021
|
const target = readlinkOrNull(absolutePath);
|
|
5901
6022
|
if (target === null) {
|
|
5902
6023
|
console.error(` Warning: ${p} is an unreadable symbolic link, skipping.`);
|
|
@@ -5924,17 +6045,22 @@ async function collectFiles(paths, hqRoot, syncRoot, filter, hooks = {}) {
|
|
|
5924
6045
|
continue;
|
|
5925
6046
|
}
|
|
5926
6047
|
if (lstat.isDirectory()) {
|
|
5927
|
-
if (!filter(absolutePath, true))
|
|
6048
|
+
if (!filter(absolutePath, true)) {
|
|
6049
|
+
hooks.onFilteredNamedPath?.(p, absolutePath, true);
|
|
5928
6050
|
continue;
|
|
6051
|
+
}
|
|
5929
6052
|
await walkDir(absolutePath, syncRoot, filter, results, hooks, cooperativeWork);
|
|
5930
6053
|
}
|
|
5931
6054
|
else if (lstat.isFile()) {
|
|
5932
6055
|
const relativePath = vaultKeyForLocalPath(syncRoot, absolutePath);
|
|
6056
|
+
if (!filter(absolutePath)) {
|
|
6057
|
+
hooks.onFilteredNamedPath?.(p, absolutePath, false);
|
|
6058
|
+
continue;
|
|
6059
|
+
}
|
|
5933
6060
|
// Cloud-only placeholders are local stand-ins; never upload them,
|
|
5934
6061
|
// except library asset placeholders, which sync as small text files.
|
|
5935
|
-
if (
|
|
5936
|
-
(
|
|
5937
|
-
isSyncedCloudPlaceholderFile(absolutePath))) {
|
|
6062
|
+
if (!isCloudPlaceholderName(path.basename(absolutePath)) ||
|
|
6063
|
+
isSyncedCloudPlaceholderFile(absolutePath)) {
|
|
5938
6064
|
noteCooperativeRowVisit("collect", relativePath);
|
|
5939
6065
|
results.push({ kind: "file", absolutePath, relativePath });
|
|
5940
6066
|
}
|
|
@@ -6232,6 +6358,13 @@ function resolveDeleteScopeRoots(paths, hqRoot, syncRoot, explicitRoots = []) {
|
|
|
6232
6358
|
* already opts out of safety gates (currency check, owned-only).
|
|
6233
6359
|
* This guard honors that explicit opt-out.
|
|
6234
6360
|
*
|
|
6361
|
+
* `inScope` in a watcher (scoped) push is NOT the delete roots alone. The
|
|
6362
|
+
* watcher names each deleted key as its own delete root, so counting only
|
|
6363
|
+
* rows under those roots makes the ratio ~100% and refuses every batch of
|
|
6364
|
+
* MIN_ABS or more deletes, which the next pull then restores. When the local
|
|
6365
|
+
* ratio trips, a scoped push re-counts the denominator over the whole journal
|
|
6366
|
+
* (`bulkBaselineJournal`), the same rows a full pass's root would cover.
|
|
6367
|
+
*
|
|
6235
6368
|
* `"owned-only"` and `"currency-gated"` both go through the guard. The
|
|
6236
6369
|
* `direction === "up"` filter in `owned-only` is no defense once a user
|
|
6237
6370
|
* has ever run full bidirectional sync (every uploaded file gets
|
|
@@ -6388,7 +6521,10 @@ export async function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSyn
|
|
|
6388
6521
|
// True for a COMPANY-scoped push (personalMode !== true). This distinguishes
|
|
6389
6522
|
// the canonical knowledge-marker layout from the personal-vault layout; it
|
|
6390
6523
|
// does not reinterpret ordinary vault-relative path segments.
|
|
6391
|
-
companyScoped = true, skipSecretPaths = false, excludedPaths = new Set()
|
|
6524
|
+
companyScoped = true, skipSecretPaths = false, excludedPaths = new Set(), holdRemoteDeletes = false,
|
|
6525
|
+
// Scoped (watcher) pushes only: reads the whole journal so the breaker's
|
|
6526
|
+
// denominator matches a full pass. Called only when the scoped ratio trips.
|
|
6527
|
+
bulkBaselineJournal) {
|
|
6392
6528
|
const plan = { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
6393
6529
|
if (scopeRoots.length === 0)
|
|
6394
6530
|
return plan;
|
|
@@ -6517,6 +6653,15 @@ companyScoped = true, skipSecretPaths = false, excludedPaths = new Set()) {
|
|
|
6517
6653
|
continue;
|
|
6518
6654
|
}
|
|
6519
6655
|
}
|
|
6656
|
+
if (holdRemoteDeletes && !presentLocally) {
|
|
6657
|
+
plan.refusedStale.push({
|
|
6658
|
+
key: relativeKey,
|
|
6659
|
+
journalEtag: entry.remoteEtag ?? "<unknown>",
|
|
6660
|
+
remoteEtag: "<not-checked>",
|
|
6661
|
+
reason: "state-rebuilt-deletes-held",
|
|
6662
|
+
});
|
|
6663
|
+
continue;
|
|
6664
|
+
}
|
|
6520
6665
|
const materializedKnowledgeDir = presentLocally &&
|
|
6521
6666
|
isMaterializedKnowledgeDirectory(localPath, relativeKey, entry, companyScoped);
|
|
6522
6667
|
if (presentLocally && !materializedKnowledgeDir)
|
|
@@ -6770,11 +6915,30 @@ companyScoped = true, skipSecretPaths = false, excludedPaths = new Set()) {
|
|
|
6770
6915
|
// Bulk-asymmetry circuit-breaker. See `BULK_ASYMMETRY_*` constants for
|
|
6771
6916
|
// rationale + bypass paths. Skipped under policy:"all" (caller asserted
|
|
6772
6917
|
// intent) and under `HQ_SYNC_DELETE_BULK_OVERRIDE` (operator rollback knob).
|
|
6773
|
-
|
|
6918
|
+
const bulkRatioTrips = () => inScopeJournalEntries > 0 &&
|
|
6919
|
+
bulkCandidatePicks / inScopeJournalEntries >= BULK_ASYMMETRY_RATIO;
|
|
6920
|
+
const bulkGuardApplies = policy !== "all" &&
|
|
6774
6921
|
!isBulkAsymmetryOverride() &&
|
|
6775
|
-
bulkCandidatePicks >= BULK_ASYMMETRY_MIN_ABS
|
|
6776
|
-
|
|
6777
|
-
|
|
6922
|
+
bulkCandidatePicks >= BULK_ASYMMETRY_MIN_ABS;
|
|
6923
|
+
if (bulkGuardApplies && bulkRatioTrips() && bulkBaselineJournal) {
|
|
6924
|
+
// The rows a full pass would count, with the same hard filters as the
|
|
6925
|
+
// Stage-1 loop. Rows it could not lstat (broken links) are not removed,
|
|
6926
|
+
// so this can only exceed a full pass's denominator by those rows.
|
|
6927
|
+
let baseline = 0;
|
|
6928
|
+
for (const [key, entry] of journalRows(bulkBaselineJournal()).entries()) {
|
|
6929
|
+
if (excludedPaths.has(key))
|
|
6930
|
+
continue;
|
|
6931
|
+
if (isNeverSyncedVaultPath(key))
|
|
6932
|
+
continue;
|
|
6933
|
+
if (skipSecretPaths && isSecretBearingVaultPath(key))
|
|
6934
|
+
continue;
|
|
6935
|
+
if (isTombstone(entry) && entry.removedReason !== "local-delete")
|
|
6936
|
+
continue;
|
|
6937
|
+
baseline++;
|
|
6938
|
+
}
|
|
6939
|
+
inScopeJournalEntries = Math.max(inScopeJournalEntries, baseline);
|
|
6940
|
+
}
|
|
6941
|
+
if (bulkGuardApplies && bulkRatioTrips()) {
|
|
6778
6942
|
// Move every staged candidate (both already-bucketed toDelete from
|
|
6779
6943
|
// owned-only and queued headCandidates from currency-gated) into
|
|
6780
6944
|
// refusedStale with reason "bulk-asymmetry", and re-label the intent-less
|