@indigoai-us/hq-cloud 6.14.8 → 6.14.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +8 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +1 -1
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +15 -1
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +12 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +26 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +68 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +89 -14
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +105 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +3 -1
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +7 -7
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +166 -82
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +292 -27
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal.d.ts +7 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +22 -1
- package/dist/journal.js.map +1 -1
- package/dist/manifest-reconcile.d.ts +27 -0
- package/dist/manifest-reconcile.d.ts.map +1 -0
- package/dist/manifest-reconcile.js +120 -0
- package/dist/manifest-reconcile.js.map +1 -0
- package/dist/manifest-reconcile.test.d.ts +2 -0
- package/dist/manifest-reconcile.test.d.ts.map +1 -0
- package/dist/manifest-reconcile.test.js +97 -0
- package/dist/manifest-reconcile.test.js.map +1 -0
- package/dist/personal-vault.d.ts +6 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +12 -0
- package/dist/personal-vault.js.map +1 -1
- package/dist/s3.d.ts +15 -2
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +54 -19
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +12 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/types.d.ts +13 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +7 -0
- package/src/bin/sync-runner-planning.ts +16 -2
- package/src/bin/sync-runner.test.ts +82 -1
- package/src/bin/sync-runner.ts +43 -0
- package/src/cli/share.test.ts +129 -3
- package/src/cli/share.ts +114 -15
- package/src/cli/sync-scope.test.ts +3 -1
- package/src/cli/sync.test.ts +352 -26
- package/src/cli/sync.ts +226 -103
- package/src/journal.ts +28 -0
- package/src/manifest-reconcile.test.ts +107 -0
- package/src/manifest-reconcile.ts +160 -0
- package/src/personal-vault.ts +15 -0
- package/src/s3.test.ts +18 -0
- package/src/s3.ts +67 -30
- package/src/types.ts +13 -0
package/src/cli/sync.ts
CHANGED
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
headRemoteFile,
|
|
27
27
|
primeObjectTransport,
|
|
28
28
|
toPosixKey,
|
|
29
|
+
classifyVaultKey,
|
|
29
30
|
} from "../s3.js";
|
|
30
31
|
import type { DownloadModeWarning, RemoteFile } from "../s3.js";
|
|
31
32
|
import {
|
|
@@ -220,7 +221,13 @@ export type SyncProgressEvent =
|
|
|
220
221
|
path: string;
|
|
221
222
|
journalEtag: string;
|
|
222
223
|
remoteEtag: string;
|
|
223
|
-
reason:
|
|
224
|
+
reason:
|
|
225
|
+
| "stale-etag"
|
|
226
|
+
| "legacy-no-etag"
|
|
227
|
+
| "bulk-asymmetry"
|
|
228
|
+
| "divergent-local"
|
|
229
|
+
| "missing-delete-intent"
|
|
230
|
+
| "intent-changed";
|
|
224
231
|
}
|
|
225
232
|
| {
|
|
226
233
|
/**
|
|
@@ -348,12 +355,10 @@ export type SyncProgressEvent =
|
|
|
348
355
|
}
|
|
349
356
|
| {
|
|
350
357
|
/**
|
|
351
|
-
* Emitted by the PULL leg once per remote key skipped because
|
|
352
|
-
*
|
|
353
|
-
*
|
|
354
|
-
*
|
|
355
|
-
* Such keys are bucket poisoning from a client with a stale doubled
|
|
356
|
-
* local tree writing via direct S3 (incident 2026-07-11). Like
|
|
358
|
+
* Emitted by the PULL leg once per remote key skipped because the server
|
|
359
|
+
* will never presign it (for example, a `companies/…`-prefixed key in a
|
|
360
|
+
* company-scoped vault or a key with control characters). Such keys are
|
|
361
|
+
* bucket poisoning from an outdated direct-S3 client. Like
|
|
357
362
|
* `skip-size-limit`, this is deliberately NOT `type: "error"` — one bad
|
|
358
363
|
* object must never error a whole company sync (erroring gated
|
|
359
364
|
* heartbeat toolset refreshes fleet-wide in the incident). Surfaced for
|
|
@@ -363,6 +368,8 @@ export type SyncProgressEvent =
|
|
|
363
368
|
*/
|
|
364
369
|
type: "skip-invalid-scoped-key";
|
|
365
370
|
path: string;
|
|
371
|
+
/** Server-compatible invalid-key code, such as INVALID_KEY_CONTROL_CHARS. */
|
|
372
|
+
errorCode?: string;
|
|
366
373
|
};
|
|
367
374
|
|
|
368
375
|
export interface SyncOptions {
|
|
@@ -816,7 +823,7 @@ async function syncWithOperationLockHeld(
|
|
|
816
823
|
const plan = planPull(run);
|
|
817
824
|
|
|
818
825
|
emitPullPlan(run.emit, plan);
|
|
819
|
-
|
|
826
|
+
reportInvalidRemoteKeys(run, plan);
|
|
820
827
|
|
|
821
828
|
const scopePlan = planScopeShrink(run);
|
|
822
829
|
const scopeRun = executeScopeShrink(run, scopePlan);
|
|
@@ -951,30 +958,30 @@ function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
|
|
|
951
958
|
}
|
|
952
959
|
|
|
953
960
|
/**
|
|
954
|
-
* Surface
|
|
955
|
-
* `
|
|
961
|
+
* Surface permanently-unpresignable remote keys the planner skipped (see the
|
|
962
|
+
* `invalidRemoteKeys` doc on PullPlan). One warning event per key on the
|
|
956
963
|
* existing sync event channel (deliberately NOT `type: "error"` — see the
|
|
957
964
|
* `skip-invalid-scoped-key` event doc), plus one best-effort cloud telemetry
|
|
958
965
|
* event on the channel sync already uses (emitCloudTelemetry). Per policy
|
|
959
966
|
* hq-alert-baseline-calibration the documented healthy baseline is ZERO
|
|
960
|
-
*
|
|
961
|
-
*
|
|
967
|
+
* invalid-key occurrences fleet-wide, so consumers can alert on any non-zero
|
|
968
|
+
* count with no threshold tuning.
|
|
962
969
|
*/
|
|
963
|
-
function
|
|
964
|
-
if (plan.
|
|
965
|
-
|
|
966
|
-
|
|
970
|
+
function reportInvalidRemoteKeys(run: PullRunContext, plan: PullPlan): void {
|
|
971
|
+
if (plan.invalidRemoteKeys.length === 0) return;
|
|
972
|
+
const countsByErrorCode = new Map<string, number>();
|
|
973
|
+
for (const { path, errorCode } of plan.invalidRemoteKeys) {
|
|
974
|
+
run.emit({ type: "skip-invalid-scoped-key", path, errorCode });
|
|
975
|
+
countsByErrorCode.set(errorCode, (countsByErrorCode.get(errorCode) ?? 0) + 1);
|
|
976
|
+
}
|
|
977
|
+
for (const [errorCode, count] of countsByErrorCode) {
|
|
978
|
+
void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
|
|
979
|
+
eventName: "sync_invalid_scoped_key_skipped",
|
|
980
|
+
source: "hq-sync",
|
|
981
|
+
companyUid: run.ctx.uid,
|
|
982
|
+
properties: { count, errorCode, leg: "pull" },
|
|
983
|
+
}, { claims: run.options.telemetryClaims });
|
|
967
984
|
}
|
|
968
|
-
void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
|
|
969
|
-
eventName: "sync_invalid_scoped_key_skipped",
|
|
970
|
-
source: "hq-sync",
|
|
971
|
-
companyUid: run.ctx.uid,
|
|
972
|
-
properties: {
|
|
973
|
-
count: plan.invalidScopedKeys.length,
|
|
974
|
-
errorCode: "INVALID_KEY_COMPANIES_SCOPED",
|
|
975
|
-
leg: "pull",
|
|
976
|
-
},
|
|
977
|
-
}, { claims: run.options.telemetryClaims });
|
|
978
985
|
}
|
|
979
986
|
|
|
980
987
|
/**
|
|
@@ -1201,11 +1208,19 @@ function executeFileTombstoneDelete(
|
|
|
1201
1208
|
counters: PullCounters,
|
|
1202
1209
|
): void {
|
|
1203
1210
|
const tombstoneKey = item.remoteFile.key;
|
|
1204
|
-
const tombstonePath =
|
|
1205
|
-
if (
|
|
1211
|
+
const tombstonePath = item.localPath;
|
|
1212
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, tombstoneKey, tombstonePath)) {
|
|
1213
|
+
run.emit({
|
|
1214
|
+
type: "error",
|
|
1215
|
+
path: tombstoneKey,
|
|
1216
|
+
message: "tombstone-suppress unlink skipped: local parent escaped the sync root",
|
|
1217
|
+
});
|
|
1218
|
+
return;
|
|
1219
|
+
}
|
|
1206
1220
|
try {
|
|
1207
1221
|
const lstat = fs.lstatSync(tombstonePath);
|
|
1208
|
-
if (
|
|
1222
|
+
if (!matchesLocalSnapshot(tombstonePath, item.localSnapshot)) {
|
|
1223
|
+
reportLocalSnapshotConflict(run, counters, tombstoneKey);
|
|
1209
1224
|
return;
|
|
1210
1225
|
}
|
|
1211
1226
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
@@ -1251,7 +1266,13 @@ async function executeConflictItem(
|
|
|
1251
1266
|
// the `.conflict-` mirror loop AND the journal false-stamp (remote etag over
|
|
1252
1267
|
// divergent local content) that the keep path produced for these files.
|
|
1253
1268
|
if (isCloudAuthoritative(vaultKeyForLocalPath(run.hqRoot, localPath))) {
|
|
1254
|
-
downloadItems.push({
|
|
1269
|
+
downloadItems.push({
|
|
1270
|
+
action: "download",
|
|
1271
|
+
remoteFile,
|
|
1272
|
+
localPath,
|
|
1273
|
+
isNew: false,
|
|
1274
|
+
localSnapshot: item.localSnapshot,
|
|
1275
|
+
});
|
|
1255
1276
|
run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1256
1277
|
return null;
|
|
1257
1278
|
}
|
|
@@ -1314,6 +1335,7 @@ async function executeConflictItem(
|
|
|
1314
1335
|
"down",
|
|
1315
1336
|
remoteFile.etag,
|
|
1316
1337
|
item.localMtime.getTime(),
|
|
1338
|
+
item.localSnapshot.kind === "symlink" ? "symlink" : "file",
|
|
1317
1339
|
);
|
|
1318
1340
|
run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1319
1341
|
counters.filesSkipped++;
|
|
@@ -1403,6 +1425,7 @@ async function executeConflictItem(
|
|
|
1403
1425
|
"down",
|
|
1404
1426
|
remoteFile.etag,
|
|
1405
1427
|
item.localMtime.getTime(),
|
|
1428
|
+
item.localSnapshot.kind === "symlink" ? "symlink" : "file",
|
|
1406
1429
|
);
|
|
1407
1430
|
// Journal-honesty: we recorded the remote etag so this conflict can't
|
|
1408
1431
|
// re-fire (#137), but the KEPT local copy diverges from that remote — it
|
|
@@ -1420,6 +1443,7 @@ async function executeConflictItem(
|
|
|
1420
1443
|
remoteFile,
|
|
1421
1444
|
localPath,
|
|
1422
1445
|
isNew: false,
|
|
1446
|
+
localSnapshot: item.localSnapshot,
|
|
1423
1447
|
});
|
|
1424
1448
|
return null;
|
|
1425
1449
|
}
|
|
@@ -1493,6 +1517,13 @@ async function downloadOne(
|
|
|
1493
1517
|
run.ctx,
|
|
1494
1518
|
remoteFile.key,
|
|
1495
1519
|
localPath,
|
|
1520
|
+
{
|
|
1521
|
+
beforeReplace: () => {
|
|
1522
|
+
if (!matchesLocalSnapshot(localPath, downloadItem.localSnapshot)) {
|
|
1523
|
+
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
1524
|
+
}
|
|
1525
|
+
},
|
|
1526
|
+
},
|
|
1496
1527
|
);
|
|
1497
1528
|
reportModeWarnings(run, modeWarnings);
|
|
1498
1529
|
const author = metadata?.["created-by"] ?? null;
|
|
@@ -1514,6 +1545,7 @@ async function downloadOne(
|
|
|
1514
1545
|
remoteFile.etag,
|
|
1515
1546
|
localLstat.mtimeMs,
|
|
1516
1547
|
createdBySub,
|
|
1548
|
+
isLocalSymlink ? "symlink" : "file",
|
|
1517
1549
|
);
|
|
1518
1550
|
|
|
1519
1551
|
const priorEntry = getEntry(run.journal, remoteFile.key);
|
|
@@ -1530,7 +1562,9 @@ async function downloadOne(
|
|
|
1530
1562
|
counters.bytesDownloaded += size;
|
|
1531
1563
|
counters.changedPaths.push(remoteFile.key);
|
|
1532
1564
|
} catch (err) {
|
|
1533
|
-
if (
|
|
1565
|
+
if (err instanceof LocalSnapshotChangedError) {
|
|
1566
|
+
reportLocalSnapshotConflict(run, counters, remoteFile.key);
|
|
1567
|
+
} else if (isAccessDenied(err)) {
|
|
1534
1568
|
counters.filesSkipped++;
|
|
1535
1569
|
} else {
|
|
1536
1570
|
run.emit({
|
|
@@ -1621,13 +1655,13 @@ async function verifyPlannedJournalTombstones(
|
|
|
1621
1655
|
// never legitimately exist in a company vault, so treating it as verified
|
|
1622
1656
|
// is safe: the executor cleans the local copy + journal entry instead of
|
|
1623
1657
|
// the push leg ever re-uploading it (tombstone-not-reupload).
|
|
1624
|
-
const autoVerified:
|
|
1625
|
-
const toHeadVerify:
|
|
1626
|
-
for (const
|
|
1627
|
-
if (run.options.personalMode !== true && key.startsWith("companies/")) {
|
|
1628
|
-
autoVerified.push(
|
|
1658
|
+
const autoVerified: PullPlan["tombstones"] = [];
|
|
1659
|
+
const toHeadVerify: PullPlan["tombstones"] = [];
|
|
1660
|
+
for (const tombstone of plan.tombstones) {
|
|
1661
|
+
if (run.options.personalMode !== true && tombstone.key.startsWith("companies/")) {
|
|
1662
|
+
autoVerified.push(tombstone);
|
|
1629
1663
|
} else {
|
|
1630
|
-
toHeadVerify.push(
|
|
1664
|
+
toHeadVerify.push(tombstone);
|
|
1631
1665
|
}
|
|
1632
1666
|
}
|
|
1633
1667
|
plan.tombstones = toHeadVerify;
|
|
@@ -1636,22 +1670,22 @@ async function verifyPlannedJournalTombstones(
|
|
|
1636
1670
|
return;
|
|
1637
1671
|
}
|
|
1638
1672
|
|
|
1639
|
-
await primeObjectTransport(run.ctx, "get", plan.tombstones);
|
|
1673
|
+
await primeObjectTransport(run.ctx, "get", plan.tombstones.map((t) => t.key));
|
|
1640
1674
|
|
|
1641
1675
|
const HEAD_VERIFY_CONCURRENCY = 5;
|
|
1642
|
-
const verified:
|
|
1676
|
+
const verified: PullPlan["tombstones"] = [];
|
|
1643
1677
|
for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
|
|
1644
1678
|
const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
|
|
1645
1679
|
const results = await Promise.all(
|
|
1646
|
-
batch.map(async (
|
|
1680
|
+
batch.map(async (tombstone) => {
|
|
1647
1681
|
try {
|
|
1648
|
-
const head = await headRemoteFile(run.ctx, key);
|
|
1649
|
-
return head === null ?
|
|
1682
|
+
const head = await headRemoteFile(run.ctx, tombstone.key);
|
|
1683
|
+
return head === null ? tombstone : null;
|
|
1650
1684
|
} catch (err) {
|
|
1651
1685
|
if (isAccessDenied(err)) return null;
|
|
1652
1686
|
run.emit({
|
|
1653
1687
|
type: "error",
|
|
1654
|
-
path: key,
|
|
1688
|
+
path: tombstone.key,
|
|
1655
1689
|
message: `tombstone HEAD verify failed (deferring): ${
|
|
1656
1690
|
err instanceof Error ? err.message : String(err)
|
|
1657
1691
|
}`,
|
|
@@ -1660,8 +1694,8 @@ async function verifyPlannedJournalTombstones(
|
|
|
1660
1694
|
}
|
|
1661
1695
|
}),
|
|
1662
1696
|
);
|
|
1663
|
-
for (const
|
|
1664
|
-
if (
|
|
1697
|
+
for (const tombstone of results) {
|
|
1698
|
+
if (tombstone !== null) verified.push(tombstone);
|
|
1665
1699
|
}
|
|
1666
1700
|
}
|
|
1667
1701
|
plan.tombstones = [...autoVerified, ...verified];
|
|
@@ -1672,13 +1706,21 @@ function executeJournalTombstoneDeletes(
|
|
|
1672
1706
|
plan: PullPlan,
|
|
1673
1707
|
counters: PullCounters,
|
|
1674
1708
|
): void {
|
|
1675
|
-
for (const
|
|
1676
|
-
const localPath =
|
|
1677
|
-
if (
|
|
1709
|
+
for (const tombstone of plan.tombstones) {
|
|
1710
|
+
const { key, localPath, localSnapshot } = tombstone;
|
|
1711
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, key, localPath)) {
|
|
1712
|
+
run.emit({
|
|
1713
|
+
type: "error",
|
|
1714
|
+
path: key,
|
|
1715
|
+
message: "tombstone unlink skipped: local parent escaped the sync root",
|
|
1716
|
+
});
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1678
1719
|
let removedSomething = false;
|
|
1679
1720
|
try {
|
|
1680
1721
|
const lstat = fs.lstatSync(localPath);
|
|
1681
|
-
if (
|
|
1722
|
+
if (!matchesLocalSnapshot(localPath, localSnapshot)) {
|
|
1723
|
+
reportLocalSnapshotConflict(run, counters, key);
|
|
1682
1724
|
continue;
|
|
1683
1725
|
}
|
|
1684
1726
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
@@ -1879,29 +1921,64 @@ function isDownloadWritePathStillContained(
|
|
|
1879
1921
|
return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
|
|
1880
1922
|
}
|
|
1881
1923
|
|
|
1882
|
-
function
|
|
1883
|
-
journal: SyncJournal,
|
|
1884
|
-
key: string,
|
|
1885
|
-
localPath: string,
|
|
1886
|
-
lstat: fs.Stats,
|
|
1887
|
-
): boolean {
|
|
1888
|
-
const journalEntry = journal.files[key];
|
|
1889
|
-
if (!journalEntry?.hash) {
|
|
1890
|
-
return lstat.isSymbolicLink() || lstat.isFile();
|
|
1891
|
-
}
|
|
1892
|
-
|
|
1924
|
+
function snapshotLocalState(localPath: string): LocalSnapshot | null {
|
|
1893
1925
|
try {
|
|
1926
|
+
const lstat = fs.lstatSync(localPath);
|
|
1894
1927
|
if (lstat.isSymbolicLink()) {
|
|
1895
|
-
return hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
1928
|
+
return { kind: "symlink", hash: hashSymlinkTarget(fs.readlinkSync(localPath)) };
|
|
1896
1929
|
}
|
|
1897
1930
|
if (lstat.isFile()) {
|
|
1898
|
-
return hashFile(localPath)
|
|
1931
|
+
return { kind: "file", hash: hashFile(localPath) };
|
|
1899
1932
|
}
|
|
1900
|
-
|
|
1901
|
-
return
|
|
1933
|
+
if (lstat.isDirectory()) return { kind: "directory" };
|
|
1934
|
+
return { kind: "other" };
|
|
1935
|
+
} catch (err: unknown) {
|
|
1936
|
+
const code =
|
|
1937
|
+
err && typeof err === "object" && "code" in err
|
|
1938
|
+
? (err as { code?: string }).code
|
|
1939
|
+
: undefined;
|
|
1940
|
+
return code === "ENOENT" ? { kind: "absent" } : null;
|
|
1902
1941
|
}
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
function matchesLocalSnapshot(localPath: string, expected: LocalSnapshot): boolean {
|
|
1945
|
+
const actual = snapshotLocalState(localPath);
|
|
1946
|
+
if (actual === null || actual.kind !== expected.kind) return false;
|
|
1947
|
+
return (
|
|
1948
|
+
(actual.kind !== "file" && actual.kind !== "symlink") ||
|
|
1949
|
+
actual.hash === (expected as Extract<LocalSnapshot, { hash: string }>).hash
|
|
1950
|
+
);
|
|
1951
|
+
}
|
|
1903
1952
|
|
|
1904
|
-
|
|
1953
|
+
function hasCurrentLocalDeleteIntent(
|
|
1954
|
+
entry: SyncJournal["files"][string] | undefined,
|
|
1955
|
+
): boolean {
|
|
1956
|
+
const intent = entry?.localDeleteIntent;
|
|
1957
|
+
return !!(
|
|
1958
|
+
intent &&
|
|
1959
|
+
intent.version === 1 &&
|
|
1960
|
+
entry.remoteEtag &&
|
|
1961
|
+
entry.kind &&
|
|
1962
|
+
intent.remoteEtag === entry.remoteEtag &&
|
|
1963
|
+
intent.localHash === entry.hash &&
|
|
1964
|
+
intent.localKind === entry.kind
|
|
1965
|
+
);
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
function reportLocalSnapshotConflict(
|
|
1969
|
+
run: PullRunContext,
|
|
1970
|
+
counters: PullCounters,
|
|
1971
|
+
key: string,
|
|
1972
|
+
): void {
|
|
1973
|
+
counters.conflicts++;
|
|
1974
|
+
counters.conflictPaths.push(key);
|
|
1975
|
+
run.emit({ type: "conflict", path: key, direction: "pull", resolution: "skip" });
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
class LocalSnapshotChangedError extends Error {
|
|
1979
|
+
constructor(key: string) {
|
|
1980
|
+
super(`local state changed while downloading ${key}; replanning required`);
|
|
1981
|
+
}
|
|
1905
1982
|
}
|
|
1906
1983
|
|
|
1907
1984
|
/**
|
|
@@ -1910,8 +1987,18 @@ function tombstoneTargetDiverged(
|
|
|
1910
1987
|
* decide what to do. `localHash` is carried on `conflict` items so the
|
|
1911
1988
|
* executor can hand it to `resolveConflict` without re-hashing.
|
|
1912
1989
|
*/
|
|
1990
|
+
type LocalSnapshot =
|
|
1991
|
+
| { kind: "absent" | "directory" | "other" }
|
|
1992
|
+
| { kind: "file" | "symlink"; hash: string };
|
|
1993
|
+
|
|
1913
1994
|
type PullPlanItem =
|
|
1914
|
-
| {
|
|
1995
|
+
| {
|
|
1996
|
+
action: "download";
|
|
1997
|
+
remoteFile: RemoteFile;
|
|
1998
|
+
localPath: string;
|
|
1999
|
+
isNew: boolean;
|
|
2000
|
+
localSnapshot: LocalSnapshot;
|
|
2001
|
+
}
|
|
1915
2002
|
| { action: "skip-ignored"; remoteFile: RemoteFile; localPath: string }
|
|
1916
2003
|
| { action: "skip-personal-mode"; remoteFile: RemoteFile; localPath: string }
|
|
1917
2004
|
| { action: "skip-unchanged"; remoteFile: RemoteFile; localPath: string }
|
|
@@ -1940,7 +2027,12 @@ type PullPlanItem =
|
|
|
1940
2027
|
// (keys ABSENT from the LIST, HEAD-verified before delete), these are
|
|
1941
2028
|
// suppressed purely on the FILE_TOMBSTONE authority and skip HEAD-verify (the
|
|
1942
2029
|
// remote object is present by definition).
|
|
1943
|
-
| {
|
|
2030
|
+
| {
|
|
2031
|
+
action: "tombstone-delete";
|
|
2032
|
+
remoteFile: RemoteFile;
|
|
2033
|
+
localPath: string;
|
|
2034
|
+
localSnapshot: LocalSnapshot;
|
|
2035
|
+
}
|
|
1944
2036
|
| {
|
|
1945
2037
|
action: "conflict";
|
|
1946
2038
|
remoteFile: RemoteFile;
|
|
@@ -1962,6 +2054,7 @@ type PullPlanItem =
|
|
|
1962
2054
|
// and silently fail to stamp the journal — leaving the
|
|
1963
2055
|
// conflict to re-fire forever.
|
|
1964
2056
|
localSize: number;
|
|
2057
|
+
localSnapshot: LocalSnapshot;
|
|
1965
2058
|
};
|
|
1966
2059
|
|
|
1967
2060
|
interface PullPlan {
|
|
@@ -1983,7 +2076,11 @@ interface PullPlan {
|
|
|
1983
2076
|
* propagating the peer's push-side delete cross-machine (Bug #9).
|
|
1984
2077
|
* Carried on the plan so the executor can iterate without re-walking.
|
|
1985
2078
|
*/
|
|
1986
|
-
tombstones:
|
|
2079
|
+
tombstones: Array<{
|
|
2080
|
+
key: string;
|
|
2081
|
+
localPath: string;
|
|
2082
|
+
localSnapshot: LocalSnapshot;
|
|
2083
|
+
}>;
|
|
1987
2084
|
/**
|
|
1988
2085
|
* Count of `tombstone-delete` items — remote keys present in the LIST but
|
|
1989
2086
|
* suppressed by a FILE_TOMBSTONE (delete-resync). Surfaced on the plan event's
|
|
@@ -1991,14 +2088,13 @@ interface PullPlan {
|
|
|
1991
2088
|
*/
|
|
1992
2089
|
filesToTombstoneDelete: number;
|
|
1993
2090
|
/**
|
|
1994
|
-
* Remote keys skipped because
|
|
1995
|
-
*
|
|
1996
|
-
*
|
|
1997
|
-
*
|
|
1998
|
-
* telemetry. Healthy baseline is an empty array (policy
|
|
2091
|
+
* Remote keys skipped because the server will never presign them. Counted in
|
|
2092
|
+
* `filesExcludedByPolicy`; carried with the server-compatible error code so
|
|
2093
|
+
* the caller can log one warning per key and emit telemetry. Healthy
|
|
2094
|
+
* baseline is an empty array (policy
|
|
1999
2095
|
* hq-alert-baseline-calibration: zero expected, any occurrence is signal).
|
|
2000
2096
|
*/
|
|
2001
|
-
|
|
2097
|
+
invalidRemoteKeys: Array<{ path: string; errorCode: string }>;
|
|
2002
2098
|
}
|
|
2003
2099
|
|
|
2004
2100
|
/**
|
|
@@ -2075,9 +2171,9 @@ function computePullPlan(
|
|
|
2075
2171
|
excludePrefixes: readonly ScopePrefixInput[] = [],
|
|
2076
2172
|
): PullPlan {
|
|
2077
2173
|
const items: PullPlanItem[] = [];
|
|
2078
|
-
//
|
|
2079
|
-
//
|
|
2080
|
-
const
|
|
2174
|
+
// Remote keys the server will never presign — collected so the caller can
|
|
2175
|
+
// warn + emit telemetry without allowing one poison object to fail a pull.
|
|
2176
|
+
const invalidRemoteKeys: Array<{ path: string; errorCode: string }> = [];
|
|
2081
2177
|
// Clean/current/non-divergent files missing locally — collected here, then
|
|
2082
2178
|
// resolved post-loop by the bulk-asymmetry decision (tombstone vs. restore).
|
|
2083
2179
|
const intentionalDeleteCandidates: Array<{
|
|
@@ -2086,6 +2182,23 @@ function computePullPlan(
|
|
|
2086
2182
|
}> = [];
|
|
2087
2183
|
|
|
2088
2184
|
for (const remoteFile of remoteFiles) {
|
|
2185
|
+
const invalidKey = classifyVaultKey(
|
|
2186
|
+
remoteFile.key,
|
|
2187
|
+
personalMode ? "personal" : "company",
|
|
2188
|
+
);
|
|
2189
|
+
if (invalidKey !== null) {
|
|
2190
|
+
invalidRemoteKeys.push({
|
|
2191
|
+
path: remoteFile.key,
|
|
2192
|
+
errorCode: invalidKey.code,
|
|
2193
|
+
});
|
|
2194
|
+
items.push({
|
|
2195
|
+
action: "skip-excluded-policy",
|
|
2196
|
+
remoteFile,
|
|
2197
|
+
localPath: companyRoot,
|
|
2198
|
+
});
|
|
2199
|
+
continue;
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2089
2202
|
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
2090
2203
|
if (localPath === null) {
|
|
2091
2204
|
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
@@ -2103,23 +2216,6 @@ function computePullPlan(
|
|
|
2103
2216
|
continue;
|
|
2104
2217
|
}
|
|
2105
2218
|
|
|
2106
|
-
// Scope-invalid key filter (incident 2026-07-11). A company vault bucket
|
|
2107
|
-
// is already company-scoped, so a literal `companies/…` key can only be
|
|
2108
|
-
// bucket poisoning — a client with a stale doubled local tree
|
|
2109
|
-
// (companies/{slug}/companies/{slug}/…) pushing via the direct-S3 STS
|
|
2110
|
-
// transport, which bypasses the server validator. The presign download
|
|
2111
|
-
// API rejects these keys (INVALID_KEY_COMPANIES_SCOPED), so attempting to
|
|
2112
|
-
// download them errors the WHOLE company sync. Skip them here — one bad
|
|
2113
|
-
// object must never take down a company sync — and record them on the
|
|
2114
|
-
// plan so the caller can warn + emit telemetry (baseline is zero; any
|
|
2115
|
-
// occurrence is signal). Mirrors the personalMode/malformed-key filters
|
|
2116
|
-
// above. Upload-side counterpart: validateVaultUploadKey in s3.ts.
|
|
2117
|
-
if (!personalMode && remoteFile.key.startsWith("companies/")) {
|
|
2118
|
-
invalidScopedKeys.push(remoteFile.key);
|
|
2119
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2120
|
-
continue;
|
|
2121
|
-
}
|
|
2122
|
-
|
|
2123
2219
|
if (
|
|
2124
2220
|
personalMode &&
|
|
2125
2221
|
remoteFile.key.startsWith("companies/") &&
|
|
@@ -2221,6 +2317,7 @@ function computePullPlan(
|
|
|
2221
2317
|
const localTombstoneSuppresses =
|
|
2222
2318
|
journalEntry !== undefined &&
|
|
2223
2319
|
isTombstone(journalEntry) &&
|
|
2320
|
+
hasCurrentLocalDeleteIntent(journalEntry) &&
|
|
2224
2321
|
journalEntry.removedAt !== undefined &&
|
|
2225
2322
|
!isRemoteRecreateAfterTombstone(remoteFile, {
|
|
2226
2323
|
deletedAt: journalEntry.removedAt,
|
|
@@ -2239,6 +2336,7 @@ function computePullPlan(
|
|
|
2239
2336
|
// dangling link on every run.
|
|
2240
2337
|
// 3. A regular file: indistinguishable from current behaviour.
|
|
2241
2338
|
let localLstat: fs.Stats | null = null;
|
|
2339
|
+
let plannedLocalSnapshot: LocalSnapshot = { kind: "absent" };
|
|
2242
2340
|
let localPathBlockedByFileAncestor = false;
|
|
2243
2341
|
try {
|
|
2244
2342
|
localLstat = fs.lstatSync(localPath);
|
|
@@ -2345,6 +2443,10 @@ function computePullPlan(
|
|
|
2345
2443
|
: hashFile(localPath);
|
|
2346
2444
|
}
|
|
2347
2445
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
2446
|
+
plannedLocalSnapshot = {
|
|
2447
|
+
kind: isLocalSymlink ? "symlink" : "file",
|
|
2448
|
+
hash: localHash,
|
|
2449
|
+
};
|
|
2348
2450
|
const remoteChanged =
|
|
2349
2451
|
!!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
|
|
2350
2452
|
|
|
@@ -2367,9 +2469,15 @@ function computePullPlan(
|
|
|
2367
2469
|
localHash,
|
|
2368
2470
|
localMtime: localLstat!.mtime,
|
|
2369
2471
|
localSize: isLocalSymlink ? 0 : localLstat!.size,
|
|
2472
|
+
localSnapshot: plannedLocalSnapshot,
|
|
2370
2473
|
});
|
|
2371
2474
|
} else {
|
|
2372
|
-
items.push({
|
|
2475
|
+
items.push({
|
|
2476
|
+
action: "tombstone-delete",
|
|
2477
|
+
remoteFile,
|
|
2478
|
+
localPath,
|
|
2479
|
+
localSnapshot: plannedLocalSnapshot,
|
|
2480
|
+
});
|
|
2373
2481
|
}
|
|
2374
2482
|
continue;
|
|
2375
2483
|
}
|
|
@@ -2395,6 +2503,7 @@ function computePullPlan(
|
|
|
2395
2503
|
// every sync.
|
|
2396
2504
|
localMtime: localLstat!.mtime,
|
|
2397
2505
|
localSize: isLocalSymlink ? 0 : localLstat!.size,
|
|
2506
|
+
localSnapshot: plannedLocalSnapshot,
|
|
2398
2507
|
});
|
|
2399
2508
|
continue;
|
|
2400
2509
|
}
|
|
@@ -2454,6 +2563,7 @@ function computePullPlan(
|
|
|
2454
2563
|
localHash,
|
|
2455
2564
|
localMtime: localLstat!.mtime,
|
|
2456
2565
|
localSize: localLstat!.size,
|
|
2566
|
+
localSnapshot: plannedLocalSnapshot,
|
|
2457
2567
|
});
|
|
2458
2568
|
continue;
|
|
2459
2569
|
}
|
|
@@ -2483,7 +2593,12 @@ function computePullPlan(
|
|
|
2483
2593
|
if (journalEntry?.removedReason === "local-delete") {
|
|
2484
2594
|
continue;
|
|
2485
2595
|
}
|
|
2486
|
-
items.push({
|
|
2596
|
+
items.push({
|
|
2597
|
+
action: "tombstone-delete",
|
|
2598
|
+
remoteFile,
|
|
2599
|
+
localPath,
|
|
2600
|
+
localSnapshot: plannedLocalSnapshot,
|
|
2601
|
+
});
|
|
2487
2602
|
continue;
|
|
2488
2603
|
}
|
|
2489
2604
|
|
|
@@ -2496,15 +2611,20 @@ function computePullPlan(
|
|
|
2496
2611
|
!localExists &&
|
|
2497
2612
|
journalEntry !== undefined &&
|
|
2498
2613
|
!isTombstone(journalEntry) &&
|
|
2499
|
-
|
|
2500
|
-
journalEntry.remoteEtag !== undefined &&
|
|
2614
|
+
hasCurrentLocalDeleteIntent(journalEntry) &&
|
|
2501
2615
|
normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag
|
|
2502
2616
|
) {
|
|
2503
2617
|
intentionalDeleteCandidates.push({ remoteFile, localPath });
|
|
2504
2618
|
continue;
|
|
2505
2619
|
}
|
|
2506
2620
|
|
|
2507
|
-
items.push({
|
|
2621
|
+
items.push({
|
|
2622
|
+
action: "download",
|
|
2623
|
+
remoteFile,
|
|
2624
|
+
localPath,
|
|
2625
|
+
isNew: !localExists,
|
|
2626
|
+
localSnapshot: plannedLocalSnapshot,
|
|
2627
|
+
});
|
|
2508
2628
|
}
|
|
2509
2629
|
|
|
2510
2630
|
// ── Intentional-delete bulk-asymmetry decision (pull-leg producer) ─────────
|
|
@@ -2526,6 +2646,7 @@ function computePullPlan(
|
|
|
2526
2646
|
remoteFile: cand.remoteFile,
|
|
2527
2647
|
localPath: cand.localPath,
|
|
2528
2648
|
isNew: true,
|
|
2649
|
+
localSnapshot: { kind: "absent" },
|
|
2529
2650
|
});
|
|
2530
2651
|
} else {
|
|
2531
2652
|
tombstoneEntry(journal, cand.remoteFile.key, "local-delete");
|
|
@@ -2585,7 +2706,7 @@ function computePullPlan(
|
|
|
2585
2706
|
// `propagateDeletes` plan in share.ts.
|
|
2586
2707
|
const remoteKeySet = new Set<string>();
|
|
2587
2708
|
for (const rf of remoteFiles) remoteKeySet.add(rf.key);
|
|
2588
|
-
const tombstones:
|
|
2709
|
+
const tombstones: PullPlan["tombstones"] = [];
|
|
2589
2710
|
for (const key of Object.keys(journal.files)) {
|
|
2590
2711
|
// Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
|
|
2591
2712
|
// carry backslash separators; the remote LIST is always forward-slash, so a
|
|
@@ -2665,7 +2786,9 @@ function computePullPlan(
|
|
|
2665
2786
|
// we can't read local state, so we can't safely decide.
|
|
2666
2787
|
if (code !== "ENOENT") continue;
|
|
2667
2788
|
}
|
|
2668
|
-
|
|
2789
|
+
const localSnapshot = snapshotLocalState(localPath);
|
|
2790
|
+
if (localSnapshot === null) continue;
|
|
2791
|
+
tombstones.push({ key, localPath, localSnapshot });
|
|
2669
2792
|
}
|
|
2670
2793
|
|
|
2671
2794
|
return {
|
|
@@ -2680,7 +2803,7 @@ function computePullPlan(
|
|
|
2680
2803
|
filesOutOfScope,
|
|
2681
2804
|
filesToTombstoneDelete,
|
|
2682
2805
|
tombstones,
|
|
2683
|
-
|
|
2806
|
+
invalidRemoteKeys,
|
|
2684
2807
|
};
|
|
2685
2808
|
}
|
|
2686
2809
|
|
|
@@ -2765,7 +2888,7 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
|
|
|
2765
2888
|
);
|
|
2766
2889
|
} else if (event.type === "skip-invalid-scoped-key") {
|
|
2767
2890
|
console.warn(
|
|
2768
|
-
` ! ${event.path} skipped — invalid
|
|
2891
|
+
` ! ${event.path} skipped — invalid vault key (${event.errorCode ?? "INVALID_KEY_COMPANIES_SCOPED"})`,
|
|
2769
2892
|
);
|
|
2770
2893
|
}
|
|
2771
2894
|
}
|
package/src/journal.ts
CHANGED
|
@@ -425,6 +425,7 @@ export function updateEntry(
|
|
|
425
425
|
remoteEtag?: string,
|
|
426
426
|
mtimeMs?: number,
|
|
427
427
|
createdBySub?: string,
|
|
428
|
+
kind?: "file" | "symlink",
|
|
428
429
|
): void {
|
|
429
430
|
const entry: JournalEntry = {
|
|
430
431
|
hash,
|
|
@@ -444,6 +445,9 @@ export function updateEntry(
|
|
|
444
445
|
if (createdBySub !== undefined && createdBySub !== "") {
|
|
445
446
|
entry.createdBySub = createdBySub;
|
|
446
447
|
}
|
|
448
|
+
if (kind !== undefined) {
|
|
449
|
+
entry.kind = kind;
|
|
450
|
+
}
|
|
447
451
|
journal.files[relativePath] = entry;
|
|
448
452
|
journal.lastSync = new Date().toISOString();
|
|
449
453
|
}
|
|
@@ -593,6 +597,30 @@ export function tombstoneEntry(
|
|
|
593
597
|
entry.removedReason = reason;
|
|
594
598
|
}
|
|
595
599
|
|
|
600
|
+
/**
|
|
601
|
+
* Mark a journal entry for a user-authorized local delete. Callers must
|
|
602
|
+
* capture the local hash and kind before removing the object; an absent path
|
|
603
|
+
* alone is never deletion intent.
|
|
604
|
+
*/
|
|
605
|
+
export function markLocalDeleteIntent(
|
|
606
|
+
journal: SyncJournal,
|
|
607
|
+
relativePath: string,
|
|
608
|
+
localHash: string,
|
|
609
|
+
localKind: "file" | "symlink",
|
|
610
|
+
): boolean {
|
|
611
|
+
const entry = journal.files[relativePath];
|
|
612
|
+
if (!entry?.remoteEtag || entry.hash !== localHash || entry.kind !== localKind) {
|
|
613
|
+
return false;
|
|
614
|
+
}
|
|
615
|
+
entry.localDeleteIntent = {
|
|
616
|
+
version: 1,
|
|
617
|
+
remoteEtag: entry.remoteEtag,
|
|
618
|
+
localHash,
|
|
619
|
+
localKind,
|
|
620
|
+
};
|
|
621
|
+
return true;
|
|
622
|
+
}
|
|
623
|
+
|
|
596
624
|
/** True if the entry is a tombstone (set by `tombstoneEntry`). */
|
|
597
625
|
export function isTombstone(entry: JournalEntry | undefined): boolean {
|
|
598
626
|
return !!entry && typeof entry.removedAt === "string";
|